diff --git a/.lift.toml b/.lift.toml new file mode 100644 index 00000000..ad1fe422 --- /dev/null +++ b/.lift.toml @@ -0,0 +1,5 @@ +# Ignore results from vendor directories +ignoreFiles = """ +vendor/ +lua_configuration/ +""" diff --git a/Dockerfile b/Dockerfile index 588b7da3..8523bfb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . +COPY lua_configuration /lua_configuration USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/Dockerfile_multiarch b/Dockerfile_multiarch index 7033106a..2e7d8732 100644 --- a/Dockerfile_multiarch +++ b/Dockerfile_multiarch @@ -21,6 +21,7 @@ RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GOOS=linux GOARCH=amd64 FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . +COPY lua_configuration /lua_configuration USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/api/v1alpha1/rollout_types.go b/api/v1alpha1/rollout_types.go index b0fb7407..90ad7984 100644 --- a/api/v1alpha1/rollout_types.go +++ b/api/v1alpha1/rollout_types.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -65,11 +66,6 @@ type WorkloadRef struct { Name string `json:"name"` } -/*type ControllerRevisionRef struct { - TargetRevisionName string `json:"targetRevisionName"` - SourceRevisionName string `json:"sourceRevisionName"` -}*/ - // RolloutStrategy defines strategy to apply during next rollout type RolloutStrategy struct { // Paused indicates that the Rollout is paused. @@ -112,42 +108,19 @@ type CanaryStep struct { Pause RolloutPause `json:"pause,omitempty"` // Matches define conditions used for matching the incoming HTTP requests to canary service. // Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied. - Matches []RouteMatch `json:"matches,omitempty"` + // If Gateway API, current only support one match. + // And cannot support both weight and matches, if both are configured, then matches takes precedence. + Matches []HttpRouteMatch `json:"matches,omitempty"` } -type RouteMatch struct { +type HttpRouteMatch struct { // Headers specifies HTTP request header matchers. Multiple match values are // ANDed together, meaning, a request must match all the specified headers // to select the route. // +kubebuilder:validation:MaxItems=16 - Headers []HeaderMatch `json:"headers"` -} - -// HeaderMatch describes how to select a route by matching request -// headers. -type HeaderMatch struct { - // Type specifies how to match against the value of the header. - // +optional - // +kubebuilder:default=Exact - Type *HeaderMatchType `json:"type,omitempty"` - // Name is the name of the HTTP Header to be matched. Name matching MUST be - // case insensitive. - Name string `json:"name"` - // Value is the value of HTTP Header to be matched. - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=4096 - Value string `json:"value"` + Headers []gatewayv1alpha2.HTTPHeaderMatch `json:"headers,omitempty"` } -// +kubebuilder:validation:Enum=Exact;RegularExpression -type HeaderMatchType string - -// HeaderMatchType constants. -const ( - HeaderMatchExact HeaderMatchType = "Exact" - HeaderMatchRegularExpression HeaderMatchType = "RegularExpression" -) - // RolloutPause defines a pause stage for a rollout type RolloutPause struct { // Duration the amount of time to wait before moving to the next step. @@ -170,7 +143,8 @@ type TrafficRouting struct { // IngressTrafficRouting configuration for ingress controller to control traffic routing type IngressTrafficRouting struct { - // ClassType refers to the class type of an `Ingress`, e.g. Nginx. Default is Nginx + // ClassType refers to the type of `Ingress`. + // current support nginx, aliyun-alb, aliyun-mse. default is nginx. // +optional ClassType string `json:"classType,omitempty"` // Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3643854c..fe915a0d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ package v1alpha1 import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -189,7 +190,7 @@ func (in *CanaryStep) DeepCopyInto(out *CanaryStep) { in.Pause.DeepCopyInto(&out.Pause) if in.Matches != nil { in, out := &in.Matches, &out.Matches - *out = make([]RouteMatch, len(*in)) + *out = make([]HttpRouteMatch, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -296,37 +297,39 @@ func (in *HTTPRouteInfo) DeepCopy() *HTTPRouteInfo { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IngressInfo) DeepCopyInto(out *IngressInfo) { +func (in *HttpRouteMatch) DeepCopyInto(out *HttpRouteMatch) { *out = *in - in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]v1alpha2.HTTPHeaderMatch, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressInfo. -func (in *IngressInfo) DeepCopy() *IngressInfo { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpRouteMatch. +func (in *HttpRouteMatch) DeepCopy() *HttpRouteMatch { if in == nil { return nil } - out := new(IngressInfo) + out := new(HttpRouteMatch) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HeaderMatch) DeepCopyInto(out *HeaderMatch) { +func (in *IngressInfo) DeepCopyInto(out *IngressInfo) { *out = *in - if in.Type != nil { - in, out := &in.Type, &out.Type - *out = new(HeaderMatchType) - **out = **in - } + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HeaderMatch. -func (in *HeaderMatch) DeepCopy() *HeaderMatch { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressInfo. +func (in *IngressInfo) DeepCopy() *IngressInfo { if in == nil { return nil } - out := new(HeaderMatch) + out := new(IngressInfo) in.DeepCopyInto(out) return out } @@ -730,28 +733,6 @@ func (in *ServiceInfo) DeepCopy() *ServiceInfo { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RouteMatch) DeepCopyInto(out *RouteMatch) { - *out = *in - if in.Headers != nil { - in, out := &in.Headers, &out.Headers - *out = make([]HeaderMatch, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteMatch. -func (in *RouteMatch) DeepCopy() *RouteMatch { - if in == nil { - return nil - } - out := new(RouteMatch) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrafficRouting) DeepCopyInto(out *TrafficRouting) { *out = *in diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 11ac3d80..b76fd4a9 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -102,7 +102,10 @@ spec: description: Matches define conditions used for matching the incoming HTTP requests to canary service. Each match is independent, i.e. this rule will be matched - if **any** one of the matches is satisfied. + if **any** one of the matches is satisfied. If Gateway + API, current only support one match. And cannot support + both weight and matches, if both are configured, then + matches takes precedence. items: properties: headers: @@ -111,18 +114,43 @@ spec: meaning, a request must match all the specified headers to select the route. items: - description: HeaderMatch describes how to select - a route by matching request headers. + description: HTTPHeaderMatch describes how to + select a HTTP route by matching HTTP request + headers. properties: name: - description: Name is the name of the HTTP + description: "Name is the name of the HTTP Header to be matched. Name matching MUST - be case insensitive. + be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify equivalent + header names, only the first entry with + an equivalent name MUST be considered + for a match. Subsequent entries with an + equivalent header name MUST be ignored. + Due to the case-insensitivity of header + names, \"foo\" and \"Foo\" are considered + equivalent. \n When a header is repeated + in an HTTP request, it is implementation-specific + behavior as to how this is represented. + Generally, proxies should follow the guidance + from the RFC: https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 + regarding processing a repeated header, + with special handling for \"Set-Cookie\"." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ type: string type: default: Exact - description: Type specifies how to match - against the value of the header. + description: "Type specifies how to match + against the value of the header. \n Support: + Core (Exact) \n Support: Custom (RegularExpression) + \n Since RegularExpression HeaderMatchType + has custom conformance, implementations + can support POSIX, PCRE or any other dialects + of regular expressions. Please read the + implementation's documentation to determine + the supported dialect." enum: - Exact - RegularExpression @@ -139,8 +167,6 @@ spec: type: object maxItems: 16 type: array - required: - - headers type: object type: array pause: @@ -199,8 +225,9 @@ spec: to route traffic, e.g. Nginx, Alb. properties: classType: - description: ClassType refers to the class type - of an `Ingress`, e.g. Nginx. Default is Nginx + description: ClassType refers to the type of `Ingress`. + current support nginx, aliyun-alb, aliyun-mse. + default is nginx. type: string name: description: Name refers to the name of an `Ingress` diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index cc69d728..fe7d811f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -144,6 +144,14 @@ rules: - get - patch - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index d0aafe3e..b7850131 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.16 require ( github.com/davecgh/go-spew v1.1.1 + github.com/evanphx/json-patch v4.11.0+incompatible github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/openkruise/kruise-api v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.22.6 k8s.io/apiextensions-apiserver v0.22.6 @@ -16,6 +18,7 @@ require ( k8s.io/component-base v0.22.6 k8s.io/klog/v2 v2.10.0 k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e + layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf sigs.k8s.io/controller-runtime v0.10.3 sigs.k8s.io/gateway-api v0.4.3 sigs.k8s.io/yaml v1.2.0 diff --git a/go.sum b/go.sum index 85dd4d99..039758a4 100644 --- a/go.sum +++ b/go.sum @@ -501,6 +501,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= +github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -675,6 +677,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1031,6 +1034,8 @@ k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e h1:ldQh+neBabomh7+89dTpiFAB8tGdfVmuIzAHbvtl+9I= k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0= +layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/lua_configuration/trafficrouting_ingress/aliyun-alb.lua b/lua_configuration/trafficrouting_ingress/aliyun-alb.lua new file mode 100644 index 00000000..c35e5825 --- /dev/null +++ b/lua_configuration/trafficrouting_ingress/aliyun-alb.lua @@ -0,0 +1,35 @@ +annotations = {} +if ( obj.annotations ) +then + annotations = obj.annotations +end +annotations["alb.ingress.kubernetes.io/canary"] = "true" +annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil +annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil +annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil +annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil +annotations["alb.ingress.kubernetes.io/canary-weight"] = nil +if ( obj.weight ~= "-1" ) +then + annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight +end +if ( not obj.matches ) +then + return annotations +end +for _,match in ipairs(obj.matches) do + local header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["alb.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end +end +return annotations diff --git a/lua_configuration/trafficrouting_ingress/aliyun-mse.lua b/lua_configuration/trafficrouting_ingress/aliyun-mse.lua new file mode 100644 index 00000000..a8195c7f --- /dev/null +++ b/lua_configuration/trafficrouting_ingress/aliyun-mse.lua @@ -0,0 +1,35 @@ +annotations = {} +if ( obj.annotations ) +then + annotations = obj.annotations +end +annotations["nginx.ingress.kubernetes.io/canary"] = "true" +annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil +annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil +if ( obj.weight ~= "-1" ) +then + annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight +end +if ( not obj.matches ) +then + return annotations +end +for _,match in ipairs(obj.matches) do + local header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end +end +return annotations diff --git a/lua_configuration/trafficrouting_ingress/nginx.lua b/lua_configuration/trafficrouting_ingress/nginx.lua new file mode 100644 index 00000000..a8195c7f --- /dev/null +++ b/lua_configuration/trafficrouting_ingress/nginx.lua @@ -0,0 +1,35 @@ +annotations = {} +if ( obj.annotations ) +then + annotations = obj.annotations +end +annotations["nginx.ingress.kubernetes.io/canary"] = "true" +annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil +annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil +annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil +if ( obj.weight ~= "-1" ) +then + annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight +end +if ( not obj.matches ) +then + return annotations +end +for _,match in ipairs(obj.matches) do + local header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end +end +return annotations diff --git a/pkg/controller/rollout/rollout_controller.go b/pkg/controller/rollout/rollout_controller.go index 97519889..7472bd36 100644 --- a/pkg/controller/rollout/rollout_controller.go +++ b/pkg/controller/rollout/rollout_controller.go @@ -74,6 +74,7 @@ type RolloutReconciler struct { //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/pkg/controller/rollout/status.go b/pkg/controller/rollout/status.go index c74ce129..9a4e7934 100644 --- a/pkg/controller/rollout/status.go +++ b/pkg/controller/rollout/status.go @@ -99,6 +99,8 @@ func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout newStatus.Phase = rolloutv1alpha1.RolloutPhaseProgressing cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") util.SetRolloutCondition(&newStatus, *cond) + newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} + newStatus.Message = "Rollout is in Progressing" } else if newStatus.CanaryStatus == nil { // The following logic is to make PaaS be able to judge whether the rollout is ready // at the first deployment of the Rollout/Workload. For example: generally, a PaaS diff --git a/pkg/controller/rollout/trafficrouting.go b/pkg/controller/rollout/trafficrouting.go index f1919fe9..c9957b1d 100644 --- a/pkg/controller/rollout/trafficrouting.go +++ b/pkg/controller/rollout/trafficrouting.go @@ -24,7 +24,7 @@ import ( rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/gateway" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/nginx" + "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/ingress" "github.com/openkruise/rollouts/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -40,9 +40,10 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } - - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + // current only support one trafficRouting + trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } canaryStatus := r.newStatus.CanaryStatus if r.newStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { @@ -102,7 +103,7 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { } // After restore stable service configuration, give the ingress provider 3 seconds to take effect - if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { + if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { klog.Infof("update rollout(%s/%s) stable service(%s) done, and wait 3 seconds", r.rollout.Namespace, r.rollout.Name, r.stableService) return false, nil } @@ -113,31 +114,30 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) return false, err } - var desiredWeight int32 - if len(r.rollout.Spec.Strategy.Canary.Steps) > 0 { - desiredWeight = *r.rollout.Spec.Strategy.Canary.Steps[r.newStatus.CanaryStatus.CurrentStepIndex-1].Weight - } - steps := len(r.rollout.Spec.Strategy.Canary.Steps) + currentStep := r.rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1] + totalStep := len(r.rollout.Spec.Strategy.Canary.Steps) cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic weight(%d)", canaryStatus.CurrentStepIndex, steps, desiredWeight) - verify, err := trController.EnsureRoutes(context.TODO(), desiredWeight) + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic (%s)", canaryStatus.CurrentStepIndex, totalStep, util.DumpJSON(currentStep)) + verify, err := trController.EnsureRoutes(context.TODO(), currentStep.Weight, currentStep.Matches) if err != nil { return false, err } else if !verify { - klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%d)", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) + klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%s)", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(currentStep)) return false, nil } - klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d) success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) + klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%s) success", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(currentStep)) return true, nil } +// restore stable service configuration, remove selector pod-template-hash func (r *rolloutContext) restoreStableService() (bool, error) { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } - - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + // current only support one trafficRouting + trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } //fetch stable service stableService := &corev1.Service{} @@ -167,7 +167,7 @@ func (r *rolloutContext) restoreStableService() (bool, error) { } // After restore stable service configuration, give the ingress provider 3 seconds to take effect if r.newStatus.CanaryStatus.LastUpdateTime != nil { - if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { + if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) { klog.Infof("restore rollout(%s/%s) stable service(%s) done, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.stableService) return false, nil } @@ -176,13 +176,17 @@ func (r *rolloutContext) restoreStableService() (bool, error) { return true, nil } +// 1. restore ingress or gateway configuration +// 2. remove canary service func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } + // current only support one trafficRouting + trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0] klog.Infof("rollout(%s/%s) start finalising traffic routing", r.rollout.Namespace, r.rollout.Name) - if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + if trafficRouting.GracePeriodSeconds <= 0 { + trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds } if r.newStatus.CanaryStatus == nil { r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} @@ -193,7 +197,7 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) return false, err } - verify, err := trController.EnsureRoutes(context.TODO(), 0) + verify, err := trController.Finalise(context.TODO()) if err != nil { return false, err } else if !verify { @@ -201,11 +205,6 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { return false, nil } - // DoFinalising, such as delete nginx canary ingress - if err = trController.Finalise(context.TODO()); err != nil { - return false, err - } - // 2. remove canary service if r.newStatus.CanaryStatus.CanaryService == "" { return true, nil @@ -226,20 +225,20 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { } func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.Controller, error) { - canary := roCtx.rollout.Spec.Strategy.Canary - if canary.TrafficRoutings[0].Ingress != nil { + trafficRouting := roCtx.rollout.Spec.Strategy.Canary.TrafficRoutings[0] + if trafficRouting.Ingress != nil { gvk := schema.GroupVersionKind{Group: rolloutv1alpha1.GroupVersion.Group, Version: rolloutv1alpha1.GroupVersion.Version, Kind: "Rollout"} - return nginx.NewNginxTrafficRouting(r.Client, r.newStatus, nginx.Config{ + return ingress.NewIngressTrafficRouting(r.Client, ingress.Config{ RolloutName: r.rollout.Name, RolloutNs: r.rollout.Namespace, CanaryService: r.canaryService, StableService: r.stableService, - TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress, + TrafficConf: trafficRouting.Ingress, OwnerRef: *metav1.NewControllerRef(r.rollout, gvk), }) } - if canary.TrafficRoutings[0].Gateway != nil { - return gateway.NewGatewayTrafficRouting(r.Client, r.newStatus, gateway.Config{ + if trafficRouting.Gateway != nil { + return gateway.NewGatewayTrafficRouting(r.Client, gateway.Config{ RolloutName: r.rollout.Name, RolloutNs: r.rollout.Namespace, CanaryService: r.canaryService, diff --git a/pkg/controller/rollout/trafficrouting/gateway/gateway.go b/pkg/controller/rollout/trafficrouting/gateway/gateway.go index 35bc06b5..f965017f 100644 --- a/pkg/controller/rollout/trafficrouting/gateway/gateway.go +++ b/pkg/controller/rollout/trafficrouting/gateway/gateway.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -40,17 +41,14 @@ type Config struct { type gatewayController struct { client.Client - //stableIngress *netv1.Ingress - conf Config - newStatus *rolloutv1alpha1.RolloutStatus + conf Config } // NewGatewayTrafficRouting The Gateway API is a part of the SIG Network. -func NewGatewayTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) { +func NewGatewayTrafficRouting(client client.Client, conf Config) (trafficrouting.Controller, error) { r := &gatewayController{ - Client: client, - conf: conf, - newStatus: newStatus, + Client: client, + conf: conf, } return r, nil } @@ -60,25 +58,17 @@ func (r *gatewayController) Initialize(ctx context.Context) error { return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, route) } -func (r *gatewayController) EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error) { +func (r *gatewayController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) { var httpRoute gatewayv1alpha2.HTTPRoute err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, &httpRoute) if err != nil { - // When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress, - // and The removal of httpRoute is expected. - if desiredWeight == 0 && errors.IsNotFound(err) { - klog.Infof("rollout(%s/%s) verify canary HTTPRoute has been deleted", r.conf.RolloutNs, r.conf.RolloutName) - return true, nil - } return false, err } - // desired route - desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, desiredWeight) + desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, weight, matches) if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { return true, nil } - // set route routeClone := &gatewayv1alpha2.HTTPRoute{} if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { @@ -92,24 +82,24 @@ func (r *gatewayController) EnsureRoutes(ctx context.Context, desiredWeight int3 klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) return false, err } - klog.Infof("rollout(%s/%s) set HTTPRoute(name:%s weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, *r.conf.TrafficConf.HTTPRouteName, desiredWeight) + klog.Infof("rollout(%s/%s) set HTTPRoute(name:%s weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, *r.conf.TrafficConf.HTTPRouteName, *weight) return false, nil } -func (r *gatewayController) Finalise(ctx context.Context) error { +func (r *gatewayController) Finalise(ctx context.Context) (bool, error) { httpRoute := &gatewayv1alpha2.HTTPRoute{} err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, httpRoute) if err != nil { if errors.IsNotFound(err) { - return nil + return true, nil } klog.Errorf("rollout(%s/%s) get HTTPRoute failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) - return err + return false, err } // desired rule - desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, -1) + desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, utilpointer.Int32(-1), nil) if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { - return nil + return true, nil } routeClone := &gatewayv1alpha2.HTTPRoute{} if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { @@ -121,13 +111,67 @@ func (r *gatewayController) Finalise(ctx context.Context) error { return r.Client.Update(context.TODO(), routeClone) }); err != nil { klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) - return err + return false, err } klog.Infof("rollout(%s/%s) TrafficRouting Finalise success", r.conf.RolloutNs, r.conf.RolloutName) - return nil + return false, nil +} + +func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) []gatewayv1alpha2.HTTPRouteRule { + var desired []gatewayv1alpha2.HTTPRouteRule + // Only when finalize method parameter weight=-1, then we need to remove the canary route policy and restore to the original configuration + if weight != nil && *weight == -1 { + for i := range rules { + rule := rules[i] + filterOutServiceBackendRef(&rule, r.conf.CanaryService) + _, stableRef := getServiceBackendRef(rule, r.conf.StableService) + if stableRef != nil { + stableRef.Weight = utilpointer.Int32(1) + setServiceBackendRef(&rule, *stableRef) + } + if len(rule.BackendRefs) != 0 { + desired = append(desired, rule) + } + } + return desired + // according to the Gateway API definition, weight and headers cannot be supported at the same time. + // A/B Testing, according to headers. current only support one match + } else if len(matches) > 0 { + return r.buildCanaryHeaderHttpRoutes(rules, matches[0].Headers) + } + // canary release, according to percentage of traffic routing + return r.buildCanaryWeightHttpRoutes(rules, weight) } -func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, canaryPercent int32) []gatewayv1alpha2.HTTPRouteRule { +func (r *gatewayController) buildCanaryHeaderHttpRoutes(rules []gatewayv1alpha2.HTTPRouteRule, headers []gatewayv1alpha2.HTTPHeaderMatch) []gatewayv1alpha2.HTTPRouteRule { + var desired []gatewayv1alpha2.HTTPRouteRule + var canarys []gatewayv1alpha2.HTTPRouteRule + for i := range rules { + rule := rules[i] + if _, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService); canaryRef != nil { + continue + } + desired = append(desired, rule) + if _, stableRef := getServiceBackendRef(rule, r.conf.StableService); stableRef == nil { + continue + } + // according to stable rule to create canary rule + canaryRule := rule.DeepCopy() + _, canaryRef := getServiceBackendRef(*canaryRule, r.conf.StableService) + canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService) + canaryRule.BackendRefs = []gatewayv1alpha2.HTTPBackendRef{*canaryRef} + // set canary headers in httpRoute + for j := range canaryRule.Matches { + match := &canaryRule.Matches[j] + match.Headers = append(match.Headers, headers...) + } + canarys = append(canarys, *canaryRule) + } + desired = append(desired, canarys...) + return desired +} + +func (r *gatewayController) buildCanaryWeightHttpRoutes(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32) []gatewayv1alpha2.HTTPRouteRule { var desired []gatewayv1alpha2.HTTPRouteRule for i := range rules { rule := rules[i] @@ -136,24 +180,16 @@ func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRo desired = append(desired, rule) continue } - // If canaryPercent = -1, delete canary backendRef - if canaryPercent == -1 { - filterOutServiceBackendRef(&rule, r.conf.CanaryService) - stableRef.Weight = nil - setServiceBackendRef(&rule, *stableRef) - // canaryPercent[0,100] - } else { - _, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService) - if canaryRef == nil { - canaryRef = stableRef.DeepCopy() - canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService) - } - stableWeight, canaryWeight := generateCanaryWeight(canaryPercent) - stableRef.Weight = &stableWeight - canaryRef.Weight = &canaryWeight - setServiceBackendRef(&rule, *stableRef) - setServiceBackendRef(&rule, *canaryRef) + _, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService) + if canaryRef == nil { + canaryRef = stableRef.DeepCopy() + canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService) } + stableWeight, canaryWeight := generateCanaryWeight(*weight) + stableRef.Weight = &stableWeight + canaryRef.Weight = &canaryWeight + setServiceBackendRef(&rule, *stableRef) + setServiceBackendRef(&rule, *canaryRef) desired = append(desired, rule) } return desired diff --git a/pkg/controller/rollout/trafficrouting/gateway/gateway_test.go b/pkg/controller/rollout/trafficrouting/gateway/gateway_test.go index a0326734..f065119d 100644 --- a/pkg/controller/rollout/trafficrouting/gateway/gateway_test.go +++ b/pkg/controller/rollout/trafficrouting/gateway/gateway_test.go @@ -20,6 +20,8 @@ import ( "reflect" "testing" + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" utilpointer "k8s.io/utils/pointer" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) @@ -32,6 +34,13 @@ var ( Spec: gatewayv1alpha2.HTTPRouteSpec{ Rules: []gatewayv1alpha2.HTTPRouteRule{ { + Matches: []gatewayv1alpha2.HTTPRouteMatch{ + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/web"), + }, + }, + }, BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ { BackendRef: gatewayv1alpha2.BackendRef{ @@ -50,6 +59,17 @@ var ( Path: &gatewayv1alpha2.HTTPPathMatch{ Value: utilpointer.String("/store"), }, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "version", + Value: "v2", + }, + }, + }, + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/v2/store"), + }, }, }, BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ @@ -88,7 +108,7 @@ var ( Matches: []gatewayv1alpha2.HTTPRouteMatch{ { Path: &gatewayv1alpha2.HTTPPathMatch{ - Value: utilpointer.String("/storev2"), + Value: utilpointer.String("/storage"), }, }, }, @@ -113,67 +133,124 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { cases := []struct { name string getRouteRules func() []gatewayv1alpha2.HTTPRouteRule - canaryPercent int32 + getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) desiredRules func() []gatewayv1alpha2.HTTPRouteRule }{ { - name: "canary weight: 20", + name: "test1 headers", getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule { rules := routeDemo.DeepCopy().Spec.Rules return rules }, - canaryPercent: 20, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + iType := gatewayv1alpha2.HeaderMatchRegularExpression + return nil, []rolloutsv1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + { + Name: "canary", + Value: "true", + }, + }, + }, + } + }, desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { rules := routeDemo.DeepCopy().Spec.Rules - rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ - { - BackendRef: gatewayv1alpha2.BackendRef{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Kind: &kindSvc, - Name: "store-svc", - Port: &portNum, + iType := gatewayv1alpha2.HeaderMatchRegularExpression + rules = append(rules, gatewayv1alpha2.HTTPRouteRule{ + Matches: []gatewayv1alpha2.HTTPRouteMatch{ + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/store"), + }, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "version", + Value: "v2", + }, + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + { + Name: "canary", + Value: "true", + }, + }, + }, + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/v2/store"), + }, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + { + Name: "canary", + Value: "true", + }, }, - Weight: utilpointer.Int32(80), }, }, - { - BackendRef: gatewayv1alpha2.BackendRef{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Kind: &kindSvc, - Name: "store-svc-canary", - Port: &portNum, + BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ + { + BackendRef: gatewayv1alpha2.BackendRef{ + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Kind: &kindSvc, + Name: "store-svc-canary", + Port: &portNum, + }, }, - Weight: utilpointer.Int32(20), }, }, - } - rules[3].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ - { - BackendRef: gatewayv1alpha2.BackendRef{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Kind: &kindSvc, - Name: "store-svc", - Port: &portNum, + }) + rules = append(rules, gatewayv1alpha2.HTTPRouteRule{ + Matches: []gatewayv1alpha2.HTTPRouteMatch{ + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/storage"), + }, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + { + Name: "canary", + Value: "true", + }, }, - Weight: utilpointer.Int32(80), }, }, - { - BackendRef: gatewayv1alpha2.BackendRef{ - BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ - Kind: &kindSvc, - Name: "store-svc-canary", - Port: &portNum, + BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ + { + BackendRef: gatewayv1alpha2.BackendRef{ + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Kind: &kindSvc, + Name: "store-svc-canary", + Port: &portNum, + }, }, - Weight: utilpointer.Int32(20), }, }, - } + }) return rules }, }, { - name: "canary weight: 0", + name: "canary weight: 20", getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule { rules := routeDemo.DeepCopy().Spec.Rules rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ @@ -222,7 +299,9 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { } return rules }, - canaryPercent: 0, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + return utilpointer.Int32(20), nil + }, desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { rules := routeDemo.DeepCopy().Spec.Rules rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ @@ -233,7 +312,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { Name: "store-svc", Port: &portNum, }, - Weight: utilpointer.Int32(100), + Weight: utilpointer.Int32(80), }, }, { @@ -243,7 +322,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { Name: "store-svc-canary", Port: &portNum, }, - Weight: utilpointer.Int32(0), + Weight: utilpointer.Int32(20), }, }, } @@ -255,7 +334,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { Name: "store-svc", Port: &portNum, }, - Weight: utilpointer.Int32(100), + Weight: utilpointer.Int32(80), }, }, { @@ -265,7 +344,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { Name: "store-svc-canary", Port: &portNum, }, - Weight: utilpointer.Int32(0), + Weight: utilpointer.Int32(20), }, }, } @@ -320,11 +399,47 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { }, }, } + iType := gatewayv1alpha2.HeaderMatchRegularExpression + rules = append(rules, gatewayv1alpha2.HTTPRouteRule{ + Matches: []gatewayv1alpha2.HTTPRouteMatch{ + { + Path: &gatewayv1alpha2.HTTPPathMatch{ + Value: utilpointer.String("/storage"), + }, + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + { + Name: "canary", + Value: "true", + }, + }, + }, + }, + BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ + { + BackendRef: gatewayv1alpha2.BackendRef{ + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Kind: &kindSvc, + Name: "store-svc-canary", + Port: &portNum, + }, + }, + }, + }, + }) return rules }, - canaryPercent: -1, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + return utilpointer.Int32(-1), nil + }, desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { rules := routeDemo.DeepCopy().Spec.Rules + rules[3].BackendRefs[0].Weight = utilpointer.Int32(1) + rules[1].BackendRefs[0].Weight = utilpointer.Int32(1) return rules }, }, @@ -339,9 +454,11 @@ func TestBuildDesiredHTTPRoute(t *testing.T) { for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { controller := &gatewayController{conf: conf} - desired := controller.buildDesiredHTTPRoute(cs.getRouteRules(), cs.canaryPercent) - if !reflect.DeepEqual(desired, cs.desiredRules()) { - t.Fatalf("expect: %v, but get %v", cs.desiredRules(), desired) + weight, matches := cs.getRoutes() + current := controller.buildDesiredHTTPRoute(cs.getRouteRules(), weight, matches) + desired := cs.desiredRules() + if !reflect.DeepEqual(current, desired) { + t.Fatalf("expect: %v, but get %v", util.DumpJSON(desired), util.DumpJSON(current)) } }) } diff --git a/pkg/controller/rollout/trafficrouting/ingress/ingress.go b/pkg/controller/rollout/trafficrouting/ingress/ingress.go new file mode 100644 index 00000000..9a1c22e3 --- /dev/null +++ b/pkg/controller/rollout/trafficrouting/ingress/ingress.go @@ -0,0 +1,298 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingress + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + jsonpatch "github.com/evanphx/json-patch" + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + "github.com/openkruise/rollouts/pkg/util/luamanager" + lua "github.com/yuin/gopher-lua" + netv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ingressController struct { + client.Client + conf Config + luaManager *luamanager.LuaManager + canaryIngressName string + luaScript string +} + +type Config struct { + RolloutName string + RolloutNs string + CanaryService string + StableService string + TrafficConf *rolloutv1alpha1.IngressTrafficRouting + OwnerRef metav1.OwnerReference +} + +func NewIngressTrafficRouting(client client.Client, conf Config) (trafficrouting.Controller, error) { + r := &ingressController{ + Client: client, + conf: conf, + canaryIngressName: defaultCanaryIngressName(conf.TrafficConf.Name), + luaManager: &luamanager.LuaManager{}, + } + luaScript, err := r.getTrafficRoutingIngressLuaScript(conf.TrafficConf.ClassType) + if err != nil { + return nil, err + } + r.luaScript = luaScript + return r, nil +} + +// Initialize verify the existence of the ingress resource and generate the canary ingress +func (r *ingressController) Initialize(ctx context.Context) error { + ingress := &netv1.Ingress{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, ingress) + if err != nil { + return err + } + canaryIngress := &netv1.Ingress{} + err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error()) + return err + } else if err == nil { + return nil + } + + // build and create canary ingress + canaryIngress = r.buildCanaryIngress(ingress) + canaryIngress.Annotations, err = r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil) + if err != nil { + klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return err + } + if err = r.Create(ctx, canaryIngress); err != nil { + klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return err + } + klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, util.DumpJSON(canaryIngress)) + return nil +} + +func (r *ingressController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) { + canaryIngress := &netv1.Ingress{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: defaultCanaryIngressName(r.conf.TrafficConf.Name)}, canaryIngress) + if err != nil { + klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return false, err + } + newAnnotations, err := r.executeLuaForCanary(canaryIngress.Annotations, weight, matches) + if err != nil { + klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return false, err + } + if reflect.DeepEqual(canaryIngress.Annotations, newAnnotations) { + return true, nil + } + byte1, _ := json.Marshal(metav1.ObjectMeta{Annotations: canaryIngress.Annotations}) + byte2, _ := json.Marshal(metav1.ObjectMeta{Annotations: newAnnotations}) + patch, err := jsonpatch.CreateMergePatch(byte1, byte2) + if err != nil { + klog.Errorf("rollout(%s/%s) create merge patch failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return false, err + } + body := fmt.Sprintf(`{"metadata":%s}`, string(patch)) + if err = r.Patch(ctx, canaryIngress, client.RawPatch(types.MergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) set canary ingress(%s) annotations(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, util.DumpJSON(newAnnotations)) + return false, nil +} + +func (r *ingressController) Finalise(ctx context.Context) (bool, error) { + canaryIngress := &netv1.Ingress{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error()) + return false, err + } + if errors.IsNotFound(err) { + return true, nil + } else if !canaryIngress.DeletionTimestamp.IsZero() { + return false, nil + } + // First, set canary route 0 weight. + newAnnotations, err := r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil) + if err != nil { + klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return false, err + } + if !reflect.DeepEqual(canaryIngress.Annotations, newAnnotations) { + byte1, _ := json.Marshal(metav1.ObjectMeta{Annotations: canaryIngress.Annotations}) + byte2, _ := json.Marshal(metav1.ObjectMeta{Annotations: newAnnotations}) + patch, err := jsonpatch.CreateMergePatch(byte1, byte2) + if err != nil { + klog.Errorf("rollout(%s/%s) create merge patch failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) + return false, err + } + body := fmt.Sprintf(`{"metadata":%s}`, string(patch)) + if err = r.Patch(ctx, canaryIngress, client.RawPatch(types.MergePatchType, []byte(body))); err != nil { + klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) set canary ingress annotations(%s) success", r.conf.RolloutNs, r.conf.RolloutName, util.DumpJSON(newAnnotations)) + return false, nil + } + + // Second, delete canary ingress + if err = r.Delete(ctx, canaryIngress); err != nil { + klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) + return false, err + } + klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name) + return false, nil +} + +func (r *ingressController) buildCanaryIngress(stableIngress *netv1.Ingress) *netv1.Ingress { + desiredCanaryIngress := &netv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.canaryIngressName, + Namespace: stableIngress.Namespace, + Annotations: stableIngress.Annotations, + Labels: stableIngress.Labels, + }, + Spec: netv1.IngressSpec{ + Rules: make([]netv1.IngressRule, 0), + IngressClassName: stableIngress.Spec.IngressClassName, + TLS: stableIngress.Spec.TLS, + }, + } + hosts := sets.NewString() + // Ensure canaryIngress is owned by this Rollout for cleanup + desiredCanaryIngress.SetOwnerReferences([]metav1.OwnerReference{r.conf.OwnerRef}) + // Copy only the rules which reference the stableService from the stableIngress to the canaryIngress + // and change service backend to canaryService. Rules **not** referencing the stableIngress will be ignored. + for ir := 0; ir < len(stableIngress.Spec.Rules); ir++ { + var hasStableServiceBackendRule bool + stableRule := stableIngress.Spec.Rules[ir] + canaryRule := netv1.IngressRule{ + Host: stableRule.Host, + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{}, + }, + } + // Update all backends pointing to the stableService to point to the canaryService now + for ip := 0; ip < len(stableRule.HTTP.Paths); ip++ { + if stableRule.HTTP.Paths[ip].Backend.Service.Name == r.conf.StableService { + hasStableServiceBackendRule = true + if stableRule.Host != "" { + hosts.Insert(stableRule.Host) + } + canaryPath := netv1.HTTPIngressPath{ + Path: stableRule.HTTP.Paths[ip].Path, + PathType: stableRule.HTTP.Paths[ip].PathType, + Backend: stableRule.HTTP.Paths[ip].Backend, + } + canaryPath.Backend.Service.Name = r.conf.CanaryService + canaryRule.HTTP.Paths = append(canaryRule.HTTP.Paths, canaryPath) + } + } + // If this rule was using the specified stableService backend, append it to the canary Ingress spec + if hasStableServiceBackendRule { + desiredCanaryIngress.Spec.Rules = append(desiredCanaryIngress.Spec.Rules, canaryRule) + } + } + + return desiredCanaryIngress +} + +func defaultCanaryIngressName(name string) string { + return fmt.Sprintf("%s-canary", name) +} + +func (r *ingressController) executeLuaForCanary(annotations map[string]string, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (map[string]string, error) { + if weight == nil { + // the lua script does not have a pointer type, so we need to pass weight=-1 to indicate the case where weight is nil. + weight = utilpointer.Int32(-1) + } + type LuaData struct { + Annotations map[string]string + Weight string + Matches []rolloutv1alpha1.HttpRouteMatch + CanaryService string + } + data := &LuaData{ + Annotations: annotations, + Weight: fmt.Sprintf("%d", *weight), + Matches: matches, + CanaryService: r.canaryIngressName, + } + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + return nil, err + } + obj := &unstructured.Unstructured{Object: unObj} + l, err := r.luaManager.RunLuaScript(obj, r.luaScript) + if err != nil { + return nil, err + } + returnValue := l.Get(-1) + if returnValue.Type() == lua.LTTable { + jsonBytes, err := luamanager.Encode(returnValue) + if err != nil { + return nil, err + } + newAnnotations := map[string]string{} + err = json.Unmarshal(jsonBytes, &newAnnotations) + if err != nil { + return nil, err + } + return newAnnotations, nil + } + return nil, fmt.Errorf("expect table output from Lua script, not %s", returnValue.Type().String()) +} + +func (r *ingressController) getTrafficRoutingIngressLuaScript(iType string) (string, error) { + if iType == "" { + iType = "nginx" + } + luaScript, err := configuration.GetTrafficRoutingIngressLuaScript(r.Client, iType) + if err != nil { + return "", err + } + if luaScript != "" { + return luaScript, nil + } + key := fmt.Sprintf("lua_configuration/trafficrouting_ingress/%s.lua", iType) + script := util.GetLuaConfigurationContent(key) + if script == "" { + return "", fmt.Errorf("%s lua script is not found", iType) + } + return script, nil +} diff --git a/pkg/controller/rollout/trafficrouting/ingress/ingress_test.go b/pkg/controller/rollout/trafficrouting/ingress/ingress_test.go new file mode 100644 index 00000000..d6354998 --- /dev/null +++ b/pkg/controller/rollout/trafficrouting/ingress/ingress_test.go @@ -0,0 +1,626 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingress + +import ( + "context" + "fmt" + "reflect" + "testing" + + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + "github.com/openkruise/rollouts/pkg/util/configuration" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +var ( + scheme *runtime.Scheme + + demoConf = corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configuration.RolloutConfigurationName, + Namespace: util.GetRolloutNamespace(), + }, + Data: map[string]string{ + fmt.Sprintf("%s.nginx", configuration.LuaTrafficRoutingIngressTypePrefix): ` + annotations = obj.annotations + annotations["nginx.ingress.kubernetes.io/canary"] = "true" + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil + annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil + if ( obj.weight ~= "-1" ) + then + annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight + end + if ( not obj.matches ) + then + return annotations + end + for _,match in ipairs(obj.matches) do + header = match.headers[1] + if ( header.name == "canary-by-cookie" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name + if ( header.type == "RegularExpression" ) + then + annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value + else + annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value + end + end + end + return annotations + `, + fmt.Sprintf("%s.aliyun-alb", configuration.LuaTrafficRoutingIngressTypePrefix): ` + function split(input, delimiter) + local arr = {} + string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end) + return arr + end + + annotations = obj.annotations + annotations["alb.ingress.kubernetes.io/canary"] = "true" + annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil + annotations["alb.ingress.kubernetes.io/canary-weight"] = nil + conditionKey = string.format("alb.ingress.kubernetes.io/conditions.%s", obj.canaryService) + annotations[conditionKey] = nil + if ( obj.weight ~= "-1" ) + then + annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight + end + if ( not obj.matches ) + then + return annotations + end + conditions = {} + match = obj.matches[1] + for _,header in ipairs(match.headers) do + condition = {} + if ( header.name == "Cookie" ) + then + condition.type = "Cookie" + condition.cookieConfig = {} + cookies = split(header.value, ";") + values = {} + for _,cookieStr in ipairs(cookies) do + cookie = split(cookieStr, "=") + value = {} + value.key = cookie[1] + value.value = cookie[2] + table.insert(values, value) + end + condition.cookieConfig.values = values + elseif ( header.name == "SourceIp" ) + then + condition.type = "SourceIp" + condition.sourceIpConfig = {} + ips = split(header.value, ";") + values = {} + for _,ip in ipairs(ips) do + table.insert(values, ip) + end + condition.sourceIpConfig.values = values + else + condition.type = "Header" + condition.headerConfig = {} + condition.headerConfig.key = header.name + vals = split(header.value, ";") + values = {} + for _,val in ipairs(vals) do + table.insert(values, val) + end + condition.headerConfig.values = values + end + table.insert(conditions, condition) + end + annotations[conditionKey] = json.encode(conditions) + return annotations + `, + }, + } + + demoIngress = netv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: netv1.IngressSpec{ + IngressClassName: utilpointer.String("nginx"), + TLS: []netv1.IngressTLS{ + { + Hosts: []string{"echoserver.example.com"}, + SecretName: "echoserver-name", + }, + { + Hosts: []string{"log.example.com"}, + SecretName: "log-name", + }, + }, + Rules: []netv1.IngressRule{ + { + Host: "echoserver.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/apis/echo", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "echoserver", + Port: netv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + { + Path: "/apis/other", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "other", + Port: netv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + { + Host: "log.example.com", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/apis/logs", + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: "echoserver", + Port: netv1.ServiceBackendPort{ + Number: 8899, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = rolloutsv1alpha1.AddToScheme(scheme) +} + +func TestInitialize(t *testing.T) { + cases := []struct { + name string + getConfigmap func() *corev1.ConfigMap + getIngress func() []*netv1.Ingress + expectIngress func() *netv1.Ingress + }{ + { + name: "init test1", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + return []*netv1.Ingress{demoIngress.DeepCopy()} + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0" + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + } + + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + fakeCli.Create(context.TODO(), cs.getConfigmap()) + for _, ingress := range cs.getIngress() { + fakeCli.Create(context.TODO(), ingress) + } + controller, err := NewIngressTrafficRouting(fakeCli, config) + if err != nil { + t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error()) + return + } + err = controller.Initialize(context.TODO()) + if err != nil { + t.Fatalf("Initialize failed: %s", err.Error()) + return + } + canaryIngress := &netv1.Ingress{} + err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress) + if err != nil { + t.Fatalf("Get canary ingress failed: %s", err.Error()) + return + } + expect := cs.expectIngress() + if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) || + !reflect.DeepEqual(canaryIngress.Spec, expect.Spec) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress)) + } + }) + } +} + +func TestEnsureRoutes(t *testing.T) { + cases := []struct { + name string + getConfigmap func() *corev1.ConfigMap + getIngress func() []*netv1.Ingress + getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) + expectIngress func() *netv1.Ingress + ingressType string + }{ + { + name: "ensure routes test1", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + return nil, []rolloutsv1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123456", + }, + }, + }, + // cookies + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "canary-by-cookie", + Value: "demo", + }, + }, + }, + } + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo" + expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + { + name: "ensure routes test2", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + return utilpointer.Int32(40), nil + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "40" + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + { + name: "ensure routes test3", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + iType := gatewayv1alpha2.HeaderMatchRegularExpression + return nil, []rolloutsv1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123*", + Type: &iType, + }, + }, + }, + } + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = "123*" + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + { + name: "ensure routes test4", + ingressType: "aliyun-alb", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["alb.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["alb.ingress.kubernetes.io/canary-weight"] = "0" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) { + return nil, []rolloutsv1alpha1.HttpRouteMatch{ + // header + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "Cookie", + Value: "demo1=value1;demo2=value2", + }, + { + Name: "SourceIp", + Value: "192.168.0.0/16;172.16.0.0/16", + }, + { + Name: "headername", + Value: "headervalue1;headervalue2", + }, + }, + }, + } + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["alb.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["alb.ingress.kubernetes.io/conditions.echoserver-canary"] = `[{"cookieConfig":{"values":[{"key":"demo1","value":"value1"},{"key":"demo2","value":"value2"}]},"type":"Cookie"},{"sourceIpConfig":{"values":["192.168.0.0/16","172.16.0.0/16"]},"type":"SourceIp"},{"headerConfig":{"key":"headername","values":["headervalue1","headervalue2"]},"type":"Header"}]` + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + } + + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + fakeCli.Create(context.TODO(), cs.getConfigmap()) + for _, ingress := range cs.getIngress() { + fakeCli.Create(context.TODO(), ingress) + } + config.TrafficConf.ClassType = cs.ingressType + controller, err := NewIngressTrafficRouting(fakeCli, config) + if err != nil { + t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error()) + return + } + weight, matches := cs.getRoutes() + _, err = controller.EnsureRoutes(context.TODO(), weight, matches) + if err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + return + } + canaryIngress := &netv1.Ingress{} + err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress) + if err != nil { + t.Fatalf("Get canary ingress failed: %s", err.Error()) + return + } + expect := cs.expectIngress() + if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) || + !reflect.DeepEqual(canaryIngress.Spec, expect.Spec) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress)) + } + }) + } +} + +func TestFinalise(t *testing.T) { + cases := []struct { + name string + getConfigmap func() *corev1.ConfigMap + getIngress func() []*netv1.Ingress + expectIngress func() *netv1.Ingress + }{ + { + name: "finalise test1", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id" + canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + expectIngress: func() *netv1.Ingress { + expect := demoIngress.DeepCopy() + expect.Name = "echoserver-canary" + expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0" + expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1] + expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return expect + }, + }, + { + name: "finalise test2", + getConfigmap: func() *corev1.ConfigMap { + return demoConf.DeepCopy() + }, + getIngress: func() []*netv1.Ingress { + canary := demoIngress.DeepCopy() + canary.Name = "echoserver-canary" + canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0" + canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1] + canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary" + return []*netv1.Ingress{demoIngress.DeepCopy(), canary} + }, + expectIngress: func() *netv1.Ingress { + return nil + }, + }, + } + + config := Config{ + RolloutName: "rollout-demo", + StableService: "echoserver", + CanaryService: "echoserver-canary", + TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{ + Name: "echoserver", + }, + } + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build() + fakeCli.Create(context.TODO(), cs.getConfigmap()) + for _, ingress := range cs.getIngress() { + fakeCli.Create(context.TODO(), ingress) + } + controller, err := NewIngressTrafficRouting(fakeCli, config) + if err != nil { + t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error()) + return + } + _, err = controller.Finalise(context.TODO()) + if err != nil { + t.Fatalf("EnsureRoutes failed: %s", err.Error()) + return + } + canaryIngress := &netv1.Ingress{} + err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress) + if err != nil { + if cs.expectIngress() == nil && errors.IsNotFound(err) { + return + } + t.Fatalf("Get canary ingress failed: %s", err.Error()) + return + } + expect := cs.expectIngress() + if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) || + !reflect.DeepEqual(canaryIngress.Spec, expect.Spec) { + t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress)) + } + }) + } +} diff --git a/pkg/controller/rollout/trafficrouting/interface.go b/pkg/controller/rollout/trafficrouting/interface.go index ae40445e..8ef3d1e8 100644 --- a/pkg/controller/rollout/trafficrouting/interface.go +++ b/pkg/controller/rollout/trafficrouting/interface.go @@ -16,20 +16,28 @@ limitations under the License. package trafficrouting -import "context" +import ( + "context" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" +) // Controller common function across all TrafficRouting implementation type Controller interface { // Initialize will validate the traffic routing resource + // 1. Ingress type, verify the existence of the ingress resource and generate the canary ingress[weight=0%] + // 2. Gateway type, verify the existence of the gateway resource Initialize(ctx context.Context) error - // EnsureRoutes check and set canary desired weight. desiredWeight[0,100] + // EnsureRoutes check and set canary weight and matches. + // weight indicates percentage of traffic to canary service, and range of values[0,100] + // matches indicates A/B Testing release for headers, cookies // 1. check if canary has been set desired weight. // 2. If not, set canary desired weight // When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true - // When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress, - // and the canary weight should be set to 0 at this time. - EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error) + EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) // Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress. // Finalise is called with a 3-second delay after completing the canary. - Finalise(ctx context.Context) error + // bool indicates whether function Finalise is complete, + // for example, when ingress type, only canary ingress Not Found is considered function finalise complete + Finalise(ctx context.Context) (bool, error) } diff --git a/pkg/controller/rollout/trafficrouting/nginx/nginx.go b/pkg/controller/rollout/trafficrouting/nginx/nginx.go deleted file mode 100644 index 0d932936..00000000 --- a/pkg/controller/rollout/trafficrouting/nginx/nginx.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nginx - -import ( - "context" - "fmt" - "strconv" - - rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" - "github.com/openkruise/rollouts/pkg/util" - netv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io" - k8sIngressClassAnnotation = "kubernetes.io/ingress.class" -) - -type nginxController struct { - client.Client - //stableIngress *netv1.Ingress - conf Config - newStatus *rolloutv1alpha1.RolloutStatus -} - -type Config struct { - RolloutName string - RolloutNs string - CanaryService string - StableService string - TrafficConf *rolloutv1alpha1.IngressTrafficRouting - OwnerRef metav1.OwnerReference -} - -func NewNginxTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) { - r := &nginxController{ - Client: client, - conf: conf, - newStatus: newStatus, - } - return r, nil -} - -func (r *nginxController) Initialize(ctx context.Context) error { - ingress := &netv1.Ingress{} - return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, ingress) -} - -func (r *nginxController) EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error) { - canaryIngress := &netv1.Ingress{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) - return false, err - } - // When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress, - // and the canary weight should be set to 0 at this time. - if desiredWeight == 0 { - // canary ingress - if errors.IsNotFound(err) || !canaryIngress.DeletionTimestamp.IsZero() { - klog.Infof("rollout(%s/%s) verify canary ingress has been deleted", r.conf.RolloutNs, r.conf.RolloutName) - return true, nil - } - } else if errors.IsNotFound(err) { - // create canary ingress - stableIngress := &netv1.Ingress{} - err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, stableIngress) - if err != nil { - return false, err - } else if !stableIngress.DeletionTimestamp.IsZero() { - klog.Warningf("rollout(%s/%s) stable ingress is deleting", r.conf.RolloutNs, r.conf.RolloutName) - return false, nil - } - canaryIngress = r.buildCanaryIngress(stableIngress, desiredWeight) - if err = r.Create(ctx, canaryIngress); err != nil { - klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) - return false, err - } - data := util.DumpJSON(canaryIngress) - klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, data) - return false, nil - } - - // check whether canary weight equals desired weight - currentWeight := getIngressCanaryWeight(canaryIngress) - if desiredWeight == currentWeight { - return true, nil - } - - cloneObj := canaryIngress.DeepCopy() - body := fmt.Sprintf(`{"metadata":{"annotations":{"%s/canary-weight":"%s"}}}`, nginxIngressAnnotationDefaultPrefix, fmt.Sprintf("%d", desiredWeight)) - if err = r.Patch(ctx, cloneObj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { - klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) - return false, err - } - klog.Infof("rollout(%s/%s) set ingress routes(weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, desiredWeight) - return false, nil -} - -func (r *nginxController) Finalise(ctx context.Context) error { - canaryIngress := &netv1.Ingress{} - err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress) - if err != nil && !errors.IsNotFound(err) { - klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.defaultCanaryIngressName(), err.Error()) - return err - } - - if errors.IsNotFound(err) || !canaryIngress.DeletionTimestamp.IsZero() { - return nil - } - - // immediate delete canary ingress - if err = r.Delete(ctx, canaryIngress); err != nil { - klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error()) - return err - } - klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name) - return nil -} - -func (r *nginxController) buildCanaryIngress(stableIngress *netv1.Ingress, desiredWeight int32) *netv1.Ingress { - desiredCanaryIngress := &netv1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: r.defaultCanaryIngressName(), - Namespace: stableIngress.Namespace, - Annotations: map[string]string{}, - }, - Spec: netv1.IngressSpec{ - Rules: make([]netv1.IngressRule, 0), - }, - } - - // Preserve ingressClassName from stable ingress - if stableIngress.Spec.IngressClassName != nil { - desiredCanaryIngress.Spec.IngressClassName = stableIngress.Spec.IngressClassName - } - - // Must preserve ingress.class on canary ingress, no other annotations matter - // See: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary - if val, ok := stableIngress.Annotations[k8sIngressClassAnnotation]; ok { - desiredCanaryIngress.Annotations[k8sIngressClassAnnotation] = val - } - - // Ensure canaryIngress is owned by this Rollout for cleanup - desiredCanaryIngress.SetOwnerReferences([]metav1.OwnerReference{r.conf.OwnerRef}) - - // Copy only the rules which reference the stableService from the stableIngress to the canaryIngress - // and change service backend to canaryService. Rules **not** referencing the stableIngress will be ignored. - for ir := 0; ir < len(stableIngress.Spec.Rules); ir++ { - var hasStableServiceBackendRule bool - ingressRule := stableIngress.Spec.Rules[ir].DeepCopy() - - // Update all backends pointing to the stableService to point to the canaryService now - for ip := 0; ip < len(ingressRule.HTTP.Paths); ip++ { - if ingressRule.HTTP.Paths[ip].Backend.Service.Name == r.conf.StableService { - hasStableServiceBackendRule = true - ingressRule.HTTP.Paths[ip].Backend.Service.Name = r.conf.CanaryService - } - } - - // If this rule was using the specified stableService backend, append it to the canary Ingress spec - if hasStableServiceBackendRule { - desiredCanaryIngress.Spec.Rules = append(desiredCanaryIngress.Spec.Rules, *ingressRule) - } - } - - // Always set `canary` and `canary-weight` - `canary-by-header` and `canary-by-cookie`, if set, will always take precedence - desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true" - desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = fmt.Sprintf("%d", desiredWeight) - return desiredCanaryIngress -} - -func (r *nginxController) defaultCanaryIngressName() string { - return fmt.Sprintf("%s-canary", r.conf.TrafficConf.Name) -} - -func getIngressCanaryWeight(ing *netv1.Ingress) int32 { - weightStr, ok := ing.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] - if !ok { - return 0 - } - weight, _ := strconv.Atoi(weightStr) - return int32(weight) -} diff --git a/pkg/util/configuration/configuration.go b/pkg/util/configuration/configuration.go new file mode 100644 index 00000000..86282f5c --- /dev/null +++ b/pkg/util/configuration/configuration.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration + +import ( + "context" + "fmt" + + "github.com/openkruise/rollouts/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // kruise rollout configmap name + RolloutConfigurationName = "kruise-rollout-configuration" + + LuaTrafficRoutingIngressTypePrefix = "lua.traffic.routing.ingress" +) + +func GetTrafficRoutingIngressLuaScript(client client.Client, iType string) (string, error) { + data, err := getRolloutConfiguration(client) + if err != nil { + return "", err + } else if len(data) == 0 { + return "", nil + } + value, ok := data[fmt.Sprintf("%s.%s", LuaTrafficRoutingIngressTypePrefix, iType)] + if !ok { + return "", nil + } + return value, nil +} + +func getRolloutConfiguration(c client.Client) (map[string]string, error) { + cfg := &corev1.ConfigMap{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: util.GetRolloutNamespace(), Name: RolloutConfigurationName}, cfg) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return cfg.Data, nil +} diff --git a/pkg/util/lua_configuration.go b/pkg/util/lua_configuration.go new file mode 100644 index 00000000..8a2d9f18 --- /dev/null +++ b/pkg/util/lua_configuration.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "io/ioutil" + "os" + "path/filepath" + + "k8s.io/klog/v2" +) + +// patch -> file.Content, +// for example: lua_configuration/trafficrouting_ingress/nginx.lua -> nginx.lua content +var luaConfigurationList map[string]string + +func init() { + luaConfigurationList = map[string]string{} + _ = filepath.Walk("./lua_configuration", func(path string, f os.FileInfo, err error) error { + if err != nil { + klog.Errorf("filepath walk ./lua_configuration failed: %s", err.Error()) + return err + } + if f.IsDir() { + return nil + } + var data []byte + data, err = ioutil.ReadFile(filepath.Clean(path)) + if err != nil { + klog.Errorf("Read file %s failed: %s", path, err.Error()) + return err + } + luaConfigurationList[path] = string(data) + return nil + }) + klog.Infof("Init Lua Configuration(%s)", DumpJSON(luaConfigurationList)) +} + +func GetLuaConfigurationContent(key string) string { + return luaConfigurationList[key] +} diff --git a/pkg/util/luamanager/json.go b/pkg/util/luamanager/json.go new file mode 100644 index 00000000..75051d30 --- /dev/null +++ b/pkg/util/luamanager/json.go @@ -0,0 +1,190 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package luamanager + +import ( + "encoding/json" + "errors" + + lua "github.com/yuin/gopher-lua" +) + +const JsonLibName = "json" + +// OpenJson is the module loader function. +func OpenJson(L *lua.LState) int { + mod := L.RegisterModule(JsonLibName, jsonFuncs) + L.Push(mod) + return 1 +} + +var jsonFuncs = map[string]lua.LGFunction{ + "decode": jsonDecode, + "encode": jsonEncode, +} + +func jsonDecode(L *lua.LState) int { + str := L.CheckString(1) + + value, err := Decode(L, []byte(str)) + if err != nil { + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + L.Push(value) + return 1 +} + +func jsonEncode(L *lua.LState) int { + value := L.CheckAny(1) + + data, err := Encode(value) + if err != nil { + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + L.Push(lua.LString(string(data))) + return 1 +} + +var ( + errNested = errors.New("cannot encode recursively nested tables to JSON") + errSparseArray = errors.New("cannot encode sparse array") + errInvalidKeys = errors.New("cannot encode mixed or invalid key types") +) + +type invalidTypeError lua.LValueType + +func (i invalidTypeError) Error() string { + return `cannot encode ` + lua.LValueType(i).String() + ` to JSON` +} + +// Encode returns the JSON encoding of value. +func Encode(value lua.LValue) ([]byte, error) { + return json.Marshal(jsonValue{ + LValue: value, + visited: make(map[*lua.LTable]bool), + }) +} + +type jsonValue struct { + lua.LValue + visited map[*lua.LTable]bool +} + +func (j jsonValue) MarshalJSON() (data []byte, err error) { + switch converted := j.LValue.(type) { + case lua.LBool: + data, err = json.Marshal(bool(converted)) + case lua.LNumber: + data, err = json.Marshal(float64(converted)) + case *lua.LNilType: + data = []byte(`null`) + case lua.LString: + data, err = json.Marshal(string(converted)) + case *lua.LTable: + if j.visited[converted] { + return nil, errNested + } + j.visited[converted] = true + + key, value := converted.Next(lua.LNil) + + switch key.Type() { + case lua.LTNil: // empty table + data = []byte(`[]`) + case lua.LTNumber: + arr := make([]jsonValue, 0, converted.Len()) + expectedKey := lua.LNumber(1) + for key != lua.LNil { + if key.Type() != lua.LTNumber { + err = errInvalidKeys + return + } + if expectedKey != key { + err = errSparseArray + return + } + arr = append(arr, jsonValue{value, j.visited}) + expectedKey++ + key, value = converted.Next(key) + } + data, err = json.Marshal(arr) + case lua.LTString: + obj := make(map[string]jsonValue) + for key != lua.LNil { + if key.Type() != lua.LTString { + err = errInvalidKeys + return + } + obj[key.String()] = jsonValue{value, j.visited} + key, value = converted.Next(key) + } + data, err = json.Marshal(obj) + default: + err = errInvalidKeys + } + default: + err = invalidTypeError(j.LValue.Type()) + } + return +} + +// Decode converts the JSON encoded data to Lua values. +func Decode(L *lua.LState, data []byte) (lua.LValue, error) { + var value interface{} + err := json.Unmarshal(data, &value) + if err != nil { + return nil, err + } + return DecodeValue(L, value), nil +} + +// DecodeValue converts the value to a Lua value. +// +// This function only converts values that the encoding/json package decodes to. +// All other values will return lua.LNil. +func DecodeValue(L *lua.LState, value interface{}) lua.LValue { + switch converted := value.(type) { + case bool: + return lua.LBool(converted) + case float64: + return lua.LNumber(converted) + case string: + return lua.LString(converted) + case json.Number: + return lua.LString(converted) + case []interface{}: + arr := L.CreateTable(len(converted), 0) + for _, item := range converted { + arr.Append(DecodeValue(L, item)) + } + return arr + case map[string]interface{}: + tbl := L.CreateTable(0, len(converted)) + for key, item := range converted { + tbl.RawSetH(lua.LString(key), DecodeValue(L, item)) + } + return tbl + case nil: + return lua.LNil + } + + return lua.LNil +} diff --git a/pkg/util/luamanager/lua.go b/pkg/util/luamanager/lua.go new file mode 100644 index 00000000..874bb87e --- /dev/null +++ b/pkg/util/luamanager/lua.go @@ -0,0 +1,93 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package luamanager + +import ( + "context" + "encoding/json" + "time" + + lua "github.com/yuin/gopher-lua" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type LuaManager struct{} + +func (m *LuaManager) RunLuaScript(obj *unstructured.Unstructured, script string) (*lua.LState, error) { + l := lua.NewState(lua.Options{SkipOpenLibs: true}) + defer l.Close() + for _, pair := range []struct { + n string + f lua.LGFunction + }{ + {lua.MathLibName, lua.OpenMath}, + {lua.BaseLibName, lua.OpenBase}, + {lua.TabLibName, lua.OpenTable}, + {lua.StringLibName, lua.OpenString}, + {JsonLibName, OpenJson}, + } { + if err := l.CallByParam(lua.P{ + Fn: l.NewFunction(pair.f), + NRet: 0, + Protect: true, + }, lua.LString(pair.n)); err != nil { + return nil, err + } + } + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + l.SetContext(ctx) + objectValue := decodeValue(l, obj.Object) + l.SetGlobal("obj", objectValue) + err := l.DoString(script) + return l, err +} + +func decodeValue(L *lua.LState, value interface{}) lua.LValue { + switch converted := value.(type) { + case bool: + return lua.LBool(converted) + case float64: + return lua.LNumber(converted) + case string: + return lua.LString(converted) + case json.Number: + return lua.LString(converted) + case int: + return lua.LNumber(converted) + case int32: + return lua.LNumber(converted) + case int64: + return lua.LNumber(converted) + case []interface{}: + arr := L.CreateTable(len(converted), 0) + for _, item := range converted { + arr.Append(decodeValue(L, item)) + } + return arr + case map[string]interface{}: + tbl := L.CreateTable(0, len(converted)) + for key, item := range converted { + tbl.RawSetH(lua.LString(key), decodeValue(L, item)) + } + return tbl + case nil: + return lua.LNil + } + + return lua.LNil +} diff --git a/pkg/util/luamanager/lua_test.go b/pkg/util/luamanager/lua_test.go new file mode 100644 index 00000000..88f60e91 --- /dev/null +++ b/pkg/util/luamanager/lua_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package luamanager + +import ( + "encoding/json" + "fmt" + "testing" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + lua "github.com/yuin/gopher-lua" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + luajson "layeh.com/gopher-json" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +func TestRunLuaScript(t *testing.T) { + cases := []struct { + name string + getObj func() *unstructured.Unstructured + getLuaScript func() string + expectResult func() string + }{ + { + name: "lua script test 1", + getObj: func() *unstructured.Unstructured { + type LuaData struct { + Annotations map[string]string + Weight string + Matches []rolloutv1alpha1.HttpRouteMatch + } + data := &LuaData{ + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + Weight: "0", + Matches: []rolloutv1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "user_id", + Value: "123456", + }, + }, + }, + }, + } + unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data) + if err != nil { + fmt.Println("to unstructured failed", err.Error()) + } + obj := &unstructured.Unstructured{Object: unObj} + return obj + }, + getLuaScript: func() string { + return ` + annotations = obj.annotations + annotations["weight"] = obj.weight + if ( not obj.matches ) + then + return annotations + end + for _,match in pairs(obj.matches) + do + if ( not (match or match.headers) ) + then + return annotations + end + for _,header in pairs(match.headers) + do + if ( not header ) + then + return annotations + end + annotations["header.name"] = header.name + annotations["header.value"] = header.value + end + end + return annotations + ` + }, + expectResult: func() string { + obj := map[string]string{ + "weight": "0", + "header.name": "user_id", + "header.value": "123456", + "kubernetes.io/ingress.class": "nginx", + } + return util.DumpJSON(obj) + }, + }, + } + + luaManager := &LuaManager{} + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fmt.Println(cs.name) + l, err := luaManager.RunLuaScript(cs.getObj(), cs.getLuaScript()) + if err != nil { + t.Fatalf("RunLuaScript failed: %s", err.Error()) + return + } + returnValue := l.Get(-1) + if returnValue.Type() != lua.LTTable { + t.Fatalf("expect table output from Lua script, not %s", returnValue.Type().String()) + } + jsonBytes, err := luajson.Encode(returnValue) + if err != nil { + t.Fatalf("encode failed: %s", err.Error()) + return + } + newObj := map[string]string{} + err = json.Unmarshal(jsonBytes, &newObj) + if err != nil { + t.Fatalf("Unmarshal failed: %s", err.Error()) + return + } + if util.DumpJSON(newObj) != cs.expectResult() { + t.Fatalf("expect(%s), but get (%s)", cs.expectResult(), util.DumpJSON(newObj)) + } + }) + } +} diff --git a/pkg/util/meta.go b/pkg/util/meta.go new file mode 100644 index 00000000..605055c2 --- /dev/null +++ b/pkg/util/meta.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "os" + +func GetRolloutNamespace() string { + if ns := os.Getenv("POD_NAMESPACE"); len(ns) > 0 { + return ns + } + return "kruise-rollout" +} diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index 60d57657..80f34b5c 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -193,9 +193,6 @@ func validateRolloutSpecCanaryTraffic(traffic *appsv1alpha1.TrafficRouting, fldP if traffic.Ingress.Name == "" { errList = append(errList, field.Invalid(fldPath.Child("Ingress"), traffic.Ingress, "TrafficRouting.Ingress.Ingress cannot be empty")) } - if traffic.Ingress.ClassType != "" && traffic.Ingress.ClassType != "nginx" { - errList = append(errList, field.Invalid(fldPath.Child("Ingress"), traffic.Ingress, "TrafficRouting.Ingress.ClassType only support nginx")) - } } if traffic.Gateway != nil { if traffic.Gateway.HTTPRouteName == nil || *traffic.Gateway.HTTPRouteName == "" { @@ -214,8 +211,8 @@ func validateRolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fldPath *fi for i := range steps { s := &steps[i] - if isTraffic && s.Weight == nil { - return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight cannot be empty for traffic routing`)} + if isTraffic && s.Weight == nil && len(s.Matches) == 0 { + return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight or matches cannot be empty for traffic routing`)} } else if s.Weight == nil && s.Replicas == nil { return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight and replicas cannot be empty at the same time`)} } @@ -231,7 +228,7 @@ func validateRolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fldPath *fi for i := 1; i < stepCount; i++ { prev := &steps[i-1] curr := &steps[i] - if isTraffic && *curr.Weight < *prev.Weight { + if isTraffic && curr.Weight != nil && prev.Weight != nil && *curr.Weight < *prev.Weight { return field.ErrorList{field.Invalid(fldPath.Child("Weight"), steps, `Steps.Weight must be a non decreasing sequence`)} } diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go index 1fe8f3c3..2f2b413c 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go @@ -232,15 +232,6 @@ func TestRolloutValidateCreate(t *testing.T) { // return []client.Object{object} // }, //}, - { - Name: "Wrong Traffic type", - Succeed: false, - GetObject: func() []client.Object { - object := rollout.DeepCopy() - object.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.ClassType = "Whatever" - return []client.Object{object} - }, - }, /**************************************************************** The following cases are conflict cases ***************************************************************/ diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 79b293fd..3527dfa1 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -1638,10 +1638,325 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) }) + + It("V1->V2: A/B testing, header & cookies", func() { + finder := util.NewControllerFinder(k8sClient) + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + headerType := gatewayv1alpha2.HeaderMatchRegularExpression + replica1 := intstr.FromInt(1) + replica2 := intstr.FromInt(2) + rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + { + Matches: []rolloutsv1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Type: &headerType, + Name: "user_id", + Value: "123456", + }, + }, + }, + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "canary-by-cookie", + Value: "demo", + }, + }, + }, + }, + Pause: rolloutsv1alpha1.RolloutPause{}, + Replicas: &replica1, + }, + { + Weight: utilpointer.Int32(30), + Replicas: &replica2, + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + } + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + workload.Spec.Replicas = utilpointer.Int32(3) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("user_id")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-pattern", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("123456")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-cookie", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("demo")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("")) + // canary deployment + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1)) + Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 1)) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + // wait step 2 complete + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-pattern", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-cookie", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("30")) + // canary deployment + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 2)) + Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 2)) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + klog.Infof("rollout(%s) completed, and check", namespace) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) + + It("V1->V2: A/B testing, aliyun-alb, header & cookies", func() { + finder := util.NewControllerFinder(k8sClient) + configmap := &v1.ConfigMap{} + Expect(ReadYamlToObject("./test_data/rollout/rollout-configuration.yaml", configmap)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(context.TODO(), configmap)).NotTo(HaveOccurred()) + defer k8sClient.Delete(context.TODO(), configmap) + + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + replica1 := intstr.FromInt(1) + replica2 := intstr.FromInt(2) + rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + { + Matches: []rolloutsv1alpha1.HttpRouteMatch{ + { + Headers: []gatewayv1alpha2.HTTPHeaderMatch{ + { + Name: "Cookie", + Value: "demo1=value1;demo2=value2", + }, + { + Name: "SourceIp", + Value: "192.168.0.0/16;172.16.0.0/16", + }, + { + Name: "headername", + Value: "headervalue1;headervalue2", + }, + }, + }, + }, + Pause: rolloutsv1alpha1.RolloutPause{}, + Replicas: &replica1, + }, + { + Weight: utilpointer.Int32(30), + Replicas: &replica2, + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + } + rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.ClassType = "aliyun-alb" + CreateObject(rollout) + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + ingress.Annotations = map[string]string{} + ingress.Spec.IngressClassName = utilpointer.String("alb") + CreateObject(ingress) + // workload + workload := &apps.Deployment{} + Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) + workload.Spec.Replicas = utilpointer.Int32(3) + CreateObject(workload) + WaitDeploymentAllPodsReady(workload) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + UpdateDeployment(workload) + By("Update deployment image from(version1) -> to(version2)") + time.Sleep(time.Second * 3) + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + // check stable, canary service & ingress + // stable service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + labIngressAnnotationDefaultPrefix := "alb.ingress.kubernetes.io" + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", labIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", labIngressAnnotationDefaultPrefix)]).Should(Equal("")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/conditions.echoserver-canary", labIngressAnnotationDefaultPrefix)]).Should(Equal(`[{"cookieConfig":{"values":[{"key":"demo1","value":"value1"},{"key":"demo2","value":"value2"}]},"type":"Cookie"},{"sourceIpConfig":{"values":["192.168.0.0/16","172.16.0.0/16"]},"type":"SourceIp"},{"headerConfig":{"key":"headername","values":["headervalue1","headervalue2"]},"type":"Header"}]`)) + // canary deployment + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1)) + Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 1)) + time.Sleep(time.Hour) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + // wait step 2 complete + WaitRolloutCanaryStepPaused(rollout.Name, 2) + // canary ingress + cIngress = &netv1.Ingress{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", labIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/conditions.echoserver-canary", labIngressAnnotationDefaultPrefix)]).Should(Equal("")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", labIngressAnnotationDefaultPrefix)]).Should(Equal("30")) + // canary deployment + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 2)) + Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 2)) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + // wait rollout complete + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + klog.Infof("rollout(%s) completed, and check", namespace) + // check service & ingress & deployment + // ingress + Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred()) + cIngress = &netv1.Ingress{} + Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) + // service + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) + cService = &v1.Service{} + Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) + // deployment + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Spec.Paused).Should(BeFalse()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + for _, env := range workload.Spec.Template.Spec.Containers[0].Env { + if env.Name == "NODE_NAME" { + Expect(env.Value).Should(Equal("version2")) + } + } + // check progressing succeed + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) + }) }) KruiseDescribe("Canary rollout with Gateway API", func() { - It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3", func() { + It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3, Gateway api", func() { By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/gateway/rollout-test.yaml", rollout)).ToNot(HaveOccurred()) @@ -2054,7 +2369,7 @@ var _ = SIGDescribe("Rollout", func() { } }) - It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + It("V1->V2: cloneSet, Percentage, 20%,40% and continuous release v3", func() { By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) diff --git a/test/e2e/test_data/rollout/rollout-configuration.yaml b/test/e2e/test_data/rollout/rollout-configuration.yaml new file mode 100644 index 00000000..7499d78e --- /dev/null +++ b/test/e2e/test_data/rollout/rollout-configuration.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kruise-rollout-configuration + namespace: kruise-rollout +data: + "lua.traffic.routing.ingress.aliyun-alb": | + function split(input, delimiter) + local arr = {} + string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end) + return arr + end + annotations = obj.annotations + if ( not annotations ) + then + annotations = {} + end + + annotations["alb.ingress.kubernetes.io/canary"] = "true" + annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil + annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil + annotations["alb.ingress.kubernetes.io/canary-weight"] = nil + conditionKey = string.format("alb.ingress.kubernetes.io/conditions.%s", obj.canaryService) + annotations[conditionKey] = nil + if ( obj.weight ~= "-1" ) + then + annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight + end + if ( not obj.matches ) + then + return annotations + end + conditions = {} + match = obj.matches[1] + for _,header in ipairs(match.headers) do + local condition = {} + if ( header.name == "Cookie" ) + then + condition.type = "Cookie" + condition.cookieConfig = {} + cookies = split(header.value, ";") + values = {} + for _,cookieStr in ipairs(cookies) do + cookie = split(cookieStr, "=") + value = {} + value.key = cookie[1] + value.value = cookie[2] + table.insert(values, value) + end + condition.cookieConfig.values = values + elseif ( header.name == "SourceIp" ) + then + condition.type = "SourceIp" + condition.sourceIpConfig = {} + ips = split(header.value, ";") + values = {} + for _,ip in ipairs(ips) do + table.insert(values, ip) + end + condition.sourceIpConfig.values = values + else + condition.type = "Header" + condition.headerConfig = {} + condition.headerConfig.key = header.name + vals = split(header.value, ";") + values = {} + for _,val in ipairs(vals) do + table.insert(values, val) + end + condition.headerConfig.values = values + end + table.insert(conditions, condition) + end + annotations[conditionKey] = json.encode(conditions) + return annotations