From 29fd4156165063ebd785c70abc90a35370a22795 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 17 Sep 2021 13:20:33 -0500 Subject: [PATCH] Update to v1.0.0 of Kubernetes NGINX ingress controller --- _run/common-kind.mk | 2 + _run/ingress-nginx-class.yaml | 8 + _run/ingress-nginx.yaml | 165 ++++++++++-------- provider/cluster/kube/client_ingress.go | 8 +- .../provider_migrate_to_hostname_operator.md | 16 +- 5 files changed, 120 insertions(+), 79 deletions(-) create mode 100644 _run/ingress-nginx-class.yaml diff --git a/_run/common-kind.mk b/_run/common-kind.mk index 9dfc58de2a..038eb89425 100644 --- a/_run/common-kind.mk +++ b/_run/common-kind.mk @@ -33,6 +33,7 @@ PROVIDER_HOST ?= $(PROVIDER_HOSTNAME):$(KIND_HTTP_PORT) PROVIDER_ENDPOINT ?= http://$(PROVIDER_HOST) INGRESS_CONFIG_PATH ?= ../ingress-nginx.yaml +INGRESS_CLASS_CONFIG_PATH ?= ../ingress-nginx-class.yaml CALICO_MANIFEST ?= https://docs.projectcalico.org/v3.8/manifests/calico.yaml .PHONY: app-http-port @@ -65,6 +66,7 @@ kind-cluster-create: $(KIND) --image "$(KIND_IMG)" kubectl label nodes $(KIND_NAME)-control-plane akash.network/role=ingress kubectl apply -f "$(INGRESS_CONFIG_PATH)" + kubectl apply -f "$(INGRESS_CLASS_CONFIG_PATH)" "$(AKASH_ROOT)/script/setup-kind.sh" .PHONY: kind-cluster-calico-create diff --git a/_run/ingress-nginx-class.yaml b/_run/ingress-nginx-class.yaml new file mode 100644 index 0000000000..94756324ce --- /dev/null +++ b/_run/ingress-nginx-class.yaml @@ -0,0 +1,8 @@ +apiVersion: "networking.k8s.io/v1" +kind: "IngressClass" +metadata: + name: "akash-ingress-class" + labels: + akash.network: "true" +spec: + controller: "k8s.io/ingress-nginx" diff --git a/_run/ingress-nginx.yaml b/_run/ingress-nginx.yaml index 8099e32879..3ba4bd9347 100644 --- a/_run/ingress-nginx.yaml +++ b/_run/ingress-nginx.yaml @@ -13,24 +13,25 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx namespace: ingress-nginx +automountServiceAccountToken: true --- # Source: ingress-nginx/templates/controller-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller @@ -42,10 +43,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm name: ingress-nginx rules: @@ -75,8 +76,7 @@ rules: - list - watch - apiGroups: - - extensions - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingresses verbs: @@ -91,14 +91,13 @@ rules: - create - patch - apiGroups: - - extensions - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingresses/status verbs: - update - apiGroups: - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingressclasses verbs: @@ -111,10 +110,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm name: ingress-nginx roleRef: @@ -131,10 +130,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx @@ -166,8 +165,7 @@ rules: - list - watch - apiGroups: - - extensions - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingresses verbs: @@ -175,14 +173,13 @@ rules: - list - watch - apiGroups: - - extensions - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingresses/status verbs: - update - apiGroups: - - networking.k8s.io # k8s 1.14+ + - networking.k8s.io resources: - ingressclasses verbs: @@ -194,7 +191,7 @@ rules: resources: - configmaps resourceNames: - - ingress-controller-leader-nginx + - ingress-controller-leader verbs: - get - update @@ -217,10 +214,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx @@ -239,10 +236,10 @@ apiVersion: v1 kind: Service metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller-admission @@ -253,6 +250,7 @@ spec: - name: https-webhook port: 443 targetPort: webhook + appProtocol: https selector: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx @@ -264,10 +262,10 @@ kind: Service metadata: annotations: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller @@ -279,10 +277,12 @@ spec: port: 80 protocol: TCP targetPort: http + appProtocol: http - name: https port: 443 protocol: TCP targetPort: https + appProtocol: https selector: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx @@ -293,10 +293,10 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller name: ingress-nginx-controller @@ -308,10 +308,6 @@ spec: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller revisionHistoryLimit: 10 - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate minReadySeconds: 0 template: metadata: @@ -323,7 +319,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: controller - image: k8s.gcr.io/ingress-nginx/controller:v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a + image: k8s.gcr.io/ingress-nginx/controller:v1.0.0@sha256:0851b34f69f69352bf168e6ccf30e1e20714a264ab1ecd1933e4d8c0fc3215c6 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -333,13 +329,11 @@ spec: args: - /nginx-ingress-controller - --election-id=ingress-controller-leader - - --ingress-class=nginx + - --controller-class=k8s.io/ingress-nginx - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - --validating-webhook=:8443 - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key - - --publish-status-address=localhost - - --enable-ssl-passthrough=true securityContext: capabilities: drop: @@ -360,25 +354,25 @@ spec: - name: LD_PRELOAD value: /usr/local/lib/libmimalloc.so livenessProbe: + failureThreshold: 5 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 successThreshold: 1 - failureThreshold: 5 + timeoutSeconds: 1 readinessProbe: + failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 successThreshold: 1 - failureThreshold: 3 + timeoutSeconds: 1 ports: - name: http containerPort: 80 @@ -388,7 +382,7 @@ spec: containerPort: 443 protocol: TCP hostPort: 443 - - name: webhook + - name: webhook containerPort: 8443 protocol: TCP volumeMounts: @@ -402,17 +396,31 @@ spec: nodeSelector: kubernetes.io/os: linux akash.network/role: ingress - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal serviceAccountName: ingress-nginx - terminationGracePeriodSeconds: 0 + terminationGracePeriodSeconds: 300 volumes: - name: webhook-cert secret: secretName: ingress-nginx-admission --- +# Source: ingress-nginx/templates/controller-ingressclass.yaml +# We don't support namespaced ingressClass yet +# So a ClusterRole and a ClusterRoleBinding is required +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + helm.sh/chart: ingress-nginx-4.0.1 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 1.0.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: nginx + namespace: ingress-nginx +spec: + controller: k8s.io/ingress-nginx +--- # Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml # before changing this value, check the required kubernetes version # https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites @@ -420,10 +428,10 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook name: ingress-nginx-admission @@ -434,7 +442,7 @@ webhooks: - apiGroups: - networking.k8s.io apiVersions: - - v1beta1 + - v1 operations: - CREATE - UPDATE @@ -444,29 +452,28 @@ webhooks: sideEffects: None admissionReviewVersions: - v1 - - v1beta1 clientConfig: service: namespace: ingress-nginx name: ingress-nginx-controller-admission - path: /networking/v1beta1/ingresses + path: /networking/v1/ingresses --- # Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: ingress-nginx-admission + namespace: ingress-nginx annotations: helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook - namespace: ingress-nginx --- # Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -477,10 +484,10 @@ metadata: helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook rules: @@ -501,10 +508,10 @@ metadata: helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook roleRef: @@ -521,17 +528,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ingress-nginx-admission + namespace: ingress-nginx annotations: helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook - namespace: ingress-nginx rules: - apiGroups: - '' @@ -546,17 +553,17 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ingress-nginx-admission + namespace: ingress-nginx annotations: helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook - namespace: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -571,32 +578,32 @@ apiVersion: batch/v1 kind: Job metadata: name: ingress-nginx-admission-create + namespace: ingress-nginx annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook - namespace: ingress-nginx spec: template: metadata: name: ingress-nginx-admission-create labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook spec: containers: - name: create - image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068 imagePullPolicy: IfNotPresent args: - create @@ -610,6 +617,8 @@ spec: fieldPath: metadata.namespace restartPolicy: OnFailure serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux securityContext: runAsNonRoot: true runAsUser: 2000 @@ -619,32 +628,32 @@ apiVersion: batch/v1 kind: Job metadata: name: ingress-nginx-admission-patch + namespace: ingress-nginx annotations: helm.sh/hook: post-install,post-upgrade helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook - namespace: ingress-nginx spec: template: metadata: name: ingress-nginx-admission-patch labels: - helm.sh/chart: ingress-nginx-3.23.0 + helm.sh/chart: ingress-nginx-4.0.1 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: 0.44.0 + app.kubernetes.io/version: 1.0.0 app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: admission-webhook spec: containers: - name: patch - image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068 imagePullPolicy: IfNotPresent args: - patch @@ -660,6 +669,8 @@ spec: fieldPath: metadata.namespace restartPolicy: OnFailure serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux securityContext: runAsNonRoot: true runAsUser: 2000 diff --git a/provider/cluster/kube/client_ingress.go b/provider/cluster/kube/client_ingress.go index 461e5e5770..3267e7890f 100644 --- a/provider/cluster/kube/client_ingress.go +++ b/provider/cluster/kube/client_ingress.go @@ -16,6 +16,10 @@ import ( "strings" ) +const ( + akashIngressClassName = "akash-ingress-class" +) + func kubeNginxIngressAnnotations(directive ctypes.ConnectHostnameToDeploymentDirective) map[string]string { // For kubernetes/ingress-nginx // https://github.com/kubernetes/ingress-nginx @@ -73,6 +77,7 @@ func (c *client) ConnectHostnameToDeployment(ctx context.Context, directive ctyp labels[akashManagedLabelName] = "true" appendLeaseLabels(directive.LeaseID, labels) + ingressClassName := akashIngressClassName obj := &netv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: ingressName, @@ -80,7 +85,8 @@ func (c *client) ConnectHostnameToDeployment(ctx context.Context, directive ctyp Annotations: kubeNginxIngressAnnotations(directive), }, Spec: netv1.IngressSpec{ - Rules: rules, + IngressClassName: &ingressClassName, + Rules: rules, }, } diff --git a/script/provider_migrate_to_hostname_operator.md b/script/provider_migrate_to_hostname_operator.md index 31ba04aa2a..875b2317e4 100644 --- a/script/provider_migrate_to_hostname_operator.md +++ b/script/provider_migrate_to_hostname_operator.md @@ -29,12 +29,26 @@ This is meant to be a stopgap measure. *Step 3*: Run `python3 provider_migrate_to_hostname_operator.py backup`. This creates two files. The first file is `provider_hosts.pickle` which is the data used to rebuild the objects later. The second file is `ingresses_backup.json` which is just a raw backup of each ingress object as retrieved from Kubernetes -*Step 4*: Make sure the provider host CRD stored in `pkg/apis/akash.network/v1/provider_hosts_crd.yaml` is applied to your kubernetes cluster by running +*Step 4*: + +Apply provider host CRD stored in `pkg/apis/akash.network/v1/provider_hosts_crd.yaml` is applied to your kubernetes cluster by running ``` kubectl apply -f pkg/apis/akash.network/v1/provider_hosts_crd.yaml ``` +Apply the newest ingress controller stored in `_run/ingress-nginx.yaml` + +``` +kubectl apply -f _run/ingress-nginx.yaml +``` + +Apply the newest ingress class stored in `_run/ingress-nginx.yaml` + +``` +kubectl apply -f _run/ingress-nginx-class.yaml +``` + *Step 5*: Run `python3 provider_migrate_to_hostname_operator.py create`. This parses the data and adds the provider hosts entries in kubernetes. *Step 6*: Run `python3 provider_migrate_to_hostname_operator.py purge`. This removes all the ingress objects from kubernetes.