better handle inverse-proxy configmap for better GC (#2391)

* add label and namespace to resource created inside pod

* fix

* done

* update existing configmap for better GC

* update dependency to make sure configmap got created before run script

* self fix

* all done

* reuse existing

* reuse

Co-authored-by: renmingu <40223865+renmingu@users.noreply.github.com>

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml

@@ -32,8 +32,6 @@ spec:
     - name: Console
       value: 'https://console.cloud.google.com/ai-platform/pipelines/clusters'
   componentKinds:
-    - group: apiextensions.k8s.io/v1beta1
-      kind: CustomResourceDefinition
     - group: v1
       kind: ServiceAccount
     - group: rbac.authorization.k8s.io/v1

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/proxy.yaml

@@ -58,4 +58,11 @@ spec:
         - image: {{ .Values.images.proxyagent }}
           imagePullPolicy: IfNotPresent
           name: proxy-agent
-      serviceAccountName: proxy-agent-runner 
+      serviceAccountName: proxy-agent-runner
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inverse-proxy-config
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}

manifests/kustomize/OWNERS

@@ -1,6 +1,7 @@
 approvers:
   - Bobgy
   - IronPan
+  - rmgogogo
 reviewers:
   - Bobgy
   - IronPan

manifests/kustomize/base/proxy/kustomization.yaml

@@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
+- proxy-configmap.yaml
 - proxy-deployment.yaml
 - proxy-role.yaml
 - proxy-rolebinding.yaml
-- proxy-sa.yaml
+- proxy-sa.yaml

manifests/kustomize/base/proxy/proxy-configmap.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inverse-proxy-config

proxy/OWNERS

@@ -1,5 +1,7 @@
 approvers:
   - IronPan
+  - rmgogogo
 reviewers:
   - IronPan
-  - ojarjur
+  - rmgogogo
+  - ojarjur

proxy/attempt-register-vm-on-proxy.sh

@@ -35,11 +35,16 @@ function run-proxy-agent {
         --health-check-unhealthy-threshold=${HEALTH_CHECK_UNHEALTHY_THRESHOLD}
 }
 
-# Check if the cluster already have proxy agent installed by checking ConfigMap.
-if kubectl get configmap inverse-proxy-config; then
-  # If ConfigMap already exist, reuse the existing endpoint (a.k.a BACKEND_ID) and same ProxyUrl.
+# Check if already has Hostname value.
+# It's possible the pod got restarted, in such case we continue use the existing
+# hostname. In proxy server side, it doesn't check VM name even pod got moved to
+# new VM.
+HOSTNAME=$(kubectl get configmap inverse-proxy-config -o json | jq -r ".data.Hostname // empty")
+if [[ -n "${HOSTNAME}" ]]; then
+  echo "Reuse existing hostname"
   PROXY_URL=$(kubectl get configmap inverse-proxy-config -o json | jq -r ".data.ProxyUrl")
   BACKEND_ID=$(kubectl get configmap inverse-proxy-config -o json | jq -r ".data.BackendId")
+  # If ConfigMap already exist, reuse the existing endpoint (a.k.a BACKEND_ID) and same ProxyUrl.
   run-proxy-agent
   exit 0
 fi
@@ -72,9 +77,14 @@ echo "Hostname: ${HOSTNAME}"
 echo "Backend id: ${BACKEND_ID}"
 
 # Store the registration information in a ConfigMap
-kubectl create configmap inverse-proxy-config \
-        --from-literal=ProxyUrl=${PROXY_URL} \
-        --from-literal=BackendId=${BACKEND_ID} \
-        --from-literal=Hostname=${HOSTNAME}
+PATCH_TEMP='{"data": {"Hostname":"'${HOSTNAME}'","ProxyUrl":"'${PROXY_URL}'","BackendId":"'${BACKEND_ID}'"}}'
+PATCH_JSON=$(printf "${PATCH_TEMP}" "${HOSTNAME}" "${PROXY_URL}" "${BACKEND_ID}")
+echo "PACTH_JSON: ${PATCH_JSON}"
+
+kubectl patch configmap/inverse-proxy-config \
+    --type merge \
+    --patch "${PATCH_JSON}"
+
+echo "Patched configmap/inverse-proxy-config"
 
 run-proxy-agent