kube-apiserver: manifest and config updates

kube-apiserver: Change kube-apiserver liveness probe config setup
Prior OSS attempt: kubernetes#94076
Bug: b/164955956
Change-Id: Ia2f75ac5e97f4eb6aa4dc9b00fc2bb3112db9360

kube-apiserver: Set profiling false for kube-apiserver
Change-Id: I052a53d72acfc3c5aa8d03f695beb93cadc0be87

kube-apiserver: allow adding memory limit to kube-apiserver via the KUBE_APISERVER_MEMORY_LIMIT environment variable
bug: 181991030
Change-Id: I1a01766e6f2415eba31d6d865f7f39d7855f969e

kube-apiserver: Drop removed insecure --address and --insecure-port flags
GKE config script version of kubernetes#106859
Bug: 209962139
Change-Id: I551cfc12fd94a927383fcff89013a3a78fb61592

kube-apiserver: Readd KUBE_APISERVER_EVENT_TTL_SEC env for kube-apiserver --event-ttl
Bug: b/176806841
Change-Id: I5c651a555287af0e891e9592dfb7c172b6bb24a2

kube-apiserver: Re-enable the deprecated behavior of CommonName on X.509 as a diagnoser to detect the fleet level impact on the deprecation
Change-Id: If4962b6ecb3294878bfa37ab0ebcd4441fae7140

kube-apiserver: Update permissions of /etc/srv/kubernetes/abac-authz-policy.jsonl if kube-apiserver is running as non-root.
BUG=199669294
Change-Id: Ic9ae9add0a35b7df99781e470c917b2427bee289

kube-apiserver: Remove reference to deleted target-ram-mb flag
Change-Id: I57d0c80eb2285ec9d93b075b1845f1805e0fd748

kube-apiserver: Re-enable SHA1 signatures with GODEBUG=x509sha1=1
Bug: b/227456358
Bug: b/226424430
Change-Id: I5165e6c2fe73e8e1b2a617ced591133228b6d275

kube-apiserver: use --api-audiences as --service-account-api-audiences is deprecated

Copy of kubernetes#103078

Change-Id: I88c2f2eb8bde4378b115e01cbbe9700c27f03955

Expose UDS with profiling data for kube-apiserver.

Added in kubernetes#114191.

See go/no-gke-tcp-pprof for policy.

Refs b/273485199

Change-Id: Ic772c9249468fdf7f516d62bd2a0bfbfe933c1bb

Allow specifying terminationGracePeriodSeconds for kube-apiserver.

Refs b/252987333

Change-Id: I20699501f429630fe74531cb086091eb9ed3611c

Run kube-apiserver with cloud-provider=external

Bug: 299159412
Change-Id: I8db8e72377f1c63a3874ee9f0567be264c40ba58

gke/cluster/gce/config-default.sh

@@ -558,3 +558,6 @@ export WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${WINDOWS_NODE_PROBLEM_DETECT
 # TLS_CIPHER_SUITES defines cipher suites allowed to be used by kube-apiserver.
 # If this variable is unset or empty, kube-apiserver will allow its default set of cipher suites.
 export TLS_CIPHER_SUITES=""
+
+# Optional: KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS controls how many seconds kube-apiserver will have to terminate gracefully.
+export KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS="${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}"

gke/cluster/gce/config-test.sh

@@ -610,3 +610,6 @@ export WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${WINDOWS_NODE_PROBLEM_DETECT
 # TLS_CIPHER_SUITES defines cipher suites allowed to be used by kube-apiserver.
 # If this variable is unset or empty, kube-apiserver will allow its default set of cipher suites.
 export TLS_CIPHER_SUITES=""
+
+# Optional: KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS controls how many seconds kube-apiserver will have to terminate gracefully.
+export KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS="${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}"

gke/cluster/gce/gci/configure-helper.sh

@@ -2011,7 +2011,6 @@ function start-konnectivity-server {
 #   DOCKER_REGISTRY
 #   FLEXVOLUME_HOSTPATH_MOUNT
 #   FLEXVOLUME_HOSTPATH_VOLUME
-#   INSECURE_PORT_MAPPING
 function compute-master-manifest-variables {
   CLOUD_CONFIG_OPT=""
   CLOUD_CONFIG_VOLUME=""
@@ -2032,15 +2031,6 @@ function compute-master-manifest-variables {
     FLEXVOLUME_HOSTPATH_MOUNT="{ \"name\": \"flexvolumedir\", \"mountPath\": \"${VOLUME_PLUGIN_DIR}\", \"readOnly\": true},"
     FLEXVOLUME_HOSTPATH_VOLUME="{ \"name\": \"flexvolumedir\", \"hostPath\": {\"path\": \"${VOLUME_PLUGIN_DIR}\"}},"
   fi
-
-  INSECURE_PORT_MAPPING=""
-  if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" == "true" ]]; then
-    # INSECURE_PORT_MAPPING is used by sed
-    # shellcheck disable=SC2089
-    INSECURE_PORT_MAPPING='{ "name": "local", "containerPort": 8080, "hostPort": 8080},'
-  fi
-  # shellcheck disable=SC2090
-  export INSECURE_PORT_MAPPING
 }
 
 # A helper function that bind mounts kubelet dirs for running mount in a chroot
@@ -3293,9 +3283,7 @@ function main() {
     GCE_GLBC_TOKEN="$(secure_random 32)"
   fi
   ADDON_MANAGER_TOKEN="$(secure_random 32)"
-  if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then
-    KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)"
-  fi
+  KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)"
   if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then
     KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)"
   fi

gke/cluster/gce/gci/configure-kubeapiserver.sh

@@ -48,38 +48,33 @@ function configure-etcd-params {
   if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then
     params_ref+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s"
   fi
+
+  if [[ -n "${KUBE_APISERVER_EVENT_TTL_SEC:-}" ]]; then
+    params_ref+=" --event-ttl=${KUBE_APISERVER_EVENT_TTL_SEC}s"
+  fi
 }
 
 # Starts kubernetes apiserver.
 # It prepares the log file, loads the docker image, calculates variables, sets them
 # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
 #
 # Assumed vars (which are calculated in function compute-master-manifest-variables)
-#   CLOUD_CONFIG_OPT
-#   CLOUD_CONFIG_VOLUME
-#   CLOUD_CONFIG_MOUNT
 #   DOCKER_REGISTRY
-#   INSECURE_PORT_MAPPING
 function start-kube-apiserver {
   echo "Start kubernetes api-server"
   prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" "${KUBE_API_SERVER_RUNASUSER:-0}"
   prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" "${KUBE_API_SERVER_RUNASUSER:-0}"
 
   # Calculate variables and assemble the command line.
-  local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
-  params+=" --address=127.0.0.1"
+  local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-}"
   params+=" --allow-privileged=true"
-  params+=" --cloud-provider=gce"
+  params+=" --cloud-provider=external"
   params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}"
 
   # params is passed by reference, so no "$"
   configure-etcd-params params
 
   params+=" --secure-port=443"
-  if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then
-    # Default is :8080
-    params+=" --insecure-port=0"
-  fi
   params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}"
   params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}"
   if [[ -n "${OLD_MASTER_IP:-}" ]]; then
@@ -137,16 +132,12 @@ function start-kube-apiserver {
       params=$(append-param-if-not-present "${params}" "max-requests-inflight" 1500)
       params=$(append-param-if-not-present "${params}" "max-mutating-requests-inflight" 500)
     fi
-    # Set amount of memory available for apiserver based on number of nodes.
-    # TODO: Once we start setting proper requests and limits for apiserver
-    # we should reuse the same logic here instead of current heuristic.
-    params=$(append-param-if-not-present "${params}" "target-ram-mb" $((NUM_NODES * 60)))
   fi
   if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
     params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
   fi
   params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}"
-  params+=" --service-account-api-audiences=${SERVICEACCOUNT_ISSUER}"
+  params+=" --api-audiences=${SERVICEACCOUNT_ISSUER}"
   params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}"
 
   local audit_policy_config_mount=""
@@ -314,6 +305,7 @@ function start-kube-apiserver {
 
     params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl"
     authorization_mode+=",ABAC"
+    chown "${KUBE_API_SERVER_RUNASUSER:-0}":"${KUBE_API_SERVER_RUNASGROUP:-0}" /etc/srv/kubernetes/abac-authz-policy.jsonl
   fi
 
   local webhook_config_mount=""
@@ -361,10 +353,21 @@ function start-kube-apiserver {
     fi
     container_env+="{\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"}"
   fi
+
+  # b/227456358
+  if [[ -n "${container_env}" ]]; then
+    container_env="${container_env}, "
+  fi
+  container_env+='{"name": "GODEBUG", "value": "x509sha1=1"}'
+
   if [[ -n "${container_env}" ]]; then
     container_env="\"env\":[${container_env}],"
   fi
 
+  # Create directory for debug socket.
+  mkdir -p /etc/srv/kubernetes/kube-apiserver/
+  chown -R "${KUBE_API_SERVER_RUNASUSER:-0}":"${KUBE_API_SERVER_RUNASGROUP:-0}" /etc/srv/kubernetes/kube-apiserver/
+
   local -r src_file="${src_dir}/kube-apiserver.manifest"
 
   # params is passed by reference, so no "$"
@@ -375,22 +378,24 @@ function start-kube-apiserver {
     healthcheck_ip=$(hostname -i)
   fi
 
+  local termination_grace_period_seconds=""
+  if [[ -n "${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}" ]]; then
+    termination_grace_period_seconds="\"terminationGracePeriodSeconds\": ${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS},"
+  fi
+
   params="$(convert-manifest-params "${params}")"
   # Evaluate variables.
   local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}"
   sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
-  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
-  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
   sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
   sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
   sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
+  sed -i -e "s@{{kube_apiserver_memory_limit}}@${KUBE_APISERVER_MEMORY_LIMIT:-1Ti}@g" "${src_file}"
   sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}"
+  sed -i -e "s@{{liveness_probe_timeout}}@${KUBE_APISERVER_LIVENESS_PROBE_TIMEOUT_SEC:-15}@g" "${src_file}"
   sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
-  sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}"
-  sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
-  sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
   sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}"
   sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"
@@ -406,6 +411,7 @@ function start-kube-apiserver {
   sed -i -e "s@{{konnectivity_socket_mount}}@${default_konnectivity_socket_mnt}@g" "${src_file}"
   sed -i -e "s@{{konnectivity_socket_volume}}@${default_konnectivity_socket_vol}@g" "${src_file}"
   sed -i -e "s@{{healthcheck_ip}}@${healthcheck_ip}@g" "${src_file}"
+  sed -i -e "s@{{termination_grace_period_seconds}}@${termination_grace_period_seconds}@g" "${src_file}"
 
   if [[ -n "${KUBE_API_SERVER_RUNASUSER:-}" && -n "${KUBE_API_SERVER_RUNASGROUP:-}" && -n "${KUBE_PKI_READERS_GROUP:-}" ]]; then
     sed -i -e "s@{{runAsUser}}@\"runAsUser\": ${KUBE_API_SERVER_RUNASUSER},@g" "${src_file}"

gke/cluster/gce/gci/testdata/kube-apiserver/base.template

@@ -9,7 +9,6 @@ readonly APISERVER_SERVER_KEY_PATH=/foo/bar
 readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
 readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
 readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
-readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080},"
 readonly DOCKER_REGISTRY="k8s.gcr.io"
 readonly ENABLE_LEGACY_ABAC=false
 readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests

gke/cluster/gce/manifests/kube-apiserver.manifest

@@ -21,20 +21,27 @@
 "priorityClassName": "system-node-critical",
 "priority": 2000001000,
 "hostNetwork": true,
+{{termination_grace_period_seconds}}
 "containers":[
     {
     "name": "kube-apiserver",
     {{containerSecurityContext}}
     "image": "{{pillar['kube_docker_registry']}}/kube-apiserver-amd64:{{pillar['kube-apiserver_docker_tag']}}",
     "resources": {
       "requests": {
+        "memory": "0",
         "cpu": "250m"
+      },
+      "limits": {
+        "memory": "{{kube_apiserver_memory_limit}}"
       }
     },
     "command": [
                  "/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true",
                  "/usr/local/bin/kube-apiserver",
                  "--allow-privileged={{pillar['allow_privileged']}}",
+                 "--profiling=false",
+                 "--debug-socket-path=/etc/srv/kubernetes/kube-apiserver/debug.socket",
                  {{params}}
                ],
     {{container_env}}
@@ -46,7 +53,9 @@
         "path": "/livez?exclude=etcd&exclude=kms-provider-0&exclude=kms-provider-1"
       },
       "initialDelaySeconds": {{liveness_probe_initial_delay}},
-      "timeoutSeconds": 15
+      "timeoutSeconds": {{liveness_probe_timeout}},
+      "periodSeconds": 5,
+      "failureThreshold": 6
     },
     "readinessProbe": {
       "httpGet": {
@@ -59,16 +68,13 @@
       "timeoutSeconds": 15
     },
     "ports":[
-      {{insecure_port_mapping}}
       { "name": "https",
       "containerPort": {{secure_port}},
       "hostPort": {{secure_port}}}
     ],
     "volumeMounts": [
         {{kms_socket_mount}}
         {{encryption_provider_mount}}
-        {{cloud_config_mount}}
-        {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
         {{webhook_authn_config_mount}}
         {{csc_config_mount}}
@@ -79,6 +85,9 @@
         { "name": "srvkube",
         "mountPath": "/etc/srv/kubernetes",
         "readOnly": true},
+        { "name": "srvkubeapiserver",
+        "mountPath": "/etc/srv/kubernetes/kube-apiserver",
+        "readOnly": false},
         { "name": "logfile",
         "mountPath": "/var/log/kube-apiserver.log",
         "readOnly": false},
@@ -109,8 +118,6 @@
 "volumes":[
   {{kms_socket_volume}}
   {{encryption_provider_volume}}
-  {{cloud_config_volume}}
-  {{additional_cloud_config_volume}}
   {{webhook_config_volume}}
   {{webhook_authn_config_volume}}
   {{csc_config_volume}}
@@ -122,6 +129,11 @@
     "hostPath": {
         "path": "/etc/srv/kubernetes"}
   },
+  { "name": "srvkubeapiserver",
+    "hostPath": {
+        "path": "/etc/srv/kubernetes/kube-apiserver",
+        "type": "DirectoryOrCreate"}
+  },
   { "name": "logfile",
     "hostPath": {
         "path": "/var/log/kube-apiserver.log",

gke/cluster/gce/util.sh

@@ -1402,6 +1402,11 @@ EOF
     if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then
       cat >>"$file" <<EOF
 KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC}")
+EOF
+    fi
+    if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_TIMEOUT_SEC:-}" ]; then
+      cat >>$file <<EOF
+KUBE_APISERVER_LIVENESS_PROBE_TIMEOUT_SEC: $(yaml-quote ${KUBE_APISERVER_LIVENESS_PROBE_TIMEOUT_SEC})
 EOF
     fi
     if [ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]; then