diff --git a/gke/cluster/gce/config-default.sh b/gke/cluster/gce/config-default.sh index b9ad8c107975a..a203838e54a21 100755 --- a/gke/cluster/gce/config-default.sh +++ b/gke/cluster/gce/config-default.sh @@ -558,3 +558,6 @@ export WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${WINDOWS_NODE_PROBLEM_DETECT # TLS_CIPHER_SUITES defines cipher suites allowed to be used by kube-apiserver. # If this variable is unset or empty, kube-apiserver will allow its default set of cipher suites. export TLS_CIPHER_SUITES="" + +# Optional: KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS controls how many seconds kube-apiserver will have to terminate gracefully. +export KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS="${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}" \ No newline at end of file diff --git a/gke/cluster/gce/config-test.sh b/gke/cluster/gce/config-test.sh index e6f5280d1f7a4..30e0a5604f18d 100755 --- a/gke/cluster/gce/config-test.sh +++ b/gke/cluster/gce/config-test.sh @@ -610,3 +610,6 @@ export WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${WINDOWS_NODE_PROBLEM_DETECT # TLS_CIPHER_SUITES defines cipher suites allowed to be used by kube-apiserver. # If this variable is unset or empty, kube-apiserver will allow its default set of cipher suites. export TLS_CIPHER_SUITES="" + +# Optional: KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS controls how many seconds kube-apiserver will have to terminate gracefully. +export KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS="${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}" \ No newline at end of file diff --git a/gke/cluster/gce/gci/configure-helper.sh b/gke/cluster/gce/gci/configure-helper.sh index 270003a764b89..2d670eb2e8123 100644 --- a/gke/cluster/gce/gci/configure-helper.sh +++ b/gke/cluster/gce/gci/configure-helper.sh @@ -2011,7 +2011,6 @@ function start-konnectivity-server { # DOCKER_REGISTRY # FLEXVOLUME_HOSTPATH_MOUNT # FLEXVOLUME_HOSTPATH_VOLUME -# INSECURE_PORT_MAPPING function compute-master-manifest-variables { CLOUD_CONFIG_OPT="" CLOUD_CONFIG_VOLUME="" @@ -2032,15 +2031,6 @@ function compute-master-manifest-variables { FLEXVOLUME_HOSTPATH_MOUNT="{ \"name\": \"flexvolumedir\", \"mountPath\": \"${VOLUME_PLUGIN_DIR}\", \"readOnly\": true}," FLEXVOLUME_HOSTPATH_VOLUME="{ \"name\": \"flexvolumedir\", \"hostPath\": {\"path\": \"${VOLUME_PLUGIN_DIR}\"}}," fi - - INSECURE_PORT_MAPPING="" - if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" == "true" ]]; then - # INSECURE_PORT_MAPPING is used by sed - # shellcheck disable=SC2089 - INSECURE_PORT_MAPPING='{ "name": "local", "containerPort": 8080, "hostPort": 8080},' - fi - # shellcheck disable=SC2090 - export INSECURE_PORT_MAPPING } # A helper function that bind mounts kubelet dirs for running mount in a chroot @@ -3293,9 +3283,7 @@ function main() { GCE_GLBC_TOKEN="$(secure_random 32)" fi ADDON_MANAGER_TOKEN="$(secure_random 32)" - if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then - KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)" - fi + KUBE_BOOTSTRAP_TOKEN="$(secure_random 32)" if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then KONNECTIVITY_SERVER_TOKEN="$(secure_random 32)" fi diff --git a/gke/cluster/gce/gci/configure-kubeapiserver.sh b/gke/cluster/gce/gci/configure-kubeapiserver.sh index 2fc9461aad962..f82c12517303a 100644 --- a/gke/cluster/gce/gci/configure-kubeapiserver.sh +++ b/gke/cluster/gce/gci/configure-kubeapiserver.sh @@ -48,6 +48,10 @@ function configure-etcd-params { if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then params_ref+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" fi + + if [[ -n "${KUBE_APISERVER_EVENT_TTL_SEC:-}" ]]; then + params_ref+=" --event-ttl=${KUBE_APISERVER_EVENT_TTL_SEC}s" + fi } # Starts kubernetes apiserver. @@ -59,7 +63,6 @@ function configure-etcd-params { # CLOUD_CONFIG_VOLUME # CLOUD_CONFIG_MOUNT # DOCKER_REGISTRY -# INSECURE_PORT_MAPPING function start-kube-apiserver { echo "Start kubernetes api-server" prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" "${KUBE_API_SERVER_RUNASUSER:-0}" @@ -67,7 +70,6 @@ function start-kube-apiserver { # Calculate variables and assemble the command line. local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" - params+=" --address=127.0.0.1" params+=" --allow-privileged=true" params+=" --cloud-provider=gce" params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}" @@ -76,10 +78,6 @@ function start-kube-apiserver { configure-etcd-params params params+=" --secure-port=443" - if [[ "${ENABLE_APISERVER_INSECURE_PORT:-false}" != "true" ]]; then - # Default is :8080 - params+=" --insecure-port=0" - fi params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" if [[ -n "${OLD_MASTER_IP:-}" ]]; then @@ -137,16 +135,12 @@ function start-kube-apiserver { params=$(append-param-if-not-present "${params}" "max-requests-inflight" 1500) params=$(append-param-if-not-present "${params}" "max-mutating-requests-inflight" 500) fi - # Set amount of memory available for apiserver based on number of nodes. - # TODO: Once we start setting proper requests and limits for apiserver - # we should reuse the same logic here instead of current heuristic. - params=$(append-param-if-not-present "${params}" "target-ram-mb" $((NUM_NODES * 60))) fi if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" fi params+=" --service-account-issuer=${SERVICEACCOUNT_ISSUER}" - params+=" --service-account-api-audiences=${SERVICEACCOUNT_ISSUER}" + params+=" --api-audiences=${SERVICEACCOUNT_ISSUER}" params+=" --service-account-signing-key-file=${SERVICEACCOUNT_KEY_PATH}" local audit_policy_config_mount="" @@ -314,6 +308,7 @@ function start-kube-apiserver { params+=" --authorization-policy-file=/etc/srv/kubernetes/abac-authz-policy.jsonl" authorization_mode+=",ABAC" + chown "${KUBE_API_SERVER_RUNASUSER:-0}":"${KUBE_API_SERVER_RUNASGROUP:-0}" /etc/srv/kubernetes/abac-authz-policy.jsonl fi local webhook_config_mount="" @@ -361,10 +356,21 @@ function start-kube-apiserver { fi container_env+="{\"name\": \"KUBE_PATCH_CONVERSION_DETECTOR\", \"value\": \"${ENABLE_PATCH_CONVERSION_DETECTOR}\"}" fi + + # b/227456358 + if [[ -n "${container_env}" ]]; then + container_env="${container_env}, " + fi + container_env+='{"name": "GODEBUG", "value": "x509sha1=1"}' + if [[ -n "${container_env}" ]]; then container_env="\"env\":[${container_env}]," fi + # Create directory for debug socket. + mkdir -p /etc/srv/kubernetes/kube-apiserver/ + chown -R "${KUBE_API_SERVER_RUNASUSER:-0}":"${KUBE_API_SERVER_RUNASGROUP:-0}" /etc/srv/kubernetes/kube-apiserver/ + local -r src_file="${src_dir}/kube-apiserver.manifest" # params is passed by reference, so no "$" @@ -375,6 +381,11 @@ function start-kube-apiserver { healthcheck_ip=$(hostname -i) fi + local termination_grace_period_seconds="" + if [[ -n "${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS:-}" ]]; then + termination_grace_period_seconds="\"terminationGracePeriodSeconds\": ${KUBE_APISERVER_TERMINATION_GRACE_PERIOD_SECONDS}," + fi + params="$(convert-manifest-params "${params}")" # Evaluate variables. local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" @@ -386,9 +397,10 @@ function start-kube-apiserver { sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}" sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" + sed -i -e "s@{{kube_apiserver_memory_limit}}@${KUBE_APISERVER_MEMORY_LIMIT:-1Ti}@g" "${src_file}" sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" + sed -i -e "s@{{liveness_probe_timeout}}@${KUBE_APISERVER_LIVENESS_PROBE_TIMEOUT_SEC:-15}@g" "${src_file}" sed -i -e "s@{{secure_port}}@443@g" "${src_file}" - sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}" sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" @@ -406,6 +418,7 @@ function start-kube-apiserver { sed -i -e "s@{{konnectivity_socket_mount}}@${default_konnectivity_socket_mnt}@g" "${src_file}" sed -i -e "s@{{konnectivity_socket_volume}}@${default_konnectivity_socket_vol}@g" "${src_file}" sed -i -e "s@{{healthcheck_ip}}@${healthcheck_ip}@g" "${src_file}" + sed -i -e "s@{{termination_grace_period_seconds}}@${termination_grace_period_seconds}@g" "${src_file}" if [[ -n "${KUBE_API_SERVER_RUNASUSER:-}" && -n "${KUBE_API_SERVER_RUNASGROUP:-}" && -n "${KUBE_PKI_READERS_GROUP:-}" ]]; then sed -i -e "s@{{runAsUser}}@\"runAsUser\": ${KUBE_API_SERVER_RUNASUSER},@g" "${src_file}" diff --git a/gke/cluster/gce/gci/testdata/kube-apiserver/base.template b/gke/cluster/gce/gci/testdata/kube-apiserver/base.template index 3ec566991a72d..c2736eb3da964 100644 --- a/gke/cluster/gce/gci/testdata/kube-apiserver/base.template +++ b/gke/cluster/gce/gci/testdata/kube-apiserver/base.template @@ -9,7 +9,6 @@ readonly APISERVER_SERVER_KEY_PATH=/foo/bar readonly APISERVER_CLIENT_CERT_PATH=/foo/bar readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," -readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080}," readonly DOCKER_REGISTRY="k8s.gcr.io" readonly ENABLE_LEGACY_ABAC=false readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests diff --git a/gke/cluster/gce/manifests/kube-apiserver.manifest b/gke/cluster/gce/manifests/kube-apiserver.manifest index 10c3411e640d2..8a202b4a91856 100644 --- a/gke/cluster/gce/manifests/kube-apiserver.manifest +++ b/gke/cluster/gce/manifests/kube-apiserver.manifest @@ -21,6 +21,7 @@ "priorityClassName": "system-node-critical", "priority": 2000001000, "hostNetwork": true, +{{termination_grace_period_seconds}} "containers":[ { "name": "kube-apiserver", @@ -28,13 +29,19 @@ "image": "{{pillar['kube_docker_registry']}}/kube-apiserver-amd64:{{pillar['kube-apiserver_docker_tag']}}", "resources": { "requests": { + "memory": "0", "cpu": "250m" + }, + "limits": { + "memory": "{{kube_apiserver_memory_limit}}" } }, "command": [ "/go-runner", "--log-file=/var/log/kube-apiserver.log", "--also-stdout=false", "--redirect-stderr=true", "/usr/local/bin/kube-apiserver", "--allow-privileged={{pillar['allow_privileged']}}", + "--profiling=false", + "--debug-socket-path=/etc/srv/kubernetes/kube-apiserver/debug.socket", {{params}} ], {{container_env}} @@ -46,7 +53,9 @@ "path": "/livez?exclude=etcd&exclude=kms-provider-0&exclude=kms-provider-1" }, "initialDelaySeconds": {{liveness_probe_initial_delay}}, - "timeoutSeconds": 15 + "timeoutSeconds": {{liveness_probe_timeout}}, + "periodSeconds": 5, + "failureThreshold": 6 }, "readinessProbe": { "httpGet": { @@ -59,7 +68,6 @@ "timeoutSeconds": 15 }, "ports":[ - {{insecure_port_mapping}} { "name": "https", "containerPort": {{secure_port}}, "hostPort": {{secure_port}}} @@ -79,6 +87,9 @@ { "name": "srvkube", "mountPath": "/etc/srv/kubernetes", "readOnly": true}, + { "name": "srvkubeapiserver", + "mountPath": "/etc/srv/kubernetes/kube-apiserver", + "readOnly": false}, { "name": "logfile", "mountPath": "/var/log/kube-apiserver.log", "readOnly": false}, @@ -122,6 +133,11 @@ "hostPath": { "path": "/etc/srv/kubernetes"} }, + { "name": "srvkubeapiserver", + "hostPath": { + "path": "/etc/srv/kubernetes/kube-apiserver", + "type": "DirectoryOrCreate"} + }, { "name": "logfile", "hostPath": { "path": "/var/log/kube-apiserver.log", diff --git a/gke/cluster/gce/util.sh b/gke/cluster/gce/util.sh index 03fc63a551b7e..f6a64dd56298f 100755 --- a/gke/cluster/gce/util.sh +++ b/gke/cluster/gce/util.sh @@ -1402,6 +1402,11 @@ EOF if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then cat >>"$file" <>$file <