Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add kubens.service, drop-ins, and kubensenter prefix to kubelet.service #3274

Merged
merged 2 commits into from
Aug 26, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ test-unit:
# make update
update:
hack/update-codegen.sh
hack/update-templates.sh

go-deps:
go mod tidy
Expand All @@ -72,6 +73,7 @@ install-tools:
verify: install-tools
golangci-lint run --build-tags=$(GOTAGS)
hack/verify-codegen.sh
hack/verify-templates.sh

# Template for defining build targets for binaries.
define target_template =
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/ashcrow/osrelease v0.0.0-20180626175927-9b292693c55c
github.com/clarketm/json v1.14.1
github.com/containers/image/v5 v5.22.0
github.com/containers/kubensmnt v1.2.0
github.com/containers/storage v1.42.0
github.com/coreos/fcct v0.5.0
github.com/coreos/go-semver v0.3.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,8 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD
github.com/containers/image/v5 v5.11.0/go.mod h1:dCbUB4w6gmxIEOCsE0tZQppr8iBoXb4Evr74ZKlmwoI=
github.com/containers/image/v5 v5.22.0 h1:KemxPmD4D2YYOFZN2SgoTk7nBFcnwPiPW0MqjYtknSE=
github.com/containers/image/v5 v5.22.0/go.mod h1:D8Ksv2RNB8qLJ7xe1P3rgJJOSQpahA6amv2Ax++/YO4=
github.com/containers/kubensmnt v1.2.0 h1:BDtkaOFQ5fN7FnB9kC6peMW50KkwI1KI8E9ROBFeQIg=
github.com/containers/kubensmnt v1.2.0/go.mod h1:1/HG09N/a1+WSD3zkurzeWtqlKRSfUUnlIF/08zloqk=
github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU=
github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
Expand Down
64 changes: 64 additions & 0 deletions hack/update-templates.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/bash

set -o errexit
set -o nounset
set -o pipefail

dropin_yaml() {
local service=$1
local name=$2
local source=$3
echo "# Autogenerated by hack/update-templates.sh; do not edit"
echo "name: $service"
echo "dropins:"
echo " - name: $name"
echo " contents: |"
sed 's/^/ /' $source

}

write_dropin_yaml() {
local service=$1
local name=$2
local source=$3
local dst=$4
echo "Generating template for drop-in $service::$name => $dst" >&2
dropin_yaml "$service" "$name" "$source" > "$dst"
}

service_yaml() {
local name=$1
local enabled=$2
local source=$3
echo "# Autogenerated by hack/update-templates.sh; do not edit"
echo "name: $name"
echo "enabled: $enabled"
echo "contents: |"
sed 's/^/ /' $source
}

write_service_yaml() {
local name=$1
local enabled=$2
local source=$3
local dst=$4
echo "Generating template for service $name (enabled=$enabled) => $dst" >&2
service_yaml "$name" "$enabled" "$source" > "$dst"
}

SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
UNITS=${SCRIPT_ROOT}/templates/common/_base/units

KUBENS_VENDOR=${SCRIPT_ROOT}/vendor/github.com/containers/kubensmnt/utils/systemd
echo "Updating templates from vendor/github.com/containers/kubensmnt/utils/systemd..." >&2
write_service_yaml kubens.service "false" ${KUBENS_VENDOR}/kubens.service ${UNITS}/kubens.service.yaml
write_dropin_yaml kubelet.service 01-kubens.conf ${KUBENS_VENDOR}/kubens-dropin.conf ${UNITS}/kubelet.service-kubens.yaml
write_dropin_yaml crio.service 01-kubens.conf ${KUBENS_VENDOR}/kubens-dropin.conf ${UNITS}/crio.service-kubens.yaml

log_cp() {
echo "Copying $1 => $2"
cp $1 $2
}

echo "Copying worker templates from equivalent master templates"
log_cp templates/master/01-master-kubelet/_base/files/kubenswrapper.yaml templates/worker/01-worker-kubelet/_base/files/kubenswrapper.yaml
34 changes: 34 additions & 0 deletions hack/verify-templates.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash

set -o errexit
set -o nounset
set -o pipefail

SCRIPT_ROOT=$(dirname "${BASH_SOURCE}")/..

DIFFROOT="${SCRIPT_ROOT}/templates"
TMP_DIFFROOT="${SCRIPT_ROOT}/_tmp/templates"
_tmp="${SCRIPT_ROOT}/_tmp"

cleanup() {
rm -rf "${_tmp}"
}
trap "cleanup" EXIT SIGINT

cleanup

mkdir -p "${TMP_DIFFROOT}"
cp -R "${DIFFROOT}"/* "${TMP_DIFFROOT}"

"${SCRIPT_ROOT}/hack/update-templates.sh"
echo "diffing ${DIFFROOT} against freshly generated codegen"
ret=0
diff -Naupr "${TMP_DIFFROOT}" "${DIFFROOT}" || ret=$?
cp -R "${TMP_DIFFROOT}"/* "${DIFFROOT}"
if [[ $ret -eq 0 ]]
then
echo "${DIFFROOT} up to date."
else
echo "${DIFFROOT} is out of date. Please run make update"
exit 1
fi
18 changes: 18 additions & 0 deletions templates/common/_base/units/crio.service-kubens.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Autogenerated by hack/update-templates.sh; do not edit
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this being here mean we'll have the kubens enabled by default? is that intended?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This drop-in does nothing except set the environment variable that will be laid down by kubens.service if it's running, so it's safe fro this to be present all the time.

The kubens.service itself is not being enabled by default (see https://github.com/openshift/machine-config-operator/pull/3274/files/d8f42ae7a8e66970a5fcbdaac6fd8712d1466ba7#diff-c38a0be931e5bc991f959879c6af89aee96be94967b4544f63ace1d52e5f88ccR3)

name: crio.service
dropins:
- name: 01-kubens.conf
contents: |
# vim:set ft=systemd :
#
# This drop-in will enable any service built with this
# github.com/containers/kubemntns library to properly join the mount namespace
# managed by kubens.service
#

[Unit]
After=kubens.service

[Service]
EnvironmentFile=-/run/kubens/env

18 changes: 18 additions & 0 deletions templates/common/_base/units/kubelet.service-kubens.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Autogenerated by hack/update-templates.sh; do not edit
name: kubelet.service
dropins:
- name: 01-kubens.conf
contents: |
# vim:set ft=systemd :
#
# This drop-in will enable any service built with this
# github.com/containers/kubemntns library to properly join the mount namespace
# managed by kubens.service
#

[Unit]
After=kubens.service

[Service]
EnvironmentFile=-/run/kubens/env

29 changes: 29 additions & 0 deletions templates/common/_base/units/kubens.service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Autogenerated by hack/update-templates.sh; do not edit
name: kubens.service
enabled: false
contents: |
[Unit]
Description=Manages a mount namespace for kubernetes-specific mounts

[Service]
Type=oneshot
RemainAfterExit=yes
RuntimeDirectory=kubens
Environment=RUNTIME_DIRECTORY=%t/kubens
Environment=BIND_POINT=%t/kubens/mnt
Environment=ENVFILE=%t/kubens/env

# Set up the runtime directory as an unbindable mountpoint
ExecStartPre=bash -c "findmnt ${RUNTIME_DIRECTORY} || mount --make-unbindable --bind ${RUNTIME_DIRECTORY} ${RUNTIME_DIRECTORY}"
# Ensure the bind point exists
ExecStartPre=touch ${BIND_POINT}
# Use 'unshare' to create the new mountpoint, then 'mount --make-rshared' so it cascades internally
ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared /
# Finally, set an env pointer for ease-of-use
ExecStartPost=bash -c 'echo "KUBENSMNT=${BIND_POINT}" > "${ENVFILE}"'

# On stop, a recursive unmount cleans up the namespace and bind-mounted unbindable parent directory
ExecStop=umount -R ${RUNTIME_DIRECTORY}

[Install]
WantedBy=multi-user.target
12 changes: 12 additions & 0 deletions templates/master/01-master-kubelet/_base/files/kubenswrapper.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Note: This compatibility wrapper is needed to bridge the gap from OCP 4.11->4.12 when the new 'kubensenter' script is being introduced.
# It can be removed (and the kubelet.service should call kubensenter directly) when 4.11 is no longer a release we must upgrade from.
mode: 0755
path: "/usr/local/bin/kubenswrapper"
contents:
inline: |
#!/bin/sh
if [ -x /usr/bin/kubensenter ]; then
exec /usr/bin/kubensenter "$@"
else
exec "$@"
fi
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
12 changes: 12 additions & 0 deletions templates/worker/01-worker-kubelet/_base/files/kubenswrapper.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Note: This compatibility wrapper is needed to bridge the gap from OCP 4.11->4.12 when the new 'kubensenter' script is being introduced.
# It can be removed (and the kubelet.service should call kubensenter directly) when 4.11 is no longer a release we must upgrade from.
mode: 0755
path: "/usr/local/bin/kubenswrapper"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why /usr/local instead of /usr?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ignition can't write to /usr. See #3137

contents:
inline: |
#!/bin/sh
if [ -x /usr/bin/kubensenter ]; then
exec /usr/bin/kubensenter "$@"
else
exec "$@"
fi
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ contents: |
EnvironmentFile=-/etc/kubernetes/kubelet-env
EnvironmentFile=/etc/node-sizing.env

ExecStart=/usr/bin/hyperkube \
kubelet \
ExecStart=/usr/local/bin/kubenswrapper \
/usr/bin/kubelet \
--config=/etc/kubernetes/kubelet.conf \
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
--kubeconfig=/var/lib/kubelet/kubeconfig \
Expand Down
4 changes: 4 additions & 0 deletions tools.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
//go:build tools
// +build tools

// tools is a dummy package that will be ignored for builds, but included for dependencies.
Expand All @@ -13,6 +14,9 @@ import (
_ "k8s.io/code-generator/cmd/defaulter-gen"
_ "k8s.io/code-generator/cmd/informer-gen"
_ "k8s.io/code-generator/cmd/lister-gen"

// TODO: Investigate openapi-gen
// _ "k8s.io/code-generator/cmd/openapi-gen"

_ "github.com/containers/kubensmnt/utils/systemd"
)
Loading