Skip to content

Commit

Permalink
UPSTREAM: <carry>: openshift-kube-apiserver: add kube-apiserver patches
Browse files Browse the repository at this point in the history
Origin-commit: 170dd7d25cca990fd7683eaf424d00bcd776c39c

Origin-commit: 35ef039cb099dc609c576cf594aadd849212a00b

UPSTREAM: <carry>: openshift-kube-apiserver: enabled conversion gen for admission configs
  • Loading branch information
deads2k authored and soltysh committed Sep 7, 2021
1 parent f7069c2 commit 313331e
Show file tree
Hide file tree
Showing 170 changed files with 15,752 additions and 31 deletions.
19 changes: 2 additions & 17 deletions cmd/kube-apiserver/app/patch_openshift.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,14 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
clientgoinformers "k8s.io/client-go/informers"
"k8s.io/kubernetes/openshift-kube-apiserver/openshiftkubeapiserver"
"k8s.io/kubernetes/pkg/master"
)

var OpenShiftKubeAPIServerConfigPatch openshiftkubeapiserver.KubeAPIServerConfigFunc = nil

type KubeAPIServerServerFunc func(server *master.Master) error

func PatchKubeAPIServerConfig(config *genericapiserver.Config, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) (genericapiserver.DelegationTarget, error) {
func PatchKubeAPIServerConfig(config *genericapiserver.Config, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) error {
if OpenShiftKubeAPIServerConfigPatch == nil {
return genericapiserver.NewEmptyDelegate(), nil
}

return OpenShiftKubeAPIServerConfigPatch(config, versionedInformers, pluginInitializers)
}

var OpenShiftKubeAPIServerServerPatch KubeAPIServerServerFunc = nil

func PatchKubeAPIServerServer(server *master.Master) error {
if OpenShiftKubeAPIServerServerPatch == nil {
return nil
}

return OpenShiftKubeAPIServerServerPatch(server)
return OpenShiftKubeAPIServerConfigPatch(config, versionedInformers, pluginInitializers)
}

var StartingDelegate genericapiserver.DelegationTarget = genericapiserver.NewEmptyDelegate()
32 changes: 18 additions & 14 deletions cmd/kube-apiserver/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"strings"
"time"

"k8s.io/kubernetes/openshift-kube-apiserver/configdefault"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/admissionenablement"
"k8s.io/kubernetes/openshift-kube-apiserver/enablement"
"k8s.io/kubernetes/openshift-kube-apiserver/openshiftkubeapiserver"

Expand Down Expand Up @@ -128,31 +128,38 @@ cluster's shared state through which all other components interact.`,
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
fs := cmd.Flags()
cliflag.PrintFlags(fs)

if len(s.OpenShiftConfig) > 0 {
enablement.ForceOpenShift()
openshiftConfig, err := enablement.GetOpenshiftConfig(s.OpenShiftConfig)
if err != nil {
klog.Fatal(err)
}
enablement.ForceOpenShift(openshiftConfig)

// this forces a patch to be called
// TODO we're going to try to remove bits of the patching.
configPatchFn, serverPatchContext := openshiftkubeapiserver.NewOpenShiftKubeAPIServerConfigPatch(genericapiserver.NewEmptyDelegate(), openshiftConfig)
configPatchFn := openshiftkubeapiserver.NewOpenShiftKubeAPIServerConfigPatch(openshiftConfig)
OpenShiftKubeAPIServerConfigPatch = configPatchFn
OpenShiftKubeAPIServerServerPatch = serverPatchContext.PatchServer

args, err := openshiftkubeapiserver.ConfigToFlags(openshiftConfig)
if err != nil {
return err
}

// hopefully this resets the flags?
if err := cmd.ParseFlags(args); err != nil {
return err
}

enablement.ForceGlobalInitializationForOpenShift(s)
// print merged flags (merged from OpenshiftConfig)
cliflag.PrintFlags(cmd.Flags())

enablement.ForceGlobalInitializationForOpenShift()
admissionenablement.InstallOpenShiftAdmissionPlugins(s)

} else {
// print default flags
cliflag.PrintFlags(cmd.Flags())
}

err := checkNonZeroInsecurePort(fs)
Expand Down Expand Up @@ -242,7 +249,7 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
if err != nil {
return nil, err
}
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, StartingDelegate)
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
Expand All @@ -252,10 +259,6 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan
return nil, err
}

if err := PatchKubeAPIServerServer(kubeAPIServer); err != nil {
return nil, err
}

// aggregator comes last in the chain
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, proxyTransport, pluginInitializer)
if err != nil {
Expand Down Expand Up @@ -546,6 +549,8 @@ func buildGenericConfig(
// on a fast local network
genericConfig.LoopbackClientConfig.DisableCompression = true

enablement.SetLoopbackClientConfig(genericConfig.LoopbackClientConfig)

kubeClientConfig := genericConfig.LoopbackClientConfig
clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
if err != nil {
Expand Down Expand Up @@ -585,14 +590,13 @@ func buildGenericConfig(
return
}

StartingDelegate, err = PatchKubeAPIServerConfig(genericConfig, versionedInformers, &pluginInitializers)
if err != nil {
if err := PatchKubeAPIServerConfig(genericConfig, versionedInformers, &pluginInitializers); err != nil {
lastErr = fmt.Errorf("failed to patch: %v", err)
return
}

if enablement.IsOpenShift() {
configdefault.SetAdmissionDefaults(s, versionedInformers, clientgoExternalClient)
admissionenablement.SetAdmissionDefaults(s, versionedInformers, clientgoExternalClient)
}
err = s.Admission.ApplyTo(
genericConfig,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package admissionenablement

import (
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/customresourcevalidation/customresourcevalidationregistration"
)

func InstallOpenShiftAdmissionPlugins(o *options.ServerRunOptions) {
existingAdmissionOrder := o.Admission.GenericAdmission.RecommendedPluginOrder
o.Admission.GenericAdmission.RecommendedPluginOrder = NewOrderedKubeAdmissionPlugins(existingAdmissionOrder)
RegisterOpenshiftKubeAdmissionPlugins(o.Admission.GenericAdmission.Plugins)
customresourcevalidationregistration.RegisterCustomResourceValidation(o.Admission.GenericAdmission.Plugins)
existingDefaultOff := o.Admission.GenericAdmission.DefaultOffPlugins
o.Admission.GenericAdmission.DefaultOffPlugins = NewDefaultOffPluginsFunc(existingDefaultOff)()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package admissionenablement

import (
"time"

"github.com/openshift/library-go/pkg/apiserver/admission/admissiontimeout"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/namespaceconditions"
)

func SetAdmissionDefaults(o *options.ServerRunOptions, informers informers.SharedInformerFactory, kubeClient kubernetes.Interface) {
// set up the decorators we need. This is done late and out of order because our decorators currently require informers which are not
// present until we start running
namespaceLabelDecorator := namespaceconditions.NamespaceLabelConditions{
NamespaceClient: kubeClient.CoreV1(),
NamespaceLister: informers.Core().V1().Namespaces().Lister(),

SkipLevelZeroNames: SkipRunLevelZeroPlugins,
SkipLevelOneNames: SkipRunLevelOnePlugins,
}
o.Admission.GenericAdmission.Decorators = append(o.Admission.GenericAdmission.Decorators,
admission.Decorators{
admission.DecoratorFunc(namespaceLabelDecorator.WithNamespaceLabelConditions),
admission.DecoratorFunc(admissiontimeout.AdmissionTimeout{Timeout: 13 * time.Second}.WithTimeout),
},
)
}
111 changes: 111 additions & 0 deletions openshift-kube-apiserver/admission/admissionenablement/register.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package admissionenablement

import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"

"github.com/openshift/apiserver-library-go/pkg/admission/imagepolicy"
imagepolicyapiv1 "github.com/openshift/apiserver-library-go/pkg/admission/imagepolicy/apis/imagepolicy/v1"
quotaclusterresourcequota "github.com/openshift/apiserver-library-go/pkg/admission/quota/clusterresourcequota"
"github.com/openshift/apiserver-library-go/pkg/securitycontextconstraints/sccadmission"
authorizationrestrictusers "k8s.io/kubernetes/openshift-kube-apiserver/admission/authorization/restrictusers"
quotaclusterresourceoverride "k8s.io/kubernetes/openshift-kube-apiserver/admission/autoscaling/clusterresourceoverride"
quotarunonceduration "k8s.io/kubernetes/openshift-kube-apiserver/admission/autoscaling/runonceduration"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/customresourcevalidation/customresourcevalidationregistration"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/network/externalipranger"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/network/restrictedendpoints"
ingressadmission "k8s.io/kubernetes/openshift-kube-apiserver/admission/route"
projectnodeenv "k8s.io/kubernetes/openshift-kube-apiserver/admission/scheduler/nodeenv"
schedulerpodnodeconstraints "k8s.io/kubernetes/openshift-kube-apiserver/admission/scheduler/podnodeconstraints"
)

func RegisterOpenshiftKubeAdmissionPlugins(plugins *admission.Plugins) {
authorizationrestrictusers.Register(plugins)
imagepolicy.Register(plugins)
ingressadmission.Register(plugins)
projectnodeenv.Register(plugins)
quotaclusterresourceoverride.Register(plugins)
quotaclusterresourcequota.Register(plugins)
quotarunonceduration.Register(plugins)
schedulerpodnodeconstraints.Register(plugins)
sccadmission.Register(plugins)
sccadmission.RegisterSCCExecRestrictions(plugins)
externalipranger.RegisterExternalIP(plugins)
restrictedendpoints.RegisterRestrictedEndpoints(plugins)
}

var (

// these are admission plugins that cannot be applied until after the kubeapiserver starts.
// TODO if nothing comes to mind in 3.10, kill this
SkipRunLevelZeroPlugins = sets.NewString()
// these are admission plugins that cannot be applied until after the openshiftapiserver apiserver starts.
SkipRunLevelOnePlugins = sets.NewString(
imagepolicyapiv1.PluginName, // "image.openshift.io/ImagePolicy"
"quota.openshift.io/ClusterResourceQuota",
"security.openshift.io/SecurityContextConstraint",
"security.openshift.io/SCCExecRestrictions",
)

// openshiftAdmissionPluginsForKubeBeforeMutating are the admission plugins to add after kube admission, before mutating webhooks
openshiftAdmissionPluginsForKubeBeforeMutating = []string{
"autoscaling.openshift.io/ClusterResourceOverride",
"authorization.openshift.io/RestrictSubjectBindings",
"autoscaling.openshift.io/RunOnceDuration",
"scheduling.openshift.io/PodNodeConstraints",
"scheduling.openshift.io/OriginPodNodeEnvironment",
"network.openshift.io/ExternalIPRanger",
"network.openshift.io/RestrictedEndpointsAdmission",
imagepolicyapiv1.PluginName, // "image.openshift.io/ImagePolicy"
"security.openshift.io/SecurityContextConstraint",
"security.openshift.io/SCCExecRestrictions",
"route.openshift.io/IngressAdmission",
}

// openshiftAdmissionPluginsForKubeAfterResourceQuota are the plugins to add after ResourceQuota plugin
openshiftAdmissionPluginsForKubeAfterResourceQuota = []string{
"quota.openshift.io/ClusterResourceQuota",
}

// additionalDefaultOnPlugins is a list of plugins we turn on by default that core kube does not.
additionalDefaultOnPlugins = sets.NewString(
"NodeRestriction",
"OwnerReferencesPermissionEnforcement",
"PersistentVolumeLabel",
"PodNodeSelector",
"PodTolerationRestriction",
"Priority",
imagepolicyapiv1.PluginName, // "image.openshift.io/ImagePolicy"
"StorageObjectInUseProtection",
)
)

func NewOrderedKubeAdmissionPlugins(kubeAdmissionOrder []string) []string {
ret := []string{}
for _, curr := range kubeAdmissionOrder {
if curr == mutatingwebhook.PluginName {
ret = append(ret, openshiftAdmissionPluginsForKubeBeforeMutating...)
ret = append(ret, customresourcevalidationregistration.AllCustomResourceValidators...)
}

ret = append(ret, curr)

if curr == resourcequota.PluginName {
ret = append(ret, openshiftAdmissionPluginsForKubeAfterResourceQuota...)
}
}
return ret
}

func NewDefaultOffPluginsFunc(kubeDefaultOffAdmission sets.String) func() sets.String {
return func() sets.String {
kubeOff := sets.NewString(kubeDefaultOffAdmission.UnsortedList()...)
kubeOff.Delete(additionalDefaultOnPlugins.List()...)
kubeOff.Delete(openshiftAdmissionPluginsForKubeBeforeMutating...)
kubeOff.Delete(openshiftAdmissionPluginsForKubeAfterResourceQuota...)
kubeOff.Delete(customresourcevalidationregistration.AllCustomResourceValidators...)
return kubeOff
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package admissionenablement

import (
"reflect"
"testing"

"k8s.io/apiserver/pkg/admission"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/kubernetes/pkg/kubeapiserver/options"

"github.com/openshift/library-go/pkg/apiserver/admission/admissionregistrationtesting"
"k8s.io/kubernetes/openshift-kube-apiserver/admission/customresourcevalidation/customresourcevalidationregistration"
)

func TestAdmissionRegistration(t *testing.T) {
orderedAdmissionChain := NewOrderedKubeAdmissionPlugins(options.AllOrderedPlugins)
defaultOffPlugins := NewDefaultOffPluginsFunc(options.DefaultOffAdmissionPlugins())()
registerAllAdmissionPlugins := func(plugins *admission.Plugins) {
genericapiserver.RegisterAllAdmissionPlugins(plugins)
options.RegisterAllAdmissionPlugins(plugins)
RegisterOpenshiftKubeAdmissionPlugins(plugins)
customresourcevalidationregistration.RegisterCustomResourceValidation(plugins)
}
plugins := admission.NewPlugins()
registerAllAdmissionPlugins(plugins)

err := admissionregistrationtesting.AdmissionRegistrationTest(plugins, orderedAdmissionChain, defaultOffPlugins)
if err != nil {
t.Fatal(err)
}
}

// TestResourceQuotaBeforeClusterResourceQuota simply test wheather ResourceQuota plugin is before ClusterResourceQuota plugin
func TestResourceQuotaBeforeClusterResourceQuota(t *testing.T) {
orderedAdmissionChain := NewOrderedKubeAdmissionPlugins(options.AllOrderedPlugins)

expectedOrderedAdmissionSubChain := []string{"ResourceQuota", "quota.openshift.io/ClusterResourceQuota", "AlwaysDeny"}
actualOrderedAdmissionChain := extractSubChain(orderedAdmissionChain, expectedOrderedAdmissionSubChain[0])

if !reflect.DeepEqual(actualOrderedAdmissionChain, expectedOrderedAdmissionSubChain) {
t.Fatalf("expected %v, got %v ", expectedOrderedAdmissionSubChain, actualOrderedAdmissionChain)
}
}

func extractSubChain(admissionChain []string, takeFrom string) []string {
indexOfTake := 0
for index, admission := range admissionChain {
if admission == takeFrom {
indexOfTake = index
break
}
}
return admissionChain[indexOfTake:]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package restrictusers

import (
userv1 "github.com/openshift/api/user/v1"
)

type fakeGroupCache struct {
groups []userv1.Group
}

func (g fakeGroupCache) GroupsFor(user string) ([]*userv1.Group, error) {
ret := []*userv1.Group{}
for i := range g.groups {
group := &g.groups[i]
for _, currUser := range group.Users {
if user == currUser {
ret = append(ret, group)
break
}
}

}
return ret, nil
}

func (g fakeGroupCache) HasSynced() bool {
return true
}
Loading

0 comments on commit 313331e

Please sign in to comment.