From 6869a99cebc4fec86327541cf22bb3722ab50567 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 28 May 2020 16:27:27 +0300 Subject: [PATCH 1/4] nfd-master: fix one docstring --- pkg/nfd-master/nfd-master.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index f60b72136e..11f040aa8c 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -332,8 +332,9 @@ func (s *labelerServer) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*p return &pb.SetLabelsReply{}, nil } -// advertiseFeatureLabels advertises the feature labels to a Kubernetes node -// via the API server. +// updateNodeFeatures ensures the Kubernetes node object is up to date, +// creating new labels and extended resources where necessary and removing +// outdated ones. Also updates the corresponding annotations. func updateNodeFeatures(helper apihelper.APIHelpers, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources) error { cli, err := helper.GetClient() if err != nil { From 46697700208df55715e41e7559b5a66dc3066d39 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 28 May 2020 17:12:48 +0300 Subject: [PATCH 2/4] nfd-master: implement --prune flag A new sub-command like flag for cleaning up a cluster. When --prune is specified nfd-master removes all NFD related labels, annotations and extended resources from all nodes of the cluster and exits. This should help undeployment of NFD and be useful for development. --- cmd/nfd-master/main.go | 5 ++- pkg/apihelper/apihelpers.go | 3 ++ pkg/apihelper/k8shelpers.go | 4 +++ pkg/apihelper/mock_APIHelpers.go | 23 ++++++++++++ pkg/nfd-master/nfd-master.go | 60 ++++++++++++++++++++++++++++---- 5 files changed, 87 insertions(+), 8 deletions(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 894cc97782..43ca655683 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -63,7 +63,7 @@ func argsParse(argv []string) (master.Args, error) { usage := fmt.Sprintf(`%s. Usage: - %s [--no-publish] [--label-whitelist=] [--port=] + %s [--prune] [--no-publish] [--label-whitelist=] [--port=] [--ca-file=] [--cert-file=] [--key-file=] [--verify-node-name] [--extra-label-ns=] [--resource-labels=] %s -h | --help @@ -72,6 +72,8 @@ func argsParse(argv []string) (master.Args, error) { Options: -h --help Show this screen. --version Output version and exit. + --prune Prune all NFD related attributes from all nodes + of the cluster and exit. --port= Port on which to listen for connections. [Default: 8080] --ca-file= Root certificate for verifying connections @@ -119,6 +121,7 @@ func argsParse(argv []string) (master.Args, error) { args.VerifyNodeName = arguments["--verify-node-name"].(bool) args.ExtraLabelNs = strings.Split(arguments["--extra-label-ns"].(string), ",") args.ResourceLabels = strings.Split(arguments["--resource-labels"].(string), ",") + args.Prune = arguments["--prune"].(bool) return args, nil } diff --git a/pkg/apihelper/apihelpers.go b/pkg/apihelper/apihelpers.go index 81240b06ad..59e1b285e0 100644 --- a/pkg/apihelper/apihelpers.go +++ b/pkg/apihelper/apihelpers.go @@ -29,6 +29,9 @@ type APIHelpers interface { // GetNode returns the Kubernetes node on which this container is running. GetNode(*k8sclient.Clientset, string) (*api.Node, error) + // GetNodes returns all the nodes in the cluster + GetNodes(*k8sclient.Clientset) (*api.NodeList, error) + // UpdateNode updates the node via the API server using a client. UpdateNode(*k8sclient.Clientset, *api.Node) error diff --git a/pkg/apihelper/k8shelpers.go b/pkg/apihelper/k8shelpers.go index 695c400f3c..17e8f7e388 100644 --- a/pkg/apihelper/k8shelpers.go +++ b/pkg/apihelper/k8shelpers.go @@ -53,6 +53,10 @@ func (h K8sHelpers) GetNode(cli *k8sclient.Clientset, nodeName string) (*api.Nod return node, nil } +func (h K8sHelpers) GetNodes(cli *k8sclient.Clientset) (*api.NodeList, error) { + return cli.CoreV1().Nodes().List(meta_v1.ListOptions{}) +} + func (h K8sHelpers) UpdateNode(c *k8sclient.Clientset, n *api.Node) error { // Send the updated node to the apiserver. _, err := c.CoreV1().Nodes().Update(n) diff --git a/pkg/apihelper/mock_APIHelpers.go b/pkg/apihelper/mock_APIHelpers.go index f6c4f54159..d79f56326b 100644 --- a/pkg/apihelper/mock_APIHelpers.go +++ b/pkg/apihelper/mock_APIHelpers.go @@ -62,6 +62,29 @@ func (_m *MockAPIHelpers) GetNode(_a0 *kubernetes.Clientset, _a1 string) (*v1.No return r0, r1 } +// GetNodes provides a mock function with given fields: _a0 +func (_m *MockAPIHelpers) GetNodes(_a0 *kubernetes.Clientset) (*v1.NodeList, error) { + ret := _m.Called(_a0) + + var r0 *v1.NodeList + if rf, ok := ret.Get(0).(func(*kubernetes.Clientset) *v1.NodeList); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1.NodeList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*kubernetes.Clientset) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // PatchStatus provides a mock function with given fields: _a0, _a1, _a2 func (_m *MockAPIHelpers) PatchStatus(_a0 *kubernetes.Clientset, _a1 string, _a2 interface{}) error { ret := _m.Called(_a0, _a1, _a2) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 11f040aa8c..7ee456bbfb 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -73,6 +73,7 @@ type Args struct { LabelWhiteList *regexp.Regexp NoPublish bool Port int + Prune bool VerifyNodeName bool ResourceLabels []string } @@ -84,9 +85,10 @@ type NfdMaster interface { } type nfdMaster struct { - args Args - server *grpc.Server - ready chan bool + args Args + server *grpc.Server + ready chan bool + apihelper apihelper.APIHelpers } // statusOp is a json marshaling helper used for patching node status @@ -121,6 +123,9 @@ func NewNfdMaster(args Args) (NfdMaster, error) { } } + // Initialize Kubernetes API helpers + nfd.apihelper = apihelper.K8sHelpers{} + return nfd, nil } @@ -130,11 +135,12 @@ func (m *nfdMaster) Run() error { stdoutLogger.Printf("Node Feature Discovery Master %s", version.Get()) stdoutLogger.Printf("NodeName: '%s'", nodeName) - // Initialize Kubernetes API helpers - helper := apihelper.APIHelpers(apihelper.K8sHelpers{}) + if m.args.Prune { + return m.prune() + } if !m.args.NoPublish { - err := updateMasterNode(helper) + err := updateMasterNode(m.apihelper) if err != nil { return fmt.Errorf("failed to update master node: %v", err) } @@ -176,7 +182,7 @@ func (m *nfdMaster) Run() error { serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsConfig))) } m.server = grpc.NewServer(serverOpts...) - pb.RegisterLabelerServer(m.server, &labelerServer{args: m.args, apiHelper: helper}) + pb.RegisterLabelerServer(m.server, &labelerServer{args: m.args, apiHelper: m.apihelper}) stdoutLogger.Printf("gRPC server serving on port: %d", m.args.Port) return m.server.Serve(lis) } @@ -201,6 +207,46 @@ func (m *nfdMaster) WaitForReady(timeout time.Duration) bool { return false } +// Prune erases all NFD related properties from the node objects of the cluster. +func (m *nfdMaster) prune() error { + cli, err := m.apihelper.GetClient() + if err != nil { + return err + } + + nodes, err := m.apihelper.GetNodes(cli) + if err != nil { + return err + } + + for _, node := range nodes.Items { + stdoutLogger.Printf("pruning node %q...", node.Name) + + // Prune labels and extended resources + err := updateNodeFeatures(m.apihelper, node.Name, Labels{}, Annotations{}, ExtendedResources{}) + if err != nil { + return fmt.Errorf("failed to prune labels from node %q: %v", node.Name, err) + } + + // Prune annotations + node, err := m.apihelper.GetNode(cli, node.Name) + if err != nil { + return err + } + for a := range node.Annotations { + if strings.HasPrefix(a, AnnotationNs) { + delete(node.Annotations, a) + } + } + err = m.apihelper.UpdateNode(cli, node) + if err != nil { + return fmt.Errorf("failed to prune annotations from node %q: %v", node.Name, err) + } + + } + return nil +} + // Advertise NFD master information func updateMasterNode(helper apihelper.APIHelpers) error { cli, err := helper.GetClient() From 458dd8dc5802a84446d215c584bbd81cb59c3455 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 28 May 2020 17:43:43 +0300 Subject: [PATCH 3/4] nfd-master: add --kubeconfig flag Useful with --prune and for development purposes. --- cmd/nfd-master/main.go | 4 ++++ pkg/apihelper/k8shelpers.go | 12 +++++++++++- pkg/nfd-master/nfd-master.go | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 43ca655683..caf1e03f0f 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -66,6 +66,7 @@ func argsParse(argv []string) (master.Args, error) { %s [--prune] [--no-publish] [--label-whitelist=] [--port=] [--ca-file=] [--cert-file=] [--key-file=] [--verify-node-name] [--extra-label-ns=] [--resource-labels=] + [--kubeconfig=] %s -h | --help %s --version @@ -74,6 +75,8 @@ func argsParse(argv []string) (master.Args, error) { --version Output version and exit. --prune Prune all NFD related attributes from all nodes of the cluster and exit. + --kubeconfig= Kubeconfig to use [Default: ] + of the cluster and exit. --port= Port on which to listen for connections. [Default: 8080] --ca-file= Root certificate for verifying connections @@ -122,6 +125,7 @@ func argsParse(argv []string) (master.Args, error) { args.ExtraLabelNs = strings.Split(arguments["--extra-label-ns"].(string), ",") args.ResourceLabels = strings.Split(arguments["--resource-labels"].(string), ",") args.Prune = arguments["--prune"].(bool) + args.Kubeconfig = arguments["--kubeconfig"].(string) return args, nil } diff --git a/pkg/apihelper/k8shelpers.go b/pkg/apihelper/k8shelpers.go index 17e8f7e388..460994eced 100644 --- a/pkg/apihelper/k8shelpers.go +++ b/pkg/apihelper/k8shelpers.go @@ -24,18 +24,28 @@ import ( "k8s.io/apimachinery/pkg/types" k8sclient "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" ) // Implements APIHelpers type K8sHelpers struct { + Kubeconfig string } func (h K8sHelpers) GetClient() (*k8sclient.Clientset, error) { // Set up an in-cluster K8S client. - config, err := restclient.InClusterConfig() + var config *restclient.Config + var err error + + if h.Kubeconfig == "" { + config, err = restclient.InClusterConfig() + } else { + config, err = clientcmd.BuildConfigFromFlags("", h.Kubeconfig) + } if err != nil { return nil, err } + clientset, err := k8sclient.NewForConfig(config) if err != nil { return nil, err diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 7ee456bbfb..66ae9799bf 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -70,6 +70,7 @@ type Args struct { CertFile string ExtraLabelNs []string KeyFile string + Kubeconfig string LabelWhiteList *regexp.Regexp NoPublish bool Port int @@ -124,7 +125,7 @@ func NewNfdMaster(args Args) (NfdMaster, error) { } // Initialize Kubernetes API helpers - nfd.apihelper = apihelper.K8sHelpers{} + nfd.apihelper = apihelper.K8sHelpers{Kubeconfig: args.Kubeconfig} return nfd, nil } From 36aa100c8f99870a9cfcf86ddfccf9a5fb234fce Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 28 May 2020 17:57:29 +0300 Subject: [PATCH 4/4] README: add --prune and --kubeconfig flags The bare minimum doc update. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f7fd7dbfc8..8075d7430e 100644 --- a/README.md +++ b/README.md @@ -49,18 +49,20 @@ Command line flags of nfd-master: ``` $ docker run --rm nfd-master --help ... -nfd-master. - - Usage: - nfd-master [--no-publish] [--label-whitelist=] [--port=] +Usage: + nfd-master [--prune] [--no-publish] [--label-whitelist=] [--port=] [--ca-file=] [--cert-file=] [--key-file=] [--verify-node-name] [--extra-label-ns=] [--resource-labels=] + [--kubeconfig=] nfd-master -h | --help nfd-master --version Options: -h --help Show this screen. --version Output version and exit. + --prune Prune all NFD related attributes from all nodes + of the cluster and exit. + --kubeconfig= Kubeconfig to use [Default: ] --port= Port on which to listen for connections. [Default: 8080] --ca-file= Root certificate for verifying connections