diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 9f7de5a8..feb19623 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -20,6 +20,7 @@ import ( "k8s.io/client-go/tools/reference" "github.com/linki/chaoskube/metrics" + "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) @@ -45,6 +46,8 @@ type Chaoskube struct { MinimumAge time.Duration // an instance of logrus.StdLogger to write log messages to Logger log.FieldLogger + // a terminator that termiantes victim pods + Terminator terminator.Terminator // dry run will not allow any pod terminations DryRun bool // grace period to terminate the pods @@ -74,8 +77,9 @@ var ( // * a list of weekdays, times of day and/or days of a year when chaos mode is disabled // * a time zone to apply to the aforementioned time-based filters // * a logger implementing logrus.FieldLogger to send log output to +// * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, gracePeriod time.Duration) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -92,7 +96,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel MinimumAge: minimumAge, Logger: logger, DryRun: dryRun, - GracePeriod: gracePeriod, + Terminator: terminator, EventRecorder: recorder, Now: time.Now, } @@ -196,7 +200,7 @@ func (c *Chaoskube) Candidates() ([]v1.Pod, error) { return pods, nil } -// DeletePod deletes the given pod. +// DeletePod deletes the given pod with the selected terminator. // It will not delete the pod if dry-run mode is enabled. func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.Logger.WithFields(log.Fields{ @@ -204,12 +208,13 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { "name": victim.Name, }).Info("terminating pod") + // return early if we're running in dryRun mode. if c.DryRun { return nil } start := time.Now() - err := c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, deleteOptions(c.GracePeriod)) + err := c.Terminator.Terminate(victim) metrics.TerminationDurationSeconds.Observe(time.Since(start).Seconds()) if err != nil { return err @@ -222,7 +227,7 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { return err } - c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was deleted by chaoskube to introduce chaos.") + c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") return nil } @@ -337,11 +342,3 @@ func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) return filteredList } - -func deleteOptions(gracePeriod time.Duration) *metav1.DeleteOptions { - if gracePeriod < 0 { - return nil - } - - return &metav1.DeleteOptions{GracePeriodSeconds: (*int64)(&gracePeriod)} -} diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index ecc5e9e0..4f18630e 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -14,13 +14,15 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes/fake" + "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" "github.com/stretchr/testify/suite" ) type Suite struct { - suite.Suite + testutil.TestSuite } var ( @@ -43,7 +45,8 @@ func (suite *Suite) TestNew() { excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} excludedDaysOfYear = []time.Time{time.Now()} minimumAge = time.Duration(42) - gracePeriod = 10 * time.Second + dryRun = true + terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) ) chaoskube := New( @@ -57,8 +60,8 @@ func (suite *Suite) TestNew() { time.UTC, minimumAge, logger, - false, - gracePeriod, + dryRun, + terminator, ) suite.Require().NotNil(chaoskube) @@ -72,8 +75,8 @@ func (suite *Suite) TestNew() { suite.Equal(time.UTC, chaoskube.Timezone) suite.Equal(minimumAge, chaoskube.MinimumAge) suite.Equal(logger, chaoskube.Logger) - suite.Equal(false, chaoskube.DryRun) - suite.Equal(gracePeriod, chaoskube.GracePeriod) + suite.Equal(dryRun, chaoskube.DryRun) + suite.Equal(terminator, chaoskube.Terminator) } // TestRunContextCanceled tests that a canceled context will exit the Run function. @@ -97,6 +100,7 @@ func (suite *Suite) TestRunContextCanceled() { chaoskube.Run(ctx, nil) } +// TestCandidates tests that the various pod filters are applied correctly. func (suite *Suite) TestCandidates() { foo := map[string]string{"namespace": "default", "name": "foo"} bar := map[string]string{"namespace": "testing", "name": "bar"} @@ -145,6 +149,7 @@ func (suite *Suite) TestCandidates() { } } +// TestVictim tests that a random victim is chosen from selected candidates. func (suite *Suite) TestVictim() { foo := map[string]string{"namespace": "default", "name": "foo"} bar := map[string]string{"namespace": "testing", "name": "bar"} @@ -200,6 +205,7 @@ func (suite *Suite) TestNoVictimReturnsError() { suite.EqualError(err, "pod not found") } +// TestDeletePod tests that a given pod is deleted and dryRun is respected. func (suite *Suite) TestDeletePod() { foo := map[string]string{"namespace": "default", "name": "foo"} bar := map[string]string{"namespace": "testing", "name": "bar"} @@ -229,11 +235,12 @@ func (suite *Suite) TestDeletePod() { err := chaoskube.DeletePod(victim) suite.Require().NoError(err) - suite.assertLog(log.InfoLevel, "terminating pod", log.Fields{"namespace": "default", "name": "foo"}) + suite.AssertLog(logOutput, log.InfoLevel, "terminating pod", log.Fields{"namespace": "default", "name": "foo"}) suite.assertCandidates(chaoskube, tt.remainingPods) } } +// TestDeletePodNotFound tests missing target pod will return an error. func (suite *Suite) TestDeletePodNotFound() { chaoskube := suite.setup( labels.Everything(), @@ -504,7 +511,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { err := chaoskube.TerminateVictim() suite.Require().NoError(err) - suite.assertLog(log.DebugLevel, msgVictimNotFound, log.Fields{}) + suite.AssertLog(logOutput, log.DebugLevel, msgVictimNotFound, log.Fields{}) } // helper functions @@ -513,38 +520,14 @@ func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string pods, err := chaoskube.Candidates() suite.Require().NoError(err) - suite.assertPods(pods, expected) + suite.AssertPods(pods, expected) } func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]string) { victim, err := chaoskube.Victim() suite.Require().NoError(err) - suite.assertPod(victim, expected) -} - -func (suite *Suite) assertPods(pods []v1.Pod, expected []map[string]string) { - suite.Require().Len(pods, len(expected)) - - for i, pod := range pods { - suite.assertPod(pod, expected[i]) - } -} - -func (suite *Suite) assertPod(pod v1.Pod, expected map[string]string) { - suite.Equal(expected["namespace"], pod.Namespace) - suite.Equal(expected["name"], pod.Name) -} - -func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) { - suite.Require().NotEmpty(logOutput.Entries) - - lastEntry := logOutput.LastEntry() - suite.Equal(level, lastEntry.Level) - suite.Equal(msg, lastEntry.Message) - for k := range fields { - suite.Equal(fields[k], lastEntry.Data[k]) - } + suite.AssertPod(victim, expected) } func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { @@ -578,8 +561,11 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { logOutput.Reset() + client := fake.NewSimpleClientset() + nullLogger, _ := test.NewNullLogger() + return New( - fake.NewSimpleClientset(), + client, labelSelector, annotations, namespaces, @@ -590,7 +576,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele minimumAge, logger, dryRun, - gracePeriod, + terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), ) } @@ -707,29 +693,3 @@ func (suite *Suite) TestMinimumAge() { suite.Len(pods, tt.candidates) } } - -func (suite *Suite) TestDeleteOptions() { - for _, tt := range []struct { - gracePeriod time.Duration - expected *metav1.DeleteOptions - }{ - { - -1, - nil, - }, - { - 0, - &metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(0)}, - }, - { - 300, - &metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(300)}, - }, - } { - suite.Equal(tt.expected, deleteOptions(tt.gracePeriod)) - } -} - -func int64Ptr(value int64) *int64 { - return &value -} diff --git a/internal/testutil/assert.go b/internal/testutil/assert.go new file mode 100644 index 00000000..56001366 --- /dev/null +++ b/internal/testutil/assert.go @@ -0,0 +1,38 @@ +package testutil + +import ( + log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + + "k8s.io/api/core/v1" + + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite +} + +func (suite *TestSuite) AssertPods(pods []v1.Pod, expected []map[string]string) { + suite.Require().Len(pods, len(expected)) + + for i, pod := range pods { + suite.AssertPod(pod, expected[i]) + } +} + +func (suite *TestSuite) AssertPod(pod v1.Pod, expected map[string]string) { + suite.Equal(expected["namespace"], pod.Namespace) + suite.Equal(expected["name"], pod.Name) +} + +func (suite *TestSuite) AssertLog(output *test.Hook, level log.Level, msg string, fields log.Fields) { + suite.Require().NotEmpty(output.Entries) + + lastEntry := output.LastEntry() + suite.Equal(level, lastEntry.Level) + suite.Equal(msg, lastEntry.Message) + for k := range fields { + suite.Equal(fields[k], lastEntry.Data[k]) + } +} diff --git a/main.go b/main.go index 8ee13baa..6df43e5f 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "github.com/linki/chaoskube/chaoskube" + "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) @@ -167,7 +168,7 @@ func main() { minimumAge, log.StandardLogger(), dryRun, - gracePeriod, + terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), ) if metricsAddress != "" { diff --git a/terminator/delete_pod.go b/terminator/delete_pod.go new file mode 100644 index 00000000..2a8f02c1 --- /dev/null +++ b/terminator/delete_pod.go @@ -0,0 +1,45 @@ +package terminator + +import ( + "time" + + log "github.com/sirupsen/logrus" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// DeletePodTerminator simply asks k8s to delete the victim pod. +type DeletePodTerminator struct { + client kubernetes.Interface + logger log.FieldLogger + gracePeriod time.Duration +} + +// NewDeletePodTerminator creates and returns a DeletePodTerminator object. +func NewDeletePodTerminator(client kubernetes.Interface, logger log.FieldLogger, gracePeriod time.Duration) *DeletePodTerminator { + return &DeletePodTerminator{ + client: client, + logger: logger.WithField("terminator", "DeletePod"), + gracePeriod: gracePeriod, + } +} + +// Terminate sends a request to Kubernetes to delete the pod. +func (t *DeletePodTerminator) Terminate(victim v1.Pod) error { + t.logger.WithFields(log.Fields{ + "namespace": victim.Namespace, + "name": victim.Name, + }).Debug("calling deletePod endpoint") + + return t.client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, deleteOptions(t.gracePeriod)) +} + +func deleteOptions(gracePeriod time.Duration) *metav1.DeleteOptions { + if gracePeriod < 0 { + return nil + } + + return &metav1.DeleteOptions{GracePeriodSeconds: (*int64)(&gracePeriod)} +} diff --git a/terminator/delete_pod_test.go b/terminator/delete_pod_test.go new file mode 100644 index 00000000..9146503a --- /dev/null +++ b/terminator/delete_pod_test.go @@ -0,0 +1,95 @@ +package terminator + +import ( + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/util" + + "github.com/stretchr/testify/suite" +) + +type DeletePodTerminatorSuite struct { + testutil.TestSuite +} + +var ( + logger, logOutput = test.NewNullLogger() +) + +func (suite *DeletePodTerminatorSuite) SetupTest() { + logger.SetLevel(log.DebugLevel) + logOutput.Reset() +} + +func (suite *DeletePodTerminatorSuite) TestInterface() { + suite.Implements((*Terminator)(nil), new(DeletePodTerminator)) +} + +func (suite *DeletePodTerminatorSuite) TestTerminate() { + logOutput.Reset() + client := fake.NewSimpleClientset() + terminator := NewDeletePodTerminator(client, logger, 10*time.Second) + + pods := []v1.Pod{ + util.NewPod("default", "foo", v1.PodRunning), + util.NewPod("testing", "bar", v1.PodRunning), + } + + for _, pod := range pods { + _, err := client.CoreV1().Pods(pod.Namespace).Create(&pod) + suite.Require().NoError(err) + } + + victim := util.NewPod("default", "foo", v1.PodRunning) + + err := terminator.Terminate(victim) + suite.Require().NoError(err) + + suite.AssertLog(logOutput, log.DebugLevel, "calling deletePod endpoint", log.Fields{"namespace": "default", "name": "foo"}) + + remainingPods, err := client.CoreV1().Pods(v1.NamespaceAll).List(metav1.ListOptions{}) + suite.Require().NoError(err) + + suite.AssertPods(remainingPods.Items, []map[string]string{ + {"namespace": "testing", "name": "bar"}, + }) +} + +func (suite *DeletePodTerminatorSuite) TestDeleteOptions() { + for _, tt := range []struct { + gracePeriod time.Duration + expected *metav1.DeleteOptions + }{ + { + -1, + nil, + }, + { + 0, + &metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(0)}, + }, + { + 300, + &metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(300)}, + }, + } { + suite.Equal(tt.expected, deleteOptions(tt.gracePeriod)) + } +} + +func TestDeletePodTerminatorSuite(t *testing.T) { + suite.Run(t, new(DeletePodTerminatorSuite)) +} + +func int64Ptr(value int64) *int64 { + return &value +} diff --git a/terminator/terminator.go b/terminator/terminator.go new file mode 100644 index 00000000..0e338877 --- /dev/null +++ b/terminator/terminator.go @@ -0,0 +1,11 @@ +package terminator + +import ( + "k8s.io/api/core/v1" +) + +// Terminator is the interface for implementations of pod terminators. +type Terminator interface { + // Terminate terminates the given pod. + Terminate(victim v1.Pod) error +}