Skip to content

Commit

Permalink
Merge pull request #117 from linki/jake-exec-refactor-without-exec
Browse files Browse the repository at this point in the history
Refactor out a Terminator interface to support different ways to kill pods
  • Loading branch information
linki authored Jan 20, 2019
2 parents 52177ce + 8c80f28 commit 3164017
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 76 deletions.
23 changes: 10 additions & 13 deletions chaoskube/chaoskube.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"k8s.io/client-go/tools/reference"

"github.com/linki/chaoskube/metrics"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"
)

Expand All @@ -45,6 +46,8 @@ type Chaoskube struct {
MinimumAge time.Duration
// an instance of logrus.StdLogger to write log messages to
Logger log.FieldLogger
// a terminator that termiantes victim pods
Terminator terminator.Terminator
// dry run will not allow any pod terminations
DryRun bool
// grace period to terminate the pods
Expand Down Expand Up @@ -74,8 +77,9 @@ var (
// * a list of weekdays, times of day and/or days of a year when chaos mode is disabled
// * a time zone to apply to the aforementioned time-based filters
// * a logger implementing logrus.FieldLogger to send log output to
// * what specific terminator to use to imbue chaos on victim pods
// * whether to enable/disable dry-run mode
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, gracePeriod time.Duration) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator) *Chaoskube {
broadcaster := record.NewBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
Expand All @@ -92,7 +96,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel
MinimumAge: minimumAge,
Logger: logger,
DryRun: dryRun,
GracePeriod: gracePeriod,
Terminator: terminator,
EventRecorder: recorder,
Now: time.Now,
}
Expand Down Expand Up @@ -196,20 +200,21 @@ func (c *Chaoskube) Candidates() ([]v1.Pod, error) {
return pods, nil
}

// DeletePod deletes the given pod.
// DeletePod deletes the given pod with the selected terminator.
// It will not delete the pod if dry-run mode is enabled.
func (c *Chaoskube) DeletePod(victim v1.Pod) error {
c.Logger.WithFields(log.Fields{
"namespace": victim.Namespace,
"name": victim.Name,
}).Info("terminating pod")

// return early if we're running in dryRun mode.
if c.DryRun {
return nil
}

start := time.Now()
err := c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, deleteOptions(c.GracePeriod))
err := c.Terminator.Terminate(victim)
metrics.TerminationDurationSeconds.Observe(time.Since(start).Seconds())
if err != nil {
return err
Expand All @@ -222,7 +227,7 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
return err
}

c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was deleted by chaoskube to introduce chaos.")
c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.")

return nil
}
Expand Down Expand Up @@ -337,11 +342,3 @@ func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time)

return filteredList
}

func deleteOptions(gracePeriod time.Duration) *metav1.DeleteOptions {
if gracePeriod < 0 {
return nil
}

return &metav1.DeleteOptions{GracePeriodSeconds: (*int64)(&gracePeriod)}
}
84 changes: 22 additions & 62 deletions chaoskube/chaoskube_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes/fake"

"github.com/linki/chaoskube/internal/testutil"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"

"github.com/stretchr/testify/suite"
)

type Suite struct {
suite.Suite
testutil.TestSuite
}

var (
Expand All @@ -43,7 +45,8 @@ func (suite *Suite) TestNew() {
excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
excludedDaysOfYear = []time.Time{time.Now()}
minimumAge = time.Duration(42)
gracePeriod = 10 * time.Second
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
)

chaoskube := New(
Expand All @@ -57,8 +60,8 @@ func (suite *Suite) TestNew() {
time.UTC,
minimumAge,
logger,
false,
gracePeriod,
dryRun,
terminator,
)
suite.Require().NotNil(chaoskube)

Expand All @@ -72,8 +75,8 @@ func (suite *Suite) TestNew() {
suite.Equal(time.UTC, chaoskube.Timezone)
suite.Equal(minimumAge, chaoskube.MinimumAge)
suite.Equal(logger, chaoskube.Logger)
suite.Equal(false, chaoskube.DryRun)
suite.Equal(gracePeriod, chaoskube.GracePeriod)
suite.Equal(dryRun, chaoskube.DryRun)
suite.Equal(terminator, chaoskube.Terminator)
}

// TestRunContextCanceled tests that a canceled context will exit the Run function.
Expand All @@ -97,6 +100,7 @@ func (suite *Suite) TestRunContextCanceled() {
chaoskube.Run(ctx, nil)
}

// TestCandidates tests that the various pod filters are applied correctly.
func (suite *Suite) TestCandidates() {
foo := map[string]string{"namespace": "default", "name": "foo"}
bar := map[string]string{"namespace": "testing", "name": "bar"}
Expand Down Expand Up @@ -145,6 +149,7 @@ func (suite *Suite) TestCandidates() {
}
}

// TestVictim tests that a random victim is chosen from selected candidates.
func (suite *Suite) TestVictim() {
foo := map[string]string{"namespace": "default", "name": "foo"}
bar := map[string]string{"namespace": "testing", "name": "bar"}
Expand Down Expand Up @@ -200,6 +205,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
suite.EqualError(err, "pod not found")
}

// TestDeletePod tests that a given pod is deleted and dryRun is respected.
func (suite *Suite) TestDeletePod() {
foo := map[string]string{"namespace": "default", "name": "foo"}
bar := map[string]string{"namespace": "testing", "name": "bar"}
Expand Down Expand Up @@ -229,11 +235,12 @@ func (suite *Suite) TestDeletePod() {
err := chaoskube.DeletePod(victim)
suite.Require().NoError(err)

suite.assertLog(log.InfoLevel, "terminating pod", log.Fields{"namespace": "default", "name": "foo"})
suite.AssertLog(logOutput, log.InfoLevel, "terminating pod", log.Fields{"namespace": "default", "name": "foo"})
suite.assertCandidates(chaoskube, tt.remainingPods)
}
}

// TestDeletePodNotFound tests missing target pod will return an error.
func (suite *Suite) TestDeletePodNotFound() {
chaoskube := suite.setup(
labels.Everything(),
Expand Down Expand Up @@ -504,7 +511,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
err := chaoskube.TerminateVictim()
suite.Require().NoError(err)

suite.assertLog(log.DebugLevel, msgVictimNotFound, log.Fields{})
suite.AssertLog(logOutput, log.DebugLevel, msgVictimNotFound, log.Fields{})
}

// helper functions
Expand All @@ -513,38 +520,14 @@ func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string
pods, err := chaoskube.Candidates()
suite.Require().NoError(err)

suite.assertPods(pods, expected)
suite.AssertPods(pods, expected)
}

func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]string) {
victim, err := chaoskube.Victim()
suite.Require().NoError(err)

suite.assertPod(victim, expected)
}

func (suite *Suite) assertPods(pods []v1.Pod, expected []map[string]string) {
suite.Require().Len(pods, len(expected))

for i, pod := range pods {
suite.assertPod(pod, expected[i])
}
}

func (suite *Suite) assertPod(pod v1.Pod, expected map[string]string) {
suite.Equal(expected["namespace"], pod.Namespace)
suite.Equal(expected["name"], pod.Name)
}

func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) {
suite.Require().NotEmpty(logOutput.Entries)

lastEntry := logOutput.LastEntry()
suite.Equal(level, lastEntry.Level)
suite.Equal(msg, lastEntry.Message)
for k := range fields {
suite.Equal(fields[k], lastEntry.Data[k])
}
suite.AssertPod(victim, expected)
}

func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube {
Expand Down Expand Up @@ -578,8 +561,11 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube {
logOutput.Reset()

client := fake.NewSimpleClientset()
nullLogger, _ := test.NewNullLogger()

return New(
fake.NewSimpleClientset(),
client,
labelSelector,
annotations,
namespaces,
Expand All @@ -590,7 +576,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
minimumAge,
logger,
dryRun,
gracePeriod,
terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod),
)
}

Expand Down Expand Up @@ -707,29 +693,3 @@ func (suite *Suite) TestMinimumAge() {
suite.Len(pods, tt.candidates)
}
}

func (suite *Suite) TestDeleteOptions() {
for _, tt := range []struct {
gracePeriod time.Duration
expected *metav1.DeleteOptions
}{
{
-1,
nil,
},
{
0,
&metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(0)},
},
{
300,
&metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(300)},
},
} {
suite.Equal(tt.expected, deleteOptions(tt.gracePeriod))
}
}

func int64Ptr(value int64) *int64 {
return &value
}
38 changes: 38 additions & 0 deletions internal/testutil/assert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package testutil

import (
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"

"k8s.io/api/core/v1"

"github.com/stretchr/testify/suite"
)

type TestSuite struct {
suite.Suite
}

func (suite *TestSuite) AssertPods(pods []v1.Pod, expected []map[string]string) {
suite.Require().Len(pods, len(expected))

for i, pod := range pods {
suite.AssertPod(pod, expected[i])
}
}

func (suite *TestSuite) AssertPod(pod v1.Pod, expected map[string]string) {
suite.Equal(expected["namespace"], pod.Namespace)
suite.Equal(expected["name"], pod.Name)
}

func (suite *TestSuite) AssertLog(output *test.Hook, level log.Level, msg string, fields log.Fields) {
suite.Require().NotEmpty(output.Entries)

lastEntry := output.LastEntry()
suite.Equal(level, lastEntry.Level)
suite.Equal(msg, lastEntry.Message)
for k := range fields {
suite.Equal(fields[k], lastEntry.Data[k])
}
}
3 changes: 2 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"k8s.io/client-go/tools/clientcmd"

"github.com/linki/chaoskube/chaoskube"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"
)

Expand Down Expand Up @@ -167,7 +168,7 @@ func main() {
minimumAge,
log.StandardLogger(),
dryRun,
gracePeriod,
terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod),
)

if metricsAddress != "" {
Expand Down
45 changes: 45 additions & 0 deletions terminator/delete_pod.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package terminator

import (
"time"

log "github.com/sirupsen/logrus"

"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

// DeletePodTerminator simply asks k8s to delete the victim pod.
type DeletePodTerminator struct {
client kubernetes.Interface
logger log.FieldLogger
gracePeriod time.Duration
}

// NewDeletePodTerminator creates and returns a DeletePodTerminator object.
func NewDeletePodTerminator(client kubernetes.Interface, logger log.FieldLogger, gracePeriod time.Duration) *DeletePodTerminator {
return &DeletePodTerminator{
client: client,
logger: logger.WithField("terminator", "DeletePod"),
gracePeriod: gracePeriod,
}
}

// Terminate sends a request to Kubernetes to delete the pod.
func (t *DeletePodTerminator) Terminate(victim v1.Pod) error {
t.logger.WithFields(log.Fields{
"namespace": victim.Namespace,
"name": victim.Name,
}).Debug("calling deletePod endpoint")

return t.client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, deleteOptions(t.gracePeriod))
}

func deleteOptions(gracePeriod time.Duration) *metav1.DeleteOptions {
if gracePeriod < 0 {
return nil
}

return &metav1.DeleteOptions{GracePeriodSeconds: (*int64)(&gracePeriod)}
}
Loading

0 comments on commit 3164017

Please sign in to comment.