Skip to content

Commit

Permalink
feat: add --tolerate-failures-status-check flag and associated logic …
Browse files Browse the repository at this point in the history 
…for improved ci/cd usage
  • Loading branch information
@aaron-prindle
aaron-prindle committed Nov 7, 2022
1 parent c50ca8a commit 2a66f9f
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 96 deletions.
9 changes: 9 additions & 0 deletions cmd/skaffold/app/cmd/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,15 @@ var flagRegistry = []Flag{
DefinedOn: []string{"dev", "debug", "deploy", "run", "apply"},
IsEnum: true,
},
{
Name: "tolerate-failures-status-check",
Usage: "Configures `status-check` to tolerate failures until Skaffold's statusCheckDeadline duration or the deployments progressDeadlineSeconds Otherwise deployment failures skaffold encounters will immediately fail the deployment. Defaults to 'false'",
Value: &opts.TolerateFailuresStatusCheck,
DefValue: false,
FlagAddMethod: "BoolVar",
DefinedOn: []string{"dev", "debug", "deploy", "run", "apply"},
IsEnum: true,
},
{
Name: "fast-fail-status-check",
Usage: "Configures `status-check` to fail immediately if any error occurs. Otherwise `status-check` will attempt to check all resources once and only then report errors and possibly exit. Defaults to 'true'",
Expand Down
149 changes: 75 additions & 74 deletions pkg/skaffold/config/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,80 +32,81 @@ type WaitForDeletions struct {

// SkaffoldOptions are options that are set by command line arguments not included in the config file itself
type SkaffoldOptions struct {
Apply bool
AutoBuild bool
AutoCreateConfig bool
AutoDeploy bool
AutoSync bool
AssumeYes bool
CacheArtifacts bool
ContainerDebugging bool
Cleanup bool
DetectMinikube bool
DryRun bool
EnableRPC bool
Force bool
ForceLoadImages bool
IterativeStatusCheck bool
FastFailStatusCheck bool
Notification bool
NoPrune bool
NoPruneChildren bool
ProfileAutoActivation bool
PropagateProfiles bool
RenderOnly bool
SkipTests bool
SkipConfigDefaults bool
Tail bool
WaitForConnection bool
EnablePlatformNodeAffinity bool
EnableGKEARMNodeToleration bool
DisableMultiPlatformBuild bool
CheckClusterNodePlatforms bool
MakePathsAbsolute *bool
MultiLevelRepo *bool
CloudRunProject string
CloudRunLocation string
ConfigurationFile string
HydrationDir string
InventoryNamespace string
InventoryID string
InventoryName string
GlobalConfig string
EventLogFile string
RenderOutput string
User string
CustomTag string
Namespace string
CacheFile string
Trigger string
KubeContext string
KubeConfig string
LastLogFile string
DigestSource string
Command string
MinikubeProfile string
RepoCacheDir string
TransformRulesFile string
VerifyDockerNetwork string
CustomLabels []string
TargetImages []string
Profiles []string
InsecureRegistries []string
ConfigurationFilter []string
HydratedManifests []string
Platforms []string
BuildConcurrency int
WatchPollInterval int
StatusCheck BoolOrUndefined
PushImages BoolOrUndefined
RPCPort IntOrUndefined
RPCHTTPPort IntOrUndefined
Muted Muted
PortForward PortForwardOptions
DefaultRepo StringOrUndefined
SyncRemoteCache SyncRemoteCacheOption
WaitForDeletions WaitForDeletions
Apply bool
AutoBuild bool
AutoCreateConfig bool
AutoDeploy bool
AutoSync bool
AssumeYes bool
CacheArtifacts bool
ContainerDebugging bool
Cleanup bool
DetectMinikube bool
DryRun bool
EnableRPC bool
Force bool
ForceLoadImages bool
IterativeStatusCheck bool
FastFailStatusCheck bool
TolerateFailuresStatusCheck bool
Notification bool
NoPrune bool
NoPruneChildren bool
ProfileAutoActivation bool
PropagateProfiles bool
RenderOnly bool
SkipTests bool
SkipConfigDefaults bool
Tail bool
WaitForConnection bool
EnablePlatformNodeAffinity bool
EnableGKEARMNodeToleration bool
DisableMultiPlatformBuild bool
CheckClusterNodePlatforms bool
MakePathsAbsolute *bool
MultiLevelRepo *bool
CloudRunProject string
CloudRunLocation string
ConfigurationFile string
HydrationDir string
InventoryNamespace string
InventoryID string
InventoryName string
GlobalConfig string
EventLogFile string
RenderOutput string
User string
CustomTag string
Namespace string
CacheFile string
Trigger string
KubeContext string
KubeConfig string
LastLogFile string
DigestSource string
Command string
MinikubeProfile string
RepoCacheDir string
TransformRulesFile string
VerifyDockerNetwork string
CustomLabels []string
TargetImages []string
Profiles []string
InsecureRegistries []string
ConfigurationFilter []string
HydratedManifests []string
Platforms []string
BuildConcurrency int
WatchPollInterval int
StatusCheck BoolOrUndefined
PushImages BoolOrUndefined
RPCPort IntOrUndefined
RPCHTTPPort IntOrUndefined
Muted Muted
PortForward PortForwardOptions
DefaultRepo StringOrUndefined
SyncRemoteCache SyncRemoteCacheOption
WaitForDeletions WaitForDeletions
}

type RunMode string
Expand Down
47 changes: 25 additions & 22 deletions pkg/skaffold/kubernetes/status/status_check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type Config interface {

StatusCheckDeadlineSeconds() int
FastFailStatusCheck() bool
StatusCheckTolerateFailures() bool
Muted() config.Muted
StatusCheck() *bool
}
Expand All @@ -89,32 +90,34 @@ type Monitor interface {
}

type monitor struct {
cfg Config
labeller *label.DefaultLabeller
deadlineSeconds int
muteLogs bool
failFast bool
seenResources resource.Group
singleRun singleflight.Group
namespaces *[]string
kubeContext string
manifests manifest.ManifestList
cfg Config
labeller *label.DefaultLabeller
deadlineSeconds int
muteLogs bool
failFast bool
tolerateFailures bool
seenResources resource.Group
singleRun singleflight.Group
namespaces *[]string
kubeContext string
manifests manifest.ManifestList
}

// NewStatusMonitor returns a status monitor which runs checks on selected resource rollouts.
// Currently implemented for deployments and statefulsets.
func NewStatusMonitor(cfg Config, labeller *label.DefaultLabeller, namespaces *[]string) Monitor {
return &monitor{
muteLogs: cfg.Muted().MuteStatusCheck(),
cfg: cfg,
labeller: labeller,
deadlineSeconds: cfg.StatusCheckDeadlineSeconds(),
seenResources: make(resource.Group),
singleRun: singleflight.Group{},
namespaces: namespaces,
kubeContext: cfg.GetKubeContext(),
manifests: make(manifest.ManifestList, 0),
failFast: cfg.FastFailStatusCheck(),
muteLogs: cfg.Muted().MuteStatusCheck(),
cfg: cfg,
labeller: labeller,
deadlineSeconds: cfg.StatusCheckDeadlineSeconds(),
seenResources: make(resource.Group),
singleRun: singleflight.Group{},
namespaces: namespaces,
kubeContext: cfg.GetKubeContext(),
manifests: make(manifest.ManifestList, 0),
failFast: cfg.FastFailStatusCheck(),
tolerateFailures: cfg.StatusCheckTolerateFailures(),
}
}

Expand Down Expand Up @@ -353,7 +356,7 @@ func getStatefulSets(ctx context.Context, client kubernetes.Interface, ns string
return resources, nil
}

func pollResourceStatus(ctx context.Context, cfg kubectl.Config, r *resource.Resource) {
func pollResourceStatus(ctx context.Context, cfg Config, r *resource.Resource) {
pollDuration := time.Duration(defaultPollPeriodInMilliseconds) * time.Millisecond
ticker := time.NewTicker(pollDuration)
defer ticker.Stop()
Expand Down Expand Up @@ -387,7 +390,7 @@ func pollResourceStatus(ctx context.Context, cfg kubectl.Config, r *resource.Res
// As any changes to build or deploy dependencies are not triggered, exit
// immediately rather than waiting for for statusCheckDeadlineSeconds
// TODO: https://github.com/GoogleContainerTools/skaffold/pull/4591
if r.HasEncounteredUnrecoverableError() {
if r.HasEncounteredUnrecoverableError() && !cfg.StatusCheckTolerateFailures() {
r.MarkComplete()
return
}
Expand Down
16 changes: 16 additions & 0 deletions pkg/skaffold/runner/runcontext/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,17 @@ func (ps Pipelines) TransformDenyList() []latest.ResourceFilter {
return denylist
}

func (ps Pipelines) StatusCheckTolerateFailures() bool {
failureTolerance := false
// set the group status check deadline to maximum of any individually specified value
for _, p := range ps.pipelines {
if p.Deploy.TolerateFailures {
failureTolerance = true
}
}
return failureTolerance
}

func (ps Pipelines) StatusCheckDeadlineSeconds() int {
c := 0
// set the group status check deadline to maximum of any individually specified value
Expand All @@ -155,6 +166,7 @@ func (ps Pipelines) StatusCheckDeadlineSeconds() int {
}
return c
}

func NewPipelines(pipelinesByConfig map[string]latest.Pipeline) Pipelines {
m := make(map[string]latest.Pipeline)
var pipelines []latest.Pipeline
Expand Down Expand Up @@ -191,6 +203,10 @@ func (rc *RunContext) StatusCheckDeadlineSeconds() int {
return rc.Pipelines.StatusCheckDeadlineSeconds()
}

func (rc *RunContext) StatusCheckTolerateFailures() bool {
return rc.Opts.TolerateFailuresStatusCheck || rc.Pipelines.StatusCheckTolerateFailures()
}

func (rc *RunContext) SkipTests() bool {
return rc.Opts.SkipTests
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/skaffold/schema/latest/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,11 @@ type DeployConfig struct {
// StatusCheckDeadlineSeconds *beta* is the deadline for deployments to stabilize in seconds.
StatusCheckDeadlineSeconds int `yaml:"statusCheckDeadlineSeconds,omitempty"`

// TolerateFailures configures the Skaffold "status-check" to tolerate failures
// (flapping deployments, etc.) until the statusCheckDeadlineSeconds duration or k8s object
// timeouts such as progressDeadlineSeconds, etc.
TolerateFailures bool `yaml:"tolerateFailures,omitempty"`

// KubeContext is the Kubernetes context that Skaffold should deploy to.
// For example: `minikube`.
KubeContext string `yaml:"kubeContext,omitempty"`
Expand Down

0 comments on commit 2a66f9f

Please sign in to comment.