From 1abfdbc51704bed42620cfb4049c32085337b1fd Mon Sep 17 00:00:00 2001 From: oysteingraendsen <75323242+oysteingraendsen@users.noreply.github.com> Date: Fri, 12 May 2023 19:24:33 +0200 Subject: [PATCH] feat: when using order group, abort plan/apply if any fail (#3323) * feat: when using order group, abort plan/apply if any fail * feat: add 'abort_on_execution_order_fail' flag on repo level * feat: use runProjectCmdsParallelGroups in version_command_runner * chore: add plan tests --------- Co-authored-by: nitrocode <7775707+nitrocode@users.noreply.github.com> --- .../docs/repo-level-atlantis-yaml.md | 8 +- server/core/config/raw/repo_cfg.go | 10 + server/core/config/raw/repo_cfg_test.go | 47 ++- server/core/config/valid/repo_cfg.go | 1 + server/events/apply_command_runner.go | 2 +- server/events/apply_command_runner_test.go | 293 ++++++++++++++ server/events/command/project_context.go | 2 + server/events/plan_command_runner.go | 4 +- server/events/plan_command_runner_test.go | 359 ++++++++++++++++++ server/events/project_command_builder.go | 10 + .../events/project_command_context_builder.go | 18 +- .../project_command_context_builder_test.go | 26 +- .../events/project_command_pool_executor.go | 5 + server/events/version_command_runner.go | 2 +- 14 files changed, 753 insertions(+), 34 deletions(-) diff --git a/runatlantis.io/docs/repo-level-atlantis-yaml.md b/runatlantis.io/docs/repo-level-atlantis-yaml.md index 2a0d079392..d291622790 100644 --- a/runatlantis.io/docs/repo-level-atlantis-yaml.md +++ b/runatlantis.io/docs/repo-level-atlantis-yaml.md @@ -50,6 +50,7 @@ automerge: true delete_source_branch_on_merge: true parallel_plan: true parallel_apply: true +abort_on_execution_order_fail: true projects: - name: my-project-name branch: /main/ @@ -64,6 +65,7 @@ projects: plan_requirements: [mergeable, approved, undiverged] apply_requirements: [mergeable, approved, undiverged] import_requirements: [mergeable, approved, undiverged] + execution_order_group: 1 workflow: myworkflow workflows: myworkflow: @@ -259,6 +261,7 @@ to be allowed to set this key. See [Server-Side Repo Config Use Cases](server-si ### Order of planning/applying ```yaml version: 3 +abort_on_execution_order_fail: true projects: - dir: project1 execution_order_group: 2 @@ -268,7 +271,10 @@ projects: With this config above, Atlantis runs planning/applying for project2 first, then for project1. Several projects can have same `execution_order_group`. Any order in one group isn't guaranteed. `parallel_plan` and `parallel_apply` respect these order groups, so parallel planning/applying works -in each group one by one. +in each group one by one. + +If any plan/apply fails and `abort_on_execution_order_fail` is set to true on a repo level, all the +following groups will be aborted. For this example, if project2 fails then project1 will not run. ### Custom Backend Config See [Custom Workflow Use Cases: Custom Backend Config](custom-workflows.html#custom-backend-config) diff --git a/server/core/config/raw/repo_cfg.go b/server/core/config/raw/repo_cfg.go index 5c1b46391e..e2bad4add1 100644 --- a/server/core/config/raw/repo_cfg.go +++ b/server/core/config/raw/repo_cfg.go @@ -25,6 +25,9 @@ const DefaultDeleteSourceBranchOnMerge = false // DefaultEmojiReaction is the default emoji reaction for repos const DefaultEmojiReaction = "" +// DefaultAbortOnExcecutionOrderFail being false is the default setting for abort on execution group failiures +const DefaultAbortOnExcecutionOrderFail = false + // RepoCfg is the raw schema for repo-level atlantis.yaml config. type RepoCfg struct { Version *int `yaml:"version,omitempty"` @@ -37,6 +40,7 @@ type RepoCfg struct { DeleteSourceBranchOnMerge *bool `yaml:"delete_source_branch_on_merge,omitempty"` EmojiReaction *string `yaml:"emoji_reaction,omitempty"` AllowedRegexpPrefixes []string `yaml:"allowed_regexp_prefixes,omitempty"` + AbortOnExcecutionOrderFail *bool `yaml:"abort_on_execution_order_fail,omitempty"` } func (r RepoCfg) Validate() error { @@ -88,6 +92,11 @@ func (r RepoCfg) ToValid() valid.RepoCfg { emojiReaction = *r.EmojiReaction } + abortOnExcecutionOrderFail := DefaultAbortOnExcecutionOrderFail + if r.AbortOnExcecutionOrderFail != nil { + abortOnExcecutionOrderFail = *r.AbortOnExcecutionOrderFail + } + return valid.RepoCfg{ Version: *r.Version, Projects: validProjects, @@ -99,5 +108,6 @@ func (r RepoCfg) ToValid() valid.RepoCfg { DeleteSourceBranchOnMerge: r.DeleteSourceBranchOnMerge, AllowedRegexpPrefixes: r.AllowedRegexpPrefixes, EmojiReaction: emojiReaction, + AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail, } } diff --git a/server/core/config/raw/repo_cfg_test.go b/server/core/config/raw/repo_cfg_test.go index 5a78960a99..7c138053f3 100644 --- a/server/core/config/raw/repo_cfg_test.go +++ b/server/core/config/raw/repo_cfg_test.go @@ -259,43 +259,48 @@ func TestConfig_ToValid(t *testing.T) { }, }, { - description: "automerge and parallel_apply omitted", + description: "automerge, parallel_apply and abort_on_execution_order_fail omitted", input: raw.RepoCfg{ Version: Int(2), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: false, - ParallelApply: false, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: false, + ParallelApply: false, + AbortOnExcecutionOrderFail: false, + Workflows: map[string]valid.Workflow{}, }, }, { - description: "automerge and parallel_apply true", + description: "automerge, parallel_apply and abort_on_execution_order_fail true", input: raw.RepoCfg{ - Version: Int(2), - Automerge: Bool(true), - ParallelApply: Bool(true), + Version: Int(2), + Automerge: Bool(true), + ParallelApply: Bool(true), + AbortOnExcecutionOrderFail: Bool(true), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: true, - ParallelApply: true, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: true, + ParallelApply: true, + AbortOnExcecutionOrderFail: true, + Workflows: map[string]valid.Workflow{}, }, }, { - description: "automerge and parallel_apply false", + description: "automerge, parallel_apply and abort_on_execution_order_fail false", input: raw.RepoCfg{ - Version: Int(2), - Automerge: Bool(false), - ParallelApply: Bool(false), + Version: Int(2), + Automerge: Bool(false), + ParallelApply: Bool(false), + AbortOnExcecutionOrderFail: Bool(false), }, exp: valid.RepoCfg{ - Version: 2, - Automerge: false, - ParallelApply: false, - Workflows: map[string]valid.Workflow{}, + Version: 2, + Automerge: false, + ParallelApply: false, + AbortOnExcecutionOrderFail: false, + Workflows: map[string]valid.Workflow{}, }, }, { diff --git a/server/core/config/valid/repo_cfg.go b/server/core/config/valid/repo_cfg.go index 06391b4c53..4c74eea60a 100644 --- a/server/core/config/valid/repo_cfg.go +++ b/server/core/config/valid/repo_cfg.go @@ -26,6 +26,7 @@ type RepoCfg struct { RepoLocking *bool EmojiReaction string AllowedRegexpPrefixes []string + AbortOnExcecutionOrderFail bool } func (r RepoCfg) FindProjectsByDirWorkspace(repoRelDir string, workspace string) []Project { diff --git a/server/events/apply_command_runner.go b/server/events/apply_command_runner.go index 6dc89bb30d..c8189d0349 100644 --- a/server/events/apply_command_runner.go +++ b/server/events/apply_command_runner.go @@ -161,7 +161,7 @@ func (a *ApplyCommandRunner) Run(ctx *command.Context, cmd *CommentCommand) { var result command.Result if a.isParallelEnabled(projectCmds) { ctx.Log.Info("Running applies in parallel") - result = runProjectCmdsParallelGroups(projectCmds, a.prjCmdRunner.Apply, a.parallelPoolSize) + result = runProjectCmdsParallelGroups(ctx, projectCmds, a.prjCmdRunner.Apply, a.parallelPoolSize) } else { result = runProjectCmds(projectCmds, a.prjCmdRunner.Apply) } diff --git a/server/events/apply_command_runner_test.go b/server/events/apply_command_runner_test.go index 713935d6cc..90e94067ee 100644 --- a/server/events/apply_command_runner_test.go +++ b/server/events/apply_command_runner_test.go @@ -215,3 +215,296 @@ func TestApplyCommandRunner_IsSilenced(t *testing.T) { }) } } + +func TestApplyCommandRunner_ExecutionOrder(t *testing.T) { + logger := logging.NewNoopLogger(t) + RegisterMockTestingT(t) + + cases := []struct { + Description string + ProjectContexts []command.ProjectContext + ProjectResults []command.ProjectResult + RunnerInvokeMatch []*EqMatcher + ExpComment string + }{ + { + Description: "When first apply fails, the second don't run", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Second", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + ExpComment: "Ran Apply for 2 projects:\n\n" + + "1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n\n### 1. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---\n### " + + "2. dir: `` workspace: ``\n**Apply Error**\n```\nShabang!\n```\n\n---", + }, + { + Description: "When first apply fails, the second not will run", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Second", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Never(), + }, + ExpComment: "Ran Apply for dir: `` workspace: ``\n\n**Apply Error**\n```\nShabang!\n```", + }, + { + Description: "When both in a group of two succeeds, the following two will run", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 0, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Third", + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Fourth", + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + Never(), + Never(), + }, + ExpComment: "Ran Apply for 2 projects:\n\n" + + "1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n\n### 1. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---\n### " + + "2. dir: `` workspace: ``\n**Apply Error**\n```\nShabang!\n```\n\n---", + }, + { + Description: "When one out of two fails, the following two will not run", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelApplyEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 0, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Third", + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + AbortOnExcecutionOrderFail: true, + ProjectName: "Fourth", + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + Once(), + Once(), + }, + ExpComment: "Ran Apply for 4 projects:\n\n" + + "1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n\n### 1. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---\n### " + + "2. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---\n### " + + "3. dir: `` workspace: ``\n**Apply Error**\n```\nShabang!\n```\n\n---\n### " + + "4. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---", + }, + { + Description: "Don't block when parallel is not set", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + AbortOnExcecutionOrderFail: true, + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + ExpComment: "Ran Apply for 2 projects:\n\n" + + "1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n\n### 1. dir: `` workspace: ``\n**Apply Error**\n```\nShabang!\n```\n\n---\n### " + + "2. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---", + }, + { + Description: "Don't block when abortOnExcecutionOrderFail is not set", + ProjectContexts: []command.ProjectContext{ + { + ExecutionOrderGroup: 0, + ProjectName: "First", + }, + { + ExecutionOrderGroup: 1, + ProjectName: "Second", + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Apply, + Error: errors.New("Shabang!"), + }, + { + Command: command.Apply, + ApplySuccess: "Great success!", + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + ExpComment: "Ran Apply for 2 projects:\n\n" + + "1. dir: `` workspace: ``\n1. dir: `` workspace: ``\n\n### 1. dir: `` workspace: ``\n**Apply Error**\n```\nShabang!\n```\n\n---\n### " + + "2. dir: `` workspace: ``\n```diff\nGreat success!\n```\n\n---", + }, + } + + for _, c := range cases { + t.Run(c.Description, func(t *testing.T) { + vcsClient := setup(t) + + scopeNull, _, _ := metrics.NewLoggingScope(logger, "atlantis") + + pull := &github.PullRequest{ + State: github.String("open"), + } + modelPull := models.PullRequest{BaseRepo: testdata.GithubRepo, State: models.OpenPullState, Num: testdata.Pull.Num} + + cmd := &events.CommentCommand{Name: command.Apply} + + ctx := &command.Context{ + User: testdata.User, + Log: logging.NewNoopLogger(t), + Scope: scopeNull, + Pull: modelPull, + HeadRepo: testdata.GithubRepo, + Trigger: command.CommentTrigger, + } + + When(githubGetter.GetPullRequest(testdata.GithubRepo, testdata.Pull.Num)).ThenReturn(pull, nil) + When(eventParsing.ParseGithubPull(pull)).ThenReturn(modelPull, modelPull.BaseRepo, testdata.GithubRepo, nil) + + When(projectCommandBuilder.BuildApplyCommands(ctx, cmd)).ThenReturn(c.ProjectContexts, nil) + for i := range c.ProjectContexts { + When(projectCommandRunner.Apply(c.ProjectContexts[i])).ThenReturn(c.ProjectResults[i]) + } + + applyCommandRunner.Run(ctx, cmd) + + for i := range c.ProjectContexts { + projectCommandRunner.VerifyWasCalled(c.RunnerInvokeMatch[i]).Apply(c.ProjectContexts[i]) + + } + + vcsClient.VerifyWasCalledOnce().CreateComment( + testdata.GithubRepo, modelPull.Num, c.ExpComment, "apply", + ) + }) + } +} diff --git a/server/events/command/project_context.go b/server/events/command/project_context.go index 8d299981f9..ffefded5f0 100644 --- a/server/events/command/project_context.go +++ b/server/events/command/project_context.go @@ -111,6 +111,8 @@ type ProjectContext struct { JobID string // The index of order group. Before planning/applying it will use to sort projects. Default is 0. ExecutionOrderGroup int + // If plans/applies should be aborted if any prior plan/apply fails + AbortOnExcecutionOrderFail bool } // SetProjectScopeTags adds ProjectContext tags to a new returned scope. diff --git a/server/events/plan_command_runner.go b/server/events/plan_command_runner.go index f648f1a5c7..8fbd1409e3 100644 --- a/server/events/plan_command_runner.go +++ b/server/events/plan_command_runner.go @@ -130,7 +130,7 @@ func (p *PlanCommandRunner) runAutoplan(ctx *command.Context) { var result command.Result if p.isParallelEnabled(projectCmds) { ctx.Log.Info("Running plans in parallel") - result = runProjectCmdsParallelGroups(projectCmds, p.prjCmdRunner.Plan, p.parallelPoolSize) + result = runProjectCmdsParallelGroups(ctx, projectCmds, p.prjCmdRunner.Plan, p.parallelPoolSize) } else { result = runProjectCmds(projectCmds, p.prjCmdRunner.Plan) } @@ -250,7 +250,7 @@ func (p *PlanCommandRunner) run(ctx *command.Context, cmd *CommentCommand) { var result command.Result if p.isParallelEnabled(projectCmds) { ctx.Log.Info("Running plans in parallel") - result = runProjectCmdsParallelGroups(projectCmds, p.prjCmdRunner.Plan, p.parallelPoolSize) + result = runProjectCmdsParallelGroups(ctx, projectCmds, p.prjCmdRunner.Plan, p.parallelPoolSize) } else { result = runProjectCmds(projectCmds, p.prjCmdRunner.Plan) } diff --git a/server/events/plan_command_runner_test.go b/server/events/plan_command_runner_test.go index 6e5c510566..6023421b48 100644 --- a/server/events/plan_command_runner_test.go +++ b/server/events/plan_command_runner_test.go @@ -1,8 +1,10 @@ package events_test import ( + "errors" "testing" + "github.com/google/go-github/v52/github" . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/events" @@ -150,3 +152,360 @@ func TestPlanCommandRunner_IsSilenced(t *testing.T) { }) } } + +func TestPlanCommandRunner_ExecutionOrder(t *testing.T) { + logger := logging.NewNoopLogger(t) + RegisterMockTestingT(t) + + cases := []struct { + Description string + ProjectContexts []command.ProjectContext + ProjectResults []command.ProjectResult + RunnerInvokeMatch []*EqMatcher + PrevPlanStored bool + }{ + { + Description: "When first plan fails, the second don't run", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + Workspace: "first", + ProjectName: "First", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + Workspace: "second", + ProjectName: "Second", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + }, + { + Description: "When first fails, the second will not run", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Second", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{}, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Never(), + }, + }, + { + Description: "When first fails by autorun, the second will not run", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + AutoplanEnabled: true, + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + AutoplanEnabled: true, + ExecutionOrderGroup: 1, + ProjectName: "Second", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{}, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Never(), + }, + }, + { + Description: "When both in a group of two succeeds, the following two will run", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Third", + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Fourth", + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + Never(), + Never(), + }, + }, + { + Description: "When one out of two fails, the following two will not run", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "First", + ParallelPlanEnabled: true, + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Third", + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + AbortOnExcecutionOrderFail: true, + ProjectName: "Fourth", + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + Once(), + Once(), + }, + }, + { + Description: "Don't block when parallel is not set", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "First", + AbortOnExcecutionOrderFail: true, + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Second", + AbortOnExcecutionOrderFail: true, + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + }, + { + Description: "Don't block when abortOnExcecutionOrderFail is not set", + ProjectContexts: []command.ProjectContext{ + { + CommandName: command.Plan, + ExecutionOrderGroup: 0, + ProjectName: "First", + }, + { + CommandName: command.Plan, + ExecutionOrderGroup: 1, + ProjectName: "Second", + }, + }, + ProjectResults: []command.ProjectResult{ + { + Command: command.Plan, + Error: errors.New("Shabang!"), + }, + { + Command: command.Plan, + PlanSuccess: &models.PlanSuccess{ + TerraformOutput: "true", + }, + }, + }, + RunnerInvokeMatch: []*EqMatcher{ + Once(), + Once(), + }, + }, + } + + for _, c := range cases { + t.Run(c.Description, func(t *testing.T) { + // vcsClient := setup(t) + + tmp := t.TempDir() + db, err := db.New(tmp) + Ok(t, err) + + vcsClient := setup(t, func(tc *TestConfig) { + tc.backend = db + }) + + scopeNull, _, _ := metrics.NewLoggingScope(logger, "atlantis") + + pull := &github.PullRequest{ + State: github.String("open"), + } + modelPull := models.PullRequest{BaseRepo: testdata.GithubRepo, State: models.OpenPullState, Num: testdata.Pull.Num} + + cmd := &events.CommentCommand{Name: command.Plan} + + ctx := &command.Context{ + User: testdata.User, + Log: logging.NewNoopLogger(t), + Scope: scopeNull, + Pull: modelPull, + HeadRepo: testdata.GithubRepo, + Trigger: command.CommentTrigger, + } + + When(githubGetter.GetPullRequest(testdata.GithubRepo, testdata.Pull.Num)).ThenReturn(pull, nil) + When(eventParsing.ParseGithubPull(pull)).ThenReturn(modelPull, modelPull.BaseRepo, testdata.GithubRepo, nil) + + When(projectCommandBuilder.BuildPlanCommands(ctx, cmd)).ThenReturn(c.ProjectContexts, nil) + // When(projectCommandBuilder.BuildPlanCommands(ctx, cmd)).Then(func(args []Param) ReturnValues { + // return ReturnValues{[]command.ProjectContext{{CommandName: command.Plan}}, nil} + // }) + for i := range c.ProjectContexts { + When(projectCommandRunner.Plan(c.ProjectContexts[i])).ThenReturn(c.ProjectResults[i]) + } + + planCommandRunner.Run(ctx, cmd) + type RepoModel interface{ models.Repo } + + for i := range c.ProjectContexts { + projectCommandRunner.VerifyWasCalled(c.RunnerInvokeMatch[i]).Plan(c.ProjectContexts[i]) + } + + vcsClient.VerifyWasCalledOnce().CreateComment( + AnyRepo(), EqInt(modelPull.Num), AnyString(), EqString("plan"), + ) + }) + } +} diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index 1f2b2aaaa9..bffcfd9d25 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -37,6 +37,8 @@ const ( DefaultParallelPlanEnabled = false // DefaultDeleteSourceBranchOnMerge being false is the default setting whether or not to remove a source branch on merge DefaultDeleteSourceBranchOnMerge = false + // DefaultAbortOnExcecutionOrderFail being false is the default setting for abort on execution group failiures + DefaultAbortOnExcecutionOrderFail = false ) func NewInstrumentedProjectCommandBuilder( @@ -385,6 +387,7 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex repoCfg.ParallelApply, repoCfg.ParallelPlan, verbose, + repoCfg.AbortOnExcecutionOrderFail, p.TerraformExecutor, )...) } @@ -408,10 +411,12 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex automerge := DefaultAutomergeEnabled parallelApply := DefaultParallelApplyEnabled parallelPlan := DefaultParallelPlanEnabled + abortOnExcecutionOrderFail := DefaultAbortOnExcecutionOrderFail if hasRepoCfg { automerge = repoCfg.Automerge parallelApply = repoCfg.ParallelApply parallelPlan = repoCfg.ParallelPlan + abortOnExcecutionOrderFail = repoCfg.AbortOnExcecutionOrderFail } pCfg := p.GlobalCfg.DefaultProjCfg(ctx.Log, ctx.Pull.BaseRepo.ID(), mp.Path, pWorkspace) @@ -427,6 +432,7 @@ func (p *DefaultProjectCommandBuilder) buildAllCommandsByCfg(ctx *command.Contex parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, p.TerraformExecutor, )...) } @@ -735,10 +741,12 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(ctx *command.Conte automerge := DefaultAutomergeEnabled parallelApply := DefaultParallelApplyEnabled parallelPlan := DefaultParallelPlanEnabled + abortOnExcecutionOrderFail := DefaultAbortOnExcecutionOrderFail if repoCfgPtr != nil { automerge = repoCfgPtr.Automerge parallelApply = repoCfgPtr.ParallelApply parallelPlan = repoCfgPtr.ParallelPlan + abortOnExcecutionOrderFail = *&repoCfgPtr.AbortOnExcecutionOrderFail } if len(matchingProjects) > 0 { @@ -763,6 +771,7 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(ctx *command.Conte parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, p.TerraformExecutor, )...) } @@ -786,6 +795,7 @@ func (p *DefaultProjectCommandBuilder) buildProjectCommandCtx(ctx *command.Conte parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, p.TerraformExecutor, )...) } diff --git a/server/events/project_command_context_builder.go b/server/events/project_command_context_builder.go index 6670fbd3b4..bc338a6c83 100644 --- a/server/events/project_command_context_builder.go +++ b/server/events/project_command_context_builder.go @@ -38,7 +38,7 @@ type ProjectCommandContextBuilder interface { prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, + automerge, parallelApply, parallelPlan, verbose, abortOnExcecutionOrderFail bool, terraformClient terraform.Client, ) []command.ProjectContext } @@ -58,12 +58,13 @@ func (cb *CommandScopedStatsProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, + automerge, parallelApply, parallelPlan, verbose, abortOnExcecutionOrderFail bool, + terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { cb.ProjectCounter.Inc(1) cmds := cb.ProjectCommandContextBuilder.BuildProjectContext( - ctx, cmdName, subCmdName, prjCfg, commentFlags, repoDir, automerge, parallelApply, parallelPlan, verbose, terraformClient, + ctx, cmdName, subCmdName, prjCfg, commentFlags, repoDir, automerge, parallelApply, parallelPlan, verbose, abortOnExcecutionOrderFail, terraformClient, ) projectCmds = []command.ProjectContext{} @@ -91,7 +92,8 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, + automerge, parallelApply, parallelPlan, verbose, abortOnExcecutionOrderFail bool, + terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { ctx.Log.Debug("Building project command context for %s", cmdName) @@ -139,6 +141,7 @@ func (cb *DefaultProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, ctx.Scope, ctx.PullRequestStatus, ) @@ -160,7 +163,8 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( prjCfg valid.MergedProjectCfg, commentFlags []string, repoDir string, - automerge, parallelApply, parallelPlan, verbose bool, terraformClient terraform.Client, + automerge, parallelApply, parallelPlan, verbose, abortOnExcecutionOrderFail bool, + terraformClient terraform.Client, ) (projectCmds []command.ProjectContext) { ctx.Log.Debug("PolicyChecks are enabled") @@ -181,6 +185,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, terraformClient, ) @@ -202,6 +207,7 @@ func (cb *PolicyCheckProjectCommandContextBuilder) BuildProjectContext( parallelApply, parallelPlan, verbose, + abortOnExcecutionOrderFail, ctx.Scope, ctx.PullRequestStatus, )) @@ -225,6 +231,7 @@ func newProjectCommandContext(ctx *command.Context, parallelApplyEnabled bool, parallelPlanEnabled bool, verbose bool, + abortOnExcecutionOrderFail bool, scope tally.Scope, pullStatus models.PullReqStatus, ) command.ProjectContext { @@ -287,6 +294,7 @@ func newProjectCommandContext(ctx *command.Context, PullReqStatus: pullStatus, JobID: uuid.New().String(), ExecutionOrderGroup: projCfg.ExecutionOrderGroup, + AbortOnExcecutionOrderFail: abortOnExcecutionOrderFail, } } diff --git a/server/events/project_command_context_builder_test.go b/server/events/project_command_context_builder_test.go index ec4cac9af5..8d9cb7edc3 100644 --- a/server/events/project_command_context_builder_test.go +++ b/server/events/project_command_context_builder_test.go @@ -62,7 +62,7 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, false, false, false, terraformClient) + result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, false, false, false, false, terraformClient) assert.Equal(t, models.ErroredPolicyCheckStatus, result[0].ProjectPlanStatus) }) @@ -81,7 +81,7 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, false, false, false, terraformClient) + result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, false, false, false, false, terraformClient) assert.Equal(t, models.ErroredPolicyCheckStatus, result[0].ProjectPlanStatus) }) @@ -101,9 +101,29 @@ func TestProjectCommandContextBuilder_PullStatus(t *testing.T) { }, } - result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, true, false, false, terraformClient) + result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, true, false, false, false, terraformClient) assert.True(t, result[0].ParallelApplyEnabled) assert.False(t, result[0].ParallelPlanEnabled) }) + + t.Run("when AbortOnExcecutionOrderFail is set to true", func(t *testing.T) { + projCfg.Name = "Apply Comment" + When(mockCommentBuilder.BuildPlanComment(projRepoRelDir, projWorkspace, "", []string{})).ThenReturn(expectedPlanCmt) + When(mockCommentBuilder.BuildApplyComment(projRepoRelDir, projWorkspace, "", false)).ThenReturn(expectedApplyCmt) + pullStatus.Projects = []models.ProjectStatus{ + { + Status: models.ErroredPlanStatus, + RepoRelDir: "dir2", + }, + { + Status: models.ErroredPolicyCheckStatus, + RepoRelDir: "dir1", + }, + } + + result := subject.BuildProjectContext(commandCtx, command.Plan, "", projCfg, []string{}, "some/dir", false, false, false, false, true, terraformClient) + + assert.True(t, result[0].AbortOnExcecutionOrderFail) + }) } diff --git a/server/events/project_command_pool_executor.go b/server/events/project_command_pool_executor.go index 1039c7db19..c3b19114d6 100644 --- a/server/events/project_command_pool_executor.go +++ b/server/events/project_command_pool_executor.go @@ -72,6 +72,7 @@ func splitByExecutionOrderGroup(cmds []command.ProjectContext) [][]command.Proje } func runProjectCmdsParallelGroups( + ctx *command.Context, cmds []command.ProjectContext, runnerFunc prjCmdRunnerFunc, poolSize int, @@ -81,6 +82,10 @@ func runProjectCmdsParallelGroups( for _, group := range groups { res := runProjectCmdsParallel(group, runnerFunc, poolSize) results = append(results, res.ProjectResults...) + if res.HasErrors() && group[0].AbortOnExcecutionOrderFail { + ctx.Log.Info("abort on execution order when failed") + break + } } return command.Result{ProjectResults: results} diff --git a/server/events/version_command_runner.go b/server/events/version_command_runner.go index 101ba28570..08f27de8c1 100644 --- a/server/events/version_command_runner.go +++ b/server/events/version_command_runner.go @@ -47,7 +47,7 @@ func (v *VersionCommandRunner) Run(ctx *command.Context, cmd *CommentCommand) { var result command.Result if v.isParallelEnabled(projectCmds) { ctx.Log.Info("Running version in parallel") - result = runProjectCmdsParallel(projectCmds, v.prjCmdRunner.Version, v.parallelPoolSize) + result = runProjectCmdsParallelGroups(ctx, projectCmds, v.prjCmdRunner.Version, v.parallelPoolSize) } else { result = runProjectCmds(projectCmds, v.prjCmdRunner.Version) }