Add the /plan and /apply endpoints (runatlantis#997)

* Add the /plan and /apply endpoints

* Resolve conflicts

* Fix wrong merge

* Add missing methods for mocks

* Fix linting error

* Fix linting error

* Move api plan/apply into APIController

* Extract commond code into helper functions

* Implement GetCloneURL for GitHub

Co-authored-by: Li Lin <li.lin@hashicorp.com>

cmd/server.go

@@ -77,6 +77,7 @@ const (
 	GitlabTokenFlag            = "gitlab-token"
 	GitlabUserFlag             = "gitlab-user"
 	GitlabWebhookSecretFlag    = "gitlab-webhook-secret" // nolint: gosec
+	APISecretFlag              = "api-secret"
 	HidePrevPlanComments       = "hide-prev-plan-comments"
 	LogLevelFlag               = "log-level"
 	ParallelPoolSize           = "parallel-pool-size"
@@ -259,6 +260,9 @@ var stringFlags = map[string]stringFlag{
 			"This means that an attacker could spoof calls to Atlantis and cause it to perform malicious actions. " +
 			"Should be specified via the ATLANTIS_GITLAB_WEBHOOK_SECRET environment variable.",
 	},
+	APISecretFlag: {
+		description: "Secret to validate requests made to the API",
+	},
 	LogLevelFlag: {
 		description:  "Log level. Either debug, info, warn, or error.",
 		defaultValue: DefaultLogLevel,

server/controllers/api_controller.go

@@ -0,0 +1,233 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/runatlantis/atlantis/server/core/locking"
+	"github.com/runatlantis/atlantis/server/events"
+	"github.com/runatlantis/atlantis/server/events/command"
+	"github.com/runatlantis/atlantis/server/events/models"
+	"github.com/runatlantis/atlantis/server/events/vcs"
+	"github.com/runatlantis/atlantis/server/logging"
+	"github.com/uber-go/tally"
+	"gopkg.in/go-playground/validator.v9"
+)
+
+const atlantisTokenHeader = "X-Atlantis-Token"
+
+type APIController struct {
+	APISecret                 []byte
+	Locker                    locking.Locker
+	Logger                    logging.SimpleLogging
+	Parser                    events.EventParsing
+	ProjectCommandBuilder     events.ProjectCommandBuilder
+	ProjectPlanCommandRunner  events.ProjectPlanCommandRunner
+	ProjectApplyCommandRunner events.ProjectApplyCommandRunner
+	RepoAllowlistChecker      *events.RepoAllowlistChecker
+	Scope                     tally.Scope
+	VCSClient                 vcs.Client
+}
+
+type APIRequest struct {
+	Repository string `validate:"required"`
+	Ref        string `validate:"required"`
+	Type       string `validate:"required"`
+	Projects   []string
+	Paths      []struct {
+		Directory string
+		Workspace string
+	}
+}
+
+func (a *APIRequest) getCommands(ctx *command.Context, cmdBuilder func(*command.Context, *events.CommentCommand) ([]command.ProjectContext, error)) ([]command.ProjectContext, error) {
+	cc := make([]*events.CommentCommand, 0)
+
+	for _, project := range a.Projects {
+		cc = append(cc, &events.CommentCommand{
+			ProjectName: project,
+		})
+	}
+	for _, path := range a.Paths {
+		cc = append(cc, &events.CommentCommand{
+			RepoRelDir: strings.TrimRight(path.Directory, "/"),
+			Workspace:  path.Workspace,
+		})
+	}
+
+	cmds := make([]command.ProjectContext, 0)
+	for _, commentCommand := range cc {
+		projectCmds, err := cmdBuilder(ctx, commentCommand)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build command: %v", err)
+		}
+		cmds = append(cmds, projectCmds...)
+	}
+
+	return cmds, nil
+}
+
+func (a *APIController) apiReportError(w http.ResponseWriter, code int, err error) {
+	response, _ := json.Marshal(map[string]string{
+		"error": err.Error(),
+	})
+	a.respond(w, logging.Warn, code, string(response))
+}
+
+func (a *APIController) Plan(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	request, ctx, code, err := a.apiParseAndValidate(r)
+	if err != nil {
+		a.apiReportError(w, code, err)
+		return
+	}
+
+	result, err := a.apiPlan(request, ctx)
+	if err != nil {
+		a.apiReportError(w, http.StatusInternalServerError, err)
+		return
+	}
+	defer a.Locker.UnlockByPull(ctx.HeadRepo.FullName, 0) // nolint: errcheck
+	if result.HasErrors() {
+		code = http.StatusInternalServerError
+	}
+
+	// TODO: make a better response
+	response, err := json.Marshal(result)
+	if err != nil {
+		a.apiReportError(w, http.StatusInternalServerError, err)
+		return
+	}
+	a.respond(w, logging.Debug, code, string(response))
+}
+
+func (a *APIController) Apply(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+
+	request, ctx, code, err := a.apiParseAndValidate(r)
+	if err != nil {
+		a.apiReportError(w, code, err)
+		return
+	}
+
+	// We must first make the plan for all projects
+	_, err = a.apiPlan(request, ctx)
+	if err != nil {
+		a.apiReportError(w, http.StatusInternalServerError, err)
+		return
+	}
+	defer a.Locker.UnlockByPull(ctx.HeadRepo.FullName, 0) // nolint: errcheck
+
+	// We can now prepare and run the apply step
+	result, err := a.apiApply(request, ctx)
+	if err != nil {
+		a.apiReportError(w, http.StatusInternalServerError, err)
+		return
+	}
+	if result.HasErrors() {
+		code = http.StatusInternalServerError
+	}
+
+	response, err := json.Marshal(result)
+	if err != nil {
+		a.apiReportError(w, http.StatusInternalServerError, err)
+		return
+	}
+	a.respond(w, logging.Debug, code, string(response))
+}
+
+func (a *APIController) apiPlan(request *APIRequest, ctx *command.Context) (*command.Result, error) {
+	cmds, err := request.getCommands(ctx, a.ProjectCommandBuilder.BuildPlanCommands)
+	if err != nil {
+		return nil, err
+	}
+
+	var projectResults []command.ProjectResult
+	for _, cmd := range cmds {
+		res := a.ProjectPlanCommandRunner.Plan(cmd)
+		projectResults = append(projectResults, res)
+	}
+	return &command.Result{ProjectResults: projectResults}, nil
+}
+
+func (a *APIController) apiApply(request *APIRequest, ctx *command.Context) (*command.Result, error) {
+	cmds, err := request.getCommands(ctx, a.ProjectCommandBuilder.BuildApplyCommands)
+	if err != nil {
+		return nil, err
+	}
+
+	var projectResults []command.ProjectResult
+	for _, cmd := range cmds {
+		res := a.ProjectApplyCommandRunner.Apply(cmd)
+		projectResults = append(projectResults, res)
+	}
+	return &command.Result{ProjectResults: projectResults}, nil
+}
+
+func (a *APIController) apiParseAndValidate(r *http.Request) (*APIRequest, *command.Context, int, error) {
+	if len(a.APISecret) == 0 {
+		return nil, nil, http.StatusBadRequest, fmt.Errorf("ignoring request since API is disabled")
+	}
+
+	// Validate the secret token
+	secret := r.Header.Get(atlantisTokenHeader)
+	if secret != string(a.APISecret) {
+		return nil, nil, http.StatusUnauthorized, fmt.Errorf("header %s did not match expected secret", atlantisTokenHeader)
+	}
+
+	// Parse the JSON payload
+	bytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return nil, nil, http.StatusBadRequest, fmt.Errorf("failed to read request")
+	}
+	var request APIRequest
+	if err = json.Unmarshal(bytes, &request); err != nil {
+		return nil, nil, http.StatusBadRequest, fmt.Errorf("failed to parse request: %v", err.Error())
+	}
+	if err = validator.New().Struct(request); err != nil {
+		return nil, nil, http.StatusBadRequest, fmt.Errorf("request %q is missing fields", string(bytes))
+	}
+
+	VCSHostType, err := models.NewVCSHostType(request.Type)
+	if err != nil {
+		return nil, nil, http.StatusBadRequest, err
+	}
+	cloneURL, err := a.VCSClient.GetCloneURL(VCSHostType, request.Repository)
+	if err != nil {
+		return nil, nil, http.StatusInternalServerError, err
+	}
+
+	baseRepo, err := a.Parser.ParseAPIPlanRequest(VCSHostType, request.Repository, cloneURL)
+	if err != nil {
+		return nil, nil, http.StatusBadRequest, fmt.Errorf("failed to parse request: %v", err)
+	}
+
+	// Check if the repo is allowlisted
+	if !a.RepoAllowlistChecker.IsAllowlisted(baseRepo.FullName, baseRepo.VCSHost.Hostname) {
+		return nil, nil, http.StatusForbidden, fmt.Errorf("repo not allowlisted")
+	}
+
+	return &request, &command.Context{
+		HeadRepo: baseRepo,
+		Pull: models.PullRequest{
+			Num:        0,
+			BaseBranch: request.Ref,
+			HeadBranch: request.Ref,
+			HeadCommit: request.Ref,
+			BaseRepo:   baseRepo,
+		},
+		Scope: a.Scope,
+		Log:   a.Logger,
+	}, http.StatusOK, nil
+}
+
+func (a *APIController) respond(w http.ResponseWriter, lvl logging.LogLevel, responseCode int, format string, args ...interface{}) {
+	response := fmt.Sprintf(format, args...)
+	a.Logger.Log(lvl, response)
+	w.WriteHeader(responseCode)
+	fmt.Fprintln(w, response)
+}

server/controllers/api_controller_test.go

@@ -0,0 +1,103 @@
+package controllers_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/runatlantis/atlantis/server/events/command"
+	"github.com/runatlantis/atlantis/server/events/models"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	. "github.com/petergtz/pegomock"
+	"github.com/runatlantis/atlantis/server/controllers"
+	. "github.com/runatlantis/atlantis/server/core/locking/mocks"
+	"github.com/runatlantis/atlantis/server/events"
+	. "github.com/runatlantis/atlantis/server/events/mocks"
+	. "github.com/runatlantis/atlantis/server/events/mocks/matchers"
+	. "github.com/runatlantis/atlantis/server/events/vcs/mocks"
+	"github.com/runatlantis/atlantis/server/logging"
+	"github.com/runatlantis/atlantis/server/metrics"
+	. "github.com/runatlantis/atlantis/testing"
+)
+
+const atlantisTokenHeader = "X-Atlantis-Token"
+const atlantisToken = "token"
+
+func TestAPIController_Plan(t *testing.T) {
+	ac, projectCommandBuilder, projectCommandRunner := setup(t)
+	body, _ := json.Marshal(controllers.APIRequest{
+		Repository: "Repo",
+		Ref:        "main",
+		Type:       "Gitlab",
+		Projects:   []string{"default"},
+	})
+	req, _ := http.NewRequest("POST", "", bytes.NewBuffer(body))
+	req.Header.Set(atlantisTokenHeader, atlantisToken)
+	w := httptest.NewRecorder()
+	ac.Plan(w, req)
+	ResponseContains(t, w, http.StatusOK, "")
+	projectCommandBuilder.VerifyWasCalledOnce().BuildPlanCommands(AnyPtrToEventsCommandContext(), AnyPtrToEventsCommentCommand())
+	projectCommandRunner.VerifyWasCalledOnce().Plan(AnyModelsProjectCommandContext())
+}
+
+func TestAPIController_Apply(t *testing.T) {
+	ac, projectCommandBuilder, projectCommandRunner := setup(t)
+	body, _ := json.Marshal(controllers.APIRequest{
+		Repository: "Repo",
+		Ref:        "main",
+		Type:       "Gitlab",
+		Projects:   []string{"default"},
+	})
+	req, _ := http.NewRequest("POST", "", bytes.NewBuffer(body))
+	req.Header.Set(atlantisTokenHeader, atlantisToken)
+	w := httptest.NewRecorder()
+	ac.Apply(w, req)
+	ResponseContains(t, w, http.StatusOK, "")
+	projectCommandBuilder.VerifyWasCalledOnce().BuildApplyCommands(AnyPtrToEventsCommandContext(), AnyPtrToEventsCommentCommand())
+	projectCommandRunner.VerifyWasCalledOnce().Plan(AnyModelsProjectCommandContext())
+	projectCommandRunner.VerifyWasCalledOnce().Apply(AnyModelsProjectCommandContext())
+}
+
+func setup(t *testing.T) (controllers.APIController, *MockProjectCommandBuilder, *MockProjectCommandRunner) {
+	RegisterMockTestingT(t)
+	locker := NewMockLocker()
+	logger := logging.NewNoopLogger(t)
+	scope, _, _ := metrics.NewLoggingScope(logger, "null")
+	parser := NewMockEventParsing()
+	vcsClient := NewMockClient()
+	repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*")
+	Ok(t, err)
+
+	projectCommandBuilder := NewMockProjectCommandBuilder()
+	When(projectCommandBuilder.BuildPlanCommands(AnyPtrToEventsCommandContext(), AnyPtrToEventsCommentCommand())).
+		ThenReturn([]command.ProjectContext{{
+			CommandName: command.Plan,
+		}}, nil)
+	When(projectCommandBuilder.BuildApplyCommands(AnyPtrToEventsCommandContext(), AnyPtrToEventsCommentCommand())).
+		ThenReturn([]command.ProjectContext{{
+			CommandName: command.Apply,
+		}}, nil)
+
+	projectCommandRunner := NewMockProjectCommandRunner()
+	When(projectCommandRunner.Plan(AnyModelsProjectCommandContext())).ThenReturn(command.ProjectResult{
+		PlanSuccess: &models.PlanSuccess{},
+	})
+	When(projectCommandRunner.Apply(AnyModelsProjectCommandContext())).ThenReturn(command.ProjectResult{
+		ApplySuccess: "success",
+	})
+
+	ac := controllers.APIController{
+		APISecret:                 []byte(atlantisToken),
+		Locker:                    locker,
+		Logger:                    logger,
+		Scope:                     scope,
+		Parser:                    parser,
+		ProjectCommandBuilder:     projectCommandBuilder,
+		ProjectPlanCommandRunner:  projectCommandRunner,
+		ProjectApplyCommandRunner: projectCommandRunner,
+		VCSClient:                 vcsClient,
+		RepoAllowlistChecker:      repoAllowlistChecker,
+	}
+	return ac, projectCommandBuilder, projectCommandRunner
+}

server/events/event_parser.go

@@ -216,6 +216,8 @@ type EventParsing interface {
 	// that returns a merge request.
 	ParseGitlabMergeRequest(mr *gitlab.MergeRequest, baseRepo models.Repo) models.PullRequest
 
+	ParseAPIPlanRequest(vcsHostType models.VCSHostType, path string, cloneURL string) (baseRepo models.Repo, err error)
+
 	// ParseBitbucketCloudPullEvent parses a pull request event from Bitbucket
 	// Cloud (bitbucket.org).
 	// pull is the parsed pull request.
@@ -303,6 +305,16 @@ type EventParser struct {
 	AzureDevopsUser    string
 }
 
+func (e *EventParser) ParseAPIPlanRequest(vcsHostType models.VCSHostType, repoFullName string, cloneURL string) (models.Repo, error) {
+	switch vcsHostType {
+	case models.Github:
+		return models.NewRepo(vcsHostType, repoFullName, cloneURL, e.GithubUser, e.GithubToken)
+	case models.Gitlab:
+		return models.NewRepo(vcsHostType, repoFullName, cloneURL, e.GitlabUser, e.GitlabToken)
+	}
+	return models.Repo{}, fmt.Errorf("not implemented")
+}
+
 // GetBitbucketCloudPullEventType returns the type of the pull request
 // event given the Bitbucket Cloud header.
 func (e *EventParser) GetBitbucketCloudPullEventType(eventTypeHeader string) models.PullRequestEventType {