merge_requests: support in-fork pipelines as well

Fixes: #725

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [0ver](https://0ver.org) (more or less).
 - new configuration parameter: `gitlab.maximum_jobs_queue_size`, controlling the queue buffer size
 - new label for pipelines and jobs: `source` to indicate the reason the pipeline started
 - new label for pipelines and jobs: `source_project` to indicate pipelines for merge requests on forks
+- pipelines in forks hosting merge request pipelines may now be selected using the `include_source_pipelines` option
 
 ### Changed
 

docs/configuration_syntax.md

@@ -284,6 +284,10 @@ project_defaults:
         # (optional, default: false)
         enabled: false
 
+        # Monitor merge request pipelines on source projects as well
+        # (optional, default: false)
+        include_source_pipelines: false
+
         # Only keep most 'n' recently updated merge requests
         # (optional, default: 0 -- disabled/keep every merge request)
         most_recent: 0

pkg/config/project.go

@@ -87,6 +87,9 @@ type ProjectPullRefsMergeRequests struct {
 	// Monitor pipelines related to project merge requests.
 	Enabled bool `yaml:"enabled"`
 
+	// Monitor all pipelines on a merge request, not just those in the main project
+	IncludeSourcePipelines bool `default:"false" yaml:"include_source_pipelines"`
+
 	// Only keep most 'n' recently updated merge requests.
 	MostRecent uint `default:"0" yaml:"most_recent"`
 

pkg/controller/pipelines.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 
 	log "github.com/sirupsen/logrus"
 	goGitlab "github.com/xanzy/go-gitlab"
@@ -33,24 +34,45 @@ func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error
 		"ref-kind":     ref.Kind,
 	}
 
+	var (
+		pipelines []*goGitlab.PipelineInfo
+		err       error
+	)
+
 	// We need a different syntax if the ref is a merge-request
-	var refName string
-	if ref.Kind == schemas.RefKindMergeRequest {
-		refName = fmt.Sprintf("refs/merge-requests/%s/head", ref.Name)
+	if ref.SourceProject == nil {
+		var refName string
+		if ref.Kind == schemas.RefKindMergeRequest {
+			refName = fmt.Sprintf("refs/merge-requests/%s/head", ref.Name)
+		} else {
+			refName = ref.Name
+		}
+
+		pipelines, _, err = c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{
+			// We only need the most recent pipeline
+			ListOptions: goGitlab.ListOptions{
+				PerPage: 1,
+				Page:    1,
+			},
+			Ref: &refName,
+		})
+		if err != nil {
+			return fmt.Errorf("error fetching project pipelines for %s: %v", ref.Project.Name, err)
+		}
 	} else {
-		refName = ref.Name
-	}
+		id, err := strconv.Atoi(ref.Name)
+		if err != nil {
+			return fmt.Errorf("error converting merge request refname to an integer %s: %v", ref.Name, err)
+		}
 
-	pipelines, _, err := c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{
-		// We only need the most recent pipeline
-		ListOptions: goGitlab.ListOptions{
+		pipelines, _, err = c.Gitlab.GetMergeRequestPipelines(ctx, ref.Project.Name, id, &goGitlab.ListOptions{
+			// We only need the most recent pipeline
 			PerPage: 1,
 			Page:    1,
-		},
-		Ref: &refName,
-	})
-	if err != nil {
-		return fmt.Errorf("error fetching project pipelines for %s: %v", ref.Project.Name, err)
+		})
+		if err != nil {
+			return fmt.Errorf("error fetching project merge requests pipeline for %s!%s: %v", ref.Project.Name, ref.Name, err)
+		}
 	}
 
 	if len(pipelines) == 0 {

pkg/controller/projects_test.go

@@ -25,6 +25,7 @@ func TestPullProjectsFromWildcard(t *testing.T) {
 
 	projects, _ := c.Store.Projects(ctx)
 	p1 := schemas.NewProject("bar")
+	p1.ID = 2
 
 	expectedProjects := schemas.Projects{
 		p1.Key(): p1,

pkg/controller/refs.go

@@ -59,12 +59,21 @@ func (c *Controller) GetRefs(ctx context.Context, p schemas.Project) (
 	}
 
 	if p.Pull.Refs.MergeRequests.Enabled {
-		if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(
-			ctx,
-			p,
-			schemas.RefKindMergeRequest,
-		); err != nil {
-			return
+		if p.Pull.Refs.MergeRequests.IncludeSourcePipelines {
+			if pulledRefs, err = c.Gitlab.GetMergeRequestRefsFromPipelines(
+				ctx,
+				p,
+			); err != nil {
+				return
+			}
+		} else {
+			if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(
+				ctx,
+				p,
+				schemas.RefKindMergeRequest,
+			); err != nil {
+				return
+			}
 		}
 
 		if err = mergo.Merge(&refs, pulledRefs); err != nil {

pkg/gitlab/pipelines.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 	log "github.com/sirupsen/logrus"
 	goGitlab "github.com/xanzy/go-gitlab"
 	"go.opentelemetry.io/otel"
@@ -86,6 +87,103 @@ func (c *Client) GetProjectPipelines(
 	return pipelines, resp, nil
 }
 
+// ListProjectMergeRequests ..
+func (c *Client) ListProjectMergeRequests(
+	ctx context.Context,
+	projectName string,
+	options *goGitlab.ListProjectMergeRequestsOptions,
+) (
+	[]*goGitlab.MergeRequest,
+	*goGitlab.Response,
+	error,
+) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ListProjectMergeRequests")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", projectName))
+
+	fields := log.Fields{
+		"project-name": projectName,
+	}
+
+	if options.Page == 0 {
+		options.Page = 1
+	}
+
+	if options.PerPage == 0 {
+		options.PerPage = 100
+	}
+
+	fields["page"] = options.Page
+	log.WithFields(fields).Trace("listing project merge requests")
+
+	c.rateLimit(ctx)
+
+	mrs, resp, err := c.MergeRequests.ListProjectMergeRequests(projectName, options, goGitlab.WithContext(ctx))
+	if err != nil {
+		return nil, resp, fmt.Errorf("error listing project merge requests for project %s: %s", projectName, err.Error())
+	}
+
+	c.requestsRemaining(resp)
+
+	return mrs, resp, nil
+}
+
+// GetMergeRequestPipelines ..
+func (c *Client) GetMergeRequestPipelines(
+	ctx context.Context,
+	projectName string,
+	mergeRequest int,
+	options *goGitlab.ListOptions,
+) (
+	[]*goGitlab.PipelineInfo,
+	*goGitlab.Response,
+	error,
+) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetMergeRequestPipelines")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", projectName))
+
+	fields := log.Fields{
+		"project-name": projectName,
+	}
+
+	if options.Page == 0 {
+		options.Page = 1
+	}
+
+	if options.PerPage == 0 {
+		options.PerPage = 100
+	}
+
+	fields["page"] = options.Page
+	log.WithFields(fields).Trace("listing merge request pipelines")
+
+	params := map[string]string{
+		"page":     fmt.Sprint(options.Page),
+		"per_page": fmt.Sprint(options.PerPage),
+	}
+	optionFunc := func(req *retryablehttp.Request) error {
+		for k, v := range params {
+			q := req.URL.Query()
+			q.Add(k, v)
+			req.URL.RawQuery = q.Encode()
+		}
+
+		return nil
+	}
+
+	c.rateLimit(ctx)
+
+	pipelines, resp, err := c.MergeRequests.ListMergeRequestPipelines(projectName, mergeRequest, optionFunc, goGitlab.WithContext(ctx))
+	if err != nil {
+		return nil, resp, fmt.Errorf("error merge request project pipelines for project %s!%d: %s", projectName, mergeRequest, err.Error())
+	}
+
+	c.requestsRemaining(resp)
+
+	return pipelines, resp, nil
+}
+
 // GetRefPipelineVariablesAsConcatenatedString ..
 func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context, ref schemas.Ref) (string, error) {
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetRefPipelineVariablesAsConcatenatedString")
@@ -292,6 +390,108 @@ func (c *Client) GetRefsFromPipelines(ctx context.Context, p schemas.Project, re
 	return
 }
 
+// GetMergeRequestRefsFromPipelines ..
+func (c *Client) GetMergeRequestRefsFromPipelines(ctx context.Context, p schemas.Project) (refs schemas.Refs, err error) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetMergeRequestRefsFromPipelines")
+	defer span.End()
+	span.SetAttributes(attribute.String("project_name", p.Name))
+	span.SetAttributes(attribute.String("ref_kind", "merge-request"))
+
+	refs = make(schemas.Refs)
+
+	options := &goGitlab.ListProjectMergeRequestsOptions{
+		ListOptions: goGitlab.ListOptions{
+			Page:    1,
+			PerPage: 100,
+		},
+		OrderBy: goGitlab.String("updated_at"),
+	}
+
+	var (
+		mostRecent, maxAgeSeconds uint
+		limitToMostRecent         bool
+	)
+
+	maxAgeSeconds = p.Pull.Refs.MergeRequests.MaxAgeSeconds
+	mostRecent = p.Pull.Refs.MergeRequests.MostRecent
+
+	if mostRecent > 0 {
+		limitToMostRecent = true
+	}
+
+	if maxAgeSeconds > 0 {
+		t := time.Now().Add(-time.Second * time.Duration(maxAgeSeconds))
+		options.UpdatedAfter = &t
+	}
+
+	for {
+		var (
+			mrs  []*goGitlab.MergeRequest
+			resp *goGitlab.Response
+		)
+
+		mrs, resp, err = c.ListProjectMergeRequests(ctx, p.Name, options)
+		if err != nil {
+			return
+		}
+
+		for _, mr := range mrs {
+			var refName string
+
+			refName = fmt.Sprint(mr.IID)
+
+			ref := schemas.NewRef(
+				p,
+				schemas.RefKindMergeRequest,
+				refName,
+			)
+
+			if mr.SourceProjectID == mr.TargetProjectID {
+				ref.SourceProject = &p
+			} else {
+				var (
+					goProject     *goGitlab.Project
+					sourceProject schemas.Project
+				)
+
+				goProject, err = c.GetProjectByID(ctx, mr.SourceProjectID)
+				if err != nil {
+					return
+				}
+				sourceProject = p
+				sourceProject.Name = goProject.PathWithNamespace
+
+				ref.SourceProject = &sourceProject
+			}
+
+			if _, ok := refs[ref.Key()]; !ok {
+				log.WithFields(log.Fields{
+					"project-name": ref.Project.Name,
+					"ref":          ref.Name,
+					"ref-kind":     ref.Kind,
+				}).Trace("found ref")
+
+				refs[ref.Key()] = ref
+
+				if limitToMostRecent {
+					mostRecent--
+					if mostRecent <= 0 {
+						return
+					}
+				}
+			}
+		}
+
+		if resp.CurrentPage >= resp.NextPage {
+			break
+		}
+
+		options.Page = resp.NextPage
+	}
+
+	return
+}
+
 // GetRefPipelineTestReport ..
 func (c *Client) GetRefPipelineTestReport(ctx context.Context, ref schemas.Ref) (schemas.TestReport, error) {
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetRefPipelineTestReport")

pkg/gitlab/projects.go

@@ -32,6 +32,23 @@ func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project
 	return p, err
 }
 
+// GetProjectByID ..
+func (c *Client) GetProjectByID(ctx context.Context, id int) (*goGitlab.Project, error) {
+	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:GetProjectByID")
+	defer span.End()
+	span.SetAttributes(attribute.Int("project_id", id))
+
+	log.WithFields(log.Fields{
+		"project-id": id,
+	}).Debug("reading project")
+
+	c.rateLimit(ctx)
+	p, resp, err := c.Projects.GetProject(id, &goGitlab.GetProjectOptions{}, goGitlab.WithContext(ctx))
+	c.requestsRemaining(resp)
+
+	return p, err
+}
+
 // ListProjects ..
 func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas.Project, error) {
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "gitlab:ListProjects")
@@ -136,6 +153,7 @@ func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas
 
 			p := schemas.NewProject(gp.PathWithNamespace)
 			p.ProjectParameters = w.ProjectParameters
+			p.ID = gp.ID
 			projects = append(projects, p)
 		}
 

pkg/schemas/projects.go

@@ -11,6 +11,7 @@ import (
 type Project struct {
 	config.Project
 
+	ID     int
 	Topics string
 }