diff --git a/docs/docs/91-migrations.md b/docs/docs/91-migrations.md index f80f8353e69..9a979058571 100644 --- a/docs/docs/91-migrations.md +++ b/docs/docs/91-migrations.md @@ -5,6 +5,7 @@ Some versions need some changes to the server configuration or the pipeline conf ## `next` - Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead +- Pipelines without a config file will now be skipped instead of failing ## 2.0.0 diff --git a/server/api/helper.go b/server/api/helper.go index d4d1da89983..cfe409b9b61 100644 --- a/server/api/helper.go +++ b/server/api/helper.go @@ -34,6 +34,8 @@ func handlePipelineErr(c *gin.Context, err error) { } else if errors.Is(err, &pipeline.ErrBadRequest{}) { c.String(http.StatusBadRequest, "%s", err) } else if errors.Is(err, pipeline.ErrFiltered) { + // for debugging purpose we add a header + c.Writer.Header().Add("Pipeline-Filtered", "true") c.Status(http.StatusNoContent) } else { _ = c.AbortWithError(http.StatusInternalServerError, err) diff --git a/server/forge/configFetcher.go b/server/forge/configFetcher.go index 773b52e95cb..5331469384f 100644 --- a/server/forge/configFetcher.go +++ b/server/forge/configFetcher.go @@ -106,7 +106,7 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config // could be adapted to allow the user to supply a list like we do in the defaults configs := []string{config} - fileMeta, err := cf.getFirstAvailableConfig(ctx, configs, true) + fileMeta, err := cf.getFirstAvailableConfig(ctx, configs) if err == nil { return fileMeta, err } @@ -116,7 +116,7 @@ func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config log.Trace().Msgf("ConfigFetch[%s]: user did not define own config, following default procedure", cf.repo.FullName) // for the order see shared/constants/constants.go - fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:], false) + fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) if err == nil { return fileMeta, err } @@ -141,7 +141,7 @@ func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta { return res } -func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fileMeta []*types.FileMeta, found bool) { +func (cf *configFetcher) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) { file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config) if err == nil && len(file) != 0 { @@ -150,40 +150,47 @@ func (cf *configFetcher) checkPipelineFile(c context.Context, config string) (fi return []*types.FileMeta{{ Name: config, Data: file, - }}, true + }}, nil } - return nil, false + return nil, err } -func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string, userDefined bool) ([]*types.FileMeta, error) { - userDefinedLog := "" - if userDefined { - userDefinedLog = "user defined" - } - +func (cf *configFetcher) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) { + var forgeErr []error for _, fileOrFolder := range configs { if strings.HasSuffix(fileOrFolder, "/") { // config is a folder files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) // if folder is not supported we will get a "Not implemented" error and continue - if err != nil && !errors.Is(err, types.ErrNotImplemented) { - log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge") + if err != nil { + if !(errors.Is(err, types.ErrNotImplemented) || errors.Is(err, &types.ErrConfigNotFound{})) { + log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge") + forgeErr = append(forgeErr, err) + } + continue } files = filterPipelineFiles(files) - if err == nil && len(files) != 0 { - log.Trace().Msgf("ConfigFetch[%s]: found %d %s files in '%s'", cf.repo.FullName, len(files), userDefinedLog, fileOrFolder) + if len(files) != 0 { + log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), fileOrFolder) return files, nil } } // config is a file - if fileMeta, found := cf.checkPipelineFile(c, fileOrFolder); found { - log.Trace().Msgf("ConfigFetch[%s]: found %s file: '%s'", cf.repo.FullName, userDefinedLog, fileOrFolder) + if fileMeta, err := cf.checkPipelineFile(c, fileOrFolder); err == nil { + log.Trace().Msgf("ConfigFetch[%s]: found file: '%s'", cf.repo.FullName, fileOrFolder) return fileMeta, nil + } else if !errors.Is(err, &types.ErrConfigNotFound{}) { + forgeErr = append(forgeErr, err) } } + // got unexpected errors + if len(forgeErr) != 0 { + return nil, errors.Join(forgeErr...) + } + // nothing found - return nil, fmt.Errorf("%s configs not found searched: %s", userDefinedLog, strings.Join(configs, ", ")) + return nil, &types.ErrConfigNotFound{Configs: configs} } diff --git a/server/forge/gitea/gitea.go b/server/forge/gitea/gitea.go index 501de15727a..ae6a6c1d07d 100644 --- a/server/forge/gitea/gitea.go +++ b/server/forge/gitea/gitea.go @@ -39,6 +39,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/common" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/store" @@ -289,7 +290,10 @@ func (c *Gitea) File(ctx context.Context, u *model.User, r *model.Repo, b *model return nil, err } - cfg, _, err := client.GetFile(r.Owner, r.Name, b.Commit, f) + cfg, resp, err := client.GetFile(r.Owner, r.Name, b.Commit, f) + if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}}) + } return cfg, err } @@ -314,6 +318,9 @@ func (c *Gitea) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model. if m, _ := filepath.Match(f, e.Path); m && e.Type == "blob" { data, err := c.File(ctx, u, r, b, e.Path) if err != nil { + if errors.Is(err, &types.ErrConfigNotFound{}) { + return nil, fmt.Errorf("git tree reported existence of file but we got: %s", err.Error()) + } return nil, fmt.Errorf("multi-pipeline cannot get %s: %w", e.Path, err) } diff --git a/server/forge/github/github.go b/server/forge/github/github.go index 4f5339cd8ae..197afc73b4b 100644 --- a/server/forge/github/github.go +++ b/server/forge/github/github.go @@ -18,6 +18,7 @@ package github import ( "context" "crypto/tls" + "errors" "fmt" "net/http" "net/url" @@ -32,6 +33,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/common" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/store" @@ -226,7 +228,10 @@ func (c *client) File(ctx context.Context, u *model.User, r *model.Repo, b *mode opts := new(github.RepositoryContentGetOptions) opts.Ref = b.Commit - content, _, _, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts) + content, _, resp, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}}) + } if err != nil { return nil, err } @@ -242,7 +247,10 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model opts := new(github.RepositoryContentGetOptions) opts.Ref = b.Commit - _, data, _, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts) + _, data, resp, err := client.Repositories.GetContents(ctx, r.Owner, r.Name, f, opts) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{f}}) + } if err != nil { return nil, err } @@ -254,6 +262,9 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model go func(path string) { content, err := c.File(ctx, u, r, b, path) if err != nil { + if errors.Is(err, &types.ErrConfigNotFound{}) { + err = fmt.Errorf("git tree reported existence of file but we got: %s", err.Error()) + } errc <- err } else { fc <- &forge_types.FileMeta{ diff --git a/server/forge/gitlab/gitlab.go b/server/forge/gitlab/gitlab.go index 4562ee14065..649f1c978c7 100644 --- a/server/forge/gitlab/gitlab.go +++ b/server/forge/gitlab/gitlab.go @@ -18,6 +18,7 @@ package gitlab import ( "context" "crypto/tls" + "errors" "fmt" "io" "net/http" @@ -33,6 +34,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/common" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/store" @@ -351,7 +353,10 @@ func (g *GitLab) File(ctx context.Context, user *model.User, repo *model.Repo, p if err != nil { return nil, err } - file, _, err := client.RepositoryFiles.GetRawFile(_repo.ID, fileName, &gitlab.GetRawFileOptions{Ref: &pipeline.Commit}, gitlab.WithContext(ctx)) + file, resp, err := client.RepositoryFiles.GetRawFile(_repo.ID, fileName, &gitlab.GetRawFileOptions{Ref: &pipeline.Commit}, gitlab.WithContext(ctx)) + if resp != nil && resp.StatusCode == http.StatusNotFound { + return nil, errors.Join(err, &types.ErrConfigNotFound{Configs: []string{fileName}}) + } return file, err } @@ -388,6 +393,9 @@ func (g *GitLab) Dir(ctx context.Context, user *model.User, repo *model.Repo, pi } data, err := g.File(ctx, user, repo, pipeline, batch[i].Path) if err != nil { + if errors.Is(err, &types.ErrConfigNotFound{}) { + return nil, fmt.Errorf("git tree reported existence of file but we got: %s", err.Error()) + } return nil, err } files = append(files, &forge_types.FileMeta{ diff --git a/server/forge/types/errors.go b/server/forge/types/errors.go index 7fd257cb9a1..957e30432c4 100644 --- a/server/forge/types/errors.go +++ b/server/forge/types/errors.go @@ -18,6 +18,7 @@ package types import ( "errors" "fmt" + "strings" ) // AuthError represents forge authentication error. @@ -56,3 +57,16 @@ func (*ErrIgnoreEvent) Is(target error) bool { _, ok := target.(*ErrIgnoreEvent) //nolint:errorlint return ok } + +type ErrConfigNotFound struct { + Configs []string +} + +func (m *ErrConfigNotFound) Error() string { + return fmt.Sprintf("configs not found: %s", strings.Join(m.Configs, ", ")) +} + +func (*ErrConfigNotFound) Is(target error) bool { + _, ok := target.(*ErrConfigNotFound) //nolint:errorlint + return ok +} diff --git a/server/pipeline/create.go b/server/pipeline/create.go index 7d711af346f..5182a4cbf45 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -16,14 +16,16 @@ package pipeline import ( "context" + "errors" "fmt" "regexp" "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors" + pipeline_errors "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/forge" + forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) @@ -64,18 +66,24 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline // fetch the pipeline file from the forge configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) forgeYamlConfigs, configFetchErr := configFetcher.Fetch(ctx) - - if configFetchErr != nil { + if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) + if err := _store.DeletePipeline(pipeline); err != nil { + log.Error().Str("repo", repo.FullName).Err(err).Msg("failed to delete pipeline without config") + } + + return nil, ErrFiltered + } else if configFetchErr != nil { + log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("error while fetching config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) return nil, updatePipelineWithErr(ctx, _store, pipeline, repo, repoUser, fmt.Errorf("pipeline definition not found in %s", repo.FullName)) } pipelineItems, parseErr := parsePipeline(_store, pipeline, repoUser, repo, forgeYamlConfigs, nil) - if errors.HasBlockingErrors(parseErr) { + if pipeline_errors.HasBlockingErrors(parseErr) { log.Debug().Str("repo", repo.FullName).Err(parseErr).Msg("failed to parse yaml") return nil, updatePipelineWithErr(ctx, _store, pipeline, repo, repoUser, parseErr) } else if parseErr != nil { - pipeline.Errors = errors.GetPipelineErrors(parseErr) + pipeline.Errors = pipeline_errors.GetPipelineErrors(parseErr) } if len(pipelineItems) == 0 {