From 9aad646fdc4ee58f78a01bd6cd571905d32e3f04 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 7 Mar 2024 09:33:15 -0600 Subject: [PATCH 01/65] wip: settings started --- Makefile | 18 ++--- api/admin/settings.go | 122 +++++++++++++++++++++++++++++++++ cmd/vela-server/server.go | 13 ++++ database/database.go | 2 + database/interface.go | 4 ++ database/resource.go | 12 ++++ database/settings/actions.go | 107 +++++++++++++++++++++++++++++ database/settings/interface.go | 20 ++++++ database/settings/opt.go | 55 +++++++++++++++ database/settings/settings.go | 82 ++++++++++++++++++++++ database/settings/table.go | 50 ++++++++++++++ router/admin.go | 4 ++ 12 files changed, 480 insertions(+), 9 deletions(-) create mode 100644 api/admin/settings.go create mode 100644 database/settings/actions.go create mode 100644 database/settings/interface.go create mode 100644 database/settings/opt.go create mode 100644 database/settings/settings.go create mode 100644 database/settings/table.go diff --git a/Makefile b/Makefile index 0df0c3fe9..fc4f00486 100644 --- a/Makefile +++ b/Makefile @@ -29,21 +29,21 @@ LD_FLAGS = -X github.com/go-vela/server/version.Commit=${GITHUB_SHA} -X github.c clean: tidy vet fmt fix # The `restart` target is intended to destroy and -# create the local Docker compose stack. +# create the local docker-compose stack. # # Usage: `make restart` .PHONY: restart restart: down up # The `up` target is intended to create -# the local Docker compose stack. +# the local docker-compose stack. # # Usage: `make up` .PHONY: up up: build compose-up # The `down` target is intended to destroy -# the local Docker compose stack. +# the local docker-compose stack. # # Usage: `make down` .PHONY: down @@ -233,34 +233,34 @@ bump-deps-full: check @go get -t -u ./... # The `pull` target is intended to pull all -# images for the local Docker compose stack. +# images for the local docker-compose stack. # # Usage: `make pull` .PHONY: pull pull: @echo @echo "### Pulling images for docker-compose stack" - @docker compose pull + @docker-compose pull # The `compose-up` target is intended to build and create -# all containers for the local Docker compose stack. +# all containers for the local docker-compose stack. # # Usage: `make compose-up` .PHONY: compose-up compose-up: @echo @echo "### Creating containers for docker-compose stack" - @docker compose -f docker-compose.yml up -d --build + @docker-compose -f docker-compose.yml up -d --build # The `compose-down` target is intended to destroy -# all containers for the local Docker compose stack. +# all containers for the local docker-compose stack. # # Usage: `make compose-down` .PHONY: compose-down compose-down: @echo @echo "### Destroying containers for docker-compose stack" - @docker compose -f docker-compose.yml down + @docker-compose -f docker-compose.yml down # The `spec-install` target is intended to install the # the needed dependencies to generate the api spec. diff --git a/api/admin/settings.go b/api/admin/settings.go new file mode 100644 index 000000000..b3d724917 --- /dev/null +++ b/api/admin/settings.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +//nolint:dupl // ignore similar code +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +type PlatformSettings_API struct { + ID int `json:"id"` + FooNum int `json:"foo_num"` + BarStr string `json:"bar_str"` +} + +// swagger:operation GET /api/v1/admin/settings admin GetSettings +// +// Get the currently configured platform settings. +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved platform settings from the database +// schema: +// type: array +// items: +// "$ref": "#/definitions/Settings" +// '500': +// description: Unable to retrieve platform settings from the database +// schema: +// "$ref": "#/definitions/Error" + +// GetSettings represents the API handler to +// captures platform settings stored in the database. +func GetSettings(c *gin.Context) { + // capture middleware values + ctx := c.Request.Context() + + logrus.Info("Admin: reading platform settings") + + // send API call to capture pending and running builds + s, err := database.FromContext(c).GetSettings(ctx) + if err != nil { + retErr := fmt.Errorf("unable to capture platform settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, s) +} + +func UpdateSettings(c *gin.Context) { + // capture middleware values + // todo: settings.Retrieve + // s := user.Retrieve(c) + + s, err := database.FromContext(c).GetSettings(c.Request.Context()) + if err != nil { + retErr := fmt.Errorf("unable to retrieve platform settings from the database: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // maxBuildLimit := c.Value("maxBuildLimit").(int64) + // defaultRepoEvents := c.Value("defaultRepoEvents").([]string) + // defaultRepoEventsMask := c.Value("defaultRepoEventsMask").(int64) + ctx := c.Request.Context() + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.Info("Admin: updating platform settings") + + // capture body from API request + type ss struct { + BarStr string `json:"bar_str"` + } + + input := new(ss) + // input := new(library.Settings) + + err = c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for platform settings: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // todo: do this right + updated := "default value" + if s != nil && input != nil { + updated = *s + input.BarStr + } + + // send API call to update the repo + s, err = database.FromContext(c).UpdateSettings(ctx, &updated) + if err != nil { + retErr := fmt.Errorf("unable to update platform settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, s) +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 1910c2d70..53e4333ad 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -77,6 +77,19 @@ func server(c *cli.Context) error { return err } + // create a platform settings object to store the settings + // err := e.client.Table(constantsTableSettings).Save(s2).Error + s := "init1" + ss, err := database.GetSettings(context.Background()) + if ss == nil || err != nil { + _, err = database.CreateSettings(context.Background(), &s) + if err != nil { + return err + } + } + + // when the server reboots, should it only use the defaults when the object doesnt exist yet? + queue, err := setupQueue(c) if err != nil { return err diff --git a/database/database.go b/database/database.go index 3fc1adb94..e9be5fafb 100644 --- a/database/database.go +++ b/database/database.go @@ -17,6 +17,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -60,6 +61,7 @@ type ( // sirupsen/logrus logger used in database functions logger *logrus.Entry + settings.SettingsInterface build.BuildInterface executable.BuildExecutableInterface deployment.DeploymentInterface diff --git a/database/interface.go b/database/interface.go index 4a51019b3..dbac4d1bd 100644 --- a/database/interface.go +++ b/database/interface.go @@ -16,6 +16,7 @@ import ( "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" + "github.com/go-vela/server/database/settings" ) // Interface represents the interface for integrating with the supported database providers. @@ -33,6 +34,9 @@ type Interface interface { // Resource Interface Functions + // SettingsInterface defines the interface for platform settings stored in the database. + settings.SettingsInterface + // BuildInterface defines the interface for builds stored in the database. build.BuildInterface diff --git a/database/resource.go b/database/resource.go index 2e309dedf..c978e5a5f 100644 --- a/database/resource.go +++ b/database/resource.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -24,6 +25,17 @@ import ( func (e *engine) NewResources(ctx context.Context) error { var err error + // create the database agnostic engine for settings + e.SettingsInterface, err = settings.New( + settings.WithContext(e.ctx), + settings.WithClient(e.client), + settings.WithLogger(e.logger), + settings.WithSkipCreation(e.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for builds e.BuildInterface, err = build.New( build.WithContext(e.ctx), diff --git a/database/settings/actions.go b/database/settings/actions.go new file mode 100644 index 000000000..cd3919baf --- /dev/null +++ b/database/settings/actions.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "database/sql" +) + +// PlatformSettings_DB is the database representation of platform settings. +type PlatformSettings_DB struct { + ID sql.NullInt64 `sql:"id"` + FooNum sql.NullInt64 `sql:"foo_num"` + BarStr sql.NullString `sql:"bar_str"` +} + +// CreateSettings updates a platform settings in the database. +func (e *engine) CreateSettings(ctx context.Context, s *string) (*string, error) { + e.logger.Tracef("creating platform settings in the database with %s", *s) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#SettingsFromLibrary + // s := database.SettingsFromLibrary(r) + + // todo: settings.validate() + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Settings.Validate + // err := settings.Validate() + // if err != nil { + // return nil, err + // } + + s2 := PlatformSettings_DB{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + FooNum: sql.NullInt64{Int64: 420, Valid: true}, + BarStr: sql.NullString{String: *s, Valid: true}, + } + + // send query to the database + err := e.client.Table(constantsTableSettings).Create(&s2).Error + if err != nil { + return nil, err + } + + return s, nil +} + +// GetSettings gets platform settings from the database. +func (e *engine) GetSettings(ctx context.Context) (*string, error) { + e.logger.Trace("getting platform settings from the database") + + // variable to store query results + s := new(PlatformSettings_DB) + + // send query to the database and store result in variable + err := e.client. + Table(constantsTableSettings). + // todo: how to ensure this is always a singleton at the first row + Where("id = ?", 1). + Take(s). + Error + if err != nil { + return nil, err + } + + // return the settings + // + // https://pkg.go.dev/github.com/go-vela/types/database#Settings.ToLibrary + return &s.BarStr.String, nil +} + +// UpdateSettings updates a platform settings in the database. +func (e *engine) UpdateSettings(ctx context.Context, s *string) (*string, error) { + e.logger.Trace("updating platform settings in the database") + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#SettingsFromLibrary + // s := database.SettingsFromLibrary(r) + + // todo: settings.validate() + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Settings.Validate + // err := settings.Validate() + // if err != nil { + // return nil, err + // } + + s2 := PlatformSettings_DB{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + FooNum: sql.NullInt64{Int64: 421, Valid: true}, + BarStr: sql.NullString{String: *s, Valid: true}, + } + + // send query to the database + err := e.client.Table(constantsTableSettings).Save(s2).Error + if err != nil { + return nil, err + } + + return s, nil +} diff --git a/database/settings/interface.go b/database/settings/interface.go new file mode 100644 index 000000000..32a897138 --- /dev/null +++ b/database/settings/interface.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" +) + +// SettingsInterface represents the Vela interface for settings +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type SettingsInterface interface { + // CreateSettings defines a function that creates a platform settings record. + CreateSettings(context.Context, *string) (*string, error) + // GetSettings defines a function that gets platform settings. + GetSettings(context.Context) (*string, error) + // UpdateSettings defines a function that updates platform settings. + UpdateSettings(context.Context, *string) (*string, error) +} diff --git a/database/settings/opt.go b/database/settings/opt.go new file mode 100644 index 000000000..c71c15a81 --- /dev/null +++ b/database/settings/opt.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// todo: Step -> Settings + +// EngineOpt represents a configuration option to initialize the database engine for Steps. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Steps. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the step engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Steps. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the step engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Steps. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the step engine + e.config.SkipCreation = skipCreation + + return nil + } +} + +// WithContext sets the context in the database engine for Steps. +func WithContext(ctx context.Context) EngineOpt { + return func(e *engine) error { + e.ctx = ctx + + return nil + } +} diff --git a/database/settings/settings.go b/database/settings/settings.go new file mode 100644 index 000000000..3c18a1f29 --- /dev/null +++ b/database/settings/settings.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +const ( + // todo: constantsTableSettings -> types.constants.TableSettings + constantsTableSettings = "settings" +) + +// todo: comments Build->Settings +type ( + // config represents the settings required to create the engine that implements the BuildInterface interface. + config struct { + // specifies to skip creating tables and indexes for the Build engine + SkipCreation bool + } + + // engine represents the build functionality that implements the BuildInterface interface. + engine struct { + // engine configuration settings used in build functions + config *config + + ctx context.Context + + // gorm.io/gorm database client used in build functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in build functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with builds in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Build engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of settings table and indexes in the database") + + return e, nil + } + + // create the settings table + err := e.CreateSettingsTable(e.ctx, e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constantsTableSettings, err) + } + + // todo: need indexes? + + return e, nil +} diff --git a/database/settings/table.go b/database/settings/table.go new file mode 100644 index 000000000..30e99a30d --- /dev/null +++ b/database/settings/table.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres settings table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +settings ( + id SERIAL PRIMARY KEY, + foo_num INTEGER, + bar_str VARCHAR(250) +); +` + + // CreateSqliteTable represents a query to create the Sqlite settings table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +settings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + foo_num INTEGER, + bar_str TEXT +); +` +) + +// CreateSettingsTable creates the settings table in the database. +func (e *engine) CreateSettingsTable(ctx context.Context, driver string) error { + e.logger.Tracef("creating settings table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the steps table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the steps table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/router/admin.go b/router/admin.go index cc321346e..cafa2a019 100644 --- a/router/admin.go +++ b/router/admin.go @@ -24,6 +24,10 @@ import ( // PUT /api/v1/admin/user // POST /api/v1/admin/workers/:worker/register. func AdminHandlers(base *gin.RouterGroup) { + // todo: figure out permissions for GET vs PUT etc + base.Group("/admin").GET("/settings", admin.GetSettings) + base.Group("/admin").PUT("/settings", admin.UpdateSettings) + // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) { From 9038758676adbe323b3ac4b07729f3cc358e5c33 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 19 Mar 2024 14:10:28 -0500 Subject: [PATCH 02/65] chore: revert makefile --- Makefile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index fc4f00486..0df0c3fe9 100644 --- a/Makefile +++ b/Makefile @@ -29,21 +29,21 @@ LD_FLAGS = -X github.com/go-vela/server/version.Commit=${GITHUB_SHA} -X github.c clean: tidy vet fmt fix # The `restart` target is intended to destroy and -# create the local docker-compose stack. +# create the local Docker compose stack. # # Usage: `make restart` .PHONY: restart restart: down up # The `up` target is intended to create -# the local docker-compose stack. +# the local Docker compose stack. # # Usage: `make up` .PHONY: up up: build compose-up # The `down` target is intended to destroy -# the local docker-compose stack. +# the local Docker compose stack. # # Usage: `make down` .PHONY: down @@ -233,34 +233,34 @@ bump-deps-full: check @go get -t -u ./... # The `pull` target is intended to pull all -# images for the local docker-compose stack. +# images for the local Docker compose stack. # # Usage: `make pull` .PHONY: pull pull: @echo @echo "### Pulling images for docker-compose stack" - @docker-compose pull + @docker compose pull # The `compose-up` target is intended to build and create -# all containers for the local docker-compose stack. +# all containers for the local Docker compose stack. # # Usage: `make compose-up` .PHONY: compose-up compose-up: @echo @echo "### Creating containers for docker-compose stack" - @docker-compose -f docker-compose.yml up -d --build + @docker compose -f docker-compose.yml up -d --build # The `compose-down` target is intended to destroy -# all containers for the local docker-compose stack. +# all containers for the local Docker compose stack. # # Usage: `make compose-down` .PHONY: compose-down compose-down: @echo @echo "### Destroying containers for docker-compose stack" - @docker-compose -f docker-compose.yml down + @docker compose -f docker-compose.yml down # The `spec-install` target is intended to install the # the needed dependencies to generate the api spec. From e779f5dfa3c611fb50a17190214870972f1ecbe4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 21 Mar 2024 12:57:19 -0500 Subject: [PATCH 03/65] wip: platform settings using new types pattern --- .vscode/launch.json | 13 ++++++++++++ Dockerfile.local | 20 +++++++++++++++++++ api/admin/settings.go | 17 +++++----------- api/types/settings.go | 25 +++++++++++++++++++++++ cmd/vela-server/server.go | 10 ++++++++-- database/settings/actions.go | 36 +++++++++------------------------- database/settings/interface.go | 8 +++++--- database/settings/opt.go | 18 ++++++++--------- database/settings/settings.go | 19 ++++++++++++++++++ database/settings/table.go | 4 ++-- docker-compose.yml | 7 +++++-- 11 files changed, 119 insertions(+), 58 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 Dockerfile.local create mode 100644 api/types/settings.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..dd6b4eaef --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "/workspace", + "port": 4000, + "host": "127.0.0.1", + } + ] +} diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 000000000..5d1cc7efb --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 + +FROM alpine:3.19.1@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b as certs + +RUN apk add --update --no-cache ca-certificates + +FROM golang + +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +EXPOSE 8080 + +ENV GODEBUG=netdns=go + +ADD release/vela-server /bin/ + +RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest +CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] + +# CMD ["/bin/vela-server"] diff --git a/api/admin/settings.go b/api/admin/settings.go index b3d724917..231503236 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -8,6 +8,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/util" "github.com/sirupsen/logrus" @@ -86,12 +87,7 @@ func UpdateSettings(c *gin.Context) { logrus.Info("Admin: updating platform settings") // capture body from API request - type ss struct { - BarStr string `json:"bar_str"` - } - - input := new(ss) - // input := new(library.Settings) + input := new(api.Settings) err = c.Bind(input) if err != nil { @@ -102,14 +98,11 @@ func UpdateSettings(c *gin.Context) { return } - // todo: do this right - updated := "default value" - if s != nil && input != nil { - updated = *s + input.BarStr - } + s.FooNum = input.FooNum + s.FooStr = input.FooStr // send API call to update the repo - s, err = database.FromContext(c).UpdateSettings(ctx, &updated) + s, err = database.FromContext(c).UpdateSettings(ctx, input) if err != nil { retErr := fmt.Errorf("unable to update platform settings: %w", err) diff --git a/api/types/settings.go b/api/types/settings.go new file mode 100644 index 000000000..f54b65968 --- /dev/null +++ b/api/types/settings.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" +) + +// Settings is the API representation of platform settings. +// +// swagger:model Settings +type Settings struct { + ID *int64 `json:"id,omitempty"` + FooNum *int64 `json:"foo_num,omitempty"` + FooStr *string `json:"foo_str,omitempty"` +} + +// String implements the Stringer interface for the Settings type. +func (w *Settings) String() string { + return fmt.Sprintf(`{ + ID: %d, +}`, + 1, // w.GetID(), + ) +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 53e4333ad..573373188 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -13,6 +13,7 @@ import ( "time" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" @@ -79,10 +80,15 @@ func server(c *cli.Context) error { // create a platform settings object to store the settings // err := e.client.Table(constantsTableSettings).Save(s2).Error - s := "init1" ss, err := database.GetSettings(context.Background()) if ss == nil || err != nil { - _, err = database.CreateSettings(context.Background(), &s) + s := new(api.Settings) + var fooNum int64 = 101 + var fooStr string = "bar" + s.FooNum = &fooNum + s.FooStr = &fooStr + + _, err = database.CreateSettings(context.Background(), s) if err != nil { return err } diff --git a/database/settings/actions.go b/database/settings/actions.go index cd3919baf..71db7b30d 100644 --- a/database/settings/actions.go +++ b/database/settings/actions.go @@ -4,18 +4,12 @@ package settings import ( "context" - "database/sql" -) -// PlatformSettings_DB is the database representation of platform settings. -type PlatformSettings_DB struct { - ID sql.NullInt64 `sql:"id"` - FooNum sql.NullInt64 `sql:"foo_num"` - BarStr sql.NullString `sql:"bar_str"` -} + api "github.com/go-vela/server/api/types" +) // CreateSettings updates a platform settings in the database. -func (e *engine) CreateSettings(ctx context.Context, s *string) (*string, error) { +func (e *engine) CreateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { e.logger.Tracef("creating platform settings in the database with %s", *s) // cast the library type to database type @@ -33,14 +27,8 @@ func (e *engine) CreateSettings(ctx context.Context, s *string) (*string, error) // return nil, err // } - s2 := PlatformSettings_DB{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - FooNum: sql.NullInt64{Int64: 420, Valid: true}, - BarStr: sql.NullString{String: *s, Valid: true}, - } - // send query to the database - err := e.client.Table(constantsTableSettings).Create(&s2).Error + err := e.client.Table(constantsTableSettings).Create(s).Error if err != nil { return nil, err } @@ -49,11 +37,11 @@ func (e *engine) CreateSettings(ctx context.Context, s *string) (*string, error) } // GetSettings gets platform settings from the database. -func (e *engine) GetSettings(ctx context.Context) (*string, error) { +func (e *engine) GetSettings(ctx context.Context) (*api.Settings, error) { e.logger.Trace("getting platform settings from the database") // variable to store query results - s := new(PlatformSettings_DB) + s := new(Settings) // send query to the database and store result in variable err := e.client. @@ -69,11 +57,11 @@ func (e *engine) GetSettings(ctx context.Context) (*string, error) { // return the settings // // https://pkg.go.dev/github.com/go-vela/types/database#Settings.ToLibrary - return &s.BarStr.String, nil + return s.ToAPI(), nil } // UpdateSettings updates a platform settings in the database. -func (e *engine) UpdateSettings(ctx context.Context, s *string) (*string, error) { +func (e *engine) UpdateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { e.logger.Trace("updating platform settings in the database") // cast the library type to database type @@ -91,14 +79,8 @@ func (e *engine) UpdateSettings(ctx context.Context, s *string) (*string, error) // return nil, err // } - s2 := PlatformSettings_DB{ - ID: sql.NullInt64{Int64: 1, Valid: true}, - FooNum: sql.NullInt64{Int64: 421, Valid: true}, - BarStr: sql.NullString{String: *s, Valid: true}, - } - // send query to the database - err := e.client.Table(constantsTableSettings).Save(s2).Error + err := e.client.Table(constantsTableSettings).Save(s).Error if err != nil { return nil, err } diff --git a/database/settings/interface.go b/database/settings/interface.go index 32a897138..f09b7e1b9 100644 --- a/database/settings/interface.go +++ b/database/settings/interface.go @@ -4,6 +4,8 @@ package settings import ( "context" + + api "github.com/go-vela/server/api/types" ) // SettingsInterface represents the Vela interface for settings @@ -12,9 +14,9 @@ import ( //nolint:revive // ignore name stutter type SettingsInterface interface { // CreateSettings defines a function that creates a platform settings record. - CreateSettings(context.Context, *string) (*string, error) + CreateSettings(context.Context, *api.Settings) (*api.Settings, error) // GetSettings defines a function that gets platform settings. - GetSettings(context.Context) (*string, error) + GetSettings(context.Context) (*api.Settings, error) // UpdateSettings defines a function that updates platform settings. - UpdateSettings(context.Context, *string) (*string, error) + UpdateSettings(context.Context, *api.Settings) (*api.Settings, error) } diff --git a/database/settings/opt.go b/database/settings/opt.go index c71c15a81..62c42fb6a 100644 --- a/database/settings/opt.go +++ b/database/settings/opt.go @@ -10,42 +10,40 @@ import ( "gorm.io/gorm" ) -// todo: Step -> Settings - -// EngineOpt represents a configuration option to initialize the database engine for Steps. +// EngineOpt represents a configuration option to initialize the database engine for Settings. type EngineOpt func(*engine) error -// WithClient sets the gorm.io/gorm client in the database engine for Steps. +// WithClient sets the gorm.io/gorm client in the database engine for Settings. func WithClient(client *gorm.DB) EngineOpt { return func(e *engine) error { - // set the gorm.io/gorm client in the step engine + // set the gorm.io/gorm client in the settings engine e.client = client return nil } } -// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Steps. +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Settings. func WithLogger(logger *logrus.Entry) EngineOpt { return func(e *engine) error { - // set the github.com/sirupsen/logrus logger in the step engine + // set the github.com/sirupsen/logrus logger in the settings engine e.logger = logger return nil } } -// WithSkipCreation sets the skip creation logic in the database engine for Steps. +// WithSkipCreation sets the skip creation logic in the database engine for Settings. func WithSkipCreation(skipCreation bool) EngineOpt { return func(e *engine) error { - // set to skip creating tables and indexes in the step engine + // set to skip creating tables and indexes in the settings engine e.config.SkipCreation = skipCreation return nil } } -// WithContext sets the context in the database engine for Steps. +// WithContext sets the context in the database engine for Settings. func WithContext(ctx context.Context) EngineOpt { return func(e *engine) error { e.ctx = ctx diff --git a/database/settings/settings.go b/database/settings/settings.go index 3c18a1f29..5e1d8f4d8 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -4,8 +4,10 @@ package settings import ( "context" + "database/sql" "fmt" + api "github.com/go-vela/server/api/types" "github.com/sirupsen/logrus" "gorm.io/gorm" @@ -41,6 +43,13 @@ type ( // https://pkg.go.dev/github.com/sirupsen/logrus#Entry logger *logrus.Entry } + + // Settings is the database representation of platform settings. + Settings struct { + ID sql.NullInt64 `sql:"id"` + FooNum sql.NullInt64 `sql:"foo_num"` + FooStr sql.NullString `sql:"foo_str"` + } ) // New creates and returns a Vela service for integrating with builds in the database. @@ -80,3 +89,13 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } + +// ToAPI converts the Worker type +// to an API Worker type. +func (s *Settings) ToAPI() *api.Settings { + settings := new(api.Settings) + + // settings.SetID(s.ID.Int64) + + return settings +} diff --git a/database/settings/table.go b/database/settings/table.go index 30e99a30d..0b6a499ec 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -16,7 +16,7 @@ IF NOT EXISTS settings ( id SERIAL PRIMARY KEY, foo_num INTEGER, - bar_str VARCHAR(250) + foo_str VARCHAR(250) ); ` @@ -27,7 +27,7 @@ IF NOT EXISTS settings ( id INTEGER PRIMARY KEY AUTOINCREMENT, foo_num INTEGER, - bar_str TEXT + foo_str TEXT ); ` ) diff --git a/docker-compose.yml b/docker-compose.yml index 6cd534285..90edcc735 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,13 +65,16 @@ services: # https://go-vela.github.io/docs/administration/worker/ worker: container_name: worker - image: target/vela-worker:latest + # image: target/vela-worker:latest + build: + context: ../worker + dockerfile: Dockerfile networks: - vela environment: EXECUTOR_DRIVER: linux QUEUE_DRIVER: redis - VELA_BUILD_LIMIT: 1 + VELA_BUILD_LIMIT: 3 VELA_BUILD_TIMEOUT: 30m VELA_LOG_LEVEL: trace VELA_RUNTIME_DRIVER: docker From 4628b028205a7ff73a424b242ec56d26d4365730 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 1 Apr 2024 11:16:26 -0500 Subject: [PATCH 04/65] feat: boiler plate api code for settings endpoints --- api/admin/settings.go | 66 +++++--------- mock/server/settings.go | 67 +++++++++++++++ mock/server/settings_test.go | 28 ++++++ router/admin.go | 9 +- router/middleware/settings/context.go | 37 ++++++++ router/middleware/settings/context_test.go | 92 ++++++++++++++++++++ router/middleware/settings/doc.go | 10 +++ router/middleware/settings/settings.go | 39 +++++++++ router/middleware/settings/settings_test.go | 95 +++++++++++++++++++++ 9 files changed, 395 insertions(+), 48 deletions(-) create mode 100644 mock/server/settings.go create mode 100644 mock/server/settings_test.go create mode 100644 router/middleware/settings/context.go create mode 100644 router/middleware/settings/context_test.go create mode 100644 router/middleware/settings/doc.go create mode 100644 router/middleware/settings/settings.go create mode 100644 router/middleware/settings/settings_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 231503236..ded2bc6f0 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -10,19 +10,14 @@ import ( "github.com/gin-gonic/gin" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/util" "github.com/sirupsen/logrus" ) -type PlatformSettings_API struct { - ID int `json:"id"` - FooNum int `json:"foo_num"` - BarStr string `json:"bar_str"` -} - // swagger:operation GET /api/v1/admin/settings admin GetSettings // -// Get the currently configured platform settings. +// Get the currently configured settings. // // --- // produces: @@ -31,80 +26,63 @@ type PlatformSettings_API struct { // - ApiKeyAuth: [] // responses: // '200': -// description: Successfully retrieved platform settings from the database +// description: Successfully retrieved settings from the database // schema: // type: array // items: // "$ref": "#/definitions/Settings" // '500': -// description: Unable to retrieve platform settings from the database +// description: Unable to retrieve settings from the database // schema: // "$ref": "#/definitions/Error" // GetSettings represents the API handler to -// captures platform settings stored in the database. +// captures settings stored in the database. func GetSettings(c *gin.Context) { // capture middleware values - ctx := c.Request.Context() + s := settings.Retrieve(c) - logrus.Info("Admin: reading platform settings") - - // send API call to capture pending and running builds - s, err := database.FromContext(c).GetSettings(ctx) - if err != nil { - retErr := fmt.Errorf("unable to capture platform settings: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } + logrus.Info("Admin: reading settings") c.JSON(http.StatusOK, s) } +// todo: swagger and comments func UpdateSettings(c *gin.Context) { // capture middleware values - // todo: settings.Retrieve - // s := user.Retrieve(c) - - s, err := database.FromContext(c).GetSettings(c.Request.Context()) - if err != nil { - retErr := fmt.Errorf("unable to retrieve platform settings from the database: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // maxBuildLimit := c.Value("maxBuildLimit").(int64) - // defaultRepoEvents := c.Value("defaultRepoEvents").([]string) - // defaultRepoEventsMask := c.Value("defaultRepoEventsMask").(int64) + s := settings.Retrieve(c) ctx := c.Request.Context() + // todo: comment is inaccurate // update engine logger with API metadata // // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.Info("Admin: updating platform settings") + logrus.Info("Admin: updating settings") // capture body from API request input := new(api.Settings) - err = c.Bind(input) + err := c.Bind(input) if err != nil { - retErr := fmt.Errorf("unable to decode JSON for platform settings: %w", err) + retErr := fmt.Errorf("unable to decode JSON for settings: %w", err) util.HandleError(c, http.StatusBadRequest, retErr) return } - s.FooNum = input.FooNum - s.FooStr = input.FooStr + if input.FooNum != nil { + s.FooNum = input.FooNum + } + + if input.FooStr != nil { + s.FooStr = input.FooStr + } // send API call to update the repo - s, err = database.FromContext(c).UpdateSettings(ctx, input) + s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { - retErr := fmt.Errorf("unable to update platform settings: %w", err) + retErr := fmt.Errorf("unable to update settings: %w", err) util.HandleError(c, http.StatusInternalServerError, retErr) diff --git a/mock/server/settings.go b/mock/server/settings.go new file mode 100644 index 000000000..5ba8bff09 --- /dev/null +++ b/mock/server/settings.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" +) + +const ( + // SettingsResp represents a JSON return for a single settings. + SettingsResp = ` + { + "id": 1 + }` + + // CreateSettingsResp represents a JSON return for creating a settings record. + CreateSettingsResp = ` + { + "id": 1 + }` + // UpdateSettingsResp represents a JSON return for modifying a settings field. + UpdateSettingsResp = ` + { + "id": 1 + }` + // RemoveSettingsResp represents a JSON return for deleting a settings record. + RemoveSettingsResp = ` + { + "id": 1 + }` +) + +// getSettings has a param :settings returns mock JSON for a http GET. +func getSettings(c *gin.Context) { + data := []byte(SettingsResp) + + var body api.Settings + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// updateSettings returns mock JSON for a http PUT. +func updateSettings(c *gin.Context) { + data := []byte(UpdateSettingsResp) + + var body api.Settings + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// removeSettings has a param :settings returns mock JSON for a http DELETE. +// +// Pass "0" to :settings to test receiving a http 404 response. +func removeSettings(c *gin.Context) { + data := []byte(RemoveSettingsResp) + + var body api.Settings + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go new file mode 100644 index 000000000..663368586 --- /dev/null +++ b/mock/server/settings_test.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types" +) + +func TestSettings_GetResp(t *testing.T) { + testSettings := api.Settings{} + + err := json.Unmarshal([]byte(SettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + if reflect.ValueOf(testSettings).Field(i).IsNil() { + t.Errorf("SettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} diff --git a/router/admin.go b/router/admin.go index cafa2a019..7ac55a31f 100644 --- a/router/admin.go +++ b/router/admin.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/api/admin" "github.com/go-vela/server/router/middleware/perm" + smiddleware "github.com/go-vela/server/router/middleware/settings" ) // AdminHandlers is a function that extends the provided base router group @@ -24,10 +25,6 @@ import ( // PUT /api/v1/admin/user // POST /api/v1/admin/workers/:worker/register. func AdminHandlers(base *gin.RouterGroup) { - // todo: figure out permissions for GET vs PUT etc - base.Group("/admin").GET("/settings", admin.GetSettings) - base.Group("/admin").PUT("/settings", admin.UpdateSettings) - // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) { @@ -63,5 +60,9 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin worker endpoint _admin.POST("/workers/:worker/register", admin.RegisterToken) + + // Admin settings endpoint + _admin.GET("/settings", smiddleware.Establish(), admin.GetSettings) + _admin.PUT("/settings", smiddleware.Establish(), admin.UpdateSettings) } // end of admin endpoints } diff --git a/router/middleware/settings/context.go b/router/middleware/settings/context.go new file mode 100644 index 000000000..c2b12a070 --- /dev/null +++ b/router/middleware/settings/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +const key = "settings" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the Settings associated with this context. +func FromContext(c context.Context) *api.Settings { + value := c.Value(key) + if value == nil { + return nil + } + + s, ok := value.(*api.Settings) + if !ok { + return nil + } + + return s +} + +// ToContext adds the Settings to this context if it supports +// the Setter interface. +func ToContext(c Setter, s *api.Settings) { + c.Set(key, s) +} diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go new file mode 100644 index 000000000..59e77e853 --- /dev/null +++ b/router/middleware/settings/context_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "testing" + + api "github.com/go-vela/server/api/types" + + "github.com/gin-gonic/gin" +) + +func TestSettings_FromContext(t *testing.T) { + // setup types + // num := int64(1) + want := &api.Settings{ + // ID: &num + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestSettings_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_ToContext(t *testing.T) { + // setup types + // num := int64(1) + want := &api.Settings{ + // ID: &num + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/settings/doc.go b/router/middleware/settings/doc.go new file mode 100644 index 000000000..2c83d65fc --- /dev/null +++ b/router/middleware/settings/doc.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package settings provides the ability for inserting +// Vela settings resources into or extracting Vela settings +// resources from the middleware chain for the API. +// +// Usage: +// +// import "github.com/go-vela/server/router/middleware/settings" +package settings diff --git a/router/middleware/settings/settings.go b/router/middleware/settings/settings.go new file mode 100644 index 000000000..ce63da1bd --- /dev/null +++ b/router/middleware/settings/settings.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// Retrieve gets the settings in the given context. +func Retrieve(c *gin.Context) *api.Settings { + return FromContext(c) +} + +// Establish sets the settings in the given context. +func Establish() gin.HandlerFunc { + return func(c *gin.Context) { + ctx := c.Request.Context() + + logrus.Debug("Reading settings") + + s, err := database.FromContext(c).GetSettings(ctx) + if err != nil { + retErr := fmt.Errorf("unable to read settings: %w", err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + ToContext(c, s) + c.Next() + } +} diff --git a/router/middleware/settings/settings_test.go b/router/middleware/settings/settings_test.go new file mode 100644 index 000000000..37eebc714 --- /dev/null +++ b/router/middleware/settings/settings_test.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database" + "github.com/go-vela/types/library" +) + +func TestSettings_Retrieve(t *testing.T) { + // setup types + want := new(api.Settings) + // want.SetID(1) + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := Retrieve(context) + + if got != want { + t.Errorf("Retrieve is %v, want %v", got, want) + } +} + +func TestSettings_Establish(t *testing.T) { + // setup types + b := new(library.Build) + b.SetID(1) + + want := new(api.Settings) + // want.SetID(1) + // want.SetHostname("worker_0") + // want.SetAddress("localhost") + // want.SetRoutes([]string{"foo", "bar", "baz"}) + // want.SetActive(true) + // want.SetStatus("available") + // want.SetLastStatusUpdateAt(12345) + // want.SetRunningBuilds([]*library.Build{b}) + // want.SetLastBuildStartedAt(12345) + // want.SetLastBuildFinishedAt(12345) + // want.SetLastCheckedIn(12345) + // want.SetBuildLimit(0) + + got := new(api.Settings) + + // setup database + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + + // defer func() { + // _ = db.DeleteSettings(context.TODO(), want) + // db.Close() + // }() + + // _, _ = db.CreateSettings(context.TODO(), want) + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/admin/settings", nil) + + // setup mock server + engine.Use(func(c *gin.Context) { database.ToContext(c, db) }) + engine.Use(Establish()) + engine.GET("/admin/settings", func(c *gin.Context) { + got = Retrieve(c) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(resp, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Establish returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Establish is %v, want %v", got, want) + } +} From 4cf60ebe334996cc9b2ee9dfc280681421bef5ba Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 2 Apr 2024 14:54:04 -0500 Subject: [PATCH 05/65] feat: wip settings object used for clone image --- api/admin/settings.go | 12 +----- api/types/settings.go | 78 ++++++++++++++++++++++++++++++++--- cmd/vela-server/server.go | 24 ++++------- compiler/engine.go | 6 ++- compiler/native/clone.go | 4 +- compiler/native/native.go | 20 +++++++-- database/settings/actions.go | 6 +-- database/settings/settings.go | 72 ++++++++++++++++++++++++++++---- database/settings/table.go | 6 +-- 9 files changed, 177 insertions(+), 51 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index ded2bc6f0..e74bb38cc 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -53,10 +53,6 @@ func UpdateSettings(c *gin.Context) { s := settings.Retrieve(c) ctx := c.Request.Context() - // todo: comment is inaccurate - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields logrus.Info("Admin: updating settings") // capture body from API request @@ -71,12 +67,8 @@ func UpdateSettings(c *gin.Context) { return } - if input.FooNum != nil { - s.FooNum = input.FooNum - } - - if input.FooStr != nil { - s.FooStr = input.FooStr + if input.CloneImage != nil { + s.CloneImage = input.CloneImage } // send API call to update the repo diff --git a/api/types/settings.go b/api/types/settings.go index f54b65968..d92172e62 100644 --- a/api/types/settings.go +++ b/api/types/settings.go @@ -4,22 +4,90 @@ package types import ( "fmt" + + "github.com/urfave/cli/v2" ) // Settings is the API representation of platform settings. // // swagger:model Settings type Settings struct { - ID *int64 `json:"id,omitempty"` - FooNum *int64 `json:"foo_num,omitempty"` - FooStr *string `json:"foo_str,omitempty"` + ID *int64 `json:"id,omitempty"` + CloneImage *string `json:"clone_image,omitempty"` +} + +// NewSettings returns a new Settings record. +func NewSettings(c *cli.Context) *Settings { + s := new(Settings) + + // singleton record ID should always be 1 + s.SetID(1) + + // set the clone image to use for the injected clone step + s.SetCloneImage(c.String("clone-image")) + + return s +} + +// GetID returns the ID field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetID() int64 { + // return zero value if Settings type or ID field is nil + if s == nil || s.ID == nil { + return 0 + } + + return *s.ID +} + +// GetCloneImage returns the CloneImage field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetCloneImage() string { + // return zero value if Settings type or CloneImage field is nil + if s == nil || s.CloneImage == nil { + return "" + } + + return *s.CloneImage +} + +// SetID sets the ID field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Settings) SetID(v int64) { + // return if Settings type is nil + if s == nil { + return + } + + s.ID = &v +} + +// SetCloneImage sets the CloneImage field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Settings) SetCloneImage(v string) { + // return if Settings type is nil + if s == nil { + return + } + + s.CloneImage = &v } // String implements the Stringer interface for the Settings type. -func (w *Settings) String() string { +func (s *Settings) String() string { return fmt.Sprintf(`{ ID: %d, + CloneImage: %s, }`, - 1, // w.GetID(), + s.GetID(), + s.GetCloneImage(), ) } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 573373188..d4f7fba73 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -68,25 +68,14 @@ func server(c *cli.Context) error { logrus.SetLevel(logrus.PanicLevel) } - compiler, err := setupCompiler(c) - if err != nil { - return err - } - database, err := database.FromCLIContext(c) if err != nil { return err } - // create a platform settings object to store the settings - // err := e.client.Table(constantsTableSettings).Save(s2).Error - ss, err := database.GetSettings(context.Background()) - if ss == nil || err != nil { - s := new(api.Settings) - var fooNum int64 = 101 - var fooStr string = "bar" - s.FooNum = &fooNum - s.FooStr = &fooStr + s, err := database.GetSettings(context.Background()) + if s == nil || err != nil { + s = api.NewSettings(c) _, err = database.CreateSettings(context.Background(), s) if err != nil { @@ -94,7 +83,12 @@ func server(c *cli.Context) error { } } - // when the server reboots, should it only use the defaults when the object doesnt exist yet? + compiler, err := setupCompiler(c) + if err != nil { + return err + } + + compiler = compiler.WithSettings(s) queue, err := setupQueue(c) if err != nil { diff --git a/compiler/engine.go b/compiler/engine.go index 69812a519..04b411db2 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -3,6 +3,7 @@ package compiler import ( + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" @@ -140,7 +141,10 @@ type Engine interface { // WithUser defines a function that sets // the library user type in the Engine. WithUser(*library.User) Engine - // WithUser defines a function that sets + // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(string, string) Engine + // WithPrivateGitHub defines a function that sets + // the server settings in the Engine. + WithSettings(*api.Settings) Engine } diff --git a/compiler/native/clone.go b/compiler/native/clone.go index 9075292f7..88f45f60d 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -30,7 +30,7 @@ func (c *client) CloneStage(p *yaml.Build) (*yaml.Build, error) { Steps: yaml.StepSlice{ &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.Settings.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, @@ -63,7 +63,7 @@ func (c *client) CloneStep(p *yaml.Build) (*yaml.Build, error) { // create new clone step clone := &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.Settings.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, diff --git a/compiler/native/native.go b/compiler/native/native.go index a93c6fd50..d10c577bb 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -10,6 +10,7 @@ import ( "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/library" @@ -25,13 +26,14 @@ type ModificationConfig struct { } type client struct { + *api.Settings Github registry.Service PrivateGithub registry.Service UsePrivateGithub bool ModificationService ModificationConfig - CloneImage string - TemplateDepth int - StarlarkExecLimit uint64 + // CloneImage string + TemplateDepth int + StarlarkExecLimit uint64 build *library.Build comment string @@ -70,7 +72,7 @@ func New(ctx *cli.Context) (*client, error) { c.Github = github // set the clone image to use for the injected clone step - c.CloneImage = ctx.String("clone-image") + // c.CloneImage = ctx.String("clone-image") // set the template depth to use for nested templates c.TemplateDepth = ctx.Int("max-template-depth") @@ -112,6 +114,7 @@ func (c *client) Duplicate() compiler.Engine { cc := new(client) // copy the essential fields from the existing client + cc.Settings = c.Settings cc.Github = c.Github cc.PrivateGithub = c.PrivateGithub cc.UsePrivateGithub = c.UsePrivateGithub @@ -210,3 +213,12 @@ func (c *client) WithUser(u *library.User) compiler.Engine { return c } + +// WithSettings sets the api settings type in the Engine. +func (c *client) WithSettings(s *api.Settings) compiler.Engine { + if s != nil { + c.Settings = s + } + + return c +} diff --git a/database/settings/actions.go b/database/settings/actions.go index 71db7b30d..debccc7ef 100644 --- a/database/settings/actions.go +++ b/database/settings/actions.go @@ -28,7 +28,7 @@ func (e *engine) CreateSettings(ctx context.Context, s *api.Settings) (*api.Sett // } // send query to the database - err := e.client.Table(constantsTableSettings).Create(s).Error + err := e.client.Table(TableSettings).Create(s).Error if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (e *engine) GetSettings(ctx context.Context) (*api.Settings, error) { // send query to the database and store result in variable err := e.client. - Table(constantsTableSettings). + Table(TableSettings). // todo: how to ensure this is always a singleton at the first row Where("id = ?", 1). Take(s). @@ -80,7 +80,7 @@ func (e *engine) UpdateSettings(ctx context.Context, s *api.Settings) (*api.Sett // } // send query to the database - err := e.client.Table(constantsTableSettings).Save(s).Error + err := e.client.Table(TableSettings).Save(s).Error if err != nil { return nil, err } diff --git a/database/settings/settings.go b/database/settings/settings.go index 5e1d8f4d8..9a6f22c3d 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -5,17 +5,24 @@ package settings import ( "context" "database/sql" + "errors" "fmt" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/util" "github.com/sirupsen/logrus" "gorm.io/gorm" ) const ( - // todo: constantsTableSettings -> types.constants.TableSettings - constantsTableSettings = "settings" + TableSettings = "settings" +) + +var ( + // ErrEmptyCloneImage defines the error type when a + // Settings type has an empty CloneImage field provided. + ErrEmptyCloneImage = errors.New("empty settings clone image provided") ) // todo: comments Build->Settings @@ -46,9 +53,8 @@ type ( // Settings is the database representation of platform settings. Settings struct { - ID sql.NullInt64 `sql:"id"` - FooNum sql.NullInt64 `sql:"foo_num"` - FooStr sql.NullString `sql:"foo_str"` + ID sql.NullInt64 `sql:"id"` + CloneImage sql.NullString `sql:"clone_image"` } ) @@ -82,7 +88,7 @@ func New(opts ...EngineOpt) (*engine, error) { // create the settings table err := e.CreateSettingsTable(e.ctx, e.client.Config.Dialector.Name()) if err != nil { - return nil, fmt.Errorf("unable to create %s table: %w", constantsTableSettings, err) + return nil, fmt.Errorf("unable to create %s table: %w", TableSettings, err) } // todo: need indexes? @@ -90,12 +96,64 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Build type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (s *Settings) Nullify() *Settings { + if s == nil { + return nil + } + + // check if the ID field should be false + if s.ID.Int64 == 0 { + s.ID.Valid = false + } + + // check if the CloneImage field should be false + if len(s.CloneImage.String) == 0 { + s.CloneImage.Valid = false + } + + return s +} + // ToAPI converts the Worker type // to an API Worker type. func (s *Settings) ToAPI() *api.Settings { settings := new(api.Settings) - // settings.SetID(s.ID.Int64) + settings.SetID(s.ID.Int64) + settings.SetCloneImage(s.CloneImage.String) return settings } + +// Validate verifies the necessary fields for +// the Settings type are populated correctly. +func (s *Settings) Validate() error { + // verify the CloneImage field is populated + if len(s.CloneImage.String) == 0 { + return ErrEmptyCloneImage + } + + // ensure that all Settings string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} + + return nil +} + +// FromAPI converts the API settings type +// to a database settings type. +func FromAPI(s *api.Settings) *Settings { + settings := &Settings{ + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + } + + return settings.Nullify() +} diff --git a/database/settings/table.go b/database/settings/table.go index 0b6a499ec..4a5615d2f 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -15,8 +15,7 @@ CREATE TABLE IF NOT EXISTS settings ( id SERIAL PRIMARY KEY, - foo_num INTEGER, - foo_str VARCHAR(250) + clone_image VARCHAR(250) ); ` @@ -26,8 +25,7 @@ CREATE TABLE IF NOT EXISTS settings ( id INTEGER PRIMARY KEY AUTOINCREMENT, - foo_num INTEGER, - foo_str TEXT + clone_image TEXT ); ` ) From 7e24e652ace8f03d0c894220958e2de00630c526 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 2 Apr 2024 15:59:37 -0500 Subject: [PATCH 06/65] feat: wip settings embedded into compiler --- api/admin/settings.go | 6 ++++++ api/build/compile_publish.go | 16 ++++++++++++++++ api/pipeline/compile.go | 10 +++++++++- api/pipeline/expand.go | 10 +++++++++- api/pipeline/template.go | 10 +++++++++- api/pipeline/validate.go | 10 +++++++++- cmd/vela-server/server.go | 5 +++++ compiler/native/clone.go | 4 ++-- compiler/native/native.go | 6 ++---- router/middleware/pipeline/pipeline.go | 3 +++ 10 files changed, 70 insertions(+), 10 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index e74bb38cc..c4f3fd0d2 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -81,5 +81,11 @@ func UpdateSettings(c *gin.Context) { return } + // todo: how to connect this with the actual settings record stored in the server + // todo: how to take this update and propogate it to the other server groups???????????? + // say you have two instances of the server running, we need to fetch this settings record before actually using it + // + // + c.JSON(http.StatusOK, s) } diff --git a/api/build/compile_publish.go b/api/build/compile_publish.go index ea908862f..ffc5bb320 100644 --- a/api/build/compile_publish.go +++ b/api/build/compile_publish.go @@ -276,6 +276,21 @@ func CompileAndPublish( repo.SetPipelineType(pipeline.GetType()) } + // get settings + s, err := database.GetSettings(c) + if err != nil { + // format the error message with extra information + err = fmt.Errorf("unable to get settings: %w", err) + + // log the error for traceability + logrus.Error(err.Error()) + + retErr := fmt.Errorf("%s: %w", baseErr, err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return nil, nil, retErr + } + var compiled *library.Pipeline // parse and compile the pipeline configuration file p, compiled, err = compiler. @@ -287,6 +302,7 @@ func CompileAndPublish( WithMetadata(cfg.Metadata). WithRepo(repo). WithUser(u). + WithSettings(s). Compile(pipelineFile) if err != nil { // format the error message with extra information diff --git a/api/pipeline/compile.go b/api/pipeline/compile.go index d829ab755..4de74e219 100644 --- a/api/pipeline/compile.go +++ b/api/pipeline/compile.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -75,6 +76,7 @@ func CompilePipeline(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -92,7 +94,13 @@ func CompilePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c). + Duplicate(). + WithCommit(p.GetCommit()). + WithMetadata(m). + WithRepo(r). + WithUser(u). + WithSettings(s) // compile the pipeline pipeline, _, err := compiler.CompileLite(p.GetData(), true) diff --git a/api/pipeline/expand.go b/api/pipeline/expand.go index 8fb19303f..93d51547d 100644 --- a/api/pipeline/expand.go +++ b/api/pipeline/expand.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -76,6 +77,7 @@ func ExpandPipeline(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -93,7 +95,13 @@ func ExpandPipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c). + Duplicate(). + WithCommit(p.GetCommit()). + WithMetadata(m). + WithRepo(r). + WithUser(u). + WithSettings(s) // expand the templates in the pipeline pipeline, _, err := compiler.CompileLite(p.GetData(), false) diff --git a/api/pipeline/template.go b/api/pipeline/template.go index 9716b6328..0df7cf7c9 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -14,6 +14,7 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -80,6 +81,7 @@ func GetTemplates(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + s := settings.Retrieve(c) ctx := c.Request.Context() entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -95,7 +97,13 @@ func GetTemplates(c *gin.Context) { }).Infof("reading templates from pipeline %s", entry) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c). + Duplicate(). + WithCommit(p.GetCommit()). + WithMetadata(m). + WithRepo(r). + WithUser(u). + WithSettings(s) // parse the pipeline configuration pipeline, _, err := compiler.Parse(p.GetData(), p.GetType(), new(yaml.Template)) diff --git a/api/pipeline/validate.go b/api/pipeline/validate.go index 333e7f0d6..62238a106 100644 --- a/api/pipeline/validate.go +++ b/api/pipeline/validate.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -74,6 +75,7 @@ func ValidatePipeline(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -91,7 +93,13 @@ func ValidatePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c). + Duplicate(). + WithCommit(p.GetCommit()). + WithMetadata(m). + WithRepo(r). + WithUser(u). + WithSettings(s) // validate the pipeline pipeline, _, err := compiler.CompileLite(p.GetData(), false) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index d4f7fba73..1dc98d727 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -88,6 +88,11 @@ func server(c *cli.Context) error { return err } + // todo: this wont really work... + // if we receive an update to settings we need to update this settings record somehow??? + // but we cant use the actual PUT endpoint because it is load balanced across a bunch of servers + // so... we need to refetch this settings record before we actually need it ?? ? ? ?? ? ? ? ? + compiler = compiler.WithSettings(s) queue, err := setupQueue(c) diff --git a/compiler/native/clone.go b/compiler/native/clone.go index 88f45f60d..2b422235a 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -30,7 +30,7 @@ func (c *client) CloneStage(p *yaml.Build) (*yaml.Build, error) { Steps: yaml.StepSlice{ &yaml.Step{ Detach: false, - Image: c.Settings.GetCloneImage(), + Image: c.settings.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, @@ -63,7 +63,7 @@ func (c *client) CloneStep(p *yaml.Build) (*yaml.Build, error) { // create new clone step clone := &yaml.Step{ Detach: false, - Image: c.Settings.GetCloneImage(), + Image: c.settings.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, diff --git a/compiler/native/native.go b/compiler/native/native.go index d10c577bb..9a8135acd 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -26,7 +26,6 @@ type ModificationConfig struct { } type client struct { - *api.Settings Github registry.Service PrivateGithub registry.Service UsePrivateGithub bool @@ -44,6 +43,7 @@ type client struct { metadata *types.Metadata repo *library.Repo user *library.User + settings *api.Settings } // New returns a Pipeline implementation that integrates with the supported registries. @@ -114,12 +114,10 @@ func (c *client) Duplicate() compiler.Engine { cc := new(client) // copy the essential fields from the existing client - cc.Settings = c.Settings cc.Github = c.Github cc.PrivateGithub = c.PrivateGithub cc.UsePrivateGithub = c.UsePrivateGithub cc.ModificationService = c.ModificationService - cc.CloneImage = c.CloneImage cc.TemplateDepth = c.TemplateDepth cc.StarlarkExecLimit = c.StarlarkExecLimit @@ -217,7 +215,7 @@ func (c *client) WithUser(u *library.User) compiler.Engine { // WithSettings sets the api settings type in the Engine. func (c *client) WithSettings(s *api.Settings) compiler.Engine { if s != nil { - c.Settings = s + c.settings = s } return c diff --git a/router/middleware/pipeline/pipeline.go b/router/middleware/pipeline/pipeline.go index 61f308854..a3700dfb6 100644 --- a/router/middleware/pipeline/pipeline.go +++ b/router/middleware/pipeline/pipeline.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -30,6 +31,7 @@ func Establish() gin.HandlerFunc { o := org.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + s := settings.Retrieve(c) ctx := c.Request.Context() if r == nil { @@ -80,6 +82,7 @@ func Establish() gin.HandlerFunc { WithMetadata(c.MustGet("metadata").(*types.Metadata)). WithRepo(r). WithUser(u). + WithSettings(s). Compile(config) if err != nil { retErr := fmt.Errorf("unable to compile pipeline configuration for %s: %w", entry, err) From 336dcf50bf392d2b6a39693a5c98aea0582b9e2e Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 3 Apr 2024 11:06:56 -0500 Subject: [PATCH 07/65] feat: query params to control output format, and clone image control --- api/admin/settings.go | 22 +++++++++++++++------- api/types/settings.go | 20 ++++++++++++++++++++ cmd/vela-server/server.go | 9 ++------- router/middleware/settings/context_test.go | 14 ++++++++++---- 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index c4f3fd0d2..df0f89284 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -6,6 +6,7 @@ package admin import ( "fmt" "net/http" + "strings" "github.com/gin-gonic/gin" api "github.com/go-vela/server/api/types" @@ -44,7 +45,20 @@ func GetSettings(c *gin.Context) { logrus.Info("Admin: reading settings") - c.JSON(http.StatusOK, s) + output := strings.ToLower(c.Query("output")) + + switch output { + case "env": + exported := s.ToEnv() + c.String(http.StatusOK, exported) + case "yaml": + exported := s.ToYAML() + c.String(http.StatusOK, exported) + case "json": + fallthrough + default: + c.JSON(http.StatusOK, s) + } } // todo: swagger and comments @@ -81,11 +95,5 @@ func UpdateSettings(c *gin.Context) { return } - // todo: how to connect this with the actual settings record stored in the server - // todo: how to take this update and propogate it to the other server groups???????????? - // say you have two instances of the server running, we need to fetch this settings record before actually using it - // - // - c.JSON(http.StatusOK, s) } diff --git a/api/types/settings.go b/api/types/settings.go index d92172e62..e97f784ce 100644 --- a/api/types/settings.go +++ b/api/types/settings.go @@ -91,3 +91,23 @@ func (s *Settings) String() string { s.GetCloneImage(), ) } + +// ToEnv converts the Settings type to a string format compatible with standard posix environments. +func (s *Settings) ToEnv() string { + return fmt.Sprintf(`CloneImage='%s' +FooBar='%s' +`, + s.GetCloneImage(), + "something-cool", + ) +} + +// ToYAML converts the Settings type to a YAML string. +func (s *Settings) ToYAML() string { + return fmt.Sprintf(`CloneImage: '%s' +FooBar: '%s' +`, + s.GetCloneImage(), + "something-cool", + ) +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 1dc98d727..38d661195 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -83,18 +83,13 @@ func server(c *cli.Context) error { } } + // todo: use settings initialization policy to reset settings? + compiler, err := setupCompiler(c) if err != nil { return err } - // todo: this wont really work... - // if we receive an update to settings we need to update this settings record somehow??? - // but we cant use the actual PUT endpoint because it is load balanced across a bunch of servers - // so... we need to refetch this settings record before we actually need it ?? ? ? ?? ? ? ? ? - - compiler = compiler.WithSettings(s) - queue, err := setupQueue(c) if err != nil { return err diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go index 59e77e853..4272cad36 100644 --- a/router/middleware/settings/context_test.go +++ b/router/middleware/settings/context_test.go @@ -12,9 +12,12 @@ import ( func TestSettings_FromContext(t *testing.T) { // setup types - // num := int64(1) + num := int64(1) + cloneImage := "target/vela-git" + want := &api.Settings{ - // ID: &num + ID: &num, + CloneImage: &cloneImage, } // setup context @@ -73,9 +76,12 @@ func TestSettings_FromContext_Empty(t *testing.T) { func TestSettings_ToContext(t *testing.T) { // setup types - // num := int64(1) + num := int64(1) + cloneImage := "target/vela-git" + want := &api.Settings{ - // ID: &num + ID: &num, + CloneImage: &cloneImage, } // setup context From 42726c8aca1dcc7e78260b9ea153af4671e7a58c Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 10 Apr 2024 17:19:53 -0500 Subject: [PATCH 08/65] wip: infra for queue routes and starlark exec limit --- api/admin/settings.go | 5 ++ api/types/settings.go | 48 +++++++++++++++---- database/settings/actions.go | 89 ----------------------------------- database/settings/create.go | 31 ++++++++++++ database/settings/get.go | 31 ++++++++++++ database/settings/settings.go | 31 ++++++++---- database/settings/table.go | 10 ++-- database/settings/update.go | 31 ++++++++++++ 8 files changed, 166 insertions(+), 110 deletions(-) delete mode 100644 database/settings/actions.go create mode 100644 database/settings/create.go create mode 100644 database/settings/get.go create mode 100644 database/settings/update.go diff --git a/api/admin/settings.go b/api/admin/settings.go index df0f89284..0ec9d69f2 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -85,6 +85,11 @@ func UpdateSettings(c *gin.Context) { s.CloneImage = input.CloneImage } + if input.QueueRoutes != nil { + // update routes if set + s.SetQueueRoutes(input.GetQueueRoutes()) + } + // send API call to update the repo s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { diff --git a/api/types/settings.go b/api/types/settings.go index e97f784ce..1cdcc4bb0 100644 --- a/api/types/settings.go +++ b/api/types/settings.go @@ -12,8 +12,9 @@ import ( // // swagger:model Settings type Settings struct { - ID *int64 `json:"id,omitempty"` - CloneImage *string `json:"clone_image,omitempty"` + ID *int64 `json:"id,omitempty"` + CloneImage *string `json:"clone_image,omitempty"` + QueueRoutes *[]string `json:"queue_routes,omitempty"` } // NewSettings returns a new Settings record. @@ -26,6 +27,9 @@ func NewSettings(c *cli.Context) *Settings { // set the clone image to use for the injected clone step s.SetCloneImage(c.String("clone-image")) + // set the queue routes (channels) to use for builds + s.SetQueueRoutes(c.StringSlice("queue.routes")) + return s } @@ -55,6 +59,19 @@ func (s *Settings) GetCloneImage() string { return *s.CloneImage } +// GetQueueRoutes returns the QueueRoutes field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetQueueRoutes() []string { + // return zero value if Settings type or QueueRoutes field is nil + if s == nil || s.QueueRoutes == nil { + return []string{} + } + + return *s.QueueRoutes +} + // SetID sets the ID field. // // When the provided Settings type is nil, it @@ -81,33 +98,48 @@ func (s *Settings) SetCloneImage(v string) { s.CloneImage = &v } +// SetQueueRoutes sets the QueueRoutes field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Settings) SetQueueRoutes(v []string) { + // return if Settings type is nil + if s == nil { + return + } + + s.QueueRoutes = &v +} + // String implements the Stringer interface for the Settings type. func (s *Settings) String() string { return fmt.Sprintf(`{ ID: %d, CloneImage: %s, + QueueRoutes: %v, }`, s.GetID(), s.GetCloneImage(), + s.GetQueueRoutes(), ) } // ToEnv converts the Settings type to a string format compatible with standard posix environments. func (s *Settings) ToEnv() string { - return fmt.Sprintf(`CloneImage='%s' -FooBar='%s' + return fmt.Sprintf(`VELA_CLONE_IMAGE='%s' +VELA_QUEUE_ROUTES='%v' `, s.GetCloneImage(), - "something-cool", + s.GetQueueRoutes(), ) } // ToYAML converts the Settings type to a YAML string. func (s *Settings) ToYAML() string { - return fmt.Sprintf(`CloneImage: '%s' -FooBar: '%s' + return fmt.Sprintf(`VELA_CLONE_IMAGE: '%s' +VELA_QUEUE_ROUTES: '%s' `, s.GetCloneImage(), - "something-cool", + s.GetQueueRoutes(), ) } diff --git a/database/settings/actions.go b/database/settings/actions.go deleted file mode 100644 index debccc7ef..000000000 --- a/database/settings/actions.go +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package settings - -import ( - "context" - - api "github.com/go-vela/server/api/types" -) - -// CreateSettings updates a platform settings in the database. -func (e *engine) CreateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { - e.logger.Tracef("creating platform settings in the database with %s", *s) - - // cast the library type to database type - // - // https://pkg.go.dev/github.com/go-vela/types/database#SettingsFromLibrary - // s := database.SettingsFromLibrary(r) - - // todo: settings.validate() - - // validate the necessary fields are populated - // - // https://pkg.go.dev/github.com/go-vela/types/database#Settings.Validate - // err := settings.Validate() - // if err != nil { - // return nil, err - // } - - // send query to the database - err := e.client.Table(TableSettings).Create(s).Error - if err != nil { - return nil, err - } - - return s, nil -} - -// GetSettings gets platform settings from the database. -func (e *engine) GetSettings(ctx context.Context) (*api.Settings, error) { - e.logger.Trace("getting platform settings from the database") - - // variable to store query results - s := new(Settings) - - // send query to the database and store result in variable - err := e.client. - Table(TableSettings). - // todo: how to ensure this is always a singleton at the first row - Where("id = ?", 1). - Take(s). - Error - if err != nil { - return nil, err - } - - // return the settings - // - // https://pkg.go.dev/github.com/go-vela/types/database#Settings.ToLibrary - return s.ToAPI(), nil -} - -// UpdateSettings updates a platform settings in the database. -func (e *engine) UpdateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { - e.logger.Trace("updating platform settings in the database") - - // cast the library type to database type - // - // https://pkg.go.dev/github.com/go-vela/types/database#SettingsFromLibrary - // s := database.SettingsFromLibrary(r) - - // todo: settings.validate() - - // validate the necessary fields are populated - // - // https://pkg.go.dev/github.com/go-vela/types/database#Settings.Validate - // err := settings.Validate() - // if err != nil { - // return nil, err - // } - - // send query to the database - err := e.client.Table(TableSettings).Save(s).Error - if err != nil { - return nil, err - } - - return s, nil -} diff --git a/database/settings/create.go b/database/settings/create.go new file mode 100644 index 000000000..8bea2888d --- /dev/null +++ b/database/settings/create.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// CreateSettings updates a platform settings in the database. +func (e *engine) CreateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { + e.logger.Tracef("creating platform settings in the database with %v", s.String()) + + // cast the api type to database type + settings := FromAPI(s) + + // validate the necessary fields are populated + err := settings.Validate() + if err != nil { + return nil, err + } + + // send query to the database + err = e.client.Table(TableSettings).Create(settings).Error + if err != nil { + return nil, err + } + + return s, nil +} diff --git a/database/settings/get.go b/database/settings/get.go new file mode 100644 index 000000000..854a21f8d --- /dev/null +++ b/database/settings/get.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// GetSettings gets platform settings from the database. +func (e *engine) GetSettings(ctx context.Context) (*api.Settings, error) { + e.logger.Trace("getting platform settings from the database") + + // variable to store query results + s := new(Settings) + + // send query to the database and store result in variable + err := e.client. + Table(TableSettings). + // todo: how to ensure this is always a singleton at the first row + Where("id = ?", 1). + Take(s). + Error + if err != nil { + return nil, err + } + + // return the settings + return s.ToAPI(), nil +} diff --git a/database/settings/settings.go b/database/settings/settings.go index 9a6f22c3d..f28de34c8 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -10,6 +10,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/util" + "github.com/lib/pq" "github.com/sirupsen/logrus" "gorm.io/gorm" @@ -23,6 +24,9 @@ var ( // ErrEmptyCloneImage defines the error type when a // Settings type has an empty CloneImage field provided. ErrEmptyCloneImage = errors.New("empty settings clone image provided") + // ErrEmptyQueueRoutes defines the error type when a + // Settings type has an empty QueueRoutes field provided. + ErrEmptyQueueRoutes = errors.New("empty settings queue routes provided") ) // todo: comments Build->Settings @@ -53,8 +57,9 @@ type ( // Settings is the database representation of platform settings. Settings struct { - ID sql.NullInt64 `sql:"id"` - CloneImage sql.NullString `sql:"clone_image"` + ID sql.NullInt64 `sql:"id"` + CloneImage sql.NullString `sql:"clone_image"` + QueueRoutes pq.StringArray `sql:"queue_routes" gorm:"type:varchar(1000)"` } ) @@ -99,7 +104,7 @@ func New(opts ...EngineOpt) (*engine, error) { // Nullify ensures the valid flag for // the sql.Null types are properly set. // -// When a field within the Build type is the zero +// When a field within the Settings type is the zero // value for the field, the valid flag is set to // false causing it to be NULL in the database. func (s *Settings) Nullify() *Settings { @@ -120,13 +125,14 @@ func (s *Settings) Nullify() *Settings { return s } -// ToAPI converts the Worker type -// to an API Worker type. +// ToAPI converts the Settings type +// to an API Settings type. func (s *Settings) ToAPI() *api.Settings { settings := new(api.Settings) settings.SetID(s.ID.Int64) settings.SetCloneImage(s.CloneImage.String) + settings.SetQueueRoutes(s.QueueRoutes) return settings } @@ -144,15 +150,22 @@ func (s *Settings) Validate() error { // to avoid unsafe HTML content s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} + // ensure that all QueueRoutes are sanitized + // to avoid unsafe HTML content + for i, v := range s.QueueRoutes { + s.QueueRoutes[i] = util.Sanitize(v) + } + return nil } -// FromAPI converts the API settings type -// to a database settings type. +// FromAPI converts the API Settings type +// to a database Settings type. func FromAPI(s *api.Settings) *Settings { settings := &Settings{ - ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, - CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + QueueRoutes: pq.StringArray(s.GetQueueRoutes()), } return settings.Nullify() diff --git a/database/settings/table.go b/database/settings/table.go index 4a5615d2f..7f8d6463c 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -14,8 +14,9 @@ const ( CREATE TABLE IF NOT EXISTS settings ( - id SERIAL PRIMARY KEY, - clone_image VARCHAR(250) + id SERIAL PRIMARY KEY, + clone_image VARCHAR(250), + queue_routes VARCHAR(1000) ); ` @@ -24,8 +25,9 @@ settings ( CREATE TABLE IF NOT EXISTS settings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - clone_image TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + clone_image TEXT, + queue_routes TEXT ); ` ) diff --git a/database/settings/update.go b/database/settings/update.go new file mode 100644 index 000000000..243eec3fb --- /dev/null +++ b/database/settings/update.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// UpdateSettings updates a platform settings in the database. +func (e *engine) UpdateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { + e.logger.Trace("updating platform settings in the database") + + // cast the api type to database type + settings := FromAPI(s) + + // validate the necessary fields are populated + err := settings.Validate() + if err != nil { + return nil, err + } + + // send query to the database + err = e.client.Table(TableSettings).Save(settings).Error + if err != nil { + return nil, err + } + + return s, nil +} From 04e43f08fad5e4985a8a0105933d01953b8671d1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 11 Apr 2024 15:45:48 -0500 Subject: [PATCH 09/65] wip: update queue and compiler using middleware --- api/admin/settings.go | 4 +- api/build/compile_publish.go | 16 ---- api/pipeline/compile.go | 5 +- api/pipeline/expand.go | 5 +- api/pipeline/template.go | 5 +- api/pipeline/validate.go | 5 +- cmd/vela-server/queue.go | 12 ++- cmd/vela-server/schedule.go | 4 +- cmd/vela-server/server.go | 31 +++++-- compiler/engine.go | 6 +- compiler/native/native.go | 10 +-- compiler/native/settings.go | 14 +++ docker-compose.yml | 5 +- queue/redis/settings.go | 14 +++ queue/service.go | 5 ++ router/admin.go | 5 +- router/middleware/compiler.go | 10 ++- router/middleware/pipeline/pipeline.go | 3 - router/middleware/queue.go | 6 ++ router/middleware/settings.go | 29 +++++++ router/middleware/settings/settings.go | 39 --------- router/middleware/settings/settings_test.go | 95 --------------------- 22 files changed, 124 insertions(+), 204 deletions(-) create mode 100644 compiler/native/settings.go create mode 100644 queue/redis/settings.go create mode 100644 router/middleware/settings.go delete mode 100644 router/middleware/settings/settings.go delete mode 100644 router/middleware/settings/settings_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 0ec9d69f2..96a598650 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -41,7 +41,7 @@ import ( // captures settings stored in the database. func GetSettings(c *gin.Context) { // capture middleware values - s := settings.Retrieve(c) + s := settings.FromContext(c) logrus.Info("Admin: reading settings") @@ -64,7 +64,7 @@ func GetSettings(c *gin.Context) { // todo: swagger and comments func UpdateSettings(c *gin.Context) { // capture middleware values - s := settings.Retrieve(c) + s := settings.FromContext(c) ctx := c.Request.Context() logrus.Info("Admin: updating settings") diff --git a/api/build/compile_publish.go b/api/build/compile_publish.go index 0c5f347a3..f60dcaa56 100644 --- a/api/build/compile_publish.go +++ b/api/build/compile_publish.go @@ -277,21 +277,6 @@ func CompileAndPublish( repo.SetPipelineType(pipeline.GetType()) } - // get settings - s, err := database.GetSettings(c) - if err != nil { - // format the error message with extra information - err = fmt.Errorf("unable to get settings: %w", err) - - // log the error for traceability - logrus.Error(err.Error()) - - retErr := fmt.Errorf("%s: %w", baseErr, err) - util.HandleError(c, http.StatusInternalServerError, retErr) - - return nil, nil, retErr - } - var compiled *library.Pipeline // parse and compile the pipeline configuration file p, compiled, err = compiler. @@ -303,7 +288,6 @@ func CompileAndPublish( WithMetadata(cfg.Metadata). WithRepo(repo). WithUser(u). - WithSettings(s). WithLabels(cfg.Labels). Compile(pipelineFile) if err != nil { diff --git a/api/pipeline/compile.go b/api/pipeline/compile.go index fbad1897e..d362a86d3 100644 --- a/api/pipeline/compile.go +++ b/api/pipeline/compile.go @@ -12,7 +12,6 @@ import ( "github.com/go-vela/server/router/middleware/org" pMiddleware "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -77,7 +76,6 @@ func CompilePipeline(c *gin.Context) { p := pMiddleware.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -100,8 +98,7 @@ func CompilePipeline(c *gin.Context) { WithCommit(p.GetCommit()). WithMetadata(m). WithRepo(r). - WithUser(u). - WithSettings(s) + WithUser(u) ruleData := prepareRuleData(c) diff --git a/api/pipeline/expand.go b/api/pipeline/expand.go index ae1bffc08..b1b1f98fd 100644 --- a/api/pipeline/expand.go +++ b/api/pipeline/expand.go @@ -12,7 +12,6 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -77,7 +76,6 @@ func ExpandPipeline(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -100,8 +98,7 @@ func ExpandPipeline(c *gin.Context) { WithCommit(p.GetCommit()). WithMetadata(m). WithRepo(r). - WithUser(u). - WithSettings(s) + WithUser(u) ruleData := prepareRuleData(c) diff --git a/api/pipeline/template.go b/api/pipeline/template.go index 0df7cf7c9..f917348d6 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -14,7 +14,6 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -81,7 +80,6 @@ func GetTemplates(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - s := settings.Retrieve(c) ctx := c.Request.Context() entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -102,8 +100,7 @@ func GetTemplates(c *gin.Context) { WithCommit(p.GetCommit()). WithMetadata(m). WithRepo(r). - WithUser(u). - WithSettings(s) + WithUser(u) // parse the pipeline configuration pipeline, _, err := compiler.Parse(p.GetData(), p.GetType(), new(yaml.Template)) diff --git a/api/pipeline/validate.go b/api/pipeline/validate.go index 66aa57692..5a91314a6 100644 --- a/api/pipeline/validate.go +++ b/api/pipeline/validate.go @@ -11,7 +11,6 @@ import ( "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types" @@ -75,7 +74,6 @@ func ValidatePipeline(c *gin.Context) { p := pipeline.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - s := settings.Retrieve(c) entry := fmt.Sprintf("%s/%s", r.GetFullName(), p.GetCommit()) @@ -98,8 +96,7 @@ func ValidatePipeline(c *gin.Context) { WithCommit(p.GetCommit()). WithMetadata(m). WithRepo(r). - WithUser(u). - WithSettings(s) + WithUser(u) ruleData := prepareRuleData(c) diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go index 8b8767e56..0dd4016d0 100644 --- a/cmd/vela-server/queue.go +++ b/cmd/vela-server/queue.go @@ -7,11 +7,12 @@ import ( "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/urfave/cli/v2" ) // helper function to setup the queue from the CLI arguments. -func setupQueue(c *cli.Context) (queue.Service, error) { +func setupQueue(c *cli.Context, s *api.Settings) (queue.Service, error) { logrus.Debug("Creating queue client from CLI configuration") // queue configuration @@ -19,7 +20,7 @@ func setupQueue(c *cli.Context) (queue.Service, error) { Driver: c.String("queue.driver"), Address: c.String("queue.addr"), Cluster: c.Bool("queue.cluster"), - Routes: c.StringSlice("queue.routes"), + Routes: s.GetQueueRoutes(), Timeout: c.Duration("queue.pop.timeout"), PrivateKey: c.String("queue.private-key"), PublicKey: c.String("queue.public-key"), @@ -28,5 +29,10 @@ func setupQueue(c *cli.Context) (queue.Service, error) { // setup the queue // // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New - return queue.New(_setup) + q, err := queue.New(_setup) + if err != nil { + return nil, err + } + + return q, nil } diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 3e4ef799f..dba9b6437 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -10,6 +10,7 @@ import ( "github.com/adhocore/gronx" "github.com/go-vela/server/api/build" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/queue" @@ -29,8 +30,9 @@ const ( scheduleWait = "waiting to trigger build for schedule" ) -func processSchedules(ctx context.Context, start time.Time, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedules(ctx context.Context, start time.Time, settings *api.Settings, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { logrus.Infof("processing active schedules to create builds") + // todo: replace allowList with settings.allowList // send API call to capture the list of active schedules schedules, err := database.ListActiveSchedules(ctx) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 38d661195..7cb95841c 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -4,6 +4,7 @@ package main import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -73,24 +74,23 @@ func server(c *cli.Context) error { return err } - s, err := database.GetSettings(context.Background()) - if s == nil || err != nil { - s = api.NewSettings(c) + settings, err := database.GetSettings(context.Background()) + if settings == nil || err != nil { + // ignore error and attempt to create initial settings record + settings = api.NewSettings(c) - _, err = database.CreateSettings(context.Background(), s) + _, err = database.CreateSettings(context.Background(), settings) if err != nil { return err } } - // todo: use settings initialization policy to reset settings? - compiler, err := setupCompiler(c) if err != nil { return err } - queue, err := setupQueue(c) + queue, err := setupQueue(c, settings) if err != nil { return err } @@ -111,6 +111,7 @@ func server(c *cli.Context) error { } router := router.Load( + middleware.Settings(database), middleware.Compiler(compiler), middleware.Database(database), middleware.Logger(logrus.StandardLogger(), time.RFC3339), @@ -227,7 +228,21 @@ func server(c *cli.Context) error { // sleep for a duration of time before processing schedules time.Sleep(jitter) - err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) + settings, err = database.GetSettings(context.Background()) + if settings == nil || err != nil { + if settings == nil { + err = errors.New("settings not found") + } + + logrus.WithError(err).Warn("unable to get platform settings") + + continue + } + + compiler.UpdateFromSettings(settings) + queue.UpdateFromSettings(settings) + + err = processSchedules(ctx, start, settings, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { diff --git a/compiler/engine.go b/compiler/engine.go index b727ffb99..350caad86 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -147,7 +147,7 @@ type Engine interface { // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(string, string) Engine - // WithPrivateGitHub defines a function that sets - // the server settings in the Engine. - WithSettings(*api.Settings) Engine + // UpdateFromSettings defines a function that takes api settings + // and updates the compiler Engine. + UpdateFromSettings(*api.Settings) } diff --git a/compiler/native/native.go b/compiler/native/native.go index d68ecac44..34ecd2a43 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -121,6 +121,7 @@ func (c *client) Duplicate() compiler.Engine { cc.ModificationService = c.ModificationService cc.TemplateDepth = c.TemplateDepth cc.StarlarkExecLimit = c.StarlarkExecLimit + cc.settings = c.settings return cc } @@ -213,15 +214,6 @@ func (c *client) WithUser(u *library.User) compiler.Engine { return c } -// WithSettings sets the api settings type in the Engine. -func (c *client) WithSettings(s *api.Settings) compiler.Engine { - if s != nil { - c.settings = s - } - - return c -} - // WithLabels sets the label(s) in the Engine. func (c *client) WithLabels(labels []string) compiler.Engine { if len(labels) != 0 { diff --git a/compiler/native/settings.go b/compiler/native/settings.go new file mode 100644 index 000000000..f3fb7cef3 --- /dev/null +++ b/compiler/native/settings.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +package native + +import ( + api "github.com/go-vela/server/api/types" +) + +// UpdateFromSettings sets the api settings type in the Engine. +func (c *client) UpdateFromSettings(s *api.Settings) { + if s != nil { + c.settings = s + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 90edcc735..664d1d2ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,10 +65,7 @@ services: # https://go-vela.github.io/docs/administration/worker/ worker: container_name: worker - # image: target/vela-worker:latest - build: - context: ../worker - dockerfile: Dockerfile + image: target/vela-worker:latest networks: - vela environment: diff --git a/queue/redis/settings.go b/queue/redis/settings.go new file mode 100644 index 000000000..afaeb6fdc --- /dev/null +++ b/queue/redis/settings.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +package redis + +import ( + api "github.com/go-vela/server/api/types" +) + +// UpdateFromSettings takes settings and updates the queue. +func (c *client) UpdateFromSettings(s *api.Settings) { + c.Logger.Trace("updating queue using settings") + + c.config.Channels = s.GetQueueRoutes() +} diff --git a/queue/service.go b/queue/service.go index fd36a64bc..caa753577 100644 --- a/queue/service.go +++ b/queue/service.go @@ -5,6 +5,7 @@ package queue import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/pipeline" ) @@ -37,4 +38,8 @@ type Service interface { // Route defines a function that decides which // channel a build gets placed within the queue. Route(*pipeline.Worker) (string, error) + + // UpdateFromSettings defines a function that updates + // runtime config within the queue using api settings. + UpdateFromSettings(*api.Settings) } diff --git a/router/admin.go b/router/admin.go index 7ac55a31f..d4a8cb697 100644 --- a/router/admin.go +++ b/router/admin.go @@ -6,7 +6,6 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/api/admin" "github.com/go-vela/server/router/middleware/perm" - smiddleware "github.com/go-vela/server/router/middleware/settings" ) // AdminHandlers is a function that extends the provided base router group @@ -62,7 +61,7 @@ func AdminHandlers(base *gin.RouterGroup) { _admin.POST("/workers/:worker/register", admin.RegisterToken) // Admin settings endpoint - _admin.GET("/settings", smiddleware.Establish(), admin.GetSettings) - _admin.PUT("/settings", smiddleware.Establish(), admin.UpdateSettings) + _admin.GET("/settings", admin.GetSettings) + _admin.PUT("/settings", admin.UpdateSettings) } // end of admin endpoints } diff --git a/router/middleware/compiler.go b/router/middleware/compiler.go index 503040192..143857665 100644 --- a/router/middleware/compiler.go +++ b/router/middleware/compiler.go @@ -5,13 +5,19 @@ package middleware import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" + "github.com/go-vela/server/router/middleware/settings" ) // Compiler is a middleware function that initializes the compiler and // attaches to the context of every http.Request. -func Compiler(cli compiler.Engine) gin.HandlerFunc { +func Compiler(comp compiler.Engine) gin.HandlerFunc { return func(c *gin.Context) { - compiler.WithGinContext(c, cli) + s := settings.FromContext(c) + + comp.UpdateFromSettings(s) + + compiler.WithGinContext(c, comp) + c.Next() } } diff --git a/router/middleware/pipeline/pipeline.go b/router/middleware/pipeline/pipeline.go index a3700dfb6..61f308854 100644 --- a/router/middleware/pipeline/pipeline.go +++ b/router/middleware/pipeline/pipeline.go @@ -11,7 +11,6 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -31,7 +30,6 @@ func Establish() gin.HandlerFunc { o := org.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - s := settings.Retrieve(c) ctx := c.Request.Context() if r == nil { @@ -82,7 +80,6 @@ func Establish() gin.HandlerFunc { WithMetadata(c.MustGet("metadata").(*types.Metadata)). WithRepo(r). WithUser(u). - WithSettings(s). Compile(config) if err != nil { retErr := fmt.Errorf("unable to compile pipeline configuration for %s: %w", entry, err) diff --git a/router/middleware/queue.go b/router/middleware/queue.go index 0ebbe72ca..3dcac5fe3 100644 --- a/router/middleware/queue.go +++ b/router/middleware/queue.go @@ -5,13 +5,19 @@ package middleware import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/queue" + "github.com/go-vela/server/router/middleware/settings" ) // Queue is a middleware function that initializes the queue and // attaches to the context of every http.Request. func Queue(q queue.Service) gin.HandlerFunc { return func(c *gin.Context) { + s := settings.FromContext(c) + + q.UpdateFromSettings(s) + queue.WithGinContext(c, q) + c.Next() } } diff --git a/router/middleware/settings.go b/router/middleware/settings.go new file mode 100644 index 000000000..f251ec94b --- /dev/null +++ b/router/middleware/settings.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "context" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/settings" + "github.com/sirupsen/logrus" +) + +// Settings is a middleware function that fetches the latest settings and +// attaches to the context of every http.Request. +func Settings(d database.Interface) gin.HandlerFunc { + return func(c *gin.Context) { + s, err := d.GetSettings(context.Background()) + if err != nil { + logrus.WithError(err).Warn("unable to get platform settings") + + return + } + + settings.ToContext(c, s) + + c.Next() + } +} diff --git a/router/middleware/settings/settings.go b/router/middleware/settings/settings.go deleted file mode 100644 index ce63da1bd..000000000 --- a/router/middleware/settings/settings.go +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package settings - -import ( - "fmt" - "net/http" - - "github.com/gin-gonic/gin" - api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database" - "github.com/go-vela/server/util" - "github.com/sirupsen/logrus" -) - -// Retrieve gets the settings in the given context. -func Retrieve(c *gin.Context) *api.Settings { - return FromContext(c) -} - -// Establish sets the settings in the given context. -func Establish() gin.HandlerFunc { - return func(c *gin.Context) { - ctx := c.Request.Context() - - logrus.Debug("Reading settings") - - s, err := database.FromContext(c).GetSettings(ctx) - if err != nil { - retErr := fmt.Errorf("unable to read settings: %w", err) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - ToContext(c, s) - c.Next() - } -} diff --git a/router/middleware/settings/settings_test.go b/router/middleware/settings/settings_test.go deleted file mode 100644 index 37eebc714..000000000 --- a/router/middleware/settings/settings_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package settings - -import ( - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/gin-gonic/gin" - api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/database" - "github.com/go-vela/types/library" -) - -func TestSettings_Retrieve(t *testing.T) { - // setup types - want := new(api.Settings) - // want.SetID(1) - - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - ToContext(context, want) - - // run test - got := Retrieve(context) - - if got != want { - t.Errorf("Retrieve is %v, want %v", got, want) - } -} - -func TestSettings_Establish(t *testing.T) { - // setup types - b := new(library.Build) - b.SetID(1) - - want := new(api.Settings) - // want.SetID(1) - // want.SetHostname("worker_0") - // want.SetAddress("localhost") - // want.SetRoutes([]string{"foo", "bar", "baz"}) - // want.SetActive(true) - // want.SetStatus("available") - // want.SetLastStatusUpdateAt(12345) - // want.SetRunningBuilds([]*library.Build{b}) - // want.SetLastBuildStartedAt(12345) - // want.SetLastBuildFinishedAt(12345) - // want.SetLastCheckedIn(12345) - // want.SetBuildLimit(0) - - got := new(api.Settings) - - // setup database - db, err := database.NewTest() - if err != nil { - t.Errorf("unable to create test database engine: %v", err) - } - - // defer func() { - // _ = db.DeleteSettings(context.TODO(), want) - // db.Close() - // }() - - // _, _ = db.CreateSettings(context.TODO(), want) - - // setup context - gin.SetMode(gin.TestMode) - - resp := httptest.NewRecorder() - context, engine := gin.CreateTestContext(resp) - context.Request, _ = http.NewRequest(http.MethodGet, "/admin/settings", nil) - - // setup mock server - engine.Use(func(c *gin.Context) { database.ToContext(c, db) }) - engine.Use(Establish()) - engine.GET("/admin/settings", func(c *gin.Context) { - got = Retrieve(c) - - c.Status(http.StatusOK) - }) - - // run test - engine.ServeHTTP(resp, context.Request) - - if resp.Code != http.StatusOK { - t.Errorf("Establish returned %v, want %v", resp.Code, http.StatusOK) - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("Establish is %v, want %v", got, want) - } -} From 648dd785968cf410d177093f161398cd3d01c15b Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 11 Apr 2024 16:10:58 -0500 Subject: [PATCH 10/65] wip: debugging compile and publish panic --- api/build/restart.go | 2 +- api/webhook/post.go | 11 +++++++++++ queue/redis/settings.go | 6 +++++- queue/service.go | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/build/restart.go b/api/build/restart.go index 8b32fdc56..e960ee148 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -117,7 +117,7 @@ func RestartBuild(c *gin.Context) { Source: "restart", Retries: 1, } - + logger.Tracef("restarting build using queue.channels: %v", queue.FromContext(c).GetChannels(nil)) // generate queue items _, item, err := CompileAndPublish( c, diff --git a/api/webhook/post.go b/api/webhook/post.go index d33a201e9..ece2b1999 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -307,6 +307,17 @@ func PostWebhook(c *gin.Context) { queue.FromContext(c), ) + // error handling done in CompileAndPublish + if err != nil { + retErr := fmt.Errorf("%s: unable to compile and publish", err) + util.HandleError(c, http.StatusBadRequest, retErr) + + h.SetStatus(constants.StatusSkipped) + h.SetError(retErr.Error()) + + return + } + // capture the build, repo, and user from the items b, repo, u := item.Build, item.Repo, item.User diff --git a/queue/redis/settings.go b/queue/redis/settings.go index afaeb6fdc..078d6a19e 100644 --- a/queue/redis/settings.go +++ b/queue/redis/settings.go @@ -8,7 +8,11 @@ import ( // UpdateFromSettings takes settings and updates the queue. func (c *client) UpdateFromSettings(s *api.Settings) { - c.Logger.Trace("updating queue using settings") + c.Logger.Tracef("updating queue using settings: %v", s.GetQueueRoutes()) c.config.Channels = s.GetQueueRoutes() } + +func (c *client) GetChannels(s *api.Settings) []string { + return c.config.Channels +} diff --git a/queue/service.go b/queue/service.go index caa753577..3dc04e550 100644 --- a/queue/service.go +++ b/queue/service.go @@ -42,4 +42,5 @@ type Service interface { // UpdateFromSettings defines a function that updates // runtime config within the queue using api settings. UpdateFromSettings(*api.Settings) + GetChannels(*api.Settings) []string } From ee9b90fa6f1c78a83f0bdf59e45c97a61c55a581 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 09:07:40 -0500 Subject: [PATCH 11/65] wip: structure for settings done --- api/admin/settings.go | 8 +++++ api/pipeline/compile.go | 7 +--- api/pipeline/expand.go | 7 +--- api/pipeline/template.go | 7 +--- api/pipeline/validate.go | 7 +--- api/types/settings.go | 63 +++++++++++++++++++++++++++++++++-- cmd/vela-server/queue.go | 5 ++- cmd/vela-server/schedule.go | 3 +- cmd/vela-server/server.go | 29 ++++++++-------- compiler/native/clone.go | 4 +-- compiler/native/native.go | 8 ++--- compiler/native/settings.go | 4 ++- database/settings/settings.go | 20 +++++++---- 13 files changed, 113 insertions(+), 59 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 96a598650..78cbdfdd6 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -85,6 +85,14 @@ func UpdateSettings(c *gin.Context) { s.CloneImage = input.CloneImage } + if input.TemplateDepth != nil { + s.TemplateDepth = input.TemplateDepth + } + + if input.StarlarkExecLimit != nil { + s.StarlarkExecLimit = input.StarlarkExecLimit + } + if input.QueueRoutes != nil { // update routes if set s.SetQueueRoutes(input.GetQueueRoutes()) diff --git a/api/pipeline/compile.go b/api/pipeline/compile.go index 21b7b7ed2..4585451ba 100644 --- a/api/pipeline/compile.go +++ b/api/pipeline/compile.go @@ -94,12 +94,7 @@ func CompilePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c). - Duplicate(). - WithCommit(p.GetCommit()). - WithMetadata(m). - WithRepo(r). - WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) ruleData := prepareRuleData(c) diff --git a/api/pipeline/expand.go b/api/pipeline/expand.go index 0544cdda0..17e62181c 100644 --- a/api/pipeline/expand.go +++ b/api/pipeline/expand.go @@ -94,12 +94,7 @@ func ExpandPipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c). - Duplicate(). - WithCommit(p.GetCommit()). - WithMetadata(m). - WithRepo(r). - WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) ruleData := prepareRuleData(c) diff --git a/api/pipeline/template.go b/api/pipeline/template.go index 8e3532c15..66c8cd627 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -95,12 +95,7 @@ func GetTemplates(c *gin.Context) { }).Infof("reading templates from pipeline %s", entry) // create the compiler object - compiler := compiler.FromContext(c). - Duplicate(). - WithCommit(p.GetCommit()). - WithMetadata(m). - WithRepo(r). - WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) // parse the pipeline configuration pipeline, _, err := compiler.Parse(p.GetData(), p.GetType(), new(yaml.Template)) diff --git a/api/pipeline/validate.go b/api/pipeline/validate.go index f358bef0d..b7f43664d 100644 --- a/api/pipeline/validate.go +++ b/api/pipeline/validate.go @@ -92,12 +92,7 @@ func ValidatePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c). - Duplicate(). - WithCommit(p.GetCommit()). - WithMetadata(m). - WithRepo(r). - WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) ruleData := prepareRuleData(c) diff --git a/api/types/settings.go b/api/types/settings.go index 1cdcc4bb0..34a484c35 100644 --- a/api/types/settings.go +++ b/api/types/settings.go @@ -12,9 +12,11 @@ import ( // // swagger:model Settings type Settings struct { - ID *int64 `json:"id,omitempty"` - CloneImage *string `json:"clone_image,omitempty"` - QueueRoutes *[]string `json:"queue_routes,omitempty"` + ID *int64 `json:"id,omitempty"` + CloneImage *string `json:"clone_image,omitempty"` + TemplateDepth *int `json:"template_depth,omitempty"` + StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` + QueueRoutes *[]string `json:"queue_routes,omitempty"` } // NewSettings returns a new Settings record. @@ -30,6 +32,9 @@ func NewSettings(c *cli.Context) *Settings { // set the queue routes (channels) to use for builds s.SetQueueRoutes(c.StringSlice("queue.routes")) + // set the queue routes (channels) to use for builds + s.SetQueueRoutes(c.StringSlice("queue.routes")) + return s } @@ -59,6 +64,32 @@ func (s *Settings) GetCloneImage() string { return *s.CloneImage } +// GetTemplateDepth returns the TemplateDepth field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetTemplateDepth() int { + // return zero value if Settings type or TemplateDepth field is nil + if s == nil || s.TemplateDepth == nil { + return 0 + } + + return *s.TemplateDepth +} + +// GetStarlarkExecLimit returns the StarlarkExecLimit field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetStarlarkExecLimit() uint64 { + // return zero value if Settings type or StarlarkExecLimit field is nil + if s == nil || s.StarlarkExecLimit == nil { + return 0 + } + + return *s.StarlarkExecLimit +} + // GetQueueRoutes returns the QueueRoutes field. // // When the provided Settings type is nil, or the field within @@ -98,6 +129,32 @@ func (s *Settings) SetCloneImage(v string) { s.CloneImage = &v } +// SetTemplateDepth sets the TemplateDepth field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Settings) SetTemplateDepth(v int) { + // return if Settings type is nil + if s == nil { + return + } + + s.TemplateDepth = &v +} + +// SetStarlarkExecLimit sets the StarlarkExecLimit field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Settings) SetStarlarkExecLimit(v uint64) { + // return if Settings type is nil + if s == nil { + return + } + + s.StarlarkExecLimit = &v +} + // SetQueueRoutes sets the QueueRoutes field. // // When the provided Settings type is nil, it diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go index 27674bc43..776af9ac4 100644 --- a/cmd/vela-server/queue.go +++ b/cmd/vela-server/queue.go @@ -5,14 +5,13 @@ package main import ( "github.com/sirupsen/logrus" - api "github.com/go-vela/server/api/types" "github.com/urfave/cli/v2" "github.com/go-vela/server/queue" ) // helper function to setup the queue from the CLI arguments. -func setupQueue(c *cli.Context, s *api.Settings) (queue.Service, error) { +func setupQueue(c *cli.Context) (queue.Service, error) { logrus.Debug("Creating queue client from CLI configuration") // queue configuration @@ -20,7 +19,7 @@ func setupQueue(c *cli.Context, s *api.Settings) (queue.Service, error) { Driver: c.String("queue.driver"), Address: c.String("queue.addr"), Cluster: c.Bool("queue.cluster"), - Routes: s.GetQueueRoutes(), + Routes: c.StringSlice("queue.routes"), Timeout: c.Duration("queue.pop.timeout"), PrivateKey: c.String("queue.private-key"), PublicKey: c.String("queue.public-key"), diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index c8ad40a72..d43e4739d 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "github.com/go-vela/server/api/build" - api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" @@ -30,7 +29,7 @@ const ( scheduleWait = "waiting to trigger build for schedule" ) -func processSchedules(ctx context.Context, start time.Time, settings *api.Settings, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedules(ctx context.Context, start time.Time, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { logrus.Infof("processing active schedules to create builds") // todo: replace allowList with settings.allowList diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index ae5a79a90..654da0a62 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -74,23 +74,12 @@ func server(c *cli.Context) error { return err } - settings, err := database.GetSettings(context.Background()) - if settings == nil || err != nil { - // ignore error and attempt to create initial settings record - settings = api.NewSettings(c) - - _, err = database.CreateSettings(context.Background(), settings) - if err != nil { - return err - } - } - compiler, err := setupCompiler(c) if err != nil { return err } - queue, err := setupQueue(c, settings) + queue, err := setupQueue(c) if err != nil { return err } @@ -110,6 +99,20 @@ func server(c *cli.Context) error { return err } + settings, err := database.GetSettings(context.Background()) + if settings == nil || err != nil { + // ignore error and attempt to create initial settings record + settings = api.NewSettings(c) + + _, err = database.CreateSettings(context.Background(), settings) + if err != nil { + return err + } + } + + queue.UpdateFromSettings(settings) + compiler.UpdateFromSettings(settings) + router := router.Load( middleware.Settings(database), middleware.Compiler(compiler), @@ -242,7 +245,7 @@ func server(c *cli.Context) error { compiler.UpdateFromSettings(settings) queue.UpdateFromSettings(settings) - err = processSchedules(ctx, start, settings, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) + err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { diff --git a/compiler/native/clone.go b/compiler/native/clone.go index 2b422235a..9075292f7 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -30,7 +30,7 @@ func (c *client) CloneStage(p *yaml.Build) (*yaml.Build, error) { Steps: yaml.StepSlice{ &yaml.Step{ Detach: false, - Image: c.settings.GetCloneImage(), + Image: c.CloneImage, Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, @@ -63,7 +63,7 @@ func (c *client) CloneStep(p *yaml.Build) (*yaml.Build, error) { // create new clone step clone := &yaml.Step{ Detach: false, - Image: c.settings.GetCloneImage(), + Image: c.CloneImage, Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, diff --git a/compiler/native/native.go b/compiler/native/native.go index 94199ef20..08dc391d5 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -29,9 +29,9 @@ type client struct { PrivateGithub registry.Service UsePrivateGithub bool ModificationService ModificationConfig - // CloneImage string - TemplateDepth int - StarlarkExecLimit uint64 + CloneImage string + TemplateDepth int + StarlarkExecLimit uint64 build *library.Build comment string @@ -42,7 +42,6 @@ type client struct { metadata *internal.Metadata repo *api.Repo user *api.User - settings *api.Settings labels []string } @@ -120,7 +119,6 @@ func (c *client) Duplicate() compiler.Engine { cc.ModificationService = c.ModificationService cc.TemplateDepth = c.TemplateDepth cc.StarlarkExecLimit = c.StarlarkExecLimit - cc.settings = c.settings return cc } diff --git a/compiler/native/settings.go b/compiler/native/settings.go index f3fb7cef3..d240bf557 100644 --- a/compiler/native/settings.go +++ b/compiler/native/settings.go @@ -9,6 +9,8 @@ import ( // UpdateFromSettings sets the api settings type in the Engine. func (c *client) UpdateFromSettings(s *api.Settings) { if s != nil { - c.settings = s + c.CloneImage = s.GetCloneImage() + c.TemplateDepth = int(s.GetTemplateDepth()) + c.StarlarkExecLimit = s.GetStarlarkExecLimit() } } diff --git a/database/settings/settings.go b/database/settings/settings.go index f28de34c8..984006483 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -57,9 +57,12 @@ type ( // Settings is the database representation of platform settings. Settings struct { - ID sql.NullInt64 `sql:"id"` - CloneImage sql.NullString `sql:"clone_image"` - QueueRoutes pq.StringArray `sql:"queue_routes" gorm:"type:varchar(1000)"` + ID sql.NullInt64 `sql:"id"` + CloneImage sql.NullString `sql:"clone_image"` + TemplateDepth sql.NullInt64 `sql:"template_depth"` + StarlarkExecLimit sql.NullInt64 `sql:"starlark_exec_limit"` + RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` + QueueRoutes pq.StringArray `sql:"queue_routes" gorm:"type:varchar(1000)"` } ) @@ -132,6 +135,9 @@ func (s *Settings) ToAPI() *api.Settings { settings.SetID(s.ID.Int64) settings.SetCloneImage(s.CloneImage.String) + settings.SetTemplateDepth(int(s.TemplateDepth.Int64)) + settings.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) + settings.SetCloneImage(s.CloneImage.String) settings.SetQueueRoutes(s.QueueRoutes) return settings @@ -163,9 +169,11 @@ func (s *Settings) Validate() error { // to a database Settings type. func FromAPI(s *api.Settings) *Settings { settings := &Settings{ - ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, - CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, - QueueRoutes: pq.StringArray(s.GetQueueRoutes()), + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, + QueueRoutes: pq.StringArray(s.GetQueueRoutes()), } return settings.Nullify() From 51c82bc8aec58d4e11bd7855d7584499336f8031 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 10:29:08 -0500 Subject: [PATCH 12/65] wip: restructure using nested settings --- api/admin/settings.go | 40 +++- api/build/restart.go | 2 +- api/types/settings.go | 232 ++++++++++++--------- cmd/vela-server/queue.go | 7 +- cmd/vela-server/server.go | 28 ++- compiler/engine.go | 7 +- compiler/native/clone.go | 4 +- compiler/native/compile.go | 8 +- compiler/native/expand.go | 4 +- compiler/native/expand_test.go | 12 +- compiler/native/native.go | 13 +- compiler/native/parse.go | 2 +- compiler/native/settings.go | 15 +- mock/server/settings.go | 10 + queue/redis/length.go | 2 +- queue/redis/opts.go | 2 +- queue/redis/opts_test.go | 4 +- queue/redis/pop.go | 4 +- queue/redis/pop_test.go | 2 +- queue/redis/redis.go | 4 +- queue/redis/route.go | 4 +- queue/redis/settings.go | 15 +- queue/service.go | 11 +- router/middleware/compiler.go | 2 +- router/middleware/queue.go | 2 +- router/middleware/settings/context_test.go | 16 +- 26 files changed, 283 insertions(+), 169 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 78cbdfdd6..9ca0a78eb 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -61,7 +61,38 @@ func GetSettings(c *gin.Context) { } } -// todo: swagger and comments +// swagger:operation PUT /api/v1/admin/settings admin AdminUpdateSettings +// +// Update the settings singleton in the database +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: Payload containing settings to update +// required: true +// schema: +// "$ref": "#/definitions/Settings" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the settings in the database +// schema: +// "$ref": "#/definitions/Settings" +// '404': +// description: Unable to update the settings in the database +// schema: +// "$ref": "#/definitions/Error" +// '501': +// description: Unable to update the settings in the database +// schema: +// "$ref": "#/definitions/Error" + +// UpdateSettings represents the API handler to +// update the settings singleton stored in the database. func UpdateSettings(c *gin.Context) { // capture middleware values s := settings.FromContext(c) @@ -93,11 +124,16 @@ func UpdateSettings(c *gin.Context) { s.StarlarkExecLimit = input.StarlarkExecLimit } - if input.QueueRoutes != nil { + if input.Routes != nil { // update routes if set s.SetQueueRoutes(input.GetQueueRoutes()) } + if input.RepoAllowlist != nil { + // update allowlist if set + s.SetRepoAllowlist(input.GetRepoAllowlist()) + } + // send API call to update the repo s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { diff --git a/api/build/restart.go b/api/build/restart.go index deac53495..94b9b2202 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -127,7 +127,7 @@ func RestartBuild(c *gin.Context) { Source: "restart", Retries: 1, } - logger.Tracef("restarting build using queue.channels: %v", queue.FromContext(c).GetChannels(nil)) + logger.Tracef("restarting build using queue.channels: %v", queue.FromContext(c).GetSettings().GetQueueRoutes()) // generate queue items _, item, code, err := CompileAndPublish( c, diff --git a/api/types/settings.go b/api/types/settings.go index 34a484c35..a42fd8c50 100644 --- a/api/types/settings.go +++ b/api/types/settings.go @@ -4,8 +4,6 @@ package types import ( "fmt" - - "github.com/urfave/cli/v2" ) // Settings is the API representation of platform settings. @@ -13,29 +11,9 @@ import ( // swagger:model Settings type Settings struct { ID *int64 `json:"id,omitempty"` - CloneImage *string `json:"clone_image,omitempty"` - TemplateDepth *int `json:"template_depth,omitempty"` - StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` - QueueRoutes *[]string `json:"queue_routes,omitempty"` -} - -// NewSettings returns a new Settings record. -func NewSettings(c *cli.Context) *Settings { - s := new(Settings) - - // singleton record ID should always be 1 - s.SetID(1) - - // set the clone image to use for the injected clone step - s.SetCloneImage(c.String("clone-image")) - - // set the queue routes (channels) to use for builds - s.SetQueueRoutes(c.StringSlice("queue.routes")) - - // set the queue routes (channels) to use for builds - s.SetQueueRoutes(c.StringSlice("queue.routes")) - - return s + RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` + *QueueSettings `json:"queue,omitempty"` + *CompilerSettings `json:"compiler,omitempty"` } // GetID returns the ID field. @@ -51,58 +29,6 @@ func (s *Settings) GetID() int64 { return *s.ID } -// GetCloneImage returns the CloneImage field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetCloneImage() string { - // return zero value if Settings type or CloneImage field is nil - if s == nil || s.CloneImage == nil { - return "" - } - - return *s.CloneImage -} - -// GetTemplateDepth returns the TemplateDepth field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetTemplateDepth() int { - // return zero value if Settings type or TemplateDepth field is nil - if s == nil || s.TemplateDepth == nil { - return 0 - } - - return *s.TemplateDepth -} - -// GetStarlarkExecLimit returns the StarlarkExecLimit field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetStarlarkExecLimit() uint64 { - // return zero value if Settings type or StarlarkExecLimit field is nil - if s == nil || s.StarlarkExecLimit == nil { - return 0 - } - - return *s.StarlarkExecLimit -} - -// GetQueueRoutes returns the QueueRoutes field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetQueueRoutes() []string { - // return zero value if Settings type or QueueRoutes field is nil - if s == nil || s.QueueRoutes == nil { - return []string{} - } - - return *s.QueueRoutes -} - // SetID sets the ID field. // // When the provided Settings type is nil, it @@ -116,56 +42,56 @@ func (s *Settings) SetID(v int64) { s.ID = &v } -// SetCloneImage sets the CloneImage field. +// SetCompilerSettings sets the CompilerSettings field. // -// When the provided Settings type is nil, it +// When the provided CompilerSettings type is nil, it // will set nothing and immediately return. -func (s *Settings) SetCloneImage(v string) { +func (s *Settings) SetCompilerSettings(cs *CompilerSettings) { // return if Settings type is nil if s == nil { return } - s.CloneImage = &v + s.CompilerSettings = cs } -// SetTemplateDepth sets the TemplateDepth field. +// SetQueueSettings sets the QueueSettings field. // -// When the provided Settings type is nil, it +// When the provided QueueSettings type is nil, it // will set nothing and immediately return. -func (s *Settings) SetTemplateDepth(v int) { +func (s *Settings) SetQueueSettings(qs *QueueSettings) { // return if Settings type is nil if s == nil { return } - s.TemplateDepth = &v + s.QueueSettings = qs } -// SetStarlarkExecLimit sets the StarlarkExecLimit field. +// GetRepoAllowlist returns the RepoAllowlist field. // -// When the provided Settings type is nil, it -// will set nothing and immediately return. -func (s *Settings) SetStarlarkExecLimit(v uint64) { - // return if Settings type is nil - if s == nil { - return +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Settings) GetRepoAllowlist() []string { + // return zero value if Settings type or RepoAllowlist field is nil + if s == nil || s.RepoAllowlist == nil { + return []string{} } - s.StarlarkExecLimit = &v + return *s.RepoAllowlist } -// SetQueueRoutes sets the QueueRoutes field. +// SetRepoAllowlist sets the RepoAllowlist field. // // When the provided Settings type is nil, it // will set nothing and immediately return. -func (s *Settings) SetQueueRoutes(v []string) { +func (s *Settings) SetRepoAllowlist(v []string) { // return if Settings type is nil if s == nil { return } - s.QueueRoutes = &v + s.RepoAllowlist = &v } // String implements the Stringer interface for the Settings type. @@ -200,3 +126,117 @@ VELA_QUEUE_ROUTES: '%s' s.GetQueueRoutes(), ) } + +type CompilerSettings struct { + CloneImage *string `json:"clone_image,omitempty"` + TemplateDepth *int `json:"template_depth,omitempty"` + StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` +} + +// GetCloneImage returns the CloneImage field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *CompilerSettings) GetCloneImage() string { + // return zero value if Settings type or CloneImage field is nil + if s == nil || s.CloneImage == nil { + return "" + } + + return *s.CloneImage +} + +// GetTemplateDepth returns the TemplateDepth field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *CompilerSettings) GetTemplateDepth() int { + // return zero value if Settings type or TemplateDepth field is nil + if s == nil || s.TemplateDepth == nil { + return 0 + } + + return *s.TemplateDepth +} + +// GetStarlarkExecLimit returns the StarlarkExecLimit field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *CompilerSettings) GetStarlarkExecLimit() uint64 { + // return zero value if Settings type or StarlarkExecLimit field is nil + if s == nil || s.StarlarkExecLimit == nil { + return 0 + } + + return *s.StarlarkExecLimit +} + +// SetCloneImage sets the CloneImage field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (s *CompilerSettings) SetCloneImage(v string) { + // return if Settings type is nil + if s == nil { + return + } + + s.CloneImage = &v +} + +// SetTemplateDepth sets the TemplateDepth field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (s *CompilerSettings) SetTemplateDepth(v int) { + // return if Settings type is nil + if s == nil { + return + } + + s.TemplateDepth = &v +} + +// SetStarlarkExecLimit sets the StarlarkExecLimit field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (s *CompilerSettings) SetStarlarkExecLimit(v uint64) { + // return if Settings type is nil + if s == nil { + return + } + + s.StarlarkExecLimit = &v +} + +type QueueSettings struct { + Routes *[]string `json:"routes,omitempty"` +} + +// GetQueueRoutes returns the QueueRoutes field. +// +// When the provided QueueSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *QueueSettings) GetQueueRoutes() []string { + // return zero value if Settings type or QueueRoutes field is nil + if s == nil || s.Routes == nil { + return []string{} + } + + return *s.Routes +} + +// SetQueueRoutes sets the QueueRoutes field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *QueueSettings) SetQueueRoutes(v []string) { + // return if Settings type is nil + if s == nil { + return + } + + s.Routes = &v +} diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go index 776af9ac4..9489e63a1 100644 --- a/cmd/vela-server/queue.go +++ b/cmd/vela-server/queue.go @@ -28,10 +28,5 @@ func setupQueue(c *cli.Context) (queue.Service, error) { // setup the queue // // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New - q, err := queue.New(_setup) - if err != nil { - return nil, err - } - - return q, nil + return queue.New(_setup) } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 654da0a62..fd3341169 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -14,12 +14,12 @@ import ( "time" "github.com/gin-gonic/gin" - api "github.com/go-vela/server/api/types" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/util/wait" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" @@ -69,12 +69,12 @@ func server(c *cli.Context) error { logrus.SetLevel(logrus.PanicLevel) } - database, err := database.FromCLIContext(c) + compiler, err := setupCompiler(c) if err != nil { return err } - compiler, err := setupCompiler(c) + database, err := database.FromCLIContext(c) if err != nil { return err } @@ -102,16 +102,28 @@ func server(c *cli.Context) error { settings, err := database.GetSettings(context.Background()) if settings == nil || err != nil { // ignore error and attempt to create initial settings record - settings = api.NewSettings(c) + settings := new(api.Settings) + + // singleton record ID should always be 1 + settings.SetID(1) + + // read in defaults supplied from the cli runtime + compilerSettings := compiler.GetSettings() + settings.SetCompilerSettings(compilerSettings) + + queueSettings := queue.GetSettings() + settings.SetQueueSettings(queueSettings) + // create the settings record in the database _, err = database.CreateSettings(context.Background(), settings) if err != nil { return err } } - queue.UpdateFromSettings(settings) - compiler.UpdateFromSettings(settings) + // update any internal settings + queue.SetSettings(settings) + compiler.SetSettings(settings) router := router.Load( middleware.Settings(database), @@ -242,8 +254,8 @@ func server(c *cli.Context) error { continue } - compiler.UpdateFromSettings(settings) - queue.UpdateFromSettings(settings) + compiler.SetSettings(settings) + queue.SetSettings(settings) err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) if err != nil { diff --git a/compiler/engine.go b/compiler/engine.go index 8524aef1a..608cbd11c 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -147,7 +147,10 @@ type Engine interface { // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(string, string) Engine - // UpdateFromSettings defines a function that takes api settings + // GetSettings defines a function that returns new api settings + // with the compiler Engine fields filled. + GetSettings() *api.CompilerSettings + // SetSettings defines a function that takes api settings // and updates the compiler Engine. - UpdateFromSettings(*api.Settings) + SetSettings(*api.Settings) } diff --git a/compiler/native/clone.go b/compiler/native/clone.go index 9075292f7..85016cfc6 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -30,7 +30,7 @@ func (c *client) CloneStage(p *yaml.Build) (*yaml.Build, error) { Steps: yaml.StepSlice{ &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, @@ -63,7 +63,7 @@ func (c *client) CloneStep(p *yaml.Build) (*yaml.Build, error) { // create new clone step clone := &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, diff --git a/compiler/native/compile.go b/compiler/native/compile.go index d5cb1d49b..1e00abce9 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -75,7 +75,7 @@ func (c *client) Compile(v interface{}) (*pipeline.Build, *library.Pipeline, err switch { case p.Metadata.RenderInline: - newPipeline, err := c.compileInline(p, c.TemplateDepth) + newPipeline, err := c.compileInline(p, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -110,7 +110,7 @@ func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substit _pipeline.SetType(c.repo.GetPipelineType()) if p.Metadata.RenderInline { - newPipeline, err := c.compileInline(p, c.TemplateDepth) + newPipeline, err := c.compileInline(p, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -167,7 +167,7 @@ func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substit case len(p.Steps) > 0: // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, ruleData, c.TemplateDepth) + p, err = c.ExpandSteps(p, templates, ruleData, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -318,7 +318,7 @@ func (c *client) compileSteps(p *yaml.Build, _pipeline *library.Pipeline, tmpls } // inject the templates into the steps - p, err = c.ExpandSteps(p, tmpls, r, c.TemplateDepth) + p, err = c.ExpandSteps(p, tmpls, r, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } diff --git a/compiler/native/expand.go b/compiler/native/expand.go index e69066ef7..14e96efda 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -28,7 +28,7 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // iterate through all stages for _, stage := range s.Stages { // inject the templates into the steps for the stage - p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.TemplateDepth) + p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.GetTemplateDepth()) if err != nil { return nil, err } @@ -345,7 +345,7 @@ func (c *client) mergeTemplate(bytes []byte, tmpl *yaml.Template, step *yaml.Ste return native.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables) case constants.PipelineTypeStarlark: //nolint:lll // ignore long line length due to return - return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables, c.StarlarkExecLimit) + return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables, c.GetStarlarkExecLimit()) default: //nolint:lll // ignore long line length due to return return &yaml.Build{}, fmt.Errorf("format of %s is unsupported", tmpl.Format) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index c43eb14bd..57787bfb5 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -356,7 +356,7 @@ func TestNative_ExpandSteps(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -624,7 +624,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { ruledata := new(pipeline.RuleData) ruledata.Branch = "main" - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -714,7 +714,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { t.Errorf("Creating new compiler returned err: %v", err) } - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -880,7 +880,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -982,7 +982,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err == nil { t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) } @@ -1068,7 +1068,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err == nil { t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 08dc391d5..13a56a940 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -29,9 +29,8 @@ type client struct { PrivateGithub registry.Service UsePrivateGithub bool ModificationService ModificationConfig - CloneImage string - TemplateDepth int - StarlarkExecLimit uint64 + + *api.CompilerSettings build *library.Build comment string @@ -70,14 +69,16 @@ func New(ctx *cli.Context) (*client, error) { c.Github = github + c.CompilerSettings = new(api.CompilerSettings) + // set the clone image to use for the injected clone step - // c.CloneImage = ctx.String("clone-image") + c.SetCloneImage(ctx.String("clone-image")) // set the template depth to use for nested templates - c.TemplateDepth = ctx.Int("max-template-depth") + c.SetTemplateDepth(ctx.Int("max-template-depth")) // set the starlark execution step limit for compiling starlark pipelines - c.StarlarkExecLimit = ctx.Uint64("compiler-starlark-exec-limit") + c.SetStarlarkExecLimit(ctx.Uint64("compiler-starlark-exec-limit")) if ctx.Bool("github-driver") { logrus.Tracef("setting up Private GitHub Client for %s", ctx.String("github-url")) diff --git a/compiler/native/parse.go b/compiler/native/parse.go index 9ccd525f1..0cd7a5819 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -71,7 +71,7 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ // capture the raw pipeline configuration raw = []byte(parsedRaw) - p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables, c.StarlarkExecLimit) + p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables, c.GetStarlarkExecLimit()) if err != nil { return nil, raw, err } diff --git a/compiler/native/settings.go b/compiler/native/settings.go index d240bf557..384d33edb 100644 --- a/compiler/native/settings.go +++ b/compiler/native/settings.go @@ -6,11 +6,16 @@ import ( api "github.com/go-vela/server/api/types" ) -// UpdateFromSettings sets the api settings type in the Engine. -func (c *client) UpdateFromSettings(s *api.Settings) { +// SetSettings sets the api settings type. +func (c *client) GetSettings() *api.CompilerSettings { + return c.CompilerSettings +} + +// SetSettings sets the api settings type. +func (c *client) SetSettings(s *api.Settings) { if s != nil { - c.CloneImage = s.GetCloneImage() - c.TemplateDepth = int(s.GetTemplateDepth()) - c.StarlarkExecLimit = s.GetStarlarkExecLimit() + c.SetCloneImage(s.GetCloneImage()) + c.SetTemplateDepth(s.GetTemplateDepth()) + c.SetStarlarkExecLimit(s.GetStarlarkExecLimit()) } } diff --git a/mock/server/settings.go b/mock/server/settings.go index 5ba8bff09..9699923ab 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -44,6 +44,16 @@ func getSettings(c *gin.Context) { c.JSON(http.StatusOK, body) } +// createSettings returns mock JSON for a http POST. +func createSettings(c *gin.Context) { + data := []byte(CreateSettingsResp) + + var body api.Settings + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} + // updateSettings returns mock JSON for a http PUT. func updateSettings(c *gin.Context) { data := []byte(UpdateSettingsResp) diff --git a/queue/redis/length.go b/queue/redis/length.go index c4d7bb7c3..7f5f22fbc 100644 --- a/queue/redis/length.go +++ b/queue/redis/length.go @@ -12,7 +12,7 @@ func (c *client) Length(ctx context.Context) (int64, error) { total := int64(0) - for _, channel := range c.config.Channels { + for _, channel := range c.GetQueueRoutes() { items, err := c.Redis.LLen(ctx, channel).Result() if err != nil { return 0, err diff --git a/queue/redis/opts.go b/queue/redis/opts.go index 582994fea..dfd9c7560 100644 --- a/queue/redis/opts.go +++ b/queue/redis/opts.go @@ -40,7 +40,7 @@ func WithChannels(channels ...string) ClientOpt { } // set the queue channels in the redis client - c.config.Channels = channels + c.SetQueueRoutes(channels) return nil } diff --git a/queue/redis/opts_test.go b/queue/redis/opts_test.go index aa775c121..454265b82 100644 --- a/queue/redis/opts_test.go +++ b/queue/redis/opts_test.go @@ -111,8 +111,8 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { t.Errorf("WithChannels returned err: %v", err) } - if !reflect.DeepEqual(_service.config.Channels, test.want) { - t.Errorf("WithChannels is %v, want %v", _service.config.Channels, test.want) + if !reflect.DeepEqual(_service.GetQueueRoutes(), test.want) { + t.Errorf("WithChannels is %v, want %v", _service.GetQueueRoutes(), test.want) } } } diff --git a/queue/redis/pop.go b/queue/redis/pop.go index 4d1af3921..be861c085 100644 --- a/queue/redis/pop.go +++ b/queue/redis/pop.go @@ -15,7 +15,7 @@ import ( // Pop grabs an item from the specified channel off the queue. func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) { - c.Logger.Tracef("popping item from queue %s", c.config.Channels) + c.Logger.Tracef("popping item from queue %s", c.GetQueueRoutes()) // define channels to pop from var channels []string @@ -24,7 +24,7 @@ func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) if len(routes) > 0 { channels = routes } else { - channels = c.config.Channels + channels = c.GetQueueRoutes() } // build a redis queue command to pop an item from queue diff --git a/queue/redis/pop_test.go b/queue/redis/pop_test.go index a5ae4fd0b..eabe4f5dd 100644 --- a/queue/redis/pop_test.go +++ b/queue/redis/pop_test.go @@ -65,7 +65,7 @@ func TestRedis_Pop(t *testing.T) { t.Errorf("unable to create queue service: %v", err) } // overwrite channel to be invalid - badChannel.config.Channels = nil + badChannel.SetQueueRoutes(nil) signed = sign.Sign(out, bytes, badChannel.config.PrivateKey) diff --git a/queue/redis/redis.go b/queue/redis/redis.go index 5a789ed58..d594dc035 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -9,6 +9,7 @@ import ( "time" "github.com/alicebob/miniredis/v2" + api "github.com/go-vela/server/api/types" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" ) @@ -16,8 +17,6 @@ import ( type config struct { // specifies the address to use for the Redis client Address string - // specifies a list of channels for managing builds for the Redis client - Channels []string // enables the Redis client to integrate with a Redis cluster Cluster bool // specifies the timeout to use for the Redis client @@ -32,6 +31,7 @@ type client struct { config *config Redis *redis.Client Options *redis.Options + *api.QueueSettings // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } diff --git a/queue/redis/route.go b/queue/redis/route.go index a25a537c6..18093adce 100644 --- a/queue/redis/route.go +++ b/queue/redis/route.go @@ -13,7 +13,7 @@ import ( // Route decides which route a build gets placed within the queue. func (c *client) Route(w *pipeline.Worker) (string, error) { - c.Logger.Tracef("deciding route from queue channels %s", c.config.Channels) + c.Logger.Tracef("deciding route from queue channels %s", c.GetQueueRoutes()) // create buffer to store route buf := bytes.Buffer{} @@ -37,7 +37,7 @@ func (c *client) Route(w *pipeline.Worker) (string, error) { route := strings.TrimLeft(buf.String(), ":") - for _, r := range c.config.Channels { + for _, r := range c.GetQueueRoutes() { if strings.EqualFold(route, r) { return route, nil } diff --git a/queue/redis/settings.go b/queue/redis/settings.go index 078d6a19e..bc2f44eef 100644 --- a/queue/redis/settings.go +++ b/queue/redis/settings.go @@ -6,13 +6,14 @@ import ( api "github.com/go-vela/server/api/types" ) -// UpdateFromSettings takes settings and updates the queue. -func (c *client) UpdateFromSettings(s *api.Settings) { - c.Logger.Tracef("updating queue using settings: %v", s.GetQueueRoutes()) - - c.config.Channels = s.GetQueueRoutes() +// SetSettings sets the api settings type in the Engine. +func (c *client) GetSettings() *api.QueueSettings { + return c.QueueSettings } -func (c *client) GetChannels(s *api.Settings) []string { - return c.config.Channels +// SetSettings sets the api settings type in the Engine. +func (c *client) SetSettings(s *api.Settings) { + if s != nil { + c.SetQueueRoutes(s.GetQueueRoutes()) + } } diff --git a/queue/service.go b/queue/service.go index c3064754c..6d73062c7 100644 --- a/queue/service.go +++ b/queue/service.go @@ -39,8 +39,11 @@ type Service interface { // channel a build gets placed within the queue. Route(*pipeline.Worker) (string, error) - // UpdateFromSettings defines a function that updates - // runtime config within the queue using api settings. - UpdateFromSettings(*api.Settings) - GetChannels(*api.Settings) []string + // GetSettings defines a function that returns + // queue settings. + GetSettings() *api.QueueSettings + + // SetSettings defines a function that takes api settings + // and updates the compiler Engine. + SetSettings(*api.Settings) } diff --git a/router/middleware/compiler.go b/router/middleware/compiler.go index 440f85eb5..dc218cd88 100644 --- a/router/middleware/compiler.go +++ b/router/middleware/compiler.go @@ -15,7 +15,7 @@ func Compiler(comp compiler.Engine) gin.HandlerFunc { return func(c *gin.Context) { s := settings.FromContext(c) - comp.UpdateFromSettings(s) + comp.SetSettings(s) compiler.WithGinContext(c, comp) diff --git a/router/middleware/queue.go b/router/middleware/queue.go index d2047b311..88ad41f93 100644 --- a/router/middleware/queue.go +++ b/router/middleware/queue.go @@ -15,7 +15,7 @@ func Queue(q queue.Service) gin.HandlerFunc { return func(c *gin.Context) { s := settings.FromContext(c) - q.UpdateFromSettings(s) + q.SetSettings(s) queue.WithGinContext(c, q) diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go index 4272cad36..d8dff91b3 100644 --- a/router/middleware/settings/context_test.go +++ b/router/middleware/settings/context_test.go @@ -15,11 +15,15 @@ func TestSettings_FromContext(t *testing.T) { num := int64(1) cloneImage := "target/vela-git" - want := &api.Settings{ - ID: &num, + cs := &api.CompilerSettings{ CloneImage: &cloneImage, } + want := &api.Settings{ + ID: &num, + CompilerSettings: cs, + } + // setup context gin.SetMode(gin.TestMode) context, _ := gin.CreateTestContext(nil) @@ -79,11 +83,15 @@ func TestSettings_ToContext(t *testing.T) { num := int64(1) cloneImage := "target/vela-git" - want := &api.Settings{ - ID: &num, + cs := &api.CompilerSettings{ CloneImage: &cloneImage, } + want := &api.Settings{ + ID: &num, + CompilerSettings: cs, + } + // setup context gin.SetMode(gin.TestMode) context, _ := gin.CreateTestContext(nil) From e0522350c4e544d4d02feb5b6a64bb215050ac2e Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 12:50:00 -0500 Subject: [PATCH 13/65] enhance: reorganize into subtypes using db scan --- api/admin/settings.go | 12 +- api/build/restart.go | 2 +- api/types/settings.go | 242 --------------------- api/types/settings/compiler.go | 95 ++++++++ api/types/settings/platform.go | 133 +++++++++++ api/types/settings/queue.go | 39 ++++ cmd/vela-server/server.go | 35 +-- compiler/engine.go | 5 +- compiler/native/compile.go | 2 +- compiler/native/native.go | 6 +- compiler/native/native_test.go | 6 +- compiler/native/settings.go | 8 +- database/resource_test.go | 3 + database/settings/create.go | 4 +- database/settings/get.go | 6 +- database/settings/interface.go | 8 +- database/settings/settings.go | 113 +++++++--- database/settings/table.go | 10 +- database/settings/update.go | 4 +- mock/server/settings.go | 30 ++- mock/server/settings_test.go | 61 +++++- queue/redis/length.go | 2 +- queue/redis/opts.go | 2 +- queue/redis/opts_test.go | 4 +- queue/redis/pop.go | 4 +- queue/redis/pop_test.go | 2 +- queue/redis/redis.go | 4 +- queue/redis/route.go | 4 +- queue/redis/settings.go | 10 +- queue/service.go | 6 +- router/middleware/allowlist.go | 6 +- router/middleware/allowlist_test.go | 10 +- router/middleware/compiler.go | 1 - router/middleware/settings/context.go | 8 +- router/middleware/settings/context_test.go | 18 +- 35 files changed, 540 insertions(+), 365 deletions(-) delete mode 100644 api/types/settings.go create mode 100644 api/types/settings/compiler.go create mode 100644 api/types/settings/platform.go create mode 100644 api/types/settings/queue.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 9ca0a78eb..b441a6078 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -9,9 +9,9 @@ import ( "strings" "github.com/gin-gonic/gin" - api "github.com/go-vela/server/api/types" + settings "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/settings" + sMiddleware "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/util" "github.com/sirupsen/logrus" ) @@ -41,7 +41,7 @@ import ( // captures settings stored in the database. func GetSettings(c *gin.Context) { // capture middleware values - s := settings.FromContext(c) + s := sMiddleware.FromContext(c) logrus.Info("Admin: reading settings") @@ -95,13 +95,13 @@ func GetSettings(c *gin.Context) { // update the settings singleton stored in the database. func UpdateSettings(c *gin.Context) { // capture middleware values - s := settings.FromContext(c) + s := sMiddleware.FromContext(c) ctx := c.Request.Context() logrus.Info("Admin: updating settings") // capture body from API request - input := new(api.Settings) + input := new(settings.Platform) err := c.Bind(input) if err != nil { @@ -126,7 +126,7 @@ func UpdateSettings(c *gin.Context) { if input.Routes != nil { // update routes if set - s.SetQueueRoutes(input.GetQueueRoutes()) + s.SetRoutes(input.GetRoutes()) } if input.RepoAllowlist != nil { diff --git a/api/build/restart.go b/api/build/restart.go index 94b9b2202..51c1bc0ff 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -127,7 +127,7 @@ func RestartBuild(c *gin.Context) { Source: "restart", Retries: 1, } - logger.Tracef("restarting build using queue.channels: %v", queue.FromContext(c).GetSettings().GetQueueRoutes()) + // generate queue items _, item, code, err := CompileAndPublish( c, diff --git a/api/types/settings.go b/api/types/settings.go deleted file mode 100644 index a42fd8c50..000000000 --- a/api/types/settings.go +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package types - -import ( - "fmt" -) - -// Settings is the API representation of platform settings. -// -// swagger:model Settings -type Settings struct { - ID *int64 `json:"id,omitempty"` - RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` - *QueueSettings `json:"queue,omitempty"` - *CompilerSettings `json:"compiler,omitempty"` -} - -// GetID returns the ID field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetID() int64 { - // return zero value if Settings type or ID field is nil - if s == nil || s.ID == nil { - return 0 - } - - return *s.ID -} - -// SetID sets the ID field. -// -// When the provided Settings type is nil, it -// will set nothing and immediately return. -func (s *Settings) SetID(v int64) { - // return if Settings type is nil - if s == nil { - return - } - - s.ID = &v -} - -// SetCompilerSettings sets the CompilerSettings field. -// -// When the provided CompilerSettings type is nil, it -// will set nothing and immediately return. -func (s *Settings) SetCompilerSettings(cs *CompilerSettings) { - // return if Settings type is nil - if s == nil { - return - } - - s.CompilerSettings = cs -} - -// SetQueueSettings sets the QueueSettings field. -// -// When the provided QueueSettings type is nil, it -// will set nothing and immediately return. -func (s *Settings) SetQueueSettings(qs *QueueSettings) { - // return if Settings type is nil - if s == nil { - return - } - - s.QueueSettings = qs -} - -// GetRepoAllowlist returns the RepoAllowlist field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Settings) GetRepoAllowlist() []string { - // return zero value if Settings type or RepoAllowlist field is nil - if s == nil || s.RepoAllowlist == nil { - return []string{} - } - - return *s.RepoAllowlist -} - -// SetRepoAllowlist sets the RepoAllowlist field. -// -// When the provided Settings type is nil, it -// will set nothing and immediately return. -func (s *Settings) SetRepoAllowlist(v []string) { - // return if Settings type is nil - if s == nil { - return - } - - s.RepoAllowlist = &v -} - -// String implements the Stringer interface for the Settings type. -func (s *Settings) String() string { - return fmt.Sprintf(`{ - ID: %d, - CloneImage: %s, - QueueRoutes: %v, -}`, - s.GetID(), - s.GetCloneImage(), - s.GetQueueRoutes(), - ) -} - -// ToEnv converts the Settings type to a string format compatible with standard posix environments. -func (s *Settings) ToEnv() string { - return fmt.Sprintf(`VELA_CLONE_IMAGE='%s' -VELA_QUEUE_ROUTES='%v' -`, - s.GetCloneImage(), - s.GetQueueRoutes(), - ) -} - -// ToYAML converts the Settings type to a YAML string. -func (s *Settings) ToYAML() string { - return fmt.Sprintf(`VELA_CLONE_IMAGE: '%s' -VELA_QUEUE_ROUTES: '%s' -`, - s.GetCloneImage(), - s.GetQueueRoutes(), - ) -} - -type CompilerSettings struct { - CloneImage *string `json:"clone_image,omitempty"` - TemplateDepth *int `json:"template_depth,omitempty"` - StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` -} - -// GetCloneImage returns the CloneImage field. -// -// When the provided CompilerSettings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *CompilerSettings) GetCloneImage() string { - // return zero value if Settings type or CloneImage field is nil - if s == nil || s.CloneImage == nil { - return "" - } - - return *s.CloneImage -} - -// GetTemplateDepth returns the TemplateDepth field. -// -// When the provided CompilerSettings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *CompilerSettings) GetTemplateDepth() int { - // return zero value if Settings type or TemplateDepth field is nil - if s == nil || s.TemplateDepth == nil { - return 0 - } - - return *s.TemplateDepth -} - -// GetStarlarkExecLimit returns the StarlarkExecLimit field. -// -// When the provided CompilerSettings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *CompilerSettings) GetStarlarkExecLimit() uint64 { - // return zero value if Settings type or StarlarkExecLimit field is nil - if s == nil || s.StarlarkExecLimit == nil { - return 0 - } - - return *s.StarlarkExecLimit -} - -// SetCloneImage sets the CloneImage field. -// -// When the provided CompilerSettings type is nil, it -// will set nothing and immediately return. -func (s *CompilerSettings) SetCloneImage(v string) { - // return if Settings type is nil - if s == nil { - return - } - - s.CloneImage = &v -} - -// SetTemplateDepth sets the TemplateDepth field. -// -// When the provided CompilerSettings type is nil, it -// will set nothing and immediately return. -func (s *CompilerSettings) SetTemplateDepth(v int) { - // return if Settings type is nil - if s == nil { - return - } - - s.TemplateDepth = &v -} - -// SetStarlarkExecLimit sets the StarlarkExecLimit field. -// -// When the provided CompilerSettings type is nil, it -// will set nothing and immediately return. -func (s *CompilerSettings) SetStarlarkExecLimit(v uint64) { - // return if Settings type is nil - if s == nil { - return - } - - s.StarlarkExecLimit = &v -} - -type QueueSettings struct { - Routes *[]string `json:"routes,omitempty"` -} - -// GetQueueRoutes returns the QueueRoutes field. -// -// When the provided QueueSettings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *QueueSettings) GetQueueRoutes() []string { - // return zero value if Settings type or QueueRoutes field is nil - if s == nil || s.Routes == nil { - return []string{} - } - - return *s.Routes -} - -// SetQueueRoutes sets the QueueRoutes field. -// -// When the provided Settings type is nil, it -// will set nothing and immediately return. -func (s *QueueSettings) SetQueueRoutes(v []string) { - // return if Settings type is nil - if s == nil { - return - } - - s.Routes = &v -} diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go new file mode 100644 index 000000000..01bef2f5d --- /dev/null +++ b/api/types/settings/compiler.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +type Compiler struct { + CloneImage *string `json:"clone_image,omitempty"` + TemplateDepth *int `json:"template_depth,omitempty"` + StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` +} + +// GetCloneImage returns the CloneImage field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetCloneImage() string { + // return zero value if Settings type or CloneImage field is nil + if cs == nil || cs.CloneImage == nil { + return "" + } + + return *cs.CloneImage +} + +// GetTemplateDepth returns the TemplateDepth field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetTemplateDepth() int { + // return zero value if Settings type or TemplateDepth field is nil + if cs == nil || cs.TemplateDepth == nil { + return 0 + } + + return *cs.TemplateDepth +} + +// GetStarlarkExecLimit returns the StarlarkExecLimit field. +// +// When the provided CompilerSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetStarlarkExecLimit() uint64 { + // return zero value if Settings type or StarlarkExecLimit field is nil + if cs == nil || cs.StarlarkExecLimit == nil { + return 0 + } + + return *cs.StarlarkExecLimit +} + +// SetCloneImage sets the CloneImage field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetCloneImage(v string) { + // return if Settings type is nil + if cs == nil { + return + } + + cs.CloneImage = &v +} + +// SetTemplateDepth sets the TemplateDepth field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetTemplateDepth(v int) { + // return if Settings type is nil + if cs == nil { + return + } + + cs.TemplateDepth = &v +} + +// SetStarlarkExecLimit sets the StarlarkExecLimit field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetStarlarkExecLimit(v uint64) { + // return if Settings type is nil + if cs == nil { + return + } + + cs.StarlarkExecLimit = &v +} + +func CompilerMockEmpty() Compiler { + cs := Compiler{} + cs.SetCloneImage("") + cs.SetTemplateDepth(0) + cs.SetStarlarkExecLimit(0) + return cs +} diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go new file mode 100644 index 000000000..59327dff5 --- /dev/null +++ b/api/types/settings/platform.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" +) + +// Platform is the API representation of platform settings. +// +// swagger:model Platform +type Platform struct { + ID *int64 `json:"id,omitempty"` + + *Queue `json:"queue,omitempty"` + *Compiler `json:"compiler,omitempty"` + + // misc + RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetID() int64 { + // return zero value if Settings type or ID field is nil + if s == nil || s.ID == nil { + return 0 + } + + return *s.ID +} + +// SetID sets the ID field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetID(v int64) { + // return if Settings type is nil + if s == nil { + return + } + + s.ID = &v +} + +// SetCompilerSettings sets the CompilerSettings field. +// +// When the provided CompilerSettings type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetCompilerSettings(cs Compiler) { + // return if Settings type is nil + if s == nil { + return + } + + s.Compiler = &cs +} + +// SetQueueSettings sets the QueueSettings field. +// +// When the provided QueueSettings type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetQueueSettings(qs Queue) { + // return if Settings type is nil + if s == nil { + return + } + + s.Queue = &qs +} + +// GetRepoAllowlist returns the RepoAllowlist field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetRepoAllowlist() []string { + // return zero value if Settings type or RepoAllowlist field is nil + if s == nil || s.RepoAllowlist == nil { + return []string{} + } + + return *s.RepoAllowlist +} + +// SetRepoAllowlist sets the RepoAllowlist field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetRepoAllowlist(v []string) { + // return if Settings type is nil + if s == nil { + return + } + + s.RepoAllowlist = &v +} + +// String implements the Stringer interface for the Settings type. +func (s *Platform) String() string { + return fmt.Sprintf(`{ + ID: %d, + CloneImage: %s, + QueueRoutes: %v, + other stuff: %v, +}`, + s.GetID(), + s.GetCloneImage(), + s.GetRoutes(), + s.GetRoutes(), + ) +} + +// ToEnv converts the Settings type to a string format compatible with standard posix environments. +func (s *Platform) ToEnv() string { + return fmt.Sprintf(`VELA_CLONE_IMAGE='%s' +VELA_QUEUE_ROUTES='%v' +`, + s.GetCloneImage(), + s.GetRoutes(), + ) +} + +// ToYAML converts the Settings type to a YAML string. +func (s *Platform) ToYAML() string { + return fmt.Sprintf(`VELA_CLONE_IMAGE: '%s' +VELA_QUEUE_ROUTES: '%s' +`, + s.GetCloneImage(), + s.GetRoutes(), + ) +} diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go new file mode 100644 index 000000000..755fe8940 --- /dev/null +++ b/api/types/settings/queue.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +type Queue struct { + Routes *[]string `json:"routes,omitempty"` +} + +// GetRoutes returns the Routes field. +// +// When the provided QueueSettings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (qs *Queue) GetRoutes() []string { + // return zero value if Settings type or Routes field is nil + if qs == nil || qs.Routes == nil { + return []string{} + } + + return *qs.Routes +} + +// SetRoutes sets the Routes field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (qs *Queue) SetRoutes(v []string) { + // return if Settings type is nil + if qs == nil { + return + } + + qs.Routes = &v +} + +func QueueMockEmpty() Queue { + qs := Queue{} + qs.SetRoutes([]string{}) + return qs +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index fd3341169..2877e354e 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -19,7 +19,7 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/util/wait" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" @@ -99,31 +99,34 @@ func server(c *cli.Context) error { return err } - settings, err := database.GetSettings(context.Background()) - if settings == nil || err != nil { + s, err := database.GetSettings(context.Background()) + if s == nil || err != nil { // ignore error and attempt to create initial settings record - settings := new(api.Settings) + s := new(settings.Platform) // singleton record ID should always be 1 - settings.SetID(1) + s.SetID(1) // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() - settings.SetCompilerSettings(compilerSettings) + s.SetCompilerSettings(compilerSettings) queueSettings := queue.GetSettings() - settings.SetQueueSettings(queueSettings) + s.SetQueueSettings(queueSettings) + + // set permitted repos + s.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) // create the settings record in the database - _, err = database.CreateSettings(context.Background(), settings) + _, err = database.CreateSettings(context.Background(), s) if err != nil { return err } } // update any internal settings - queue.SetSettings(settings) - compiler.SetSettings(settings) + queue.SetSettings(s) + compiler.SetSettings(s) router := router.Load( middleware.Settings(database), @@ -140,7 +143,7 @@ func server(c *cli.Context) error { middleware.QueueSigningPrivateKey(c.String("queue.private-key")), middleware.QueueSigningPublicKey(c.String("queue.public-key")), middleware.QueueAddress(c.String("queue.addr")), - middleware.Allowlist(c.StringSlice("vela-repo-allowlist")), + middleware.Allowlist(), middleware.DefaultBuildLimit(c.Int64("default-build-limit")), middleware.DefaultTimeout(c.Int64("default-build-timeout")), middleware.MaxBuildLimit(c.Int64("max-build-limit")), @@ -243,9 +246,9 @@ func server(c *cli.Context) error { // sleep for a duration of time before processing schedules time.Sleep(jitter) - settings, err = database.GetSettings(context.Background()) - if settings == nil || err != nil { - if settings == nil { + s, err = database.GetSettings(context.Background()) + if s == nil || err != nil { + if s == nil { err = errors.New("settings not found") } @@ -254,8 +257,8 @@ func server(c *cli.Context) error { continue } - compiler.SetSettings(settings) - queue.SetSettings(settings) + compiler.SetSettings(s) + queue.SetSettings(s) err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) if err != nil { diff --git a/compiler/engine.go b/compiler/engine.go index 608cbd11c..418f0e47d 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -4,6 +4,7 @@ package compiler import ( api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/internal" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" @@ -149,8 +150,8 @@ type Engine interface { WithPrivateGitHub(string, string) Engine // GetSettings defines a function that returns new api settings // with the compiler Engine fields filled. - GetSettings() *api.CompilerSettings + GetSettings() settings.Compiler // SetSettings defines a function that takes api settings // and updates the compiler Engine. - SetSettings(*api.Settings) + SetSettings(*settings.Platform) } diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 1e00abce9..c68538a52 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -209,7 +209,7 @@ func (c *client) compileInline(p *yaml.Build, depth int) (*yaml.Build, error) { // return if max template depth has been reached if depth == 0 { - retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + retErr := fmt.Errorf("max template depth of %d exceeded", c.GetTemplateDepth()) return nil, retErr } diff --git a/compiler/native/native.go b/compiler/native/native.go index 13a56a940..0a1487162 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" @@ -30,7 +31,7 @@ type client struct { UsePrivateGithub bool ModificationService ModificationConfig - *api.CompilerSettings + settings.Compiler build *library.Build comment string @@ -69,7 +70,7 @@ func New(ctx *cli.Context) (*client, error) { c.Github = github - c.CompilerSettings = new(api.CompilerSettings) + c.Compiler = settings.Compiler{} // set the clone image to use for the injected clone step c.SetCloneImage(ctx.String("clone-image")) @@ -118,6 +119,7 @@ func (c *client) Duplicate() compiler.Engine { cc.PrivateGithub = c.PrivateGithub cc.UsePrivateGithub = c.UsePrivateGithub cc.ModificationService = c.ModificationService + cc.CloneImage = c.CloneImage cc.TemplateDepth = c.TemplateDepth cc.StarlarkExecLimit = c.StarlarkExecLimit diff --git a/compiler/native/native_test.go b/compiler/native/native_test.go index 5e84a6664..7d710cc9a 100644 --- a/compiler/native/native_test.go +++ b/compiler/native/native_test.go @@ -10,6 +10,7 @@ import ( "github.com/urfave/cli/v2" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler/registry/github" "github.com/go-vela/server/internal" "github.com/go-vela/types/library" @@ -21,7 +22,8 @@ func TestNative_New(t *testing.T) { c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") want := &client{ - Github: public, + Github: public, + Compiler: settings.CompilerMockEmpty(), } // run test @@ -51,6 +53,7 @@ func TestNative_New_PrivateGithub(t *testing.T) { Github: public, PrivateGithub: private, UsePrivateGithub: true, + Compiler: settings.CompilerMockEmpty(), } // run test @@ -80,6 +83,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { Github: public, PrivateGithub: private, UsePrivateGithub: true, + Compiler: settings.CompilerMockEmpty(), } // run test diff --git a/compiler/native/settings.go b/compiler/native/settings.go index 384d33edb..578b8ec23 100644 --- a/compiler/native/settings.go +++ b/compiler/native/settings.go @@ -3,16 +3,16 @@ package native import ( - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // SetSettings sets the api settings type. -func (c *client) GetSettings() *api.CompilerSettings { - return c.CompilerSettings +func (c *client) GetSettings() settings.Compiler { + return c.Compiler } // SetSettings sets the api settings type. -func (c *client) SetSettings(s *api.Settings) { +func (c *client) SetSettings(s *settings.Platform) { if s != nil { c.SetCloneImage(s.GetCloneImage()) c.SetTemplateDepth(s.GetTemplateDepth()) diff --git a/database/resource_test.go b/database/resource_test.go index 5819739c4..c51110edf 100644 --- a/database/resource_test.go +++ b/database/resource_test.go @@ -18,6 +18,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -27,6 +28,8 @@ func TestDatabase_Engine_NewResources(t *testing.T) { _postgres, _mock := testPostgres(t) defer _postgres.Close() + // ensure the mock expects the settings queries + _mock.ExpectExec(settings.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the build queries _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/database/settings/create.go b/database/settings/create.go index 8bea2888d..a0db02cea 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -5,11 +5,11 @@ package settings import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // CreateSettings updates a platform settings in the database. -func (e *engine) CreateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { +func (e *engine) CreateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Tracef("creating platform settings in the database with %v", s.String()) // cast the api type to database type diff --git a/database/settings/get.go b/database/settings/get.go index 854a21f8d..73bd00d68 100644 --- a/database/settings/get.go +++ b/database/settings/get.go @@ -5,15 +5,15 @@ package settings import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // GetSettings gets platform settings from the database. -func (e *engine) GetSettings(ctx context.Context) (*api.Settings, error) { +func (e *engine) GetSettings(ctx context.Context) (*settings.Platform, error) { e.logger.Trace("getting platform settings from the database") // variable to store query results - s := new(Settings) + s := new(Platform) // send query to the database and store result in variable err := e.client. diff --git a/database/settings/interface.go b/database/settings/interface.go index f09b7e1b9..a7cc755cd 100644 --- a/database/settings/interface.go +++ b/database/settings/interface.go @@ -5,7 +5,7 @@ package settings import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // SettingsInterface represents the Vela interface for settings @@ -14,9 +14,9 @@ import ( //nolint:revive // ignore name stutter type SettingsInterface interface { // CreateSettings defines a function that creates a platform settings record. - CreateSettings(context.Context, *api.Settings) (*api.Settings, error) + CreateSettings(context.Context, *settings.Platform) (*settings.Platform, error) // GetSettings defines a function that gets platform settings. - GetSettings(context.Context) (*api.Settings, error) + GetSettings(context.Context) (*settings.Platform, error) // UpdateSettings defines a function that updates platform settings. - UpdateSettings(context.Context, *api.Settings) (*api.Settings, error) + UpdateSettings(context.Context, *settings.Platform) (*settings.Platform, error) } diff --git a/database/settings/settings.go b/database/settings/settings.go index 984006483..4152d7588 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -5,14 +5,16 @@ package settings import ( "context" "database/sql" + "database/sql/driver" + "encoding/json" "errors" "fmt" - api "github.com/go-vela/server/api/types" "github.com/go-vela/server/util" "github.com/lib/pq" "github.com/sirupsen/logrus" + "github.com/go-vela/server/api/types/settings" "gorm.io/gorm" ) @@ -24,9 +26,6 @@ var ( // ErrEmptyCloneImage defines the error type when a // Settings type has an empty CloneImage field provided. ErrEmptyCloneImage = errors.New("empty settings clone image provided") - // ErrEmptyQueueRoutes defines the error type when a - // Settings type has an empty QueueRoutes field provided. - ErrEmptyQueueRoutes = errors.New("empty settings queue routes provided") ) // todo: comments Build->Settings @@ -55,17 +54,64 @@ type ( logger *logrus.Entry } - // Settings is the database representation of platform settings. - Settings struct { - ID sql.NullInt64 `sql:"id"` - CloneImage sql.NullString `sql:"clone_image"` - TemplateDepth sql.NullInt64 `sql:"template_depth"` - StarlarkExecLimit sql.NullInt64 `sql:"starlark_exec_limit"` - RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` - QueueRoutes pq.StringArray `sql:"queue_routes" gorm:"type:varchar(1000)"` + // Platform is the database representation of platform settings. + Platform struct { + ID sql.NullInt64 `sql:"id"` + Compiler + Queue + + RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` + } + + // Compiler is the database representation of compiler settings. + Compiler struct { + CloneImage sql.NullString `json:"clone_image" sql:"clone_image"` + TemplateDepth sql.NullInt64 `json:"template_depth" sql:"template_depth"` + StarlarkExecLimit sql.NullInt64 `json:"starlark_exec_limit" sql:"starlark_exec_limit"` + } + + // Queue is the database representation of queue settings. + Queue struct { + Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` } ) +// Value - Implementation of valuer for database/sql for Compiler. +func (r Compiler) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Compiler. +func (r *Compiler) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for compiler: %T", v) + } +} + +// Value - Implementation of valuer for database/sql for Queue. +func (r Queue) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Queue. +func (r *Queue) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for queue: %T", v) + } +} + // New creates and returns a Vela service for integrating with builds in the database. // //nolint:revive // ignore returning unexported engine @@ -110,7 +156,7 @@ func New(opts ...EngineOpt) (*engine, error) { // When a field within the Settings type is the zero // value for the field, the valid flag is set to // false causing it to be NULL in the database. -func (s *Settings) Nullify() *Settings { +func (s *Platform) Nullify() *Platform { if s == nil { return nil } @@ -130,22 +176,23 @@ func (s *Settings) Nullify() *Settings { // ToAPI converts the Settings type // to an API Settings type. -func (s *Settings) ToAPI() *api.Settings { - settings := new(api.Settings) +func (s *Platform) ToAPI() *settings.Platform { + settings := new(settings.Platform) settings.SetID(s.ID.Int64) settings.SetCloneImage(s.CloneImage.String) settings.SetTemplateDepth(int(s.TemplateDepth.Int64)) settings.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) settings.SetCloneImage(s.CloneImage.String) - settings.SetQueueRoutes(s.QueueRoutes) + settings.SetRoutes(s.Routes) + settings.SetRepoAllowlist(s.RepoAllowlist) return settings } // Validate verifies the necessary fields for // the Settings type are populated correctly. -func (s *Settings) Validate() error { +func (s *Platform) Validate() error { // verify the CloneImage field is populated if len(s.CloneImage.String) == 0 { return ErrEmptyCloneImage @@ -156,24 +203,36 @@ func (s *Settings) Validate() error { // to avoid unsafe HTML content s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} - // ensure that all QueueRoutes are sanitized + // ensure that all Queue.Routes are sanitized + // to avoid unsafe HTML content + for i, v := range s.Routes { + s.Routes[i] = util.Sanitize(v) + } + + // ensure that all RepoAllowlist are sanitized // to avoid unsafe HTML content - for i, v := range s.QueueRoutes { - s.QueueRoutes[i] = util.Sanitize(v) + for i, v := range s.RepoAllowlist { + s.RepoAllowlist[i] = util.Sanitize(v) } + // todo: allowlist return nil } // FromAPI converts the API Settings type // to a database Settings type. -func FromAPI(s *api.Settings) *Settings { - settings := &Settings{ - ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, - CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, - TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, - StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, - QueueRoutes: pq.StringArray(s.GetQueueRoutes()), +func FromAPI(s *settings.Platform) *Platform { + settings := &Platform{ + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, + }, + Queue: Queue{ + Routes: pq.StringArray(s.GetRoutes()), + }, + RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), } return settings.Nullify() diff --git a/database/settings/table.go b/database/settings/table.go index 7f8d6463c..9cc30806e 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -15,8 +15,9 @@ CREATE TABLE IF NOT EXISTS settings ( id SERIAL PRIMARY KEY, - clone_image VARCHAR(250), - queue_routes VARCHAR(1000) + compiler JSON DEFAULT NULL, + queue JSON DEFAULT NULL, + repo_allowlist VARCHAR(1000) ); ` @@ -26,8 +27,9 @@ CREATE TABLE IF NOT EXISTS settings ( id INTEGER PRIMARY KEY AUTOINCREMENT, - clone_image TEXT, - queue_routes TEXT + compiler TEXT, + queue TEXT, + repo_allowlist VARCHAR(1000) ); ` ) diff --git a/database/settings/update.go b/database/settings/update.go index 243eec3fb..32ecf2778 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -5,11 +5,11 @@ package settings import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // UpdateSettings updates a platform settings in the database. -func (e *engine) UpdateSettings(ctx context.Context, s *api.Settings) (*api.Settings, error) { +func (e *engine) UpdateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Trace("updating platform settings in the database") // cast the api type to database type diff --git a/mock/server/settings.go b/mock/server/settings.go index 9699923ab..d6f72d123 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -7,30 +7,42 @@ import ( "net/http" "github.com/gin-gonic/gin" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) const ( // SettingsResp represents a JSON return for a single settings. SettingsResp = ` { - "id": 1 + "id": 1, + "compiler": {}, + "queue": {}, + "repo_allowlist": [] }` // CreateSettingsResp represents a JSON return for creating a settings record. CreateSettingsResp = ` { - "id": 1 + "id": 1, + "compiler": {}, + "queue": {}, + "repo_allowlist": [] }` // UpdateSettingsResp represents a JSON return for modifying a settings field. UpdateSettingsResp = ` { - "id": 1 + "id": 1, + "compiler": {}, + "queue": {}, + "repo_allowlist": [] }` // RemoveSettingsResp represents a JSON return for deleting a settings record. RemoveSettingsResp = ` { - "id": 1 + "id": 1, + "compiler": {}, + "queue": {}, + "repo_allowlist": [] }` ) @@ -38,7 +50,7 @@ const ( func getSettings(c *gin.Context) { data := []byte(SettingsResp) - var body api.Settings + var body settings.Platform _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -48,7 +60,7 @@ func getSettings(c *gin.Context) { func createSettings(c *gin.Context) { data := []byte(CreateSettingsResp) - var body api.Settings + var body settings.Platform _ = json.Unmarshal(data, &body) c.JSON(http.StatusCreated, body) @@ -58,7 +70,7 @@ func createSettings(c *gin.Context) { func updateSettings(c *gin.Context) { data := []byte(UpdateSettingsResp) - var body api.Settings + var body settings.Platform _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -70,7 +82,7 @@ func updateSettings(c *gin.Context) { func removeSettings(c *gin.Context) { data := []byte(RemoveSettingsResp) - var body api.Settings + var body settings.Platform _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go index 663368586..93397dadb 100644 --- a/mock/server/settings_test.go +++ b/mock/server/settings_test.go @@ -7,11 +7,11 @@ import ( "reflect" "testing" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) func TestSettings_GetResp(t *testing.T) { - testSettings := api.Settings{} + testSettings := settings.Platform{} err := json.Unmarshal([]byte(SettingsResp), &testSettings) if err != nil { @@ -21,8 +21,63 @@ func TestSettings_GetResp(t *testing.T) { tSettings := reflect.TypeOf(testSettings) for i := 0; i < tSettings.NumField(); i++ { - if reflect.ValueOf(testSettings).Field(i).IsNil() { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { t.Errorf("SettingsResp missing field %s", tSettings.Field(i).Name) } } } + +func TestSettings_CreateResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(CreateSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("CreateSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} + +func TestSettings_UpdateResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(UpdateSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("UpdateSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} + +func TestSettings_RemoveResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(RemoveSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("RemoveSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} diff --git a/queue/redis/length.go b/queue/redis/length.go index 7f5f22fbc..d6bc79226 100644 --- a/queue/redis/length.go +++ b/queue/redis/length.go @@ -12,7 +12,7 @@ func (c *client) Length(ctx context.Context) (int64, error) { total := int64(0) - for _, channel := range c.GetQueueRoutes() { + for _, channel := range c.GetRoutes() { items, err := c.Redis.LLen(ctx, channel).Result() if err != nil { return 0, err diff --git a/queue/redis/opts.go b/queue/redis/opts.go index dfd9c7560..81a9509f4 100644 --- a/queue/redis/opts.go +++ b/queue/redis/opts.go @@ -40,7 +40,7 @@ func WithChannels(channels ...string) ClientOpt { } // set the queue channels in the redis client - c.SetQueueRoutes(channels) + c.SetRoutes(channels) return nil } diff --git a/queue/redis/opts_test.go b/queue/redis/opts_test.go index 454265b82..70b8b4e75 100644 --- a/queue/redis/opts_test.go +++ b/queue/redis/opts_test.go @@ -111,8 +111,8 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { t.Errorf("WithChannels returned err: %v", err) } - if !reflect.DeepEqual(_service.GetQueueRoutes(), test.want) { - t.Errorf("WithChannels is %v, want %v", _service.GetQueueRoutes(), test.want) + if !reflect.DeepEqual(_service.GetRoutes(), test.want) { + t.Errorf("WithChannels is %v, want %v", _service.GetRoutes(), test.want) } } } diff --git a/queue/redis/pop.go b/queue/redis/pop.go index be861c085..867bc104f 100644 --- a/queue/redis/pop.go +++ b/queue/redis/pop.go @@ -15,7 +15,7 @@ import ( // Pop grabs an item from the specified channel off the queue. func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) { - c.Logger.Tracef("popping item from queue %s", c.GetQueueRoutes()) + c.Logger.Tracef("popping item from queue %s", c.GetRoutes()) // define channels to pop from var channels []string @@ -24,7 +24,7 @@ func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) if len(routes) > 0 { channels = routes } else { - channels = c.GetQueueRoutes() + channels = c.GetRoutes() } // build a redis queue command to pop an item from queue diff --git a/queue/redis/pop_test.go b/queue/redis/pop_test.go index eabe4f5dd..50ec88a20 100644 --- a/queue/redis/pop_test.go +++ b/queue/redis/pop_test.go @@ -65,7 +65,7 @@ func TestRedis_Pop(t *testing.T) { t.Errorf("unable to create queue service: %v", err) } // overwrite channel to be invalid - badChannel.SetQueueRoutes(nil) + badChannel.SetRoutes(nil) signed = sign.Sign(out, bytes, badChannel.config.PrivateKey) diff --git a/queue/redis/redis.go b/queue/redis/redis.go index d594dc035..c65b27bb3 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -9,7 +9,7 @@ import ( "time" "github.com/alicebob/miniredis/v2" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" ) @@ -31,7 +31,7 @@ type client struct { config *config Redis *redis.Client Options *redis.Options - *api.QueueSettings + settings.Queue // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } diff --git a/queue/redis/route.go b/queue/redis/route.go index 18093adce..e11f8f60f 100644 --- a/queue/redis/route.go +++ b/queue/redis/route.go @@ -13,7 +13,7 @@ import ( // Route decides which route a build gets placed within the queue. func (c *client) Route(w *pipeline.Worker) (string, error) { - c.Logger.Tracef("deciding route from queue channels %s", c.GetQueueRoutes()) + c.Logger.Tracef("deciding route from queue channels %s", c.GetRoutes()) // create buffer to store route buf := bytes.Buffer{} @@ -37,7 +37,7 @@ func (c *client) Route(w *pipeline.Worker) (string, error) { route := strings.TrimLeft(buf.String(), ":") - for _, r := range c.GetQueueRoutes() { + for _, r := range c.GetRoutes() { if strings.EqualFold(route, r) { return route, nil } diff --git a/queue/redis/settings.go b/queue/redis/settings.go index bc2f44eef..118081462 100644 --- a/queue/redis/settings.go +++ b/queue/redis/settings.go @@ -3,17 +3,17 @@ package redis import ( - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) // SetSettings sets the api settings type in the Engine. -func (c *client) GetSettings() *api.QueueSettings { - return c.QueueSettings +func (c *client) GetSettings() settings.Queue { + return c.Queue } // SetSettings sets the api settings type in the Engine. -func (c *client) SetSettings(s *api.Settings) { +func (c *client) SetSettings(s *settings.Platform) { if s != nil { - c.SetQueueRoutes(s.GetQueueRoutes()) + c.SetRoutes(s.GetRoutes()) } } diff --git a/queue/service.go b/queue/service.go index 6d73062c7..a976e7045 100644 --- a/queue/service.go +++ b/queue/service.go @@ -5,7 +5,7 @@ package queue import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/queue/models" "github.com/go-vela/types/pipeline" ) @@ -41,9 +41,9 @@ type Service interface { // GetSettings defines a function that returns // queue settings. - GetSettings() *api.QueueSettings + GetSettings() settings.Queue // SetSettings defines a function that takes api settings // and updates the compiler Engine. - SetSettings(*api.Settings) + SetSettings(*settings.Platform) } diff --git a/router/middleware/allowlist.go b/router/middleware/allowlist.go index edfe5feb2..4734b6d89 100644 --- a/router/middleware/allowlist.go +++ b/router/middleware/allowlist.go @@ -4,13 +4,15 @@ package middleware import ( "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/settings" ) // Allowlist is a middleware function that attaches the allowlist used // to limit which repos can be activated within the system. -func Allowlist(allowlist []string) gin.HandlerFunc { +func Allowlist() gin.HandlerFunc { return func(c *gin.Context) { - c.Set("allowlist", allowlist) + s := settings.FromContext(c) + c.Set("allowlist", s.GetRepoAllowlist()) c.Next() } } diff --git a/router/middleware/allowlist_test.go b/router/middleware/allowlist_test.go index 2bc59623d..7c7c8b5c6 100644 --- a/router/middleware/allowlist_test.go +++ b/router/middleware/allowlist_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types/settings" ) func TestMiddleware_Allowlist(t *testing.T) { @@ -24,7 +25,14 @@ func TestMiddleware_Allowlist(t *testing.T) { context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) // setup mock server - engine.Use(Allowlist(want)) + engine.Use(func(c *gin.Context) { + s := new(settings.Platform) + s.SetRepoAllowlist(want) + + c.Set("settings", s) + c.Next() + }) + engine.Use(Allowlist()) engine.GET("/health", func(c *gin.Context) { got = c.Value("allowlist").([]string) diff --git a/router/middleware/compiler.go b/router/middleware/compiler.go index dc218cd88..fbd382a4d 100644 --- a/router/middleware/compiler.go +++ b/router/middleware/compiler.go @@ -14,7 +14,6 @@ import ( func Compiler(comp compiler.Engine) gin.HandlerFunc { return func(c *gin.Context) { s := settings.FromContext(c) - comp.SetSettings(s) compiler.WithGinContext(c, comp) diff --git a/router/middleware/settings/context.go b/router/middleware/settings/context.go index c2b12a070..f5aa5fb62 100644 --- a/router/middleware/settings/context.go +++ b/router/middleware/settings/context.go @@ -5,7 +5,7 @@ package settings import ( "context" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" ) const key = "settings" @@ -16,13 +16,13 @@ type Setter interface { } // FromContext returns the Settings associated with this context. -func FromContext(c context.Context) *api.Settings { +func FromContext(c context.Context) *settings.Platform { value := c.Value(key) if value == nil { return nil } - s, ok := value.(*api.Settings) + s, ok := value.(*settings.Platform) if !ok { return nil } @@ -32,6 +32,6 @@ func FromContext(c context.Context) *api.Settings { // ToContext adds the Settings to this context if it supports // the Setter interface. -func ToContext(c Setter, s *api.Settings) { +func ToContext(c Setter, s *settings.Platform) { c.Set(key, s) } diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go index d8dff91b3..e4a678646 100644 --- a/router/middleware/settings/context_test.go +++ b/router/middleware/settings/context_test.go @@ -5,7 +5,7 @@ package settings import ( "testing" - api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/gin-gonic/gin" ) @@ -15,13 +15,13 @@ func TestSettings_FromContext(t *testing.T) { num := int64(1) cloneImage := "target/vela-git" - cs := &api.CompilerSettings{ + cs := settings.Compiler{ CloneImage: &cloneImage, } - want := &api.Settings{ - ID: &num, - CompilerSettings: cs, + want := &settings.Platform{ + ID: &num, + Compiler: &cs, } // setup context @@ -83,13 +83,13 @@ func TestSettings_ToContext(t *testing.T) { num := int64(1) cloneImage := "target/vela-git" - cs := &api.CompilerSettings{ + cs := settings.Compiler{ CloneImage: &cloneImage, } - want := &api.Settings{ - ID: &num, - CompilerSettings: cs, + want := &settings.Platform{ + ID: &num, + Compiler: &cs, } // setup context From 1f36713b5b7c68d6608bbfa3f34273d491d30582 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 12:56:24 -0500 Subject: [PATCH 14/65] fix: linter --- database/interface.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/interface.go b/database/interface.go index dbac4d1bd..e7fd06847 100644 --- a/database/interface.go +++ b/database/interface.go @@ -13,10 +13,10 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" - "github.com/go-vela/server/database/settings" ) // Interface represents the interface for integrating with the supported database providers. From 82fabcab6bce523ba76e68617d42c1f1b858ba32 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 12:59:09 -0500 Subject: [PATCH 15/65] chore: remove debug files --- .vscode/launch.json | 13 ------------- Dockerfile.local | 20 -------------------- 2 files changed, 33 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 Dockerfile.local diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index dd6b4eaef..000000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "configurations": [ - { - "name": "Connect to server", - "type": "go", - "request": "attach", - "mode": "remote", - "remotePath": "/workspace", - "port": 4000, - "host": "127.0.0.1", - } - ] -} diff --git a/Dockerfile.local b/Dockerfile.local deleted file mode 100644 index 5d1cc7efb..000000000 --- a/Dockerfile.local +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -FROM alpine:3.19.1@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b as certs - -RUN apk add --update --no-cache ca-certificates - -FROM golang - -COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - -EXPOSE 8080 - -ENV GODEBUG=netdns=go - -ADD release/vela-server /bin/ - -RUN CGO_ENABLED=0 go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest -CMD [ "/go/bin/dlv", "--listen=:4000", "--headless=true", "--log=true", "--accept-multiclient", "--api-version=2", "exec", "/bin/vela-server" ] - -# CMD ["/bin/vela-server"] From 17d6e6884d5858dc554ac7dd19f29dfc9bab5263 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:02:41 -0500 Subject: [PATCH 16/65] chore: remove debug line --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 664d1d2ad..6cd534285 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: environment: EXECUTOR_DRIVER: linux QUEUE_DRIVER: redis - VELA_BUILD_LIMIT: 3 + VELA_BUILD_LIMIT: 1 VELA_BUILD_TIMEOUT: 30m VELA_LOG_LEVEL: trace VELA_RUNTIME_DRIVER: docker From d9ac8db257ce75f78af2d2c9cb2bf4ac95548e8e Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:04:14 -0500 Subject: [PATCH 17/65] chore: cleanup, fmt, remove repo allowlist --- api/admin/settings.go | 7 ++-- api/repo/create.go | 6 ++-- api/types/settings/compiler.go | 1 + api/types/settings/queue.go | 1 + cmd/vela-server/queue.go | 1 - cmd/vela-server/schedule.go | 1 - cmd/vela-server/server.go | 8 +++-- database/resource.go | 2 ++ mock/server/server.go | 3 ++ mock/server/settings.go | 10 ------ router/admin.go | 28 +++++++++------- router/middleware/allowlist.go | 18 ---------- router/middleware/allowlist_test.go | 52 ----------------------------- router/middleware/compiler_test.go | 12 +++++++ router/middleware/queue.go | 1 - 15 files changed, 50 insertions(+), 101 deletions(-) delete mode 100644 router/middleware/allowlist.go delete mode 100644 router/middleware/allowlist_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index b441a6078..e0eac854c 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -1,6 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//nolint:dupl // ignore similar code package admin import ( @@ -9,7 +8,7 @@ import ( "strings" "github.com/gin-gonic/gin" - settings "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database" sMiddleware "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/util" @@ -146,3 +145,7 @@ func UpdateSettings(c *gin.Context) { c.JSON(http.StatusOK, s) } + +func RemoveSettings(c *gin.Context) { + c.JSON(http.StatusOK, "TODO: not implemented") +} diff --git a/api/repo/create.go b/api/repo/create.go index 824f7d0f5..7e8233b74 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/api/types" "github.com/go-vela/server/api/types/actions" "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -71,7 +72,8 @@ import ( func CreateRepo(c *gin.Context) { // capture middleware values u := user.Retrieve(c) - allowlist := c.Value("allowlist").([]string) + s := settings.FromContext(c) + defaultBuildLimit := c.Value("defaultBuildLimit").(int64) defaultTimeout := c.Value("defaultTimeout").(int64) maxBuildLimit := c.Value("maxBuildLimit").(int64) @@ -217,7 +219,7 @@ func CreateRepo(c *gin.Context) { ) // ensure repo is allowed to be activated - if !util.CheckAllowlist(r, allowlist) { + if !util.CheckAllowlist(r, s.GetRepoAllowlist()) { retErr := fmt.Errorf("unable to activate repo: %s is not on allowlist", r.GetFullName()) util.HandleError(c, http.StatusForbidden, retErr) diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go index 01bef2f5d..1f4e07429 100644 --- a/api/types/settings/compiler.go +++ b/api/types/settings/compiler.go @@ -91,5 +91,6 @@ func CompilerMockEmpty() Compiler { cs.SetCloneImage("") cs.SetTemplateDepth(0) cs.SetStarlarkExecLimit(0) + return cs } diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go index 755fe8940..185d2ae3f 100644 --- a/api/types/settings/queue.go +++ b/api/types/settings/queue.go @@ -35,5 +35,6 @@ func (qs *Queue) SetRoutes(v []string) { func QueueMockEmpty() Queue { qs := Queue{} qs.SetRoutes([]string{}) + return qs } diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go index 9489e63a1..7a3a74c41 100644 --- a/cmd/vela-server/queue.go +++ b/cmd/vela-server/queue.go @@ -4,7 +4,6 @@ package main import ( "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" "github.com/go-vela/server/queue" diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index d43e4739d..d069b05a4 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -31,7 +31,6 @@ const ( func processSchedules(ctx context.Context, start time.Time, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { logrus.Infof("processing active schedules to create builds") - // todo: replace allowList with settings.allowList // send API call to capture the list of active schedules schedules, err := database.ListActiveSchedules(ctx) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 2877e354e..ab29f3dfa 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -25,6 +25,7 @@ import ( "github.com/go-vela/server/router/middleware" ) +//nolint:funlen // ignore function length func server(c *cli.Context) error { // set log formatter switch c.String("log-formatter") { @@ -143,7 +144,6 @@ func server(c *cli.Context) error { middleware.QueueSigningPrivateKey(c.String("queue.private-key")), middleware.QueueSigningPublicKey(c.String("queue.public-key")), middleware.QueueAddress(c.String("queue.addr")), - middleware.Allowlist(), middleware.DefaultBuildLimit(c.Int64("default-build-limit")), middleware.DefaultTimeout(c.Int64("default-build-timeout")), middleware.MaxBuildLimit(c.Int64("max-build-limit")), @@ -190,6 +190,7 @@ func server(c *cli.Context) error { select { case sig := <-signalChannel: logrus.Infof("received signal: %s", sig) + err := srv.Shutdown(ctx) if err != nil { logrus.Error(err) @@ -197,10 +198,12 @@ func server(c *cli.Context) error { done() case <-gctx.Done(): logrus.Info("closing signal goroutine") + err := srv.Shutdown(ctx) if err != nil { logrus.Error(err) } + return gctx.Err() } @@ -260,7 +263,8 @@ func server(c *cli.Context) error { compiler.SetSettings(s) queue.SetSettings(s) - err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) + // todo: SCHEDULES allowlist, not repo!! + err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, s.GetRepoAllowlist()) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { diff --git a/database/resource.go b/database/resource.go index c978e5a5f..6803e0ed7 100644 --- a/database/resource.go +++ b/database/resource.go @@ -22,6 +22,8 @@ import ( ) // NewResources creates and returns the database agnostic engines for resources. +// +//nolint:funlen // ignore function length func (e *engine) NewResources(ctx context.Context) error { var err error diff --git a/mock/server/server.go b/mock/server/server.go index 0b7dcff79..324a79bda 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -29,6 +29,9 @@ func FakeHandler() http.Handler { e.PUT("/api/v1/admin/user", updateUser) e.POST("/api/v1/admin/workers/:worker/register", registerToken) e.PUT("api/v1/admin/clean", cleanResoures) + e.GET("/api/v1/admin/settings", getSettings) + e.PUT("/api/v1/admin/settings", updateSettings) + e.DELETE("/api/v1/admin/settings", removeSettings) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/mock/server/settings.go b/mock/server/settings.go index d6f72d123..54caa6817 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -56,16 +56,6 @@ func getSettings(c *gin.Context) { c.JSON(http.StatusOK, body) } -// createSettings returns mock JSON for a http POST. -func createSettings(c *gin.Context) { - data := []byte(CreateSettingsResp) - - var body settings.Platform - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusCreated, body) -} - // updateSettings returns mock JSON for a http PUT. func updateSettings(c *gin.Context) { data := []byte(UpdateSettingsResp) diff --git a/router/admin.go b/router/admin.go index 8d0da2d05..6f32b4feb 100644 --- a/router/admin.go +++ b/router/admin.go @@ -12,18 +12,21 @@ import ( // AdminHandlers is a function that extends the provided base router group // with the API handlers for admin functionality. // -// GET /api/v1/admin/builds/queue -// GET /api/v1/admin/build/:id -// PUT /api/v1/admin/build -// PUT /api/v1/admin/clean -// PUT /api/v1/admin/deployment -// PUT /api/v1/admin/hook -// PUT /api/v1/admin/repo -// PUT /api/v1/admin/secret -// PUT /api/v1/admin/service -// PUT /api/v1/admin/step -// PUT /api/v1/admin/user -// POST /api/v1/admin/workers/:worker/register. +// GET /api/v1/admin/builds/queue +// GET /api/v1/admin/build/:id +// PUT /api/v1/admin/build +// PUT /api/v1/admin/clean +// PUT /api/v1/admin/deployment +// PUT /api/v1/admin/hook +// PUT /api/v1/admin/repo +// PUT /api/v1/admin/secret +// PUT /api/v1/admin/service +// PUT /api/v1/admin/step +// PUT /api/v1/admin/user +// POST /api/v1/admin/workers/:worker/register. +// GET /api/v1/admin/settings +// PUT /api/v1/admin/settings +// DELETE /api/v1/admin/settings func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) @@ -64,5 +67,6 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin settings endpoint _admin.GET("/settings", admin.GetSettings) _admin.PUT("/settings", admin.UpdateSettings) + _admin.DELETE("/settings", admin.RemoveSettings) } // end of admin endpoints } diff --git a/router/middleware/allowlist.go b/router/middleware/allowlist.go deleted file mode 100644 index 4734b6d89..000000000 --- a/router/middleware/allowlist.go +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "github.com/gin-gonic/gin" - "github.com/go-vela/server/router/middleware/settings" -) - -// Allowlist is a middleware function that attaches the allowlist used -// to limit which repos can be activated within the system. -func Allowlist() gin.HandlerFunc { - return func(c *gin.Context) { - s := settings.FromContext(c) - c.Set("allowlist", s.GetRepoAllowlist()) - c.Next() - } -} diff --git a/router/middleware/allowlist_test.go b/router/middleware/allowlist_test.go deleted file mode 100644 index 7c7c8b5c6..000000000 --- a/router/middleware/allowlist_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/api/types/settings" -) - -func TestMiddleware_Allowlist(t *testing.T) { - // setup types - got := []string{""} - want := []string{"foobar"} - - // setup context - gin.SetMode(gin.TestMode) - - resp := httptest.NewRecorder() - context, engine := gin.CreateTestContext(resp) - context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) - - // setup mock server - engine.Use(func(c *gin.Context) { - s := new(settings.Platform) - s.SetRepoAllowlist(want) - - c.Set("settings", s) - c.Next() - }) - engine.Use(Allowlist()) - engine.GET("/health", func(c *gin.Context) { - got = c.Value("allowlist").([]string) - - c.Status(http.StatusOK) - }) - - // run test - engine.ServeHTTP(context.Writer, context.Request) - - if resp.Code != http.StatusOK { - t.Errorf("Secret returned %v, want %v", resp.Code, http.StatusOK) - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("Secret is %v, want %v", got, want) - } -} diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index 517de8881..14694c2ad 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -12,6 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/urfave/cli/v2" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/native" ) @@ -20,7 +21,9 @@ func TestMiddleware_CompilerNative(t *testing.T) { // setup types var got compiler.Engine + wantCloneImage := "target/vela-git" want, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want.SetCloneImage(wantCloneImage) // setup context gin.SetMode(gin.TestMode) @@ -31,6 +34,15 @@ func TestMiddleware_CompilerNative(t *testing.T) { // setup mock server engine.Use(Compiler(want)) + // setup mock server + engine.Use(func(c *gin.Context) { + s := new(settings.Platform) + // todo: this should fail + // s.SetCloneImage(wantCloneImage) + + c.Set("settings", s) + c.Next() + }) engine.GET("/health", func(c *gin.Context) { got = compiler.FromContext(c) diff --git a/router/middleware/queue.go b/router/middleware/queue.go index 88ad41f93..eff077aa0 100644 --- a/router/middleware/queue.go +++ b/router/middleware/queue.go @@ -14,7 +14,6 @@ import ( func Queue(q queue.Service) gin.HandlerFunc { return func(c *gin.Context) { s := settings.FromContext(c) - q.SetSettings(s) queue.WithGinContext(c, q) From 68aff5eb19ace668466bb238394879cc76be4616 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:06:59 -0500 Subject: [PATCH 18/65] golangci-lint run --fix --- api/admin/settings.go | 3 ++- compiler/native/native.go | 3 +-- database/settings/opt.go | 1 - database/settings/settings.go | 4 ++-- mock/server/settings.go | 1 + queue/redis/redis.go | 3 ++- router/admin.go | 2 +- router/middleware/settings.go | 3 ++- router/middleware/settings/context_test.go | 4 ++-- 9 files changed, 13 insertions(+), 11 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index e0eac854c..676d6ab7a 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -8,11 +8,12 @@ import ( "strings" "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database" sMiddleware "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/util" - "github.com/sirupsen/logrus" ) // swagger:operation GET /api/v1/admin/settings admin GetSettings diff --git a/compiler/native/native.go b/compiler/native/native.go index 0a1487162..426cf7df2 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -5,11 +5,10 @@ package native import ( "time" - api "github.com/go-vela/server/api/types" - "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" diff --git a/database/settings/opt.go b/database/settings/opt.go index 62c42fb6a..a9646d1bf 100644 --- a/database/settings/opt.go +++ b/database/settings/opt.go @@ -6,7 +6,6 @@ import ( "context" "github.com/sirupsen/logrus" - "gorm.io/gorm" ) diff --git a/database/settings/settings.go b/database/settings/settings.go index 4152d7588..0ee41c1e1 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -10,12 +10,12 @@ import ( "errors" "fmt" - "github.com/go-vela/server/util" "github.com/lib/pq" "github.com/sirupsen/logrus" + "gorm.io/gorm" "github.com/go-vela/server/api/types/settings" - "gorm.io/gorm" + "github.com/go-vela/server/util" ) const ( diff --git a/mock/server/settings.go b/mock/server/settings.go index 54caa6817..baed4dcb2 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -7,6 +7,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types/settings" ) diff --git a/queue/redis/redis.go b/queue/redis/redis.go index c65b27bb3..28bd1ed2a 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -9,9 +9,10 @@ import ( "time" "github.com/alicebob/miniredis/v2" - "github.com/go-vela/server/api/types/settings" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types/settings" ) type config struct { diff --git a/router/admin.go b/router/admin.go index 6f32b4feb..37b2be220 100644 --- a/router/admin.go +++ b/router/admin.go @@ -26,7 +26,7 @@ import ( // POST /api/v1/admin/workers/:worker/register. // GET /api/v1/admin/settings // PUT /api/v1/admin/settings -// DELETE /api/v1/admin/settings +// DELETE /api/v1/admin/settings. func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) diff --git a/router/middleware/settings.go b/router/middleware/settings.go index f251ec94b..7f3579714 100644 --- a/router/middleware/settings.go +++ b/router/middleware/settings.go @@ -6,9 +6,10 @@ import ( "context" "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/settings" - "github.com/sirupsen/logrus" ) // Settings is a middleware function that fetches the latest settings and diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go index e4a678646..0c17b4297 100644 --- a/router/middleware/settings/context_test.go +++ b/router/middleware/settings/context_test.go @@ -5,9 +5,9 @@ package settings import ( "testing" - "github.com/go-vela/server/api/types/settings" - "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/types/settings" ) func TestSettings_FromContext(t *testing.T) { From 05d0389038bf4283688586086df4cd7f254262da Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:10:02 -0500 Subject: [PATCH 19/65] fix: linter --- api/repo/create.go | 1 + cmd/vela-server/server.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/repo/create.go b/api/repo/create.go index 7e8233b74..7249bbfa4 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -199,6 +199,7 @@ func CreateRepo(c *gin.Context) { return } + r.SetPipelineType(input.GetPipelineType()) } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index ab29f3dfa..a927c951a 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -25,7 +25,7 @@ import ( "github.com/go-vela/server/router/middleware" ) -//nolint:funlen // ignore function length +//nolint:funlen,gocyclo // ignore function length and cyclomatic complexity func server(c *cli.Context) error { // set log formatter switch c.String("log-formatter") { @@ -195,6 +195,7 @@ func server(c *cli.Context) error { if err != nil { logrus.Error(err) } + done() case <-gctx.Done(): logrus.Info("closing signal goroutine") @@ -213,6 +214,7 @@ func server(c *cli.Context) error { // spawn goroutine for starting the server g.Go(func() error { logrus.Infof("starting server on %s", addr.Host) + err = srv.ListenAndServe() if err != nil { // log a message indicating the failure of the server @@ -225,6 +227,7 @@ func server(c *cli.Context) error { // spawn goroutine for starting the scheduler g.Go(func() error { logrus.Info("starting scheduler") + for { // track the starting time for when the server begins processing schedules // From 763f107378e6ae18ac9f2c8875600f219c02b9b6 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:21:14 -0500 Subject: [PATCH 20/65] feat: schedule allowlist and more linting --- api/admin/settings.go | 7 +++- api/schedule/create.go | 36 ++++++++-------- api/types/settings/platform.go | 29 ++++++++++++- cmd/vela-server/schedule.go | 9 ++-- cmd/vela-server/server.go | 9 ++-- database/settings/settings.go | 14 +++++-- database/settings/table.go | 18 ++++---- mock/server/settings.go | 12 ++++-- router/middleware/allowlist_schedule.go | 16 ------- router/middleware/allowlist_schedule_test.go | 44 -------------------- 10 files changed, 92 insertions(+), 102 deletions(-) delete mode 100644 router/middleware/allowlist_schedule.go delete mode 100644 router/middleware/allowlist_schedule_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 676d6ab7a..a0df70fd9 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -130,10 +130,15 @@ func UpdateSettings(c *gin.Context) { } if input.RepoAllowlist != nil { - // update allowlist if set + // update repo allowlist if set s.SetRepoAllowlist(input.GetRepoAllowlist()) } + if input.ScheduleAllowlist != nil { + // update schedule allowlist if set + s.SetScheduleAllowlist(input.GetScheduleAllowlist()) + } + // send API call to update the repo s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { diff --git a/api/schedule/create.go b/api/schedule/create.go index 7a9439b8c..96e20d64c 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types/library" @@ -77,7 +78,8 @@ func CreateSchedule(c *gin.Context) { u := user.Retrieve(c) r := repo.Retrieve(c) ctx := c.Request.Context() - allowlist := c.Value("allowlistschedule").([]string) + s := settings.FromContext(c) + minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) // capture body from API request @@ -119,7 +121,7 @@ func CreateSchedule(c *gin.Context) { }).Infof("creating new schedule %s", input.GetName()) // ensure repo is allowed to create new schedules - if !util.CheckAllowlist(r, allowlist) { + if !util.CheckAllowlist(r, s.GetScheduleAllowlist()) { retErr := fmt.Errorf("unable to create schedule %s: %s is not on allowlist", input.GetName(), r.GetFullName()) util.HandleError(c, http.StatusForbidden, retErr) @@ -127,29 +129,29 @@ func CreateSchedule(c *gin.Context) { return } - s := new(library.Schedule) + schedule := new(library.Schedule) // update fields in schedule object - s.SetCreatedBy(u.GetName()) - s.SetRepoID(r.GetID()) - s.SetName(input.GetName()) - s.SetEntry(input.GetEntry()) - s.SetCreatedAt(time.Now().UTC().Unix()) - s.SetUpdatedAt(time.Now().UTC().Unix()) - s.SetUpdatedBy(u.GetName()) + schedule.SetCreatedBy(u.GetName()) + schedule.SetRepoID(r.GetID()) + schedule.SetName(input.GetName()) + schedule.SetEntry(input.GetEntry()) + schedule.SetCreatedAt(time.Now().UTC().Unix()) + schedule.SetUpdatedAt(time.Now().UTC().Unix()) + schedule.SetUpdatedBy(u.GetName()) if input.GetBranch() == "" { - s.SetBranch(r.GetBranch()) + schedule.SetBranch(r.GetBranch()) } else { - s.SetBranch(input.GetBranch()) + schedule.SetBranch(input.GetBranch()) } // set the active field based off the input provided if input.Active == nil { // default active field to true - s.SetActive(true) + schedule.SetActive(true) } else { - s.SetActive(input.GetActive()) + schedule.SetActive(input.GetActive()) } // send API call to capture the schedule from the database @@ -178,7 +180,7 @@ func CreateSchedule(c *gin.Context) { dbSchedule.SetActive(true) // send API call to update the schedule - s, err = database.FromContext(c).UpdateSchedule(ctx, dbSchedule, true) + schedule, err = database.FromContext(c).UpdateSchedule(ctx, dbSchedule, true) if err != nil { retErr := fmt.Errorf("unable to set schedule %s to active: %w", dbSchedule.GetName(), err) @@ -188,7 +190,7 @@ func CreateSchedule(c *gin.Context) { } } else { // send API call to create the schedule - s, err = database.FromContext(c).CreateSchedule(ctx, s) + schedule, err = database.FromContext(c).CreateSchedule(ctx, schedule) if err != nil { retErr := fmt.Errorf("unable to create new schedule %s: %w", r.GetName(), err) @@ -198,7 +200,7 @@ func CreateSchedule(c *gin.Context) { } } - c.JSON(http.StatusCreated, s) + c.JSON(http.StatusCreated, schedule) } // validateEntry validates the entry for a minimum frequency. diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index 59327dff5..4a54f8e17 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -16,7 +16,8 @@ type Platform struct { *Compiler `json:"compiler,omitempty"` // misc - RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` + RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` + ScheduleAllowlist *[]string `json:"schedule_allowlist,omitempty"` } // GetID returns the ID field. @@ -97,6 +98,32 @@ func (s *Platform) SetRepoAllowlist(v []string) { s.RepoAllowlist = &v } +// GetScheduleAllowlist returns the ScheduleAllowlist field. +// +// When the provided Settings type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetScheduleAllowlist() []string { + // return zero value if Settings type or ScheduleAllowlist field is nil + if s == nil || s.ScheduleAllowlist == nil { + return []string{} + } + + return *s.ScheduleAllowlist +} + +// SetScheduleAllowlist sets the RepoAllowlist field. +// +// When the provided Settings type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetScheduleAllowlist(v []string) { + // return if Settings type is nil + if s == nil { + return + } + + s.ScheduleAllowlist = &v +} + // String implements the Stringer interface for the Settings type. func (s *Platform) String() string { return fmt.Sprintf(`{ diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index d069b05a4..da183291b 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "github.com/go-vela/server/api/build" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" @@ -29,7 +30,7 @@ const ( scheduleWait = "waiting to trigger build for schedule" ) -func processSchedules(ctx context.Context, start time.Time, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedules(ctx context.Context, start time.Time, settings *settings.Platform, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service) error { logrus.Infof("processing active schedules to create builds") // send API call to capture the list of active schedules @@ -122,7 +123,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En } // process the schedule and trigger a new build - err = processSchedule(ctx, schedule, compiler, database, metadata, queue, scm, allowList) + err = processSchedule(ctx, schedule, settings, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) @@ -134,7 +135,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En } // processSchedule will, given a schedule, process it and trigger a new build. -func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedule(ctx context.Context, s *library.Schedule, settings *settings.Platform, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service) error { // send API call to capture the repo for the schedule r, err := database.GetRepo(ctx, s.GetRepoID()) if err != nil { @@ -142,7 +143,7 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler } // ensure repo has not been removed from allow list - if !util.CheckAllowlist(r, allowList) { + if !util.CheckAllowlist(r, settings.GetScheduleAllowlist()) { return fmt.Errorf("skipping schedule: repo %s no longer on allow list", r.GetFullName()) } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index a927c951a..a897178c3 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -115,9 +115,12 @@ func server(c *cli.Context) error { queueSettings := queue.GetSettings() s.SetQueueSettings(queueSettings) - // set permitted repos + // set repos permitted to be added s.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) + // set repos permitted to use schedules + s.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) + // create the settings record in the database _, err = database.CreateSettings(context.Background(), s) if err != nil { @@ -153,7 +156,6 @@ func server(c *cli.Context) error { middleware.DefaultRepoEvents(c.StringSlice("default-repo-events")), middleware.DefaultRepoEventsMask(c.Int64("default-repo-events-mask")), middleware.DefaultRepoApproveBuild(c.String("default-repo-approve-build")), - middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), ) @@ -266,8 +268,7 @@ func server(c *cli.Context) error { compiler.SetSettings(s) queue.SetSettings(s) - // todo: SCHEDULES allowlist, not repo!! - err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, s.GetRepoAllowlist()) + err = processSchedules(ctx, start, s, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { diff --git a/database/settings/settings.go b/database/settings/settings.go index 0ee41c1e1..47f14fbdf 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -60,7 +60,8 @@ type ( Compiler Queue - RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` + RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` + ScheduleAllowlist pq.StringArray `sql:"schedule_allowlist" gorm:"type:varchar(1000)"` } // Compiler is the database representation of compiler settings. @@ -186,6 +187,7 @@ func (s *Platform) ToAPI() *settings.Platform { settings.SetCloneImage(s.CloneImage.String) settings.SetRoutes(s.Routes) settings.SetRepoAllowlist(s.RepoAllowlist) + settings.SetScheduleAllowlist(s.ScheduleAllowlist) return settings } @@ -215,7 +217,12 @@ func (s *Platform) Validate() error { s.RepoAllowlist[i] = util.Sanitize(v) } - // todo: allowlist + // ensure that all ScheduleAllowlist are sanitized + // to avoid unsafe HTML content + for i, v := range s.ScheduleAllowlist { + s.ScheduleAllowlist[i] = util.Sanitize(v) + } + return nil } @@ -232,7 +239,8 @@ func FromAPI(s *settings.Platform) *Platform { Queue: Queue{ Routes: pq.StringArray(s.GetRoutes()), }, - RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), + RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), + ScheduleAllowlist: pq.StringArray(s.GetScheduleAllowlist()), } return settings.Nullify() diff --git a/database/settings/table.go b/database/settings/table.go index 9cc30806e..0a66d154f 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -14,10 +14,11 @@ const ( CREATE TABLE IF NOT EXISTS settings ( - id SERIAL PRIMARY KEY, - compiler JSON DEFAULT NULL, - queue JSON DEFAULT NULL, - repo_allowlist VARCHAR(1000) + id SERIAL PRIMARY KEY, + compiler JSON DEFAULT NULL, + queue JSON DEFAULT NULL, + repo_allowlist VARCHAR(1000) + schedule_allowlist VARCHAR(1000) ); ` @@ -26,10 +27,11 @@ settings ( CREATE TABLE IF NOT EXISTS settings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - compiler TEXT, - queue TEXT, - repo_allowlist VARCHAR(1000) + id INTEGER PRIMARY KEY AUTOINCREMENT, + compiler TEXT, + queue TEXT, + repo_allowlist VARCHAR(1000) + schedule_allowlist VARCHAR(1000) ); ` ) diff --git a/mock/server/settings.go b/mock/server/settings.go index baed4dcb2..a00e4a0bc 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -18,7 +18,8 @@ const ( "id": 1, "compiler": {}, "queue": {}, - "repo_allowlist": [] + "repo_allowlist": [], + "schedule_allowlist": [] }` // CreateSettingsResp represents a JSON return for creating a settings record. @@ -27,7 +28,8 @@ const ( "id": 1, "compiler": {}, "queue": {}, - "repo_allowlist": [] + "repo_allowlist": [], + "schedule_allowlist": [] }` // UpdateSettingsResp represents a JSON return for modifying a settings field. UpdateSettingsResp = ` @@ -35,7 +37,8 @@ const ( "id": 1, "compiler": {}, "queue": {}, - "repo_allowlist": [] + "repo_allowlist": [], + "schedule_allowlist": [] }` // RemoveSettingsResp represents a JSON return for deleting a settings record. RemoveSettingsResp = ` @@ -43,7 +46,8 @@ const ( "id": 1, "compiler": {}, "queue": {}, - "repo_allowlist": [] + "repo_allowlist": [], + "schedule_allowlist": [] }` ) diff --git a/router/middleware/allowlist_schedule.go b/router/middleware/allowlist_schedule.go deleted file mode 100644 index d22a7aa20..000000000 --- a/router/middleware/allowlist_schedule.go +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "github.com/gin-gonic/gin" -) - -// AllowlistSchedule is a middleware function that attaches the allowlistschedule used -// to limit which repos can utilize the schedule feature within the system. -func AllowlistSchedule(allowlistschedule []string) gin.HandlerFunc { - return func(c *gin.Context) { - c.Set("allowlistschedule", allowlistschedule) - c.Next() - } -} diff --git a/router/middleware/allowlist_schedule_test.go b/router/middleware/allowlist_schedule_test.go deleted file mode 100644 index 1461b8719..000000000 --- a/router/middleware/allowlist_schedule_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/gin-gonic/gin" -) - -func TestMiddleware_AllowlistSchedule(t *testing.T) { - // setup types - got := []string{""} - want := []string{"foobar"} - - // setup context - gin.SetMode(gin.TestMode) - - resp := httptest.NewRecorder() - context, engine := gin.CreateTestContext(resp) - context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) - - // setup mock server - engine.Use(AllowlistSchedule(want)) - engine.GET("/health", func(c *gin.Context) { - got = c.Value("allowlistschedule").([]string) - - c.Status(http.StatusOK) - }) - - // run test - engine.ServeHTTP(context.Writer, context.Request) - - if resp.Code != http.StatusOK { - t.Errorf("AllowlistSchedule returned %v, want %v", resp.Code, http.StatusOK) - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("AllowlistSchedule is %v, want %v", got, want) - } -} From a612efa162ca2a97340d478400ae08599c88b7d5 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:22:29 -0500 Subject: [PATCH 21/65] fix: comma missing --- database/settings/table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/settings/table.go b/database/settings/table.go index 0a66d154f..7e97e7466 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -17,7 +17,7 @@ settings ( id SERIAL PRIMARY KEY, compiler JSON DEFAULT NULL, queue JSON DEFAULT NULL, - repo_allowlist VARCHAR(1000) + repo_allowlist VARCHAR(1000), schedule_allowlist VARCHAR(1000) ); ` @@ -30,7 +30,7 @@ settings ( id INTEGER PRIMARY KEY AUTOINCREMENT, compiler TEXT, queue TEXT, - repo_allowlist VARCHAR(1000) + repo_allowlist VARCHAR(1000), schedule_allowlist VARCHAR(1000) ); ` From fd7f354d167c13b05b5246fa76fe034035d64e3c Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 14:54:26 -0500 Subject: [PATCH 22/65] fix: compiler middleware test --- router/middleware/compiler_test.go | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index 14694c2ad..813abb98c 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -15,11 +15,13 @@ import ( "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/native" + sMiddleware "github.com/go-vela/server/router/middleware/settings" ) func TestMiddleware_CompilerNative(t *testing.T) { // setup types var got compiler.Engine + got, _ = native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) wantCloneImage := "target/vela-git" want, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) @@ -30,19 +32,25 @@ func TestMiddleware_CompilerNative(t *testing.T) { resp := httptest.NewRecorder() context, engine := gin.CreateTestContext(resp) - context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) - // setup mock server - engine.Use(Compiler(want)) - // setup mock server - engine.Use(func(c *gin.Context) { - s := new(settings.Platform) - // todo: this should fail - // s.SetCloneImage(wantCloneImage) + engine.Use(func() gin.HandlerFunc { + return func(c *gin.Context) { + s := settings.Platform{ + Compiler: &settings.Compiler{}, + } + s.SetCloneImage(wantCloneImage) + + sMiddleware.ToContext(c, &s) + + c.Next() + } + }(), + ) + + engine.Use(Compiler(got)) + + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) - c.Set("settings", s) - c.Next() - }) engine.GET("/health", func(c *gin.Context) { got = compiler.FromContext(c) From a6ae52582bc3559c1cac8b33e688807964d60f9d Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 15:28:53 -0500 Subject: [PATCH 23/65] feat: delete settings --- api/admin/settings.go | 47 ++++++++++++++++++++++++++++++---- database/settings/delete.go | 22 ++++++++++++++++ database/settings/interface.go | 2 ++ mock/server/server.go | 2 +- mock/server/settings.go | 10 ++++---- mock/server/settings_test.go | 4 +-- router/admin.go | 2 +- 7 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 database/settings/delete.go diff --git a/api/admin/settings.go b/api/admin/settings.go index a0df70fd9..d6841678a 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -31,7 +31,7 @@ import ( // schema: // type: array // items: -// "$ref": "#/definitions/Settings" +// "$ref": "#/definitions/Platform" // '500': // description: Unable to retrieve settings from the database // schema: @@ -74,14 +74,14 @@ func GetSettings(c *gin.Context) { // description: Payload containing settings to update // required: true // schema: -// "$ref": "#/definitions/Settings" +// "$ref": "#/definitions/Platform" // security: // - ApiKeyAuth: [] // responses: // '200': // description: Successfully updated the settings in the database // schema: -// "$ref": "#/definitions/Settings" +// "$ref": "#/definitions/Platform" // '404': // description: Unable to update the settings in the database // schema: @@ -152,6 +152,43 @@ func UpdateSettings(c *gin.Context) { c.JSON(http.StatusOK, s) } -func RemoveSettings(c *gin.Context) { - c.JSON(http.StatusOK, "TODO: not implemented") +// swagger:operation DELETE /api/v1/admin/settings admin DeleteSettings +// +// Delete the platform settings record +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the platform settings record +// schema: +// type: string +// '500': +// description: Unable to delete the platform settings record +// schema: +// "$ref": "#/definitions/Error" + +// DeleteSettings represents the API handler to remove +// the platform settings singleton from the configured backend. +func DeleteSettings(c *gin.Context) { + // capture middleware values + // s := sMiddleware.FromContext(c) + // ctx := c.Request.Context() + + logrus.Info("Admin: deleting settings") + + // send API call to remove the settings record + // err := database.FromContext(c).DeleteSettings(ctx, ss) + // if err != nil { + // retErr := fmt.Errorf("unable to delete platform settings: %w", err) + + // util.HandleError(c, http.StatusInternalServerError, retErr) + + // return + // } + + c.JSON(http.StatusOK, "platform settings deleted") } diff --git a/database/settings/delete.go b/database/settings/delete.go new file mode 100644 index 000000000..83f374ae2 --- /dev/null +++ b/database/settings/delete.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" +) + +// DeleteSettings removes platform settings from the database. +func (e *engine) DeleteSettings(ctx context.Context, s *settings.Platform) error { + e.logger.Trace("deleting platform settings from the database") + + worker := FromAPI(s) + + // send query to the database + return e.client. + Table(TableSettings). + Delete(worker). + Error +} diff --git a/database/settings/interface.go b/database/settings/interface.go index a7cc755cd..3725a700f 100644 --- a/database/settings/interface.go +++ b/database/settings/interface.go @@ -19,4 +19,6 @@ type SettingsInterface interface { GetSettings(context.Context) (*settings.Platform, error) // UpdateSettings defines a function that updates platform settings. UpdateSettings(context.Context, *settings.Platform) (*settings.Platform, error) + // DeleteSettings defines a function that deletes platform settings. + DeleteSettings(context.Context, *settings.Platform) error } diff --git a/mock/server/server.go b/mock/server/server.go index 324a79bda..9fe2c9f90 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -31,7 +31,7 @@ func FakeHandler() http.Handler { e.PUT("api/v1/admin/clean", cleanResoures) e.GET("/api/v1/admin/settings", getSettings) e.PUT("/api/v1/admin/settings", updateSettings) - e.DELETE("/api/v1/admin/settings", removeSettings) + e.DELETE("/api/v1/admin/settings", deleteSettings) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/mock/server/settings.go b/mock/server/settings.go index a00e4a0bc..882ec61ea 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -40,8 +40,8 @@ const ( "repo_allowlist": [], "schedule_allowlist": [] }` - // RemoveSettingsResp represents a JSON return for deleting a settings record. - RemoveSettingsResp = ` + // DeleteSettingsResp represents a JSON return for deleting a settings record. + DeleteSettingsResp = ` { "id": 1, "compiler": {}, @@ -71,11 +71,11 @@ func updateSettings(c *gin.Context) { c.JSON(http.StatusOK, body) } -// removeSettings has a param :settings returns mock JSON for a http DELETE. +// deleteSettings has a param :settings returns mock JSON for a http DELETE. // // Pass "0" to :settings to test receiving a http 404 response. -func removeSettings(c *gin.Context) { - data := []byte(RemoveSettingsResp) +func deleteSettings(c *gin.Context) { + data := []byte(DeleteSettingsResp) var body settings.Platform _ = json.Unmarshal(data, &body) diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go index 93397dadb..3b07d0e1b 100644 --- a/mock/server/settings_test.go +++ b/mock/server/settings_test.go @@ -67,7 +67,7 @@ func TestSettings_UpdateResp(t *testing.T) { func TestSettings_RemoveResp(t *testing.T) { testSettings := settings.Platform{} - err := json.Unmarshal([]byte(RemoveSettingsResp), &testSettings) + err := json.Unmarshal([]byte(DeleteSettingsResp), &testSettings) if err != nil { t.Errorf("error unmarshaling settings: %v", err) } @@ -77,7 +77,7 @@ func TestSettings_RemoveResp(t *testing.T) { for i := 0; i < tSettings.NumField(); i++ { f := reflect.ValueOf(testSettings).Field(i) if f.IsNil() { - t.Errorf("RemoveSettingsResp missing field %s", tSettings.Field(i).Name) + t.Errorf("DeleteSettingsResp missing field %s", tSettings.Field(i).Name) } } } diff --git a/router/admin.go b/router/admin.go index 37b2be220..653cd14f2 100644 --- a/router/admin.go +++ b/router/admin.go @@ -67,6 +67,6 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin settings endpoint _admin.GET("/settings", admin.GetSettings) _admin.PUT("/settings", admin.UpdateSettings) - _admin.DELETE("/settings", admin.RemoveSettings) + _admin.DELETE("/settings", admin.DeleteSettings) } // end of admin endpoints } From 50b84937517cfbf37f67053eb7e423396e63fd34 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 15:31:04 -0500 Subject: [PATCH 24/65] fix: uncomment delete code --- api/admin/settings.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index d6841678a..68e29a960 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -175,20 +175,20 @@ func UpdateSettings(c *gin.Context) { // the platform settings singleton from the configured backend. func DeleteSettings(c *gin.Context) { // capture middleware values - // s := sMiddleware.FromContext(c) - // ctx := c.Request.Context() + s := sMiddleware.FromContext(c) + ctx := c.Request.Context() logrus.Info("Admin: deleting settings") // send API call to remove the settings record - // err := database.FromContext(c).DeleteSettings(ctx, ss) - // if err != nil { - // retErr := fmt.Errorf("unable to delete platform settings: %w", err) + err := database.FromContext(c).DeleteSettings(ctx, s) + if err != nil { + retErr := fmt.Errorf("unable to delete platform settings: %w", err) - // util.HandleError(c, http.StatusInternalServerError, retErr) + util.HandleError(c, http.StatusInternalServerError, retErr) - // return - // } + return + } c.JSON(http.StatusOK, "platform settings deleted") } From 47368abcec1e6a3582e88a4185bfbca110b356e7 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 16:59:29 -0500 Subject: [PATCH 25/65] fix: create nested api types --- api/types/settings/platform.go | 10 +++++----- database/settings/settings.go | 27 +++++++++++++++------------ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index 4a54f8e17..50bfbd6d8 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -10,14 +10,14 @@ import ( // // swagger:model Platform type Platform struct { - ID *int64 `json:"id,omitempty"` + ID *int64 `json:"id"` - *Queue `json:"queue,omitempty"` - *Compiler `json:"compiler,omitempty"` + *Queue `json:"queue"` + *Compiler `json:"compiler"` // misc - RepoAllowlist *[]string `json:"repo_allowlist,omitempty"` - ScheduleAllowlist *[]string `json:"schedule_allowlist,omitempty"` + RepoAllowlist *[]string `json:"repo_allowlist"` + ScheduleAllowlist *[]string `json:"schedule_allowlist"` } // GetID returns the ID field. diff --git a/database/settings/settings.go b/database/settings/settings.go index 47f14fbdf..3cdc5ddb9 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -178,18 +178,21 @@ func (s *Platform) Nullify() *Platform { // ToAPI converts the Settings type // to an API Settings type. func (s *Platform) ToAPI() *settings.Platform { - settings := new(settings.Platform) - - settings.SetID(s.ID.Int64) - settings.SetCloneImage(s.CloneImage.String) - settings.SetTemplateDepth(int(s.TemplateDepth.Int64)) - settings.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) - settings.SetCloneImage(s.CloneImage.String) - settings.SetRoutes(s.Routes) - settings.SetRepoAllowlist(s.RepoAllowlist) - settings.SetScheduleAllowlist(s.ScheduleAllowlist) - - return settings + ss := new(settings.Platform) + ss.SetID(s.ID.Int64) + + ss.SetRepoAllowlist(s.RepoAllowlist) + ss.SetScheduleAllowlist(s.ScheduleAllowlist) + + ss.Compiler = &settings.Compiler{} + ss.SetCloneImage(s.CloneImage.String) + ss.SetTemplateDepth(int(s.TemplateDepth.Int64)) + ss.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) + + ss.Queue = &settings.Queue{} + ss.SetRoutes(s.Routes) + + return ss } // Validate verifies the necessary fields for From 453f64598a9ee219cf24bbf1066eac40b78e6f5f Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 17 Apr 2024 17:32:18 -0500 Subject: [PATCH 26/65] fix: check for nils --- api/admin/settings.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 68e29a960..19d4a9ef9 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -112,20 +112,21 @@ func UpdateSettings(c *gin.Context) { return } - if input.CloneImage != nil { - s.CloneImage = input.CloneImage + if input.Compiler != nil { + if input.CloneImage != nil { + s.CloneImage = input.CloneImage + } + + if input.TemplateDepth != nil { + s.TemplateDepth = input.TemplateDepth + } + + if input.StarlarkExecLimit != nil { + s.StarlarkExecLimit = input.StarlarkExecLimit + } } - if input.TemplateDepth != nil { - s.TemplateDepth = input.TemplateDepth - } - - if input.StarlarkExecLimit != nil { - s.StarlarkExecLimit = input.StarlarkExecLimit - } - - if input.Routes != nil { - // update routes if set + if input.Queue != nil { s.SetRoutes(input.GetRoutes()) } From dac5053e58c27d47e1a7e183fc8ab4b1c1b0dcc0 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 19 Apr 2024 15:51:00 -0500 Subject: [PATCH 27/65] fix: remove sanitize but still crop fields --- api/dashboard/create.go | 4 ++-- api/types/user.go | 17 ++++++----------- api/types/user_test.go | 18 +++++++----------- database/integration_test.go | 8 ++++---- database/repo/get_org_test.go | 1 + database/repo/get_test.go | 1 + database/repo/list_org_test.go | 1 + database/repo/list_test.go | 1 + database/repo/list_user_test.go | 1 + database/repo/repo.go | 2 +- database/repo/repo_test.go | 6 ++---- 11 files changed, 27 insertions(+), 33 deletions(-) diff --git a/api/dashboard/create.go b/api/dashboard/create.go index c0e8f0029..f14a06fe1 100644 --- a/api/dashboard/create.go +++ b/api/dashboard/create.go @@ -143,7 +143,7 @@ func CreateDashboard(c *gin.Context) { // when a user is inactive or not found in the database. It returns a sanitized slice of admins. func createAdminSet(c context.Context, caller *types.User, users []*types.User) ([]*types.User, error) { // add user creating the dashboard to admin list - admins := []*types.User{caller.Sanitize()} + admins := []*types.User{caller.Crop()} dupMap := make(map[string]bool) @@ -158,7 +158,7 @@ func createAdminSet(c context.Context, caller *types.User, users []*types.User) return nil, fmt.Errorf("unable to create dashboard: %s is not an active user", u.GetName()) } - admins = append(admins, dbUser.Sanitize()) + admins = append(admins, dbUser.Crop()) dupMap[dbUser.GetName()] = true } diff --git a/api/types/user.go b/api/types/user.go index 2bad2f3ea..a7d7691c0 100644 --- a/api/types/user.go +++ b/api/types/user.go @@ -4,8 +4,6 @@ package types import ( "fmt" - - "github.com/go-vela/types/constants" ) // User is the API representation of a user. @@ -22,18 +20,15 @@ type User struct { Dashboards *[]string `json:"dashboards,omitempty"` } -// Sanitize creates a duplicate of the User without the token values. -func (u *User) Sanitize() *User { - // create a variable since constants can not be addressable - // - // https://golang.org/ref/spec#Address_operators - value := constants.SecretMask - +// Crop creates a duplicate of the User with certain fields cropped. +// +// Generally used for cropping large fields that aren't useful for all API calls like favorites and dashboards. +func (u *User) Crop() *User { return &User{ ID: u.ID, Name: u.Name, - RefreshToken: &value, - Token: &value, + RefreshToken: u.RefreshToken, + Token: u.Token, Active: u.Active, } } diff --git a/api/types/user_test.go b/api/types/user_test.go index 1da7acb2f..91134e3e0 100644 --- a/api/types/user_test.go +++ b/api/types/user_test.go @@ -6,26 +6,22 @@ import ( "fmt" "reflect" "testing" - - "github.com/go-vela/types/constants" ) -func TestTypes_User_Sanitize(t *testing.T) { +func TestTypes_User_Crop(t *testing.T) { // setup types u := testUser() - want := new(User) - want.SetID(1) - want.SetName("octocat") - want.SetActive(true) - want.SetToken(constants.SecretMask) - want.SetRefreshToken(constants.SecretMask) + want := testUser() + want.Favorites = nil + want.Dashboards = nil + want.Admin = nil // run test - got := u.Sanitize() + got := u.Crop() if !reflect.DeepEqual(got, want) { - t.Errorf("Sanitize is %v, want %v", got, want) + t.Errorf("Crop is %v, want %v", got, want) } } diff --git a/database/integration_test.go b/database/integration_test.go index 2444795e1..a513f4a57 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -2216,7 +2216,7 @@ func newResources() *Resources { dashboardOne.SetCreatedBy("octocat") dashboardOne.SetUpdatedAt(2) dashboardOne.SetUpdatedBy("octokitty") - dashboardOne.SetAdmins([]*api.User{userOne.Sanitize(), userTwo.Sanitize()}) + dashboardOne.SetAdmins([]*api.User{userOne.Crop(), userTwo.Crop()}) dashboardOne.SetRepos([]*api.DashboardRepo{dashRepo}) dashboardTwo := new(api.Dashboard) @@ -2226,7 +2226,7 @@ func newResources() *Resources { dashboardTwo.SetCreatedBy("octocat") dashboardTwo.SetUpdatedAt(2) dashboardTwo.SetUpdatedBy("octokitty") - dashboardTwo.SetAdmins([]*api.User{userOne.Sanitize(), userTwo.Sanitize()}) + dashboardTwo.SetAdmins([]*api.User{userOne.Crop(), userTwo.Crop()}) dashboardTwo.SetRepos([]*api.DashboardRepo{dashRepo}) executableOne := new(library.BuildExecutable) @@ -2386,7 +2386,7 @@ func newResources() *Resources { repoOne := new(api.Repo) repoOne.SetID(1) - repoOne.SetOwner(userOne.Sanitize()) + repoOne.SetOwner(userOne.Crop()) repoOne.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") repoOne.SetOrg("github") repoOne.SetName("octocat") @@ -2409,7 +2409,7 @@ func newResources() *Resources { repoTwo := new(api.Repo) repoTwo.SetID(2) - repoTwo.SetOwner(userOne.Sanitize()) + repoTwo.SetOwner(userOne.Crop()) repoTwo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") repoTwo.SetOrg("github") repoTwo.SetName("octokitty") diff --git a/database/repo/get_org_test.go b/database/repo/get_org_test.go index dc168088c..149b402a9 100644 --- a/database/repo/get_org_test.go +++ b/database/repo/get_org_test.go @@ -30,6 +30,7 @@ func TestRepo_Engine_GetRepoForOrg(t *testing.T) { _owner := testOwner() _owner.SetID(1) _owner.SetName("foo") + _owner.SetToken("bar") _repo.SetOwner(_owner) diff --git a/database/repo/get_test.go b/database/repo/get_test.go index 9b3ded139..651fc8f39 100644 --- a/database/repo/get_test.go +++ b/database/repo/get_test.go @@ -30,6 +30,7 @@ func TestRepo_Engine_GetRepo(t *testing.T) { _owner := testOwner() _owner.SetID(1) _owner.SetName("foo") + _owner.SetToken("bar") _repo.SetOwner(_owner) diff --git a/database/repo/list_org_test.go b/database/repo/list_org_test.go index 54af05969..7f13b540a 100644 --- a/database/repo/list_org_test.go +++ b/database/repo/list_org_test.go @@ -56,6 +56,7 @@ func TestRepo_Engine_ListReposForOrg(t *testing.T) { _owner := testOwner() _owner.SetID(1) _owner.SetName("foo") + _owner.SetToken("bar") _repoOne.SetOwner(_owner) _repoTwo.SetOwner(_owner) diff --git a/database/repo/list_test.go b/database/repo/list_test.go index 5adbc0fa6..01df9981e 100644 --- a/database/repo/list_test.go +++ b/database/repo/list_test.go @@ -41,6 +41,7 @@ func TestRepo_Engine_ListRepos(t *testing.T) { _owner := testOwner() _owner.SetID(1) _owner.SetName("foo") + _owner.SetToken("bar") _repoOne.SetOwner(_owner) _repoTwo.SetOwner(_owner) diff --git a/database/repo/list_user_test.go b/database/repo/list_user_test.go index 1bdae5d36..b89fce07e 100644 --- a/database/repo/list_user_test.go +++ b/database/repo/list_user_test.go @@ -58,6 +58,7 @@ func TestRepo_Engine_ListReposForUser(t *testing.T) { _owner := testOwner() _owner.SetID(1) _owner.SetName("foo") + _owner.SetToken("bar") _repoOne.SetOwner(_owner) _repoTwo.SetOwner(_owner) diff --git a/database/repo/repo.go b/database/repo/repo.go index 33e1cbf36..fc1e7808e 100644 --- a/database/repo/repo.go +++ b/database/repo/repo.go @@ -293,7 +293,7 @@ func (r *Repo) ToAPI() *api.Repo { repo := new(api.Repo) repo.SetID(r.ID.Int64) - repo.SetOwner(r.Owner.ToAPI().Sanitize()) + repo.SetOwner(r.Owner.ToAPI().Crop()) repo.SetHash(r.Hash.String) repo.SetOrg(r.Org.String) repo.SetName(r.Name.String) diff --git a/database/repo/repo_test.go b/database/repo/repo_test.go index 4a179c6d8..8d50ade32 100644 --- a/database/repo/repo_test.go +++ b/database/repo/repo_test.go @@ -232,13 +232,11 @@ func testEvents() *api.Events { // testOwner is a helper function that returns a sanitized user. func testOwner() *api.User { - mask := constants.SecretMask - return &api.User{ ID: new(int64), Name: new(string), - RefreshToken: &mask, - Token: &mask, + RefreshToken: new(string), + Token: new(string), Active: new(bool), } } From be0d6725e1fd13c3cc00f4d813f1ab04bb694462 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 12:13:50 -0500 Subject: [PATCH 28/65] feat: types tests and general cleanup --- api/admin/settings.go | 16 +-- api/types/settings/compiler.go | 28 +++-- api/types/settings/compiler_test.go | 116 ++++++++++++++++++++ api/types/settings/platform.go | 158 +++++++++++++++------------- api/types/settings/platform_test.go | 145 +++++++++++++++++++++++++ api/types/settings/queue.go | 12 +++ api/types/settings/queue_test.go | 90 ++++++++++++++++ cmd/vela-server/server.go | 4 +- 8 files changed, 471 insertions(+), 98 deletions(-) create mode 100644 api/types/settings/compiler_test.go create mode 100644 api/types/settings/platform_test.go create mode 100644 api/types/settings/queue_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 19d4a9ef9..92d0229b5 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -5,7 +5,6 @@ package admin import ( "fmt" "net/http" - "strings" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -45,20 +44,7 @@ func GetSettings(c *gin.Context) { logrus.Info("Admin: reading settings") - output := strings.ToLower(c.Query("output")) - - switch output { - case "env": - exported := s.ToEnv() - c.String(http.StatusOK, exported) - case "yaml": - exported := s.ToYAML() - c.String(http.StatusOK, exported) - case "json": - fallthrough - default: - c.JSON(http.StatusOK, s) - } + c.JSON(http.StatusOK, s) } // swagger:operation PUT /api/v1/admin/settings admin AdminUpdateSettings diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go index 1f4e07429..e79717301 100644 --- a/api/types/settings/compiler.go +++ b/api/types/settings/compiler.go @@ -2,6 +2,8 @@ package settings +import "fmt" + type Compiler struct { CloneImage *string `json:"clone_image,omitempty"` TemplateDepth *int `json:"template_depth,omitempty"` @@ -10,7 +12,7 @@ type Compiler struct { // GetCloneImage returns the CloneImage field. // -// When the provided CompilerSettings type is nil, or the field within +// When the provided Compiler type is nil, or the field within // the type is nil, it returns the zero value for the field. func (cs *Compiler) GetCloneImage() string { // return zero value if Settings type or CloneImage field is nil @@ -23,7 +25,7 @@ func (cs *Compiler) GetCloneImage() string { // GetTemplateDepth returns the TemplateDepth field. // -// When the provided CompilerSettings type is nil, or the field within +// When the provided Compiler type is nil, or the field within // the type is nil, it returns the zero value for the field. func (cs *Compiler) GetTemplateDepth() int { // return zero value if Settings type or TemplateDepth field is nil @@ -36,7 +38,7 @@ func (cs *Compiler) GetTemplateDepth() int { // GetStarlarkExecLimit returns the StarlarkExecLimit field. // -// When the provided CompilerSettings type is nil, or the field within +// When the provided Compiler type is nil, or the field within // the type is nil, it returns the zero value for the field. func (cs *Compiler) GetStarlarkExecLimit() uint64 { // return zero value if Settings type or StarlarkExecLimit field is nil @@ -49,7 +51,7 @@ func (cs *Compiler) GetStarlarkExecLimit() uint64 { // SetCloneImage sets the CloneImage field. // -// When the provided CompilerSettings type is nil, it +// When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetCloneImage(v string) { // return if Settings type is nil @@ -62,7 +64,7 @@ func (cs *Compiler) SetCloneImage(v string) { // SetTemplateDepth sets the TemplateDepth field. // -// When the provided CompilerSettings type is nil, it +// When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetTemplateDepth(v int) { // return if Settings type is nil @@ -75,7 +77,7 @@ func (cs *Compiler) SetTemplateDepth(v int) { // SetStarlarkExecLimit sets the StarlarkExecLimit field. // -// When the provided CompilerSettings type is nil, it +// When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetStarlarkExecLimit(v uint64) { // return if Settings type is nil @@ -86,6 +88,20 @@ func (cs *Compiler) SetStarlarkExecLimit(v uint64) { cs.StarlarkExecLimit = &v } +// String implements the Stringer interface for the Compiler type. +func (cs *Compiler) String() string { + return fmt.Sprintf(`{ + CloneImage: %s, + TemplateDepth: %d, + StarlarkExecLimit: %d, +}`, + cs.GetCloneImage(), + cs.GetTemplateDepth(), + cs.GetStarlarkExecLimit(), + ) +} + +// CompilerMockEmpty returns an empty Compiler type. func CompilerMockEmpty() Compiler { cs := Compiler{} cs.SetCloneImage("") diff --git a/api/types/settings/compiler_test.go b/api/types/settings/compiler_test.go new file mode 100644 index 000000000..a224a9cb7 --- /dev/null +++ b/api/types/settings/compiler_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTypes_Compiler_Getters(t *testing.T) { + // setup tests + tests := []struct { + compiler *Compiler + want *Compiler + }{ + { + compiler: testCompilerSettings(), + want: testCompilerSettings(), + }, + { + compiler: new(Compiler), + want: new(Compiler), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.compiler.GetCloneImage(), test.want.GetCloneImage()) { + t.Errorf("GetCloneImage is %v, want %v", test.compiler.GetCloneImage(), test.want.GetCloneImage()) + } + + if !reflect.DeepEqual(test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) { + t.Errorf("GetTemplateDepth is %v, want %v", test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) + } + + if !reflect.DeepEqual(test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) { + t.Errorf("GetStarlarkExecLimit is %v, want %v", test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) + } + } +} + +func TestTypes_Compiler_Setters(t *testing.T) { + // setup types + var cs *Compiler + + // setup tests + tests := []struct { + compiler *Compiler + want *Compiler + }{ + { + compiler: testCompilerSettings(), + want: testCompilerSettings(), + }, + { + compiler: cs, + want: new(Compiler), + }, + } + + // run tests + for _, test := range tests { + test.compiler.SetCloneImage(test.want.GetCloneImage()) + + if !reflect.DeepEqual(test.compiler.GetCloneImage(), test.want.GetCloneImage()) { + t.Errorf("SetCloneImage is %v, want %v", test.compiler.GetCloneImage(), test.want.GetCloneImage()) + } + + test.compiler.SetTemplateDepth(test.want.GetTemplateDepth()) + + if !reflect.DeepEqual(test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) { + t.Errorf("SetTemplateDepth is %v, want %v", test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) + } + + test.compiler.SetStarlarkExecLimit(test.want.GetStarlarkExecLimit()) + + if !reflect.DeepEqual(test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) { + t.Errorf("SetStarlarkExecLimit is %v, want %v", test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) + } + } +} + +func TestTypes_Compiler_String(t *testing.T) { + // setup types + cs := testCompilerSettings() + + want := fmt.Sprintf(`{ + CloneImage: %s, + TemplateDepth: %d, + StarlarkExecLimit: %d, +}`, + cs.GetCloneImage(), + cs.GetTemplateDepth(), + cs.GetStarlarkExecLimit(), + ) + + // run test + got := cs.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testCompilerSettings is a test helper function to create a Compiler +// type with all fields set to a fake value. +func testCompilerSettings() *Compiler { + cs := new(Compiler) + + cs.SetCloneImage("target/vela-git:latest") + cs.SetTemplateDepth(1) + cs.SetStarlarkExecLimit(100) + + return cs +} diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index 50bfbd6d8..f7914aa95 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -10,22 +10,19 @@ import ( // // swagger:model Platform type Platform struct { - ID *int64 `json:"id"` - - *Queue `json:"queue"` - *Compiler `json:"compiler"` - - // misc + ID *int64 `json:"id"` + *Queue `json:"queue"` + *Compiler `json:"compiler"` RepoAllowlist *[]string `json:"repo_allowlist"` ScheduleAllowlist *[]string `json:"schedule_allowlist"` } // GetID returns the ID field. // -// When the provided Settings type is nil, or the field within +// When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. func (s *Platform) GetID() int64 { - // return zero value if Settings type or ID field is nil + // return zero value if Platform type or ID field is nil if s == nil || s.ID == nil { return 0 } @@ -33,12 +30,64 @@ func (s *Platform) GetID() int64 { return *s.ID } +// GetCompiler returns the Compiler field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetCompiler() Compiler { + // return zero value if Platform type or Compiler field is nil + if s == nil || s.Compiler == nil { + return Compiler{} + } + + return *s.Compiler +} + +// GetQueue returns the Queue field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetQueue() Queue { + // return zero value if Platform type or Queue field is nil + if s == nil || s.Queue == nil { + return Queue{} + } + + return *s.Queue +} + +// GetRepoAllowlist returns the RepoAllowlist field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetRepoAllowlist() []string { + // return zero value if Platform type or RepoAllowlist field is nil + if s == nil || s.RepoAllowlist == nil { + return []string{} + } + + return *s.RepoAllowlist +} + +// GetScheduleAllowlist returns the ScheduleAllowlist field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetScheduleAllowlist() []string { + // return zero value if Platform type or ScheduleAllowlist field is nil + if s == nil || s.ScheduleAllowlist == nil { + return []string{} + } + + return *s.ScheduleAllowlist +} + // SetID sets the ID field. // -// When the provided Settings type is nil, it +// When the provided Platform type is nil, it // will set nothing and immediately return. func (s *Platform) SetID(v int64) { - // return if Settings type is nil + // return if Platform type is nil if s == nil { return } @@ -46,12 +95,12 @@ func (s *Platform) SetID(v int64) { s.ID = &v } -// SetCompilerSettings sets the CompilerSettings field. +// SetCompiler sets the Compiler field. // -// When the provided CompilerSettings type is nil, it +// When the provided Compiler type is nil, it // will set nothing and immediately return. -func (s *Platform) SetCompilerSettings(cs Compiler) { - // return if Settings type is nil +func (s *Platform) SetCompiler(cs Compiler) { + // return if Platform type is nil if s == nil { return } @@ -59,12 +108,12 @@ func (s *Platform) SetCompilerSettings(cs Compiler) { s.Compiler = &cs } -// SetQueueSettings sets the QueueSettings field. +// SetQueue sets the Queue field. // -// When the provided QueueSettings type is nil, it +// When the provided Queue type is nil, it // will set nothing and immediately return. -func (s *Platform) SetQueueSettings(qs Queue) { - // return if Settings type is nil +func (s *Platform) SetQueue(qs Queue) { + // return if Platform type is nil if s == nil { return } @@ -72,25 +121,12 @@ func (s *Platform) SetQueueSettings(qs Queue) { s.Queue = &qs } -// GetRepoAllowlist returns the RepoAllowlist field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Platform) GetRepoAllowlist() []string { - // return zero value if Settings type or RepoAllowlist field is nil - if s == nil || s.RepoAllowlist == nil { - return []string{} - } - - return *s.RepoAllowlist -} - // SetRepoAllowlist sets the RepoAllowlist field. // -// When the provided Settings type is nil, it +// When the provided Platform type is nil, it // will set nothing and immediately return. func (s *Platform) SetRepoAllowlist(v []string) { - // return if Settings type is nil + // return if Platform type is nil if s == nil { return } @@ -98,25 +134,12 @@ func (s *Platform) SetRepoAllowlist(v []string) { s.RepoAllowlist = &v } -// GetScheduleAllowlist returns the ScheduleAllowlist field. -// -// When the provided Settings type is nil, or the field within -// the type is nil, it returns the zero value for the field. -func (s *Platform) GetScheduleAllowlist() []string { - // return zero value if Settings type or ScheduleAllowlist field is nil - if s == nil || s.ScheduleAllowlist == nil { - return []string{} - } - - return *s.ScheduleAllowlist -} - // SetScheduleAllowlist sets the RepoAllowlist field. // -// When the provided Settings type is nil, it +// When the provided Platform type is nil, it // will set nothing and immediately return. func (s *Platform) SetScheduleAllowlist(v []string) { - // return if Settings type is nil + // return if Platform type is nil if s == nil { return } @@ -124,37 +147,22 @@ func (s *Platform) SetScheduleAllowlist(v []string) { s.ScheduleAllowlist = &v } -// String implements the Stringer interface for the Settings type. +// String implements the Stringer interface for the Platform type. func (s *Platform) String() string { + cs := s.GetCompiler() + qs := s.GetQueue() + return fmt.Sprintf(`{ ID: %d, - CloneImage: %s, - QueueRoutes: %v, - other stuff: %v, + Compiler: %v, + Queue: %v, + RepoAllowlist: %v, + ScheduleAllowlist: %v, }`, s.GetID(), - s.GetCloneImage(), - s.GetRoutes(), - s.GetRoutes(), - ) -} - -// ToEnv converts the Settings type to a string format compatible with standard posix environments. -func (s *Platform) ToEnv() string { - return fmt.Sprintf(`VELA_CLONE_IMAGE='%s' -VELA_QUEUE_ROUTES='%v' -`, - s.GetCloneImage(), - s.GetRoutes(), - ) -} - -// ToYAML converts the Settings type to a YAML string. -func (s *Platform) ToYAML() string { - return fmt.Sprintf(`VELA_CLONE_IMAGE: '%s' -VELA_QUEUE_ROUTES: '%s' -`, - s.GetCloneImage(), - s.GetRoutes(), + cs.String(), + qs.String(), + s.GetRepoAllowlist(), + s.GetScheduleAllowlist(), ) } diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go new file mode 100644 index 000000000..794a01b35 --- /dev/null +++ b/api/types/settings/platform_test.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTypes_Platform_Getters(t *testing.T) { + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: testPlatformSettings(), + want: testPlatformSettings(), + }, + { + platform: new(Platform), + want: new(Platform), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.platform.GetCompiler(), test.want.GetCompiler()) { + t.Errorf("GetCompiler is %v, want %v", test.platform.GetCompiler(), test.want.GetCompiler()) + } + + if !reflect.DeepEqual(test.platform.GetQueue(), test.want.GetQueue()) { + t.Errorf("GetQueue is %v, want %v", test.platform.GetQueue(), test.want.GetQueue()) + } + + if !reflect.DeepEqual(test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) { + t.Errorf("GetRepoAllowlist is %v, want %v", test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) + } + + if !reflect.DeepEqual(test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) { + t.Errorf("GetScheduleAllowlist is %v, want %v", test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) + } + } +} + +func TestTypes_Platform_Setters(t *testing.T) { + // setup types + var s *Platform + + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: testPlatformSettings(), + want: testPlatformSettings(), + }, + { + platform: s, + want: new(Platform), + }, + } + + // run tests + for _, test := range tests { + test.platform.SetCompiler(test.want.GetCompiler()) + + if !reflect.DeepEqual(test.platform.GetCompiler(), test.want.GetCompiler()) { + t.Errorf("SetCompiler is %v, want %v", test.platform.GetCompiler(), test.want.GetCompiler()) + } + + test.platform.SetQueue(test.want.GetQueue()) + + if !reflect.DeepEqual(test.platform.GetQueue(), test.want.GetQueue()) { + t.Errorf("SetQueue is %v, want %v", test.platform.GetQueue(), test.want.GetQueue()) + } + + test.platform.SetRepoAllowlist(test.want.GetRepoAllowlist()) + + if !reflect.DeepEqual(test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) { + t.Errorf("SetRepoAllowlist is %v, want %v", test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) + } + + test.platform.SetScheduleAllowlist(test.want.GetScheduleAllowlist()) + + if !reflect.DeepEqual(test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) { + t.Errorf("SetScheduleAllowlist is %v, want %v", test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) + } + } +} + +func TestTypes_Platform_String(t *testing.T) { + // setup types + s := testPlatformSettings() + cs := s.GetCompiler() + qs := s.GetQueue() + + want := fmt.Sprintf(`{ + ID: %d, + Compiler: %v, + Queue: %v, + RepoAllowlist: %v, + ScheduleAllowlist: %v, +}`, + s.GetID(), + cs.String(), + qs.String(), + s.GetRepoAllowlist(), + s.GetScheduleAllowlist(), + ) + + // run test + got := s.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testPlatformSettings is a test helper function to create a Platform +// type with all fields set to a fake value. +func testPlatformSettings() *Platform { + // setup types + // setup compiler + cs := new(Compiler) + + cs.SetCloneImage("target/vela-git:latest") + cs.SetTemplateDepth(1) + cs.SetStarlarkExecLimit(100) + + // setup queue + qs := new(Queue) + + qs.SetRoutes([]string{"vela"}) + + // setup platform + s := new(Platform) + + s.SetCompiler(*cs) + s.SetQueue(*qs) + + return s +} diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go index 185d2ae3f..9b507dd82 100644 --- a/api/types/settings/queue.go +++ b/api/types/settings/queue.go @@ -2,6 +2,8 @@ package settings +import "fmt" + type Queue struct { Routes *[]string `json:"routes,omitempty"` } @@ -32,6 +34,16 @@ func (qs *Queue) SetRoutes(v []string) { qs.Routes = &v } +// String implements the Stringer interface for the Queue type. +func (qs *Queue) String() string { + return fmt.Sprintf(`{ + Routes: %v, +}`, + qs.GetRoutes(), + ) +} + +// QueueMockEmpty returns an empty Queue type. func QueueMockEmpty() Queue { qs := Queue{} qs.SetRoutes([]string{}) diff --git a/api/types/settings/queue_test.go b/api/types/settings/queue_test.go new file mode 100644 index 000000000..30389b881 --- /dev/null +++ b/api/types/settings/queue_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTypes_Queue_Getters(t *testing.T) { + // setup tests + tests := []struct { + queue *Queue + want *Queue + }{ + { + queue: testQueueSettings(), + want: testQueueSettings(), + }, + { + queue: new(Queue), + want: new(Queue), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.queue.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("GetRoutes is %v, want %v", test.queue.GetRoutes(), test.want.GetRoutes()) + } + } +} + +func TestTypes_Queue_Setters(t *testing.T) { + // setup types + var qs *Queue + + // setup tests + tests := []struct { + queue *Queue + want *Queue + }{ + { + queue: testQueueSettings(), + want: testQueueSettings(), + }, + { + queue: qs, + want: new(Queue), + }, + } + + // run tests + for _, test := range tests { + test.queue.SetRoutes(test.want.GetRoutes()) + + if !reflect.DeepEqual(test.queue.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("SetRoutes is %v, want %v", test.queue.GetRoutes(), test.want.GetRoutes()) + } + } +} + +func TestTypes_Queue_String(t *testing.T) { + // setup types + qs := testQueueSettings() + + want := fmt.Sprintf(`{ + Routes: %s, +}`, + qs.GetRoutes(), + ) + + // run test + got := qs.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testQueueSettings is a test helper function to create a Queue +// type with all fields set to a fake value. +func testQueueSettings() *Queue { + qs := new(Queue) + + qs.SetRoutes([]string{"vela"}) + + return qs +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index a897178c3..37bf95a5e 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -110,10 +110,10 @@ func server(c *cli.Context) error { // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() - s.SetCompilerSettings(compilerSettings) + s.SetCompiler(compilerSettings) queueSettings := queue.GetSettings() - s.SetQueueSettings(queueSettings) + s.SetQueue(queueSettings) // set repos permitted to be added s.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) From aaf0038c5ef545a771bbcbd71dc03abb671b094f Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 13:03:12 -0500 Subject: [PATCH 29/65] feat: db integration tests --- database/integration_test.go | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/database/integration_test.go b/database/integration_test.go index 36e2cb5a6..3fe4c8caf 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/google/go-cmp/cmp" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -24,6 +25,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + dbSettings "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/testutils" "github.com/go-vela/server/database/user" @@ -49,6 +51,7 @@ type Resources struct { Steps []*library.Step Users []*api.User Workers []*api.Worker + Platform []*settings.Platform } func TestDatabase_Integration(t *testing.T) { @@ -150,6 +153,8 @@ func TestDatabase_Integration(t *testing.T) { t.Run("test_workers", func(t *testing.T) { testWorkers(t, db, resources) }) + t.Run("test_settings", func(t *testing.T) { testSettings(t, db, resources) }) + err = db.Close() if err != nil { t.Errorf("unable to close database engine for %s: %v", test.name, err) @@ -2135,6 +2140,65 @@ func testWorkers(t *testing.T, db Interface, resources *Resources) { } } +func testSettings(t *testing.T, db Interface, resources *Resources) { + // create a variable to track the number of methods called for settings + methods := make(map[string]bool) + // capture the element type of the settings interface + element := reflect.TypeOf(new(dbSettings.SettingsInterface)).Elem() + // iterate through all methods found in the settings interface + for i := 0; i < element.NumMethod(); i++ { + // skip tracking the methods to create indexes and tables for settings + // since those are already called when the database engine starts + if strings.Contains(element.Method(i).Name, "Index") || + strings.Contains(element.Method(i).Name, "Table") { + continue + } + + // add the method name to the list of functions + methods[element.Method(i).Name] = false + } + + // create the settings + for _, s := range resources.Platform { + _, err := db.CreateSettings(context.TODO(), s) + if err != nil { + t.Errorf("unable to create settings %d: %v", s.GetID(), err) + } + } + methods["CreateSettings"] = true + + // update the settings + for _, s := range resources.Platform { + s.SetCloneImage("target/vela-git:abc123") + got, err := db.UpdateSettings(context.TODO(), s) + if err != nil { + t.Errorf("unable to update settings %d: %v", s.GetID(), err) + } + + if !cmp.Equal(got, s) { + t.Errorf("UpdateSettings() is %v, want %v", got, s) + } + } + methods["UpdateSettings"] = true + methods["GetSettings"] = true + + // delete the settings + for _, s := range resources.Platform { + err := db.DeleteSettings(context.TODO(), s) + if err != nil { + t.Errorf("unable to delete settings %d: %v", s.GetID(), err) + } + } + methods["DeleteSettings"] = true + + // ensure we called all the methods we expected to + for method, called := range methods { + if !called { + t.Errorf("method %s was not called for settings", method) + } + } +} + func newResources() *Resources { userOne := new(api.User) userOne.SetID(1) From dfdcec50bde01d51c3608a10da3381353c490454 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 13:15:52 -0500 Subject: [PATCH 30/65] chore: remove unneeded api endpoints --- api/admin/settings.go | 51 ++++----------------------- mock/server/server.go | 1 - mock/server/settings.go | 68 ++++++++++++++++-------------------- mock/server/settings_test.go | 36 ------------------- router/admin.go | 3 +- 5 files changed, 38 insertions(+), 121 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 92d0229b5..9d24b6de3 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -47,9 +47,9 @@ func GetSettings(c *gin.Context) { c.JSON(http.StatusOK, s) } -// swagger:operation PUT /api/v1/admin/settings admin AdminUpdateSettings +// swagger:operation PUT /api/v1/admin/settings admin UpdateSettings // -// Update the settings singleton in the database +// Update the settings singleton in the database. // // --- // produces: @@ -113,7 +113,9 @@ func UpdateSettings(c *gin.Context) { } if input.Queue != nil { - s.SetRoutes(input.GetRoutes()) + if input.Queue.Routes != nil { + s.SetRoutes(input.GetRoutes()) + } } if input.RepoAllowlist != nil { @@ -126,7 +128,7 @@ func UpdateSettings(c *gin.Context) { s.SetScheduleAllowlist(input.GetScheduleAllowlist()) } - // send API call to update the repo + // send API call to update the settings s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { retErr := fmt.Errorf("unable to update settings: %w", err) @@ -138,44 +140,3 @@ func UpdateSettings(c *gin.Context) { c.JSON(http.StatusOK, s) } - -// swagger:operation DELETE /api/v1/admin/settings admin DeleteSettings -// -// Delete the platform settings record -// -// --- -// produces: -// - application/json -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the platform settings record -// schema: -// type: string -// '500': -// description: Unable to delete the platform settings record -// schema: -// "$ref": "#/definitions/Error" - -// DeleteSettings represents the API handler to remove -// the platform settings singleton from the configured backend. -func DeleteSettings(c *gin.Context) { - // capture middleware values - s := sMiddleware.FromContext(c) - ctx := c.Request.Context() - - logrus.Info("Admin: deleting settings") - - // send API call to remove the settings record - err := database.FromContext(c).DeleteSettings(ctx, s) - if err != nil { - retErr := fmt.Errorf("unable to delete platform settings: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, "platform settings deleted") -} diff --git a/mock/server/server.go b/mock/server/server.go index 9fe2c9f90..882e7a8a4 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -31,7 +31,6 @@ func FakeHandler() http.Handler { e.PUT("api/v1/admin/clean", cleanResoures) e.GET("/api/v1/admin/settings", getSettings) e.PUT("/api/v1/admin/settings", updateSettings) - e.DELETE("/api/v1/admin/settings", deleteSettings) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/mock/server/settings.go b/mock/server/settings.go index 882ec61ea..8a7c8b92e 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -16,38 +16,44 @@ const ( SettingsResp = ` { "id": 1, - "compiler": {}, - "queue": {}, - "repo_allowlist": [], - "schedule_allowlist": [] + "compiler": { + "clone_image": "target/vela-git", + "template_depth": 3, + "starlark_exec_limit": 100 + }, + "queue": { + "routes": [ + "vela" + ] + }, + "repo_allowlist": [ + "*" + ], + "schedule_allowlist": [ + "octocat/hello-world" + ] }` - // CreateSettingsResp represents a JSON return for creating a settings record. - CreateSettingsResp = ` - { - "id": 1, - "compiler": {}, - "queue": {}, - "repo_allowlist": [], - "schedule_allowlist": [] - }` // UpdateSettingsResp represents a JSON return for modifying a settings field. UpdateSettingsResp = ` { "id": 1, - "compiler": {}, - "queue": {}, - "repo_allowlist": [], - "schedule_allowlist": [] - }` - // DeleteSettingsResp represents a JSON return for deleting a settings record. - DeleteSettingsResp = ` - { - "id": 1, - "compiler": {}, - "queue": {}, + "compiler": { + "clone_image": "target/vela-git:latest", + "template_depth": 5, + "starlark_exec_limit": 123 + }, + "queue": { + "routes": [ + "vela", + "large" + ] + }, "repo_allowlist": [], - "schedule_allowlist": [] + "schedule_allowlist": [ + "octocat/hello-world", + "octocat/*" + ] }` ) @@ -70,15 +76,3 @@ func updateSettings(c *gin.Context) { c.JSON(http.StatusOK, body) } - -// deleteSettings has a param :settings returns mock JSON for a http DELETE. -// -// Pass "0" to :settings to test receiving a http 404 response. -func deleteSettings(c *gin.Context) { - data := []byte(DeleteSettingsResp) - - var body settings.Platform - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusOK, body) -} diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go index 3b07d0e1b..7061d4409 100644 --- a/mock/server/settings_test.go +++ b/mock/server/settings_test.go @@ -28,24 +28,6 @@ func TestSettings_GetResp(t *testing.T) { } } -func TestSettings_CreateResp(t *testing.T) { - testSettings := settings.Platform{} - - err := json.Unmarshal([]byte(CreateSettingsResp), &testSettings) - if err != nil { - t.Errorf("error unmarshaling settings: %v", err) - } - - tSettings := reflect.TypeOf(testSettings) - - for i := 0; i < tSettings.NumField(); i++ { - f := reflect.ValueOf(testSettings).Field(i) - if f.IsNil() { - t.Errorf("CreateSettingsResp missing field %s", tSettings.Field(i).Name) - } - } -} - func TestSettings_UpdateResp(t *testing.T) { testSettings := settings.Platform{} @@ -63,21 +45,3 @@ func TestSettings_UpdateResp(t *testing.T) { } } } - -func TestSettings_RemoveResp(t *testing.T) { - testSettings := settings.Platform{} - - err := json.Unmarshal([]byte(DeleteSettingsResp), &testSettings) - if err != nil { - t.Errorf("error unmarshaling settings: %v", err) - } - - tSettings := reflect.TypeOf(testSettings) - - for i := 0; i < tSettings.NumField(); i++ { - f := reflect.ValueOf(testSettings).Field(i) - if f.IsNil() { - t.Errorf("DeleteSettingsResp missing field %s", tSettings.Field(i).Name) - } - } -} diff --git a/router/admin.go b/router/admin.go index 653cd14f2..2abd3b612 100644 --- a/router/admin.go +++ b/router/admin.go @@ -64,9 +64,8 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin worker endpoint _admin.POST("/workers/:worker/register", admin.RegisterToken) - // Admin settings endpoint + // Admin settings endpoints _admin.GET("/settings", admin.GetSettings) _admin.PUT("/settings", admin.UpdateSettings) - _admin.DELETE("/settings", admin.DeleteSettings) } // end of admin endpoints } From 1e0b4da51f7bc3bbdef4d3538ab0f44ee6308913 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 14:08:47 -0500 Subject: [PATCH 31/65] feat: db tests and general cleanup --- api/admin/settings.go | 2 - api/types/settings/compiler.go | 8 +- api/types/settings/platform.go | 13 ++ api/types/settings/queue.go | 8 +- cmd/vela-server/server.go | 18 ++- compiler/native/settings.go | 2 +- database/integration_test.go | 9 -- database/settings/create.go | 2 +- database/settings/create_test.go | 79 ++++++++++ database/settings/delete.go | 22 --- database/settings/get.go | 1 - database/settings/get_test.go | 89 +++++++++++ database/settings/interface.go | 2 - database/settings/{opt.go => opts.go} | 0 database/settings/opts_test.go | 208 ++++++++++++++++++++++++++ database/settings/settings.go | 17 ++- database/settings/settings_test.go | 171 +++++++++++++++++++++ database/settings/table_test.go | 58 +++++++ database/settings/update_test.go | 81 ++++++++++ queue/redis/settings.go | 2 +- 20 files changed, 735 insertions(+), 57 deletions(-) create mode 100644 database/settings/create_test.go delete mode 100644 database/settings/delete.go create mode 100644 database/settings/get_test.go rename database/settings/{opt.go => opts.go} (100%) create mode 100644 database/settings/opts_test.go create mode 100644 database/settings/settings_test.go create mode 100644 database/settings/table_test.go create mode 100644 database/settings/update_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 9d24b6de3..660e97c52 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -119,12 +119,10 @@ func UpdateSettings(c *gin.Context) { } if input.RepoAllowlist != nil { - // update repo allowlist if set s.SetRepoAllowlist(input.GetRepoAllowlist()) } if input.ScheduleAllowlist != nil { - // update schedule allowlist if set s.SetScheduleAllowlist(input.GetScheduleAllowlist()) } diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go index e79717301..a18a31401 100644 --- a/api/types/settings/compiler.go +++ b/api/types/settings/compiler.go @@ -41,7 +41,7 @@ func (cs *Compiler) GetTemplateDepth() int { // When the provided Compiler type is nil, or the field within // the type is nil, it returns the zero value for the field. func (cs *Compiler) GetStarlarkExecLimit() uint64 { - // return zero value if Settings type or StarlarkExecLimit field is nil + // return zero value if Compiler type or StarlarkExecLimit field is nil if cs == nil || cs.StarlarkExecLimit == nil { return 0 } @@ -54,7 +54,7 @@ func (cs *Compiler) GetStarlarkExecLimit() uint64 { // When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetCloneImage(v string) { - // return if Settings type is nil + // return if Compiler type is nil if cs == nil { return } @@ -67,7 +67,7 @@ func (cs *Compiler) SetCloneImage(v string) { // When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetTemplateDepth(v int) { - // return if Settings type is nil + // return if Compiler type is nil if cs == nil { return } @@ -80,7 +80,7 @@ func (cs *Compiler) SetTemplateDepth(v int) { // When the provided Compiler type is nil, it // will set nothing and immediately return. func (cs *Compiler) SetStarlarkExecLimit(v uint64) { - // return if Settings type is nil + // return if Compiler type is nil if cs == nil { return } diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index f7914aa95..c51ae409c 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -166,3 +166,16 @@ func (s *Platform) String() string { s.GetScheduleAllowlist(), ) } + +// PlatformMockEmpty returns an empty Platform type. +func PlatformMockEmpty() Platform { + s := Platform{} + + s.SetCompiler(CompilerMockEmpty()) + s.SetQueue(QueueMockEmpty()) + + s.SetRepoAllowlist([]string{}) + s.SetScheduleAllowlist([]string{}) + + return s +} diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go index 9b507dd82..9bf54dfd1 100644 --- a/api/types/settings/queue.go +++ b/api/types/settings/queue.go @@ -10,10 +10,10 @@ type Queue struct { // GetRoutes returns the Routes field. // -// When the provided QueueSettings type is nil, or the field within +// When the provided Queue type is nil, or the field within // the type is nil, it returns the zero value for the field. func (qs *Queue) GetRoutes() []string { - // return zero value if Settings type or Routes field is nil + // return zero value if Queue type or Routes field is nil if qs == nil || qs.Routes == nil { return []string{} } @@ -23,10 +23,10 @@ func (qs *Queue) GetRoutes() []string { // SetRoutes sets the Routes field. // -// When the provided Settings type is nil, it +// When the provided Queue type is nil, it // will set nothing and immediately return. func (qs *Queue) SetRoutes(v []string) { - // return if Settings type is nil + // return if Queue type is nil if qs == nil { return } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 37bf95a5e..7e8cda23e 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -102,7 +102,11 @@ func server(c *cli.Context) error { s, err := database.GetSettings(context.Background()) if s == nil || err != nil { - // ignore error and attempt to create initial settings record + // only log error and attempt to create initial settings record + if err != nil { + logrus.WithError(err).Error("unable to get platform settings") + } + s := new(settings.Platform) // singleton record ID should always be 1 @@ -255,11 +259,15 @@ func server(c *cli.Context) error { time.Sleep(jitter) s, err = database.GetSettings(context.Background()) - if s == nil || err != nil { - if s == nil { - err = errors.New("settings not found") - } + if s == nil { + err = errors.New("settings not found") + + logrus.WithError(err).Warn("unable to get platform settings") + continue + } + + if err != nil { logrus.WithError(err).Warn("unable to get platform settings") continue diff --git a/compiler/native/settings.go b/compiler/native/settings.go index 578b8ec23..12997ec9c 100644 --- a/compiler/native/settings.go +++ b/compiler/native/settings.go @@ -6,7 +6,7 @@ import ( "github.com/go-vela/server/api/types/settings" ) -// SetSettings sets the api settings type. +// GetSettings retrieves the api settings type. func (c *client) GetSettings() settings.Compiler { return c.Compiler } diff --git a/database/integration_test.go b/database/integration_test.go index 3fe4c8caf..45d31d502 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -2182,15 +2182,6 @@ func testSettings(t *testing.T, db Interface, resources *Resources) { methods["UpdateSettings"] = true methods["GetSettings"] = true - // delete the settings - for _, s := range resources.Platform { - err := db.DeleteSettings(context.TODO(), s) - if err != nil { - t.Errorf("unable to delete settings %d: %v", s.GetID(), err) - } - } - methods["DeleteSettings"] = true - // ensure we called all the methods we expected to for method, called := range methods { if !called { diff --git a/database/settings/create.go b/database/settings/create.go index a0db02cea..9691db532 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -8,7 +8,7 @@ import ( "github.com/go-vela/server/api/types/settings" ) -// CreateSettings updates a platform settings in the database. +// CreateSettings creates a platform settings record in the database. func (e *engine) CreateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Tracef("creating platform settings in the database with %v", s.String()) diff --git a/database/settings/create_test.go b/database/settings/create_test.go new file mode 100644 index 000000000..0a48b9c69 --- /dev/null +++ b/database/settings/create_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSettings_Engine_CreateSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "settings" ("compiler","queue","repo_allowlist","schedule_allowlist","id") VALUES ($1,$2,$3,$4,$5) RETURNING "id"`). + WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CreateSettings(context.TODO(), _settings) + + if test.failure { + if err == nil { + t.Errorf("CreateSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, _settings) { + t.Errorf("CreateSettings for %s returned %s, want %s", test.name, got, _settings) + } + }) + } +} diff --git a/database/settings/delete.go b/database/settings/delete.go deleted file mode 100644 index 83f374ae2..000000000 --- a/database/settings/delete.go +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package settings - -import ( - "context" - - "github.com/go-vela/server/api/types/settings" -) - -// DeleteSettings removes platform settings from the database. -func (e *engine) DeleteSettings(ctx context.Context, s *settings.Platform) error { - e.logger.Trace("deleting platform settings from the database") - - worker := FromAPI(s) - - // send query to the database - return e.client. - Table(TableSettings). - Delete(worker). - Error -} diff --git a/database/settings/get.go b/database/settings/get.go index 73bd00d68..b5d7d0775 100644 --- a/database/settings/get.go +++ b/database/settings/get.go @@ -18,7 +18,6 @@ func (e *engine) GetSettings(ctx context.Context) (*settings.Platform, error) { // send query to the database and store result in variable err := e.client. Table(TableSettings). - // todo: how to ensure this is always a singleton at the first row Where("id = ?", 1). Take(s). Error diff --git a/database/settings/get_test.go b/database/settings/get_test.go new file mode 100644 index 000000000..7f587e57a --- /dev/null +++ b/database/settings/get_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_Engine_GetSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "compiler", "queue", "repo_allowlist", "schedule_allowlist"}). + AddRow(1, `{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "settings" WHERE id = $1 LIMIT $2`).WithArgs(1, 1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateSettings(context.TODO(), _settings) + if err != nil { + t.Errorf("unable to create test settings for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *settings.Platform + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _settings, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _settings, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetSettings(context.TODO()) + + if test.failure { + if err == nil { + t.Errorf("GetSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetSettings for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/settings/interface.go b/database/settings/interface.go index 3725a700f..a7cc755cd 100644 --- a/database/settings/interface.go +++ b/database/settings/interface.go @@ -19,6 +19,4 @@ type SettingsInterface interface { GetSettings(context.Context) (*settings.Platform, error) // UpdateSettings defines a function that updates platform settings. UpdateSettings(context.Context, *settings.Platform) (*settings.Platform, error) - // DeleteSettings defines a function that deletes platform settings. - DeleteSettings(context.Context, *settings.Platform) error } diff --git a/database/settings/opt.go b/database/settings/opts.go similarity index 100% rename from database/settings/opt.go rename to database/settings/opts.go diff --git a/database/settings/opts_test.go b/database/settings/opts_test.go new file mode 100644 index 000000000..6fccdec17 --- /dev/null +++ b/database/settings/opts_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestSettings_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithContext(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + ctx context.Context + want context.Context + }{ + { + failure: false, + name: "context set to TODO", + ctx: context.TODO(), + want: context.TODO(), + }, + { + failure: false, + name: "context set to nil", + ctx: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithContext(test.ctx)(e) + + if test.failure { + if err == nil { + t.Errorf("WithContext for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithContext returned err: %v", err) + } + + if !reflect.DeepEqual(e.ctx, test.want) { + t.Errorf("WithContext is %v, want %v", e.ctx, test.want) + } + }) + } +} diff --git a/database/settings/settings.go b/database/settings/settings.go index 3cdc5ddb9..2ea1095b2 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -60,8 +60,8 @@ type ( Compiler Queue - RepoAllowlist pq.StringArray `sql:"repo_allowlist" gorm:"type:varchar(1000)"` - ScheduleAllowlist pq.StringArray `sql:"schedule_allowlist" gorm:"type:varchar(1000)"` + RepoAllowlist pq.StringArray `json:"repo_allowlist" sql:"repo_allowlist" gorm:"type:varchar(1000)"` + ScheduleAllowlist pq.StringArray `json:"schedule_allowlist" sql:"schedule_allowlist" gorm:"type:varchar(1000)"` } // Compiler is the database representation of compiler settings. @@ -73,7 +73,7 @@ type ( // Queue is the database representation of queue settings. Queue struct { - Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` + Routes pq.StringArray `json:"routes" sql:"routes" gorm:"type:varchar(1000)"` } ) @@ -146,8 +146,6 @@ func New(opts ...EngineOpt) (*engine, error) { return nil, fmt.Errorf("unable to create %s table: %w", TableSettings, err) } - // todo: need indexes? - return e, nil } @@ -203,6 +201,15 @@ func (s *Platform) Validate() error { return ErrEmptyCloneImage } + // verify compiler settings are within limits + if s.TemplateDepth.Int64 <= 0 { + return fmt.Errorf("template depth must be greater than zero, got: %d", s.TemplateDepth.Int64) + } + + if s.StarlarkExecLimit.Int64 <= 0 { + return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", s.StarlarkExecLimit.Int64) + } + // ensure that all Settings string fields // that can be returned as JSON are sanitized // to avoid unsafe HTML content diff --git a/database/settings/settings_test.go b/database/settings/settings_test.go new file mode 100644 index 000000000..0e1ac773f --- /dev/null +++ b/database/settings/settings_test.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres settings engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite settings engine: %v", err) + } + + return _engine +} + +// testSettings is a test helper function to create an api +// Platform type with all fields set to their zero values. +func testSettings() *settings.Platform { + s := settings.PlatformMockEmpty() + + return &s +} diff --git a/database/settings/table_test.go b/database/settings/table_test.go new file mode 100644 index 000000000..13eaab42b --- /dev/null +++ b/database/settings/table_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSettings_Engine_CreateSettingsTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateSettingsTable(context.TODO(), test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateSettingsTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateSettingsTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/settings/update_test.go b/database/settings/update_test.go new file mode 100644 index 000000000..944533972 --- /dev/null +++ b/database/settings/update_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSettings_Engine_UpdateSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela", "large"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "settings" SET "compiler"=$1,"queue"=$2,"repo_allowlist"=$3,"schedule_allowlist"=$4 WHERE "id" = $5`). + WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateSettings(context.TODO(), _settings) + if err != nil { + t.Errorf("unable to create test settings for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.UpdateSettings(context.TODO(), _settings) + + if test.failure { + if err == nil { + t.Errorf("UpdateSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, _settings) { + t.Errorf("UpdateSettings for %s returned %s, want %s", test.name, got, _settings) + } + }) + } +} diff --git a/queue/redis/settings.go b/queue/redis/settings.go index 118081462..70747b6c3 100644 --- a/queue/redis/settings.go +++ b/queue/redis/settings.go @@ -6,7 +6,7 @@ import ( "github.com/go-vela/server/api/types/settings" ) -// SetSettings sets the api settings type in the Engine. +// GetSettings retrieves the api settings type in the Engine. func (c *client) GetSettings() settings.Queue { return c.Queue } From b0d9bbca216845536f125f78dbe88e92345e2396 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 14:12:33 -0500 Subject: [PATCH 32/65] chore: lint --- database/settings/create.go | 2 +- database/settings/table.go | 2 +- database/settings/update.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/database/settings/create.go b/database/settings/create.go index 9691db532..d6636e5e5 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -9,7 +9,7 @@ import ( ) // CreateSettings creates a platform settings record in the database. -func (e *engine) CreateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { +func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Tracef("creating platform settings in the database with %v", s.String()) // cast the api type to database type diff --git a/database/settings/table.go b/database/settings/table.go index 7e97e7466..7b7c3a8cc 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -37,7 +37,7 @@ settings ( ) // CreateSettingsTable creates the settings table in the database. -func (e *engine) CreateSettingsTable(ctx context.Context, driver string) error { +func (e *engine) CreateSettingsTable(_ context.Context, driver string) error { e.logger.Tracef("creating settings table in the database") // handle the driver provided to create the table diff --git a/database/settings/update.go b/database/settings/update.go index 32ecf2778..c23956abd 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -9,7 +9,7 @@ import ( ) // UpdateSettings updates a platform settings in the database. -func (e *engine) UpdateSettings(ctx context.Context, s *settings.Platform) (*settings.Platform, error) { +func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { e.logger.Trace("updating platform settings in the database") // cast the api type to database type From af4acdd7ff7bb09ea1ab645503ac8d1ffb30afbc Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 29 Apr 2024 14:21:28 -0500 Subject: [PATCH 33/65] feat: flag for resetting settings record on startup --- cmd/vela-server/main.go | 6 ++++++ cmd/vela-server/server.go | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 5ce4c7bd7..689a45aa6 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -79,6 +79,12 @@ func main() { Name: "vela-secret", Usage: "secret used for server <-> agent communication", }, + &cli.BoolFlag{ + EnvVars: []string{"VELA_REINITIALIZE_SETTINGS_ON_STARTUP"}, + Name: "vela-reinitialize-settings-on-startup", + Usage: "instructs the server to reinitialize the settings singleton on startup, overwriting any updates currently stored in the database and restoring them to the configured environment values", + Value: false, + }, &cli.StringFlag{ EnvVars: []string{"VELA_SERVER_PRIVATE_KEY"}, Name: "vela-server-private-key", diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 7e8cda23e..099aba7eb 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -101,13 +101,14 @@ func server(c *cli.Context) error { } s, err := database.GetSettings(context.Background()) - if s == nil || err != nil { - // only log error and attempt to create initial settings record + if c.Bool("vela-reinitialize-settings-on-startup") || s == nil || err != nil { + // only log errors if err != nil { logrus.WithError(err).Error("unable to get platform settings") } - s := new(settings.Platform) + // create initial settings record + s = new(settings.Platform) // singleton record ID should always be 1 s.SetID(1) From d32244977c2dd5d51be5d51c70cf67e92e22dfca Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 30 Apr 2024 09:31:42 -0500 Subject: [PATCH 34/65] fix: get template depth in log, not pointer --- compiler/native/expand.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 14e96efda..26bf6199b 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -51,7 +51,7 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * // return if max template depth has been reached if depth == 0 { - retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + retErr := fmt.Errorf("max template depth of %d exceeded", c.GetTemplateDepth()) return s, retErr } From be90441311c02dab82b2f1963e9073cac4d8a75a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 May 2024 10:08:26 -0500 Subject: [PATCH 35/65] enhance: timestamps for created and updated --- api/admin/settings.go | 6 ++ api/types/settings/platform.go | 87 +++++++++++++++++++++++++++++ api/types/settings/platform_test.go | 6 ++ cmd/vela-server/server.go | 6 ++ database/secret/create_test.go | 7 ++- database/secret/secret_test.go | 28 ---------- database/secret/update_test.go | 7 ++- database/settings/create_test.go | 7 ++- database/settings/get_test.go | 7 ++- database/settings/settings.go | 26 +++++++++ database/settings/table.go | 10 +++- database/settings/update_test.go | 8 ++- mock/database/any.go | 32 +++++++++++ mock/server/settings.go | 10 +++- 14 files changed, 203 insertions(+), 44 deletions(-) create mode 100644 mock/database/any.go diff --git a/api/admin/settings.go b/api/admin/settings.go index 660e97c52..329ba24d9 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -5,6 +5,7 @@ package admin import ( "fmt" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -12,6 +13,7 @@ import ( "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database" sMiddleware "github.com/go-vela/server/router/middleware/settings" + uMiddleware "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" ) @@ -82,6 +84,7 @@ func GetSettings(c *gin.Context) { func UpdateSettings(c *gin.Context) { // capture middleware values s := sMiddleware.FromContext(c) + u := uMiddleware.FromContext(c) ctx := c.Request.Context() logrus.Info("Admin: updating settings") @@ -89,6 +92,9 @@ func UpdateSettings(c *gin.Context) { // capture body from API request input := new(settings.Platform) + input.SetUpdatedAt(time.Now().UTC().Unix()) + input.SetUpdatedBy(u.GetName()) + err := c.Bind(input) if err != nil { retErr := fmt.Errorf("unable to decode JSON for settings: %w", err) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index c51ae409c..c4201eb49 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -15,6 +15,9 @@ type Platform struct { *Compiler `json:"compiler"` RepoAllowlist *[]string `json:"repo_allowlist"` ScheduleAllowlist *[]string `json:"schedule_allowlist"` + CreatedAt *int64 `json:"created_at,omitempty"` + UpdatedAt *int64 `json:"updated_at,omitempty"` + UpdatedBy *string `json:"updated_by,omitempty"` } // GetID returns the ID field. @@ -82,6 +85,45 @@ func (s *Platform) GetScheduleAllowlist() []string { return *s.ScheduleAllowlist } +// GetCreatedAt returns the CreatedAt field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetCreatedAt() int64 { + // return zero value if Platform type or CreatedAt field is nil + if s == nil || s.CreatedAt == nil { + return 0 + } + + return *s.CreatedAt +} + +// GetUpdatedAt returns the UpdatedAt field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetUpdatedAt() int64 { + // return zero value if Platform type or UpdatedAt field is nil + if s == nil || s.UpdatedAt == nil { + return 0 + } + + return *s.UpdatedAt +} + +// GetUpdatedBy returns the UpdatedBy field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Platform) GetUpdatedBy() string { + // return zero value if Platform type or UpdatedBy field is nil + if s == nil || s.UpdatedBy == nil { + return "" + } + + return *s.UpdatedBy +} + // SetID sets the ID field. // // When the provided Platform type is nil, it @@ -147,6 +189,45 @@ func (s *Platform) SetScheduleAllowlist(v []string) { s.ScheduleAllowlist = &v } +// SetCreatedAt sets the CreatedAt field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetCreatedAt(v int64) { + // return if Platform type is nil + if s == nil { + return + } + + s.CreatedAt = &v +} + +// SetUpdatedAt sets the UpdatedAt field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetUpdatedAt(v int64) { + // return if Platform type is nil + if s == nil { + return + } + + s.UpdatedAt = &v +} + +// SetUpdatedBy sets the UpdatedBy field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (s *Platform) SetUpdatedBy(v string) { + // return if Platform type is nil + if s == nil { + return + } + + s.UpdatedBy = &v +} + // String implements the Stringer interface for the Platform type. func (s *Platform) String() string { cs := s.GetCompiler() @@ -158,12 +239,18 @@ func (s *Platform) String() string { Queue: %v, RepoAllowlist: %v, ScheduleAllowlist: %v, + CreatedAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, }`, s.GetID(), cs.String(), qs.String(), s.GetRepoAllowlist(), s.GetScheduleAllowlist(), + s.GetCreatedAt(), + s.GetUpdatedAt(), + s.GetUpdatedBy(), ) } diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go index 794a01b35..2d30a330b 100644 --- a/api/types/settings/platform_test.go +++ b/api/types/settings/platform_test.go @@ -103,12 +103,18 @@ func TestTypes_Platform_String(t *testing.T) { Queue: %v, RepoAllowlist: %v, ScheduleAllowlist: %v, + CreatedAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, }`, s.GetID(), cs.String(), qs.String(), s.GetRepoAllowlist(), s.GetScheduleAllowlist(), + s.GetCreatedAt(), + s.GetUpdatedAt(), + s.GetUpdatedBy(), ) // run test diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 099aba7eb..b0283d520 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -113,6 +113,8 @@ func server(c *cli.Context) error { // singleton record ID should always be 1 s.SetID(1) + s.SetCreatedAt(time.Now().UTC().Unix()) + // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() s.SetCompiler(compilerSettings) @@ -138,6 +140,10 @@ func server(c *cli.Context) error { compiler.SetSettings(s) router := router.Load( + // settings middleware must be first + // todo: figure out if deterministic order matters + // + // we arleady depend on establish running in order, so mayube this is fine middleware.Settings(database), middleware.Compiler(compiler), middleware.Database(database), diff --git a/database/secret/create_test.go b/database/secret/create_test.go index 7ec36795b..75bbff395 100644 --- a/database/secret/create_test.go +++ b/database/secret/create_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" + dbMock "github.com/go-vela/server/mock/database" "github.com/go-vela/types/library" ) @@ -63,21 +64,21 @@ func TestSecret_Engine_CreateSecret(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). + WithArgs("foo", "bar", nil, "baz", dbMock.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). WillReturnRows(_rows) // ensure the mock expects the org secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). + WithArgs("foo", "*", nil, "bar", dbMock.AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). WillReturnRows(_rows) // ensure the mock expects the shared secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). + WithArgs("foo", nil, "bar", "baz", dbMock.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/secret/secret_test.go b/database/secret/secret_test.go index fdc10818c..41bb4dd3a 100644 --- a/database/secret/secret_test.go +++ b/database/secret/secret_test.go @@ -3,10 +3,8 @@ package secret import ( - "database/sql/driver" "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" "github.com/sirupsen/logrus" @@ -251,29 +249,3 @@ func testEvents() *library.Events { }, } } - -// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values -// that are otherwise not easily compared. These typically would be values generated -// before adding or updating them in the database. -// -// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime -type AnyArgument struct{} - -// Match satisfies sqlmock.Argument interface. -func (a AnyArgument) Match(_ driver.Value) bool { - return true -} - -// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. -type NowTimestamp struct{} - -// Match satisfies sqlmock.Argument interface. -func (t NowTimestamp) Match(v driver.Value) bool { - ts, ok := v.(int64) - if !ok { - return false - } - now := time.Now().Unix() - - return now-ts < 10 -} diff --git a/database/secret/update_test.go b/database/secret/update_test.go index 11b6245a1..0307b788c 100644 --- a/database/secret/update_test.go +++ b/database/secret/update_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" + dbMock "github.com/go-vela/server/mock/database" "github.com/go-vela/types/library" ) @@ -60,21 +61,21 @@ func TestSecret_Engine_UpdateSecret(t *testing.T) { _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, 1, false, false, 1, "user", AnyArgument{}, "user2", 1). + WithArgs("foo", "bar", nil, "baz", dbMock.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", dbMock.AnyArgument{}, "user2", 1). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the org query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, 1, false, false, 1, "user", AnyArgument{}, "user2", 2). + WithArgs("foo", "*", nil, "bar", dbMock.AnyArgument{}, "org", nil, 1, false, false, 1, "user", dbMock.AnyArgument{}, "user2", 2). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the shared query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, 1, false, false, 1, "user", NowTimestamp{}, "user2", 3). + WithArgs("foo", nil, "bar", "baz", dbMock.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", dbMock.NowTimestamp{}, "user2", 3). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/settings/create_test.go b/database/settings/create_test.go index 0a48b9c69..db7bbcfb6 100644 --- a/database/settings/create_test.go +++ b/database/settings/create_test.go @@ -20,6 +20,9 @@ func TestSettings_Engine_CreateSettings(t *testing.T) { _settings.SetRoutes([]string{"vela"}) _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(1) + _settings.SetUpdatedBy("") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -28,9 +31,9 @@ func TestSettings_Engine_CreateSettings(t *testing.T) { _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) // ensure the mock expects the query - _mock.ExpectQuery(`INSERT INTO "settings" ("compiler","queue","repo_allowlist","schedule_allowlist","id") VALUES ($1,$2,$3,$4,$5) RETURNING "id"`). + _mock.ExpectQuery(`INSERT INTO "settings" ("compiler","queue","repo_allowlist","schedule_allowlist","created_at","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, - `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1). + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/settings/get_test.go b/database/settings/get_test.go index 7f587e57a..83196848b 100644 --- a/database/settings/get_test.go +++ b/database/settings/get_test.go @@ -22,15 +22,18 @@ func TestSettings_Engine_GetSettings(t *testing.T) { _settings.SetRoutes([]string{"vela"}) _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(1) + _settings.SetUpdatedBy("octocat") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // create expected result in mock _rows := sqlmock.NewRows( - []string{"id", "compiler", "queue", "repo_allowlist", "schedule_allowlist"}). + []string{"id", "compiler", "queue", "repo_allowlist", "schedule_allowlist", "created_at", "updated_at", "updated_by"}). AddRow(1, `{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, - `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`) + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, `octocat`) // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "settings" WHERE id = $1 LIMIT $2`).WithArgs(1, 1).WillReturnRows(_rows) diff --git a/database/settings/settings.go b/database/settings/settings.go index 2ea1095b2..94cad28ed 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -62,6 +62,10 @@ type ( RepoAllowlist pq.StringArray `json:"repo_allowlist" sql:"repo_allowlist" gorm:"type:varchar(1000)"` ScheduleAllowlist pq.StringArray `json:"schedule_allowlist" sql:"schedule_allowlist" gorm:"type:varchar(1000)"` + + CreatedAt sql.NullInt64 `sql:"created_at"` + UpdatedAt sql.NullInt64 `sql:"updated_at"` + UpdatedBy sql.NullString `sql:"updated_by"` } // Compiler is the database representation of compiler settings. @@ -170,6 +174,21 @@ func (s *Platform) Nullify() *Platform { s.CloneImage.Valid = false } + // check if the CreatedAt field should be false + if s.CreatedAt.Int64 == 0 { + s.CreatedAt.Valid = false + } + + // check if the UpdatedAt field should be false + if s.UpdatedAt.Int64 == 0 { + s.UpdatedAt.Valid = false + } + + // check if the UpdatedBy field should be false + if len(s.UpdatedBy.String) == 0 { + s.UpdatedBy.Valid = false + } + return s } @@ -190,6 +209,10 @@ func (s *Platform) ToAPI() *settings.Platform { ss.Queue = &settings.Queue{} ss.SetRoutes(s.Routes) + ss.SetCreatedAt(s.CreatedAt.Int64) + ss.SetUpdatedAt(s.UpdatedAt.Int64) + ss.SetUpdatedBy(s.UpdatedBy.String) + return ss } @@ -251,6 +274,9 @@ func FromAPI(s *settings.Platform) *Platform { }, RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), ScheduleAllowlist: pq.StringArray(s.GetScheduleAllowlist()), + CreatedAt: sql.NullInt64{Int64: s.GetCreatedAt(), Valid: true}, + UpdatedAt: sql.NullInt64{Int64: s.GetUpdatedAt(), Valid: true}, + UpdatedBy: sql.NullString{String: s.GetUpdatedBy(), Valid: true}, } return settings.Nullify() diff --git a/database/settings/table.go b/database/settings/table.go index 7b7c3a8cc..2be553b8f 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -18,7 +18,10 @@ settings ( compiler JSON DEFAULT NULL, queue JSON DEFAULT NULL, repo_allowlist VARCHAR(1000), - schedule_allowlist VARCHAR(1000) + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by VARCHAR(250) ); ` @@ -31,7 +34,10 @@ settings ( compiler TEXT, queue TEXT, repo_allowlist VARCHAR(1000), - schedule_allowlist VARCHAR(1000) + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by TEXT ); ` ) diff --git a/database/settings/update_test.go b/database/settings/update_test.go index 944533972..fa678e213 100644 --- a/database/settings/update_test.go +++ b/database/settings/update_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + dbMock "github.com/go-vela/server/mock/database" ) func TestSettings_Engine_UpdateSettings(t *testing.T) { @@ -20,14 +21,17 @@ func TestSettings_Engine_UpdateSettings(t *testing.T) { _settings.SetRoutes([]string{"vela", "large"}) _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(2) + _settings.SetUpdatedBy("octocat") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // ensure the mock expects the query - _mock.ExpectExec(`UPDATE "settings" SET "compiler"=$1,"queue"=$2,"repo_allowlist"=$3,"schedule_allowlist"=$4 WHERE "id" = $5`). + _mock.ExpectExec(`UPDATE "settings" SET "compiler"=$1,"queue"=$2,"repo_allowlist"=$3,"schedule_allowlist"=$4,"created_at"=$5,"updated_at"=$6,"updated_by"=$7 WHERE "id" = $8`). WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, - `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1). + `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, dbMock.AnyArgument{}, "octocat", 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/mock/database/any.go b/mock/database/any.go new file mode 100644 index 000000000..815ce8fdb --- /dev/null +++ b/mock/database/any.go @@ -0,0 +1,32 @@ +package mock + +import ( + "database/sql/driver" + "time" +) + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type AnyArgument struct{} + +// Match satisfies sqlmock.Argument interface. +func (a AnyArgument) Match(_ driver.Value) bool { + return true +} + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/mock/server/settings.go b/mock/server/settings.go index 8a7c8b92e..56ed13cdc 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -31,7 +31,10 @@ const ( ], "schedule_allowlist": [ "octocat/hello-world" - ] + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" }` // UpdateSettingsResp represents a JSON return for modifying a settings field. @@ -53,7 +56,10 @@ const ( "schedule_allowlist": [ "octocat/hello-world", "octocat/*" - ] + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" }` ) From 8c37621916fd1700804f5ccfe3f1860812f8f10d Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 May 2024 10:17:13 -0500 Subject: [PATCH 36/65] chore: lint --- database/secret/create_test.go | 8 ++++---- database/secret/update_test.go | 8 ++++---- database/settings/update_test.go | 5 +++-- mock/database/any.go => database/testutils/mock_args.go | 5 ++++- 4 files changed, 15 insertions(+), 11 deletions(-) rename mock/database/any.go => database/testutils/mock_args.go (93%) diff --git a/database/secret/create_test.go b/database/secret/create_test.go index 75bbff395..ed3d95591 100644 --- a/database/secret/create_test.go +++ b/database/secret/create_test.go @@ -9,7 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" - dbMock "github.com/go-vela/server/mock/database" + "github.com/go-vela/server/database/testutils" "github.com/go-vela/types/library" ) @@ -64,21 +64,21 @@ func TestSecret_Engine_CreateSecret(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "bar", nil, "baz", dbMock.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). + WithArgs("foo", "bar", nil, "baz", testutils.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). WillReturnRows(_rows) // ensure the mock expects the org secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "*", nil, "bar", dbMock.AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). + WithArgs("foo", "*", nil, "bar", testutils.AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). WillReturnRows(_rows) // ensure the mock expects the shared secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", nil, "bar", "baz", dbMock.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). + WithArgs("foo", nil, "bar", "baz", testutils.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/secret/update_test.go b/database/secret/update_test.go index 0307b788c..e53d43b4e 100644 --- a/database/secret/update_test.go +++ b/database/secret/update_test.go @@ -9,7 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" - dbMock "github.com/go-vela/server/mock/database" + "github.com/go-vela/server/database/testutils" "github.com/go-vela/types/library" ) @@ -61,21 +61,21 @@ func TestSecret_Engine_UpdateSecret(t *testing.T) { _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "bar", nil, "baz", dbMock.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", dbMock.AnyArgument{}, "user2", 1). + WithArgs("foo", "bar", nil, "baz", testutils.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", testutils.AnyArgument{}, "user2", 1). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the org query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "*", nil, "bar", dbMock.AnyArgument{}, "org", nil, 1, false, false, 1, "user", dbMock.AnyArgument{}, "user2", 2). + WithArgs("foo", "*", nil, "bar", testutils.AnyArgument{}, "org", nil, 1, false, false, 1, "user", testutils.AnyArgument{}, "user2", 2). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the shared query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", nil, "bar", "baz", dbMock.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", dbMock.NowTimestamp{}, "user2", 3). + WithArgs("foo", nil, "bar", "baz", testutils.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", testutils.NowTimestamp{}, "user2", 3). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/settings/update_test.go b/database/settings/update_test.go index fa678e213..dcaa7202b 100644 --- a/database/settings/update_test.go +++ b/database/settings/update_test.go @@ -8,7 +8,8 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - dbMock "github.com/go-vela/server/mock/database" + + "github.com/go-vela/server/database/testutils" ) func TestSettings_Engine_UpdateSettings(t *testing.T) { @@ -31,7 +32,7 @@ func TestSettings_Engine_UpdateSettings(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "settings" SET "compiler"=$1,"queue"=$2,"repo_allowlist"=$3,"schedule_allowlist"=$4,"created_at"=$5,"updated_at"=$6,"updated_by"=$7 WHERE "id" = $8`). WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, - `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, dbMock.AnyArgument{}, "octocat", 1). + `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, testutils.AnyArgument{}, "octocat", 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/mock/database/any.go b/database/testutils/mock_args.go similarity index 93% rename from mock/database/any.go rename to database/testutils/mock_args.go index 815ce8fdb..c2d8c562f 100644 --- a/mock/database/any.go +++ b/database/testutils/mock_args.go @@ -1,4 +1,6 @@ -package mock +// SPDX-License-Identifier: Apache-2.0 + +package testutils import ( "database/sql/driver" @@ -26,6 +28,7 @@ func (t NowTimestamp) Match(v driver.Value) bool { if !ok { return false } + now := time.Now().Unix() return now-ts < 10 From 9e4a3275d5c30cb57f578ff71ebc49eff44d06e0 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 May 2024 11:11:31 -0500 Subject: [PATCH 37/65] enhance: timestamps for created and updated --- api/admin/settings.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 329ba24d9..d5b194dbc 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -5,7 +5,6 @@ package admin import ( "fmt" "net/http" - "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -92,9 +91,6 @@ func UpdateSettings(c *gin.Context) { // capture body from API request input := new(settings.Platform) - input.SetUpdatedAt(time.Now().UTC().Unix()) - input.SetUpdatedBy(u.GetName()) - err := c.Bind(input) if err != nil { retErr := fmt.Errorf("unable to decode JSON for settings: %w", err) @@ -132,6 +128,8 @@ func UpdateSettings(c *gin.Context) { s.SetScheduleAllowlist(input.GetScheduleAllowlist()) } + s.SetUpdatedBy(u.GetName()) + // send API call to update the settings s, err = database.FromContext(c).UpdateSettings(ctx, s) if err != nil { From e765740cc045b37c9bf7dd432ef8e4fd51107d13 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 May 2024 11:22:15 -0500 Subject: [PATCH 38/65] enhance: validations for timestamps --- database/settings/create.go | 2 +- database/settings/settings.go | 34 ++++++++++++++++++---------------- database/settings/table.go | 22 +++++++++++----------- database/settings/update.go | 2 +- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/database/settings/create.go b/database/settings/create.go index d6636e5e5..d4b4f7901 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -22,7 +22,7 @@ func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*setti } // send query to the database - err = e.client.Table(TableSettings).Create(settings).Error + err = e.client.Table(TableSettings).Create(settings.Nullify()).Error if err != nil { return nil, err } diff --git a/database/settings/settings.go b/database/settings/settings.go index 94cad28ed..ab7ff68e9 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -28,27 +28,26 @@ var ( ErrEmptyCloneImage = errors.New("empty settings clone image provided") ) -// todo: comments Build->Settings type ( - // config represents the settings required to create the engine that implements the BuildInterface interface. + // config represents the settings required to create the engine that implements the SettingsInterface interface. config struct { - // specifies to skip creating tables and indexes for the Build engine + // specifies to skip creating tables and indexes for the Settings engine SkipCreation bool } - // engine represents the build functionality that implements the BuildInterface interface. + // engine represents the settings functionality that implements the SettingsInterface interface. engine struct { - // engine configuration settings used in build functions + // engine configuration settings used in settings functions config *config ctx context.Context - // gorm.io/gorm database client used in build functions + // gorm.io/gorm database client used in settings functions // // https://pkg.go.dev/gorm.io/gorm#DB client *gorm.DB - // sirupsen/logrus logger used in build functions + // sirupsen/logrus logger used in settings functions // // https://pkg.go.dev/github.com/sirupsen/logrus#Entry logger *logrus.Entry @@ -117,11 +116,11 @@ func (r *Queue) Scan(value interface{}) error { } } -// New creates and returns a Vela service for integrating with builds in the database. +// New creates and returns a Vela service for integrating with settingss in the database. // //nolint:revive // ignore returning unexported engine func New(opts ...EngineOpt) (*engine, error) { - // create new Build engine + // create new Settings engine e := new(engine) // create new fields @@ -175,20 +174,15 @@ func (s *Platform) Nullify() *Platform { } // check if the CreatedAt field should be false - if s.CreatedAt.Int64 == 0 { + if s.CreatedAt.Int64 < 0 { s.CreatedAt.Valid = false } // check if the UpdatedAt field should be false - if s.UpdatedAt.Int64 == 0 { + if s.UpdatedAt.Int64 < 0 { s.UpdatedAt.Valid = false } - // check if the UpdatedBy field should be false - if len(s.UpdatedBy.String) == 0 { - s.UpdatedBy.Valid = false - } - return s } @@ -256,6 +250,14 @@ func (s *Platform) Validate() error { s.ScheduleAllowlist[i] = util.Sanitize(v) } + if s.CreatedAt.Int64 < 0 { + return fmt.Errorf("created_at must be greater than zero, got: %d", s.CreatedAt.Int64) + } + + if s.UpdatedAt.Int64 < 0 { + return fmt.Errorf("updated_at must be greater than zero, got: %d", s.UpdatedAt.Int64) + } + return nil } diff --git a/database/settings/table.go b/database/settings/table.go index 2be553b8f..282f6fcca 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -19,9 +19,9 @@ settings ( queue JSON DEFAULT NULL, repo_allowlist VARCHAR(1000), schedule_allowlist VARCHAR(1000), - created_at INTEGER, - updated_at INTEGER, - updated_by VARCHAR(250) + created_at INTEGER, + updated_at INTEGER, + updated_by VARCHAR(250) ); ` @@ -30,14 +30,14 @@ settings ( CREATE TABLE IF NOT EXISTS settings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - compiler TEXT, - queue TEXT, - repo_allowlist VARCHAR(1000), - schedule_allowlist VARCHAR(1000), - created_at INTEGER, - updated_at INTEGER, - updated_by TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + compiler TEXT, + queue TEXT, + repo_allowlist VARCHAR(1000), + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by TEXT ); ` ) diff --git a/database/settings/update.go b/database/settings/update.go index c23956abd..351f1f953 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -22,7 +22,7 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti } // send query to the database - err = e.client.Table(TableSettings).Save(settings).Error + err = e.client.Table(TableSettings).Save(settings.Nullify()).Error if err != nil { return nil, err } From 83215daa26fdaf5e8af79778c8c213a76cc9bd12 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 May 2024 11:33:31 -0500 Subject: [PATCH 39/65] fix: test --- database/settings/create_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/settings/create_test.go b/database/settings/create_test.go index db7bbcfb6..e621bf76e 100644 --- a/database/settings/create_test.go +++ b/database/settings/create_test.go @@ -33,7 +33,7 @@ func TestSettings_Engine_CreateSettings(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "settings" ("compiler","queue","repo_allowlist","schedule_allowlist","created_at","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, - `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, nil, 1). + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, ``, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) From d7ff645f7b5c8b327e728cb015785ad75e1d922e Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 3 May 2024 15:33:41 -0500 Subject: [PATCH 40/65] fix: return updated settings --- database/settings/update.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/database/settings/update.go b/database/settings/update.go index 351f1f953..12cd703e5 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -13,19 +13,21 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Trace("updating platform settings in the database") // cast the api type to database type - settings := FromAPI(s) + s_ := FromAPI(s) // validate the necessary fields are populated - err := settings.Validate() + err := s_.Validate() if err != nil { return nil, err } // send query to the database - err = e.client.Table(TableSettings).Save(settings.Nullify()).Error + err = e.client.Table(TableSettings).Save(s_.Nullify()).Error if err != nil { return nil, err } + s = s_.ToAPI() + return s, nil } From 7ab84af512d8d45baf91c8b9b562a3cd09edde7a Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 3 May 2024 15:41:44 -0500 Subject: [PATCH 41/65] fix: return updated settings --- database/settings/update_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/settings/update_test.go b/database/settings/update_test.go index dcaa7202b..7b9f31a38 100644 --- a/database/settings/update_test.go +++ b/database/settings/update_test.go @@ -23,7 +23,7 @@ func TestSettings_Engine_UpdateSettings(t *testing.T) { _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) _settings.SetScheduleAllowlist([]string{"*"}) _settings.SetCreatedAt(1) - _settings.SetUpdatedAt(2) + _settings.SetUpdatedAt(1) _settings.SetUpdatedBy("octocat") _postgres, _mock := testPostgres(t) @@ -65,6 +65,7 @@ func TestSettings_Engine_UpdateSettings(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { got, err := test.database.UpdateSettings(context.TODO(), _settings) + got.SetUpdatedAt(_settings.GetUpdatedAt()) if test.failure { if err == nil { From 069da631cae861c43c80a782676c83d6d22a233e Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 3 May 2024 15:44:07 -0500 Subject: [PATCH 42/65] chore: lint --- database/settings/update.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/database/settings/update.go b/database/settings/update.go index 12cd703e5..7de291521 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -13,21 +13,21 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Trace("updating platform settings in the database") // cast the api type to database type - s_ := FromAPI(s) + dbS := FromAPI(s) // validate the necessary fields are populated - err := s_.Validate() + err := dbS.Validate() if err != nil { return nil, err } // send query to the database - err = e.client.Table(TableSettings).Save(s_.Nullify()).Error + err = e.client.Table(TableSettings).Save(dbS.Nullify()).Error if err != nil { return nil, err } - s = s_.ToAPI() + s = dbS.ToAPI() return s, nil } From f5effff6133f2167ffe93a9db1c7f0dd31d8edf4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 7 May 2024 10:51:08 -0500 Subject: [PATCH 43/65] enhance: move to db types, fix tests --- api/admin/settings.go | 6 +- database/settings/create.go | 3 +- database/settings/get.go | 3 +- database/settings/settings.go | 210 +-------------------------- database/settings/settings_test.go | 3 +- database/settings/update.go | 3 +- database/types/settings.go | 219 +++++++++++++++++++++++++++++ database/types/settings_test.go | 195 +++++++++++++++++++++++++ 8 files changed, 425 insertions(+), 217 deletions(-) create mode 100644 database/types/settings.go create mode 100644 database/types/settings_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index d5b194dbc..b49dd1285 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -102,15 +102,15 @@ func UpdateSettings(c *gin.Context) { if input.Compiler != nil { if input.CloneImage != nil { - s.CloneImage = input.CloneImage + s.SetCloneImage(*input.CloneImage) } if input.TemplateDepth != nil { - s.TemplateDepth = input.TemplateDepth + s.SetTemplateDepth(*input.TemplateDepth) } if input.StarlarkExecLimit != nil { - s.StarlarkExecLimit = input.StarlarkExecLimit + s.SetStarlarkExecLimit(*input.StarlarkExecLimit) } } diff --git a/database/settings/create.go b/database/settings/create.go index d4b4f7901..0e86f7fbc 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -6,6 +6,7 @@ import ( "context" "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" ) // CreateSettings creates a platform settings record in the database. @@ -13,7 +14,7 @@ func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Tracef("creating platform settings in the database with %v", s.String()) // cast the api type to database type - settings := FromAPI(s) + settings := types.FromAPI(s) // validate the necessary fields are populated err := settings.Validate() diff --git a/database/settings/get.go b/database/settings/get.go index b5d7d0775..0bba2e0f4 100644 --- a/database/settings/get.go +++ b/database/settings/get.go @@ -6,6 +6,7 @@ import ( "context" "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" ) // GetSettings gets platform settings from the database. @@ -13,7 +14,7 @@ func (e *engine) GetSettings(ctx context.Context) (*settings.Platform, error) { e.logger.Trace("getting platform settings from the database") // variable to store query results - s := new(Platform) + s := new(types.Platform) // send query to the database and store result in variable err := e.client. diff --git a/database/settings/settings.go b/database/settings/settings.go index ab7ff68e9..a673ed49d 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -4,30 +4,16 @@ package settings import ( "context" - "database/sql" - "database/sql/driver" - "encoding/json" - "errors" "fmt" - "github.com/lib/pq" "github.com/sirupsen/logrus" "gorm.io/gorm" - - "github.com/go-vela/server/api/types/settings" - "github.com/go-vela/server/util" ) const ( TableSettings = "settings" ) -var ( - // ErrEmptyCloneImage defines the error type when a - // Settings type has an empty CloneImage field provided. - ErrEmptyCloneImage = errors.New("empty settings clone image provided") -) - type ( // config represents the settings required to create the engine that implements the SettingsInterface interface. config struct { @@ -52,71 +38,9 @@ type ( // https://pkg.go.dev/github.com/sirupsen/logrus#Entry logger *logrus.Entry } - - // Platform is the database representation of platform settings. - Platform struct { - ID sql.NullInt64 `sql:"id"` - Compiler - Queue - - RepoAllowlist pq.StringArray `json:"repo_allowlist" sql:"repo_allowlist" gorm:"type:varchar(1000)"` - ScheduleAllowlist pq.StringArray `json:"schedule_allowlist" sql:"schedule_allowlist" gorm:"type:varchar(1000)"` - - CreatedAt sql.NullInt64 `sql:"created_at"` - UpdatedAt sql.NullInt64 `sql:"updated_at"` - UpdatedBy sql.NullString `sql:"updated_by"` - } - - // Compiler is the database representation of compiler settings. - Compiler struct { - CloneImage sql.NullString `json:"clone_image" sql:"clone_image"` - TemplateDepth sql.NullInt64 `json:"template_depth" sql:"template_depth"` - StarlarkExecLimit sql.NullInt64 `json:"starlark_exec_limit" sql:"starlark_exec_limit"` - } - - // Queue is the database representation of queue settings. - Queue struct { - Routes pq.StringArray `json:"routes" sql:"routes" gorm:"type:varchar(1000)"` - } ) -// Value - Implementation of valuer for database/sql for Compiler. -func (r Compiler) Value() (driver.Value, error) { - valueString, err := json.Marshal(r) - return string(valueString), err -} - -// Scan - Implement the database/sql scanner interface for Compiler. -func (r *Compiler) Scan(value interface{}) error { - switch v := value.(type) { - case []byte: - return json.Unmarshal(v, &r) - case string: - return json.Unmarshal([]byte(v), &r) - default: - return fmt.Errorf("wrong type for compiler: %T", v) - } -} - -// Value - Implementation of valuer for database/sql for Queue. -func (r Queue) Value() (driver.Value, error) { - valueString, err := json.Marshal(r) - return string(valueString), err -} - -// Scan - Implement the database/sql scanner interface for Queue. -func (r *Queue) Scan(value interface{}) error { - switch v := value.(type) { - case []byte: - return json.Unmarshal(v, &r) - case string: - return json.Unmarshal([]byte(v), &r) - default: - return fmt.Errorf("wrong type for queue: %T", v) - } -} - -// New creates and returns a Vela service for integrating with settingss in the database. +// New creates and returns a Vela service for integrating with settings in the database. // //nolint:revive // ignore returning unexported engine func New(opts ...EngineOpt) (*engine, error) { @@ -151,135 +75,3 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } - -// Nullify ensures the valid flag for -// the sql.Null types are properly set. -// -// When a field within the Settings type is the zero -// value for the field, the valid flag is set to -// false causing it to be NULL in the database. -func (s *Platform) Nullify() *Platform { - if s == nil { - return nil - } - - // check if the ID field should be false - if s.ID.Int64 == 0 { - s.ID.Valid = false - } - - // check if the CloneImage field should be false - if len(s.CloneImage.String) == 0 { - s.CloneImage.Valid = false - } - - // check if the CreatedAt field should be false - if s.CreatedAt.Int64 < 0 { - s.CreatedAt.Valid = false - } - - // check if the UpdatedAt field should be false - if s.UpdatedAt.Int64 < 0 { - s.UpdatedAt.Valid = false - } - - return s -} - -// ToAPI converts the Settings type -// to an API Settings type. -func (s *Platform) ToAPI() *settings.Platform { - ss := new(settings.Platform) - ss.SetID(s.ID.Int64) - - ss.SetRepoAllowlist(s.RepoAllowlist) - ss.SetScheduleAllowlist(s.ScheduleAllowlist) - - ss.Compiler = &settings.Compiler{} - ss.SetCloneImage(s.CloneImage.String) - ss.SetTemplateDepth(int(s.TemplateDepth.Int64)) - ss.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) - - ss.Queue = &settings.Queue{} - ss.SetRoutes(s.Routes) - - ss.SetCreatedAt(s.CreatedAt.Int64) - ss.SetUpdatedAt(s.UpdatedAt.Int64) - ss.SetUpdatedBy(s.UpdatedBy.String) - - return ss -} - -// Validate verifies the necessary fields for -// the Settings type are populated correctly. -func (s *Platform) Validate() error { - // verify the CloneImage field is populated - if len(s.CloneImage.String) == 0 { - return ErrEmptyCloneImage - } - - // verify compiler settings are within limits - if s.TemplateDepth.Int64 <= 0 { - return fmt.Errorf("template depth must be greater than zero, got: %d", s.TemplateDepth.Int64) - } - - if s.StarlarkExecLimit.Int64 <= 0 { - return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", s.StarlarkExecLimit.Int64) - } - - // ensure that all Settings string fields - // that can be returned as JSON are sanitized - // to avoid unsafe HTML content - s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} - - // ensure that all Queue.Routes are sanitized - // to avoid unsafe HTML content - for i, v := range s.Routes { - s.Routes[i] = util.Sanitize(v) - } - - // ensure that all RepoAllowlist are sanitized - // to avoid unsafe HTML content - for i, v := range s.RepoAllowlist { - s.RepoAllowlist[i] = util.Sanitize(v) - } - - // ensure that all ScheduleAllowlist are sanitized - // to avoid unsafe HTML content - for i, v := range s.ScheduleAllowlist { - s.ScheduleAllowlist[i] = util.Sanitize(v) - } - - if s.CreatedAt.Int64 < 0 { - return fmt.Errorf("created_at must be greater than zero, got: %d", s.CreatedAt.Int64) - } - - if s.UpdatedAt.Int64 < 0 { - return fmt.Errorf("updated_at must be greater than zero, got: %d", s.UpdatedAt.Int64) - } - - return nil -} - -// FromAPI converts the API Settings type -// to a database Settings type. -func FromAPI(s *settings.Platform) *Platform { - settings := &Platform{ - ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, - Compiler: Compiler{ - CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, - TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, - StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, - }, - Queue: Queue{ - Routes: pq.StringArray(s.GetRoutes()), - }, - RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), - ScheduleAllowlist: pq.StringArray(s.GetScheduleAllowlist()), - CreatedAt: sql.NullInt64{Int64: s.GetCreatedAt(), Valid: true}, - UpdatedAt: sql.NullInt64{Int64: s.GetUpdatedAt(), Valid: true}, - UpdatedBy: sql.NullString{String: s.GetUpdatedBy(), Valid: true}, - } - - return settings.Nullify() -} diff --git a/database/settings/settings_test.go b/database/settings/settings_test.go index 0e1ac773f..85b65de75 100644 --- a/database/settings/settings_test.go +++ b/database/settings/settings_test.go @@ -7,12 +7,11 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/api/types/settings" "github.com/sirupsen/logrus" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" - - "github.com/go-vela/server/api/types/settings" ) func TestSettings_New(t *testing.T) { diff --git a/database/settings/update.go b/database/settings/update.go index 7de291521..b72b4d543 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -6,6 +6,7 @@ import ( "context" "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" ) // UpdateSettings updates a platform settings in the database. @@ -13,7 +14,7 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Trace("updating platform settings in the database") // cast the api type to database type - dbS := FromAPI(s) + dbS := types.FromAPI(s) // validate the necessary fields are populated err := dbS.Validate() diff --git a/database/types/settings.go b/database/types/settings.go new file mode 100644 index 000000000..a8f2e0035 --- /dev/null +++ b/database/types/settings.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + + "github.com/lib/pq" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/util" +) + +var ( + // ErrEmptyCloneImage defines the error type when a + // Settings type has an empty CloneImage field provided. + ErrEmptyCloneImage = errors.New("empty settings clone image provided") +) + +type ( + + // Platform is the database representation of platform settings. + Platform struct { + ID sql.NullInt64 `sql:"id"` + Compiler + Queue + + RepoAllowlist pq.StringArray `json:"repo_allowlist" sql:"repo_allowlist" gorm:"type:varchar(1000)"` + ScheduleAllowlist pq.StringArray `json:"schedule_allowlist" sql:"schedule_allowlist" gorm:"type:varchar(1000)"` + + CreatedAt sql.NullInt64 `sql:"created_at"` + UpdatedAt sql.NullInt64 `sql:"updated_at"` + UpdatedBy sql.NullString `sql:"updated_by"` + } + + // Compiler is the database representation of compiler settings. + Compiler struct { + CloneImage sql.NullString `json:"clone_image" sql:"clone_image"` + TemplateDepth sql.NullInt64 `json:"template_depth" sql:"template_depth"` + StarlarkExecLimit sql.NullInt64 `json:"starlark_exec_limit" sql:"starlark_exec_limit"` + } + + // Queue is the database representation of queue settings. + Queue struct { + Routes pq.StringArray `json:"routes" sql:"routes" gorm:"type:varchar(1000)"` + } +) + +// Value - Implementation of valuer for database/sql for Compiler. +func (r Compiler) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Compiler. +func (r *Compiler) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for compiler: %T", v) + } +} + +// Value - Implementation of valuer for database/sql for Queue. +func (r Queue) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Queue. +func (r *Queue) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for queue: %T", v) + } +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Settings type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (s *Platform) Nullify() *Platform { + if s == nil { + return nil + } + + // check if the ID field should be false + if s.ID.Int64 == 0 { + s.ID.Valid = false + } + + // check if the CloneImage field should be false + if len(s.CloneImage.String) == 0 { + s.CloneImage.Valid = false + } + + // check if the CreatedAt field should be false + if s.CreatedAt.Int64 < 0 { + s.CreatedAt.Valid = false + } + + // check if the UpdatedAt field should be false + if s.UpdatedAt.Int64 < 0 { + s.UpdatedAt.Valid = false + } + + return s +} + +// ToAPI converts the Settings type +// to an API Settings type. +func (s *Platform) ToAPI() *settings.Platform { + ss := new(settings.Platform) + ss.SetID(s.ID.Int64) + + ss.SetRepoAllowlist(s.RepoAllowlist) + ss.SetScheduleAllowlist(s.ScheduleAllowlist) + + ss.Compiler = &settings.Compiler{} + ss.SetCloneImage(s.CloneImage.String) + ss.SetTemplateDepth(int(s.TemplateDepth.Int64)) + ss.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) + + ss.Queue = &settings.Queue{} + ss.SetRoutes(s.Routes) + + ss.SetCreatedAt(s.CreatedAt.Int64) + ss.SetUpdatedAt(s.UpdatedAt.Int64) + ss.SetUpdatedBy(s.UpdatedBy.String) + + return ss +} + +// Validate verifies the necessary fields for +// the Settings type are populated correctly. +func (s *Platform) Validate() error { + // verify the CloneImage field is populated + if len(s.CloneImage.String) == 0 { + return ErrEmptyCloneImage + } + + // verify compiler settings are within limits + if s.TemplateDepth.Int64 <= 0 { + return fmt.Errorf("template depth must be greater than zero, got: %d", s.TemplateDepth.Int64) + } + + if s.StarlarkExecLimit.Int64 <= 0 { + return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", s.StarlarkExecLimit.Int64) + } + + // ensure that all Settings string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} + + // ensure that all Queue.Routes are sanitized + // to avoid unsafe HTML content + for i, v := range s.Routes { + s.Routes[i] = util.Sanitize(v) + } + + // ensure that all RepoAllowlist are sanitized + // to avoid unsafe HTML content + for i, v := range s.RepoAllowlist { + s.RepoAllowlist[i] = util.Sanitize(v) + } + + // ensure that all ScheduleAllowlist are sanitized + // to avoid unsafe HTML content + for i, v := range s.ScheduleAllowlist { + s.ScheduleAllowlist[i] = util.Sanitize(v) + } + + if s.CreatedAt.Int64 < 0 { + return fmt.Errorf("created_at must be greater than zero, got: %d", s.CreatedAt.Int64) + } + + if s.UpdatedAt.Int64 < 0 { + return fmt.Errorf("updated_at must be greater than zero, got: %d", s.UpdatedAt.Int64) + } + + return nil +} + +// FromAPI converts the API Settings type +// to a database Settings type. +func FromAPI(s *settings.Platform) *Platform { + settings := &Platform{ + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, + }, + Queue: Queue{ + Routes: pq.StringArray(s.GetRoutes()), + }, + RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), + ScheduleAllowlist: pq.StringArray(s.GetScheduleAllowlist()), + CreatedAt: sql.NullInt64{Int64: s.GetCreatedAt(), Valid: true}, + UpdatedAt: sql.NullInt64{Int64: s.GetUpdatedAt(), Valid: true}, + UpdatedBy: sql.NullString{String: s.GetUpdatedBy(), Valid: true}, + } + + return settings.Nullify() +} diff --git a/database/types/settings_test.go b/database/types/settings_test.go new file mode 100644 index 000000000..c926411ac --- /dev/null +++ b/database/types/settings_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types/settings" +) + +func TestTypes_Platform_Nullify(t *testing.T) { + // setup types + var s *Platform + + want := &Platform{ + ID: sql.NullInt64{Int64: 0, Valid: false}, + } + + // setup tests + tests := []struct { + repo *Platform + want *Platform + }{ + { + repo: testPlatform(), + want: testPlatform(), + }, + { + repo: s, + want: nil, + }, + { + repo: new(Platform), + want: want, + }, + } + + // run tests + for _, test := range tests { + got := test.repo.Nullify() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("Nullify is %v, want %v", got, test.want) + } + } +} + +func TestTypes_Platform_ToAPI(t *testing.T) { + // setup types + want := new(api.Platform) + want.SetID(1) + want.SetRepoAllowlist([]string{"github/octocat"}) + want.SetScheduleAllowlist([]string{"*"}) + want.SetCreatedAt(0) + want.SetUpdatedAt(0) + want.SetUpdatedBy("") + + want.Compiler = new(api.Compiler) + want.SetCloneImage("target/vela-git:latest") + want.SetTemplateDepth(10) + want.SetStarlarkExecLimit(100) + + want.Queue = new(api.Queue) + want.SetRoutes([]string{"vela"}) + + // run test + got := testPlatform().ToAPI() + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestTypes_Platform_Validate(t *testing.T) { + // setup tests + tests := []struct { + failure bool + settings *Platform + }{ + { + failure: false, + settings: testPlatform(), + }, + { // no CloneImage set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + }, + }, + { // no TemplateDepth set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + }, + }, + { // no StarlarkExecLimit set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + }, + }, + }, + { // no queue fields set for settings + failure: false, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + Queue: Queue{}, + }, + }, + } + + // run tests + for _, test := range tests { + err := test.settings.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} + +func TestTypes_Platform_PlatformFromAPI(t *testing.T) { + // setup types + s := new(api.Platform) + s.SetID(1) + s.SetRepoAllowlist([]string{"github/octocat"}) + s.SetScheduleAllowlist([]string{"*"}) + s.SetCreatedAt(0) + s.SetUpdatedAt(0) + s.SetUpdatedBy("") + + s.Compiler = new(api.Compiler) + s.SetCloneImage("target/vela-git:latest") + s.SetTemplateDepth(10) + s.SetStarlarkExecLimit(100) + + s.Queue = new(api.Queue) + s.SetRoutes([]string{"vela"}) + + want := testPlatform() + + // run test + got := FromAPI(s) + + if !reflect.DeepEqual(got, want) { + t.Errorf("PlatformFromAPI is %v, want %v", got, want) + } +} + +// testPlatform is a test helper function to create a Platform +// type with all fields set to a fake value. +func testPlatform() *Platform { + return &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + Queue: Queue{ + Routes: []string{"vela"}, + }, + RepoAllowlist: []string{"github/octocat"}, + ScheduleAllowlist: []string{"*"}, + CreatedAt: sql.NullInt64{Int64: 0, Valid: true}, + UpdatedAt: sql.NullInt64{Int64: 0, Valid: true}, + UpdatedBy: sql.NullString{String: "", Valid: true}, + } +} From 20f5691a67b3bfdfc01f8e74a50db4f24c2e5be2 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 7 May 2024 10:55:00 -0500 Subject: [PATCH 44/65] fix: add jitter around initialization --- cmd/vela-server/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index b0283d520..5de403dde 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -100,6 +100,13 @@ func server(c *cli.Context) error { return err } + jitter := wait.Jitter(60*time.Second, 0.5) + + logrus.Infof("sleeping for %v before initializing settings", jitter) + + // sleep for a duration of time before processing schedules + time.Sleep(jitter) + s, err := database.GetSettings(context.Background()) if c.Bool("vela-reinitialize-settings-on-startup") || s == nil || err != nil { // only log errors @@ -114,6 +121,8 @@ func server(c *cli.Context) error { s.SetID(1) s.SetCreatedAt(time.Now().UTC().Unix()) + s.SetUpdatedAt(time.Now().UTC().Unix()) + s.SetUpdatedBy("vela-server") // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() From 7bd45ca68d3290e57df69a5c86ce2745648031d1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 7 May 2024 10:55:55 -0500 Subject: [PATCH 45/65] fix: import order --- database/settings/settings_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/settings/settings_test.go b/database/settings/settings_test.go index 85b65de75..0e1ac773f 100644 --- a/database/settings/settings_test.go +++ b/database/settings/settings_test.go @@ -7,11 +7,12 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/server/api/types/settings" "github.com/sirupsen/logrus" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" + + "github.com/go-vela/server/api/types/settings" ) func TestSettings_New(t *testing.T) { From 83f9380e120d35e8ee418ac2850434a92c1a8469 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 7 May 2024 10:58:52 -0500 Subject: [PATCH 46/65] fix: comment and jitter time --- cmd/vela-server/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 5de403dde..85bc4edba 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -100,11 +100,11 @@ func server(c *cli.Context) error { return err } - jitter := wait.Jitter(60*time.Second, 0.5) + jitter := wait.Jitter(5*time.Second, 0.5) logrus.Infof("sleeping for %v before initializing settings", jitter) - // sleep for a duration of time before processing schedules + // sleep for a duration of time before initializing settings time.Sleep(jitter) s, err := database.GetSettings(context.Background()) From 5a7fda954304cd8f7d8f37ebf94960bdb4d45a58 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 8 May 2024 15:20:04 -0500 Subject: [PATCH 47/65] fix: use refresh interval to avoid db spam --- api/types/settings/platform.go | 17 ++++++++ api/types/settings/platform_test.go | 50 +++++++++++++++++++++-- cmd/vela-server/main.go | 10 ++--- cmd/vela-server/server.go | 61 +++++++++++++++++------------ router/middleware/settings.go | 23 +++-------- 5 files changed, 110 insertions(+), 51 deletions(-) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index c4201eb49..2be662803 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -228,6 +228,23 @@ func (s *Platform) SetUpdatedBy(v string) { s.UpdatedBy = &v } +// Update takes another settings record and updates the internal fields, intended +// to be used when the refreshing settings record shared across the server. +func (s *Platform) Update(s_ *Platform) { + if s == nil { + return + } + + if s_ == nil { + return + } + + s.SetCompiler(s_.GetCompiler()) + s.SetQueue(s_.GetQueue()) + s.SetRepoAllowlist(s_.GetRepoAllowlist()) + s.SetScheduleAllowlist(s_.GetScheduleAllowlist()) +} + // String implements the Stringer interface for the Platform type. func (s *Platform) String() string { cs := s.GetCompiler() diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go index 2d30a330b..b13f4947e 100644 --- a/api/types/settings/platform_test.go +++ b/api/types/settings/platform_test.go @@ -6,6 +6,8 @@ import ( "fmt" "reflect" "testing" + + "github.com/google/go-cmp/cmp" ) func TestTypes_Platform_Getters(t *testing.T) { @@ -91,6 +93,42 @@ func TestTypes_Platform_Setters(t *testing.T) { } } +func TestTypes_Platform_Update(t *testing.T) { + // setup types + s := testPlatformSettings() + + // update fields + sUpdate := testPlatformSettings() + sUpdate.SetCompiler(Compiler{}) + sUpdate.SetQueue(Queue{}) + sUpdate.SetRepoAllowlist([]string{"foo"}) + sUpdate.SetScheduleAllowlist([]string{"bar"}) + + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: s, + want: testPlatformSettings(), + }, + { + platform: s, + want: sUpdate, + }, + } + + // run tests + for _, test := range tests { + test.platform.Update(test.want) + + if diff := cmp.Diff(test.want, test.platform); diff != "" { + t.Errorf("(Update: -want +got):\n%s", diff) + } + } +} + func TestTypes_Platform_String(t *testing.T) { // setup types s := testPlatformSettings() @@ -128,6 +166,15 @@ func TestTypes_Platform_String(t *testing.T) { // testPlatformSettings is a test helper function to create a Platform // type with all fields set to a fake value. func testPlatformSettings() *Platform { + // setup platform + s := new(Platform) + s.SetID(1) + s.SetCreatedAt(1) + s.SetUpdatedAt(1) + s.SetUpdatedBy("vela-server") + s.SetRepoAllowlist([]string{"foo", "bar"}) + s.SetScheduleAllowlist([]string{"*"}) + // setup types // setup compiler cs := new(Compiler) @@ -141,9 +188,6 @@ func testPlatformSettings() *Platform { qs.SetRoutes([]string{"vela"}) - // setup platform - s := new(Platform) - s.SetCompiler(*cs) s.SetQueue(*qs) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 689a45aa6..7fa51ca27 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -79,11 +79,11 @@ func main() { Name: "vela-secret", Usage: "secret used for server <-> agent communication", }, - &cli.BoolFlag{ - EnvVars: []string{"VELA_REINITIALIZE_SETTINGS_ON_STARTUP"}, - Name: "vela-reinitialize-settings-on-startup", - Usage: "instructs the server to reinitialize the settings singleton on startup, overwriting any updates currently stored in the database and restoring them to the configured environment values", - Value: false, + &cli.DurationFlag{ + EnvVars: []string{"VELA_PLATFORM_SETTINGS_REFRESH_INTERVAL", "VELA_SETTINGS_REFRESH_INTERVAL"}, + Name: "settings-refresh-interval", + Usage: "interval at which platform settings will be refreshed", + Value: 5 * time.Second, }, &cli.StringFlag{ EnvVars: []string{"VELA_SERVER_PRIVATE_KEY"}, diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 85bc4edba..b822c6a8a 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -4,7 +4,6 @@ package main import ( "context" - "errors" "fmt" "net/http" "net/url" @@ -100,16 +99,19 @@ func server(c *cli.Context) error { return err } - jitter := wait.Jitter(5*time.Second, 0.5) + jitter := wait.Jitter(5*time.Second, 2.0) - logrus.Infof("sleeping for %v before initializing settings", jitter) + logrus.Infof("retrieving initial platform settings after %v delay", jitter) - // sleep for a duration of time before initializing settings time.Sleep(jitter) s, err := database.GetSettings(context.Background()) - if c.Bool("vela-reinitialize-settings-on-startup") || s == nil || err != nil { + if s == nil || err != nil { // only log errors + if s == nil { + logrus.Error("unable to get platform settings") + } + if err != nil { logrus.WithError(err).Error("unable to get platform settings") } @@ -144,16 +146,13 @@ func server(c *cli.Context) error { } } - // update any internal settings + // update any internal settings, this occurs in middleware + // to keep settings refreshed for each request queue.SetSettings(s) compiler.SetSettings(s) router := router.Load( - // settings middleware must be first - // todo: figure out if deterministic order matters - // - // we arleady depend on establish running in order, so mayube this is fine - middleware.Settings(database), + middleware.Settings(s), middleware.Compiler(compiler), middleware.Database(database), middleware.Logger(logrus.StandardLogger(), time.RFC3339), @@ -233,6 +232,30 @@ func server(c *cli.Context) error { return nil }) + // spawn goroutine for refreshing settings + g.Go(func() error { + interval := c.Duration("settings-refresh-interval") + + logrus.Infof("refreshing platform settings every %v", interval) + + for { + time.Sleep(interval) + + s_, err := database.GetSettings(context.Background()) + if err != nil { + logrus.WithError(err).Warn("unable to refresh platform settings") + + continue + } + + // update the internal fields for the shared settings record + s.SetCompiler(s_.GetCompiler()) + s.SetQueue(s_.GetQueue()) + s.SetRepoAllowlist(s_.GetRepoAllowlist()) + s.SetScheduleAllowlist(s_.GetScheduleAllowlist()) + } + }) + // spawn goroutine for starting the server g.Go(func() error { logrus.Infof("starting server on %s", addr.Host) @@ -274,21 +297,7 @@ func server(c *cli.Context) error { // sleep for a duration of time before processing schedules time.Sleep(jitter) - s, err = database.GetSettings(context.Background()) - if s == nil { - err = errors.New("settings not found") - - logrus.WithError(err).Warn("unable to get platform settings") - - continue - } - - if err != nil { - logrus.WithError(err).Warn("unable to get platform settings") - - continue - } - + // update internal settings updated through refresh compiler.SetSettings(s) queue.SetSettings(s) diff --git a/router/middleware/settings.go b/router/middleware/settings.go index 7f3579714..dc6074baf 100644 --- a/router/middleware/settings.go +++ b/router/middleware/settings.go @@ -3,27 +3,16 @@ package middleware import ( - "context" - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/settings" + "github.com/go-vela/server/api/types/settings" + sMiddleware "github.com/go-vela/server/router/middleware/settings" ) -// Settings is a middleware function that fetches the latest settings and -// attaches to the context of every http.Request. -func Settings(d database.Interface) gin.HandlerFunc { +// Settings is a middleware function that attaches settings +// to the context of every http.Request. +func Settings(s *settings.Platform) gin.HandlerFunc { return func(c *gin.Context) { - s, err := d.GetSettings(context.Background()) - if err != nil { - logrus.WithError(err).Warn("unable to get platform settings") - - return - } - - settings.ToContext(c, s) + sMiddleware.ToContext(c, s) c.Next() } From b358b725fe1847dc582b57c7ea1c3aaa3e07718f Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 8 May 2024 16:06:14 -0500 Subject: [PATCH 48/65] enhance: background go func that refreshes settings on an interval, cli context in middleware --- api/admin/settings.go | 77 +++++++++++++++++++ cmd/vela-server/compiler.go | 24 ------ cmd/vela-server/queue.go | 31 -------- cmd/vela-server/server.go | 12 +-- compiler/native/clone_test.go | 4 +- compiler/native/compile_test.go | 26 +++---- compiler/native/environment_test.go | 8 +- compiler/native/expand_test.go | 16 ++-- compiler/native/initialize_test.go | 4 +- compiler/native/native.go | 4 +- compiler/native/native_test.go | 50 ++++++------ compiler/native/parse_test.go | 24 +++--- compiler/native/script_test.go | 4 +- compiler/native/substitute_test.go | 4 +- compiler/native/transform_test.go | 4 +- compiler/native/validate_test.go | 36 ++++----- database/settings/table.go | 18 ++--- database/types/settings.go | 1 - mock/server/server.go | 1 + mock/server/settings.go | 35 +++++++++ queue/queue.go | 22 ++++++ router/admin.go | 3 +- router/middleware/cli.go | 19 +++++ router/middleware/cli/context.go | 37 +++++++++ router/middleware/cli/context_test.go | 85 +++++++++++++++++++++ router/middleware/cli/doc.go | 10 +++ router/middleware/cli_test.go | 50 ++++++++++++ router/middleware/compiler_test.go | 4 +- router/middleware/pipeline/pipeline_test.go | 2 +- router/middleware/settings_test.go | 47 ++++++++++++ router/middleware/worker_test.go | 4 +- 31 files changed, 497 insertions(+), 169 deletions(-) delete mode 100644 cmd/vela-server/compiler.go delete mode 100644 cmd/vela-server/queue.go create mode 100644 router/middleware/cli.go create mode 100644 router/middleware/cli/context.go create mode 100644 router/middleware/cli/context_test.go create mode 100644 router/middleware/cli/doc.go create mode 100644 router/middleware/cli_test.go create mode 100644 router/middleware/settings_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index b49dd1285..472b288d2 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -5,12 +5,16 @@ package admin import ( "fmt" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" + cliMiddleware "github.com/go-vela/server/router/middleware/cli" sMiddleware "github.com/go-vela/server/router/middleware/settings" uMiddleware "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" @@ -142,3 +146,76 @@ func UpdateSettings(c *gin.Context) { c.JSON(http.StatusOK, s) } + +// swagger:operation DELETE /api/v1/admin/settings admin RestoreSettings +// +// Restore the currently configured settings to the environment defaults. +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully restored default settings in the database +// schema: +// type: array +// items: +// "$ref": "#/definitions/Platform" +// '500': +// description: Unable to restore settings in the database +// schema: +// "$ref": "#/definitions/Error" + +// RestoreSettings represents the API handler to +// restore settings stored in the database to the environment defaults. +func RestoreSettings(c *gin.Context) { + // capture middleware values + s := sMiddleware.FromContext(c) + u := uMiddleware.FromContext(c) + cliCtx := cliMiddleware.FromContext(c) + ctx := c.Request.Context() + + logrus.Info("Admin: restoring settings") + + compiler, err := native.FromCLIContext(cliCtx) + if err != nil { + retErr := fmt.Errorf("unable to restore settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + queue, err := queue.FromCLIContext(cliCtx) + if err != nil { + retErr := fmt.Errorf("unable to restore settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + s.SetUpdatedAt(time.Now().UTC().Unix()) + s.SetUpdatedBy(u.GetName()) + + // read in defaults supplied from the cli runtime + compilerSettings := compiler.GetSettings() + s.SetCompiler(compilerSettings) + + queueSettings := queue.GetSettings() + s.SetQueue(queueSettings) + + // send API call to update the settings + s, err = database.FromContext(c).UpdateSettings(ctx, s) + if err != nil { + retErr := fmt.Errorf("unable to update (restore) settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, s) +} diff --git a/cmd/vela-server/compiler.go b/cmd/vela-server/compiler.go deleted file mode 100644 index 872f3fbd0..000000000 --- a/cmd/vela-server/compiler.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - - "github.com/go-vela/server/compiler" - "github.com/go-vela/server/compiler/native" - "github.com/go-vela/types/constants" -) - -// helper function to setup the queue from the CLI arguments. -func setupCompiler(c *cli.Context) (compiler.Engine, error) { - logrus.Debug("Creating queue client from CLI configuration") - return setupCompilerNative(c) -} - -// helper function to setup the Kafka queue from the CLI arguments. -func setupCompilerNative(c *cli.Context) (compiler.Engine, error) { - logrus.Tracef("Creating %s compiler client from CLI configuration", constants.DriverKafka) - return native.New(c) -} diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go deleted file mode 100644 index 7a3a74c41..000000000 --- a/cmd/vela-server/queue.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - - "github.com/go-vela/server/queue" -) - -// helper function to setup the queue from the CLI arguments. -func setupQueue(c *cli.Context) (queue.Service, error) { - logrus.Debug("Creating queue client from CLI configuration") - - // queue configuration - _setup := &queue.Setup{ - Driver: c.String("queue.driver"), - Address: c.String("queue.addr"), - Cluster: c.Bool("queue.cluster"), - Routes: c.StringSlice("queue.routes"), - Timeout: c.Duration("queue.pop.timeout"), - PrivateKey: c.String("queue.private-key"), - PublicKey: c.String("queue.public-key"), - } - - // setup the queue - // - // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New - return queue.New(_setup) -} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index b822c6a8a..e38b69f43 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -19,7 +19,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" ) @@ -69,7 +71,7 @@ func server(c *cli.Context) error { logrus.SetLevel(logrus.PanicLevel) } - compiler, err := setupCompiler(c) + compiler, err := native.FromCLIContext(c) if err != nil { return err } @@ -79,7 +81,7 @@ func server(c *cli.Context) error { return err } - queue, err := setupQueue(c) + queue, err := queue.FromCLIContext(c) if err != nil { return err } @@ -152,6 +154,7 @@ func server(c *cli.Context) error { compiler.SetSettings(s) router := router.Load( + middleware.CLI(c), middleware.Settings(s), middleware.Compiler(compiler), middleware.Database(database), @@ -249,10 +252,7 @@ func server(c *cli.Context) error { } // update the internal fields for the shared settings record - s.SetCompiler(s_.GetCompiler()) - s.SetQueue(s_.GetQueue()) - s.SetRepoAllowlist(s_.GetRepoAllowlist()) - s.SetScheduleAllowlist(s_.GetScheduleAllowlist()) + s.Update(s_) } }) diff --git a/compiler/native/clone_test.go b/compiler/native/clone_test.go index 120e3524f..304f703b2 100644 --- a/compiler/native/clone_test.go +++ b/compiler/native/clone_test.go @@ -84,7 +84,7 @@ func TestNative_CloneStage(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } @@ -167,7 +167,7 @@ func TestNative_CloneStep(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 811f9855e..59cee9bc8 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -244,7 +244,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -582,7 +582,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -841,7 +841,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1086,7 +1086,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1207,7 +1207,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1328,7 +1328,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1407,7 +1407,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1595,7 +1595,7 @@ func TestNative_Compile_Clone(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1803,7 +1803,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1839,7 +1839,7 @@ func TestNative_Compile_NoStepsorStages(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1871,7 +1871,7 @@ func TestNative_Compile_StepsandStages(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -2952,7 +2952,7 @@ func Test_Compile_Inline(t *testing.T) { if err != nil { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -3823,7 +3823,7 @@ func Test_CompileLite(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index 1e090dad1..bb3f9d8da 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -60,7 +60,7 @@ func TestNative_EnvironmentStages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -205,7 +205,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { } // run test non-local - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -380,7 +380,7 @@ func TestNative_EnvironmentServices(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -538,7 +538,7 @@ func TestNative_EnvironmentSecrets(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 9f41af418..171c80d30 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -144,7 +144,7 @@ func TestNative_ExpandStages(t *testing.T) { } // run test -- missing private github - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -165,7 +165,7 @@ func TestNative_ExpandStages(t *testing.T) { } // run test - compiler, err = New(c) + compiler, err = FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -346,7 +346,7 @@ func TestNative_ExpandSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -615,7 +615,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { wantEnvironment := raw.StringSliceMap{} // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -708,7 +708,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -870,7 +870,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -972,7 +972,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -1058,7 +1058,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } diff --git a/compiler/native/initialize_test.go b/compiler/native/initialize_test.go index 33276ffdb..fb2a6548b 100644 --- a/compiler/native/initialize_test.go +++ b/compiler/native/initialize_test.go @@ -61,7 +61,7 @@ func TestNative_InitStage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -109,7 +109,7 @@ func TestNative_InitStep(t *testing.T) { }, } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 4301a2f79..16b4f54ee 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -43,10 +43,10 @@ type client struct { labels []string } -// New returns a Pipeline implementation that integrates with the supported registries. +// FromCLIContext returns a Pipeline implementation that integrates with the supported registries. // //nolint:revive // ignore returning unexported client -func New(ctx *cli.Context) (*client, error) { +func FromCLIContext(ctx *cli.Context) (*client, error) { logrus.Debug("Creating registry clients from CLI configuration") c := new(client) diff --git a/compiler/native/native_test.go b/compiler/native/native_test.go index 2a86e653e..3c1eb8075 100644 --- a/compiler/native/native_test.go +++ b/compiler/native/native_test.go @@ -26,7 +26,7 @@ func TestNative_New(t *testing.T) { } // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -56,7 +56,7 @@ func TestNative_New_PrivateGithub(t *testing.T) { } // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -86,7 +86,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { } // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -105,10 +105,10 @@ func TestNative_DuplicateStripBuild(t *testing.T) { id := int64(1) b := &api.Build{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -128,11 +128,11 @@ func TestNative_WithBuild(t *testing.T) { id := int64(1) b := &api.Build{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.build = b // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -149,11 +149,11 @@ func TestNative_WithFiles(t *testing.T) { f := []string{"foo"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.files = f // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -169,11 +169,11 @@ func TestNative_WithComment(t *testing.T) { c := cli.NewContext(nil, set, nil) comment := "ok to test" - want, _ := New(c) + want, _ := FromCLIContext(c) want.comment = comment // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -189,11 +189,11 @@ func TestNative_WithLocal(t *testing.T) { c := cli.NewContext(nil, set, nil) local := true - want, _ := New(c) + want, _ := FromCLIContext(c) want.local = true // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -209,11 +209,11 @@ func TestNative_WithLocalTemplates(t *testing.T) { c := cli.NewContext(nil, set, nil) localTemplates := []string{"example:tmpl.yml", "exmpl:template.yml"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.localTemplates = []string{"example:tmpl.yml", "exmpl:template.yml"} // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -248,11 +248,11 @@ func TestNative_WithMetadata(t *testing.T) { }, } - want, _ := New(c) + want, _ := FromCLIContext(c) want.metadata = m // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -274,11 +274,11 @@ func TestNative_WithPrivateGitHub(t *testing.T) { private, _ := github.New(url, token) - want, _ := New(c) + want, _ := FromCLIContext(c) want.PrivateGithub = private // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -296,11 +296,11 @@ func TestNative_WithRepo(t *testing.T) { id := int64(1) r := &api.Repo{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.repo = r // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -318,11 +318,11 @@ func TestNative_WithUser(t *testing.T) { id := int64(1) u := &api.User{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.user = u // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -338,11 +338,11 @@ func TestNative_WithLabels(t *testing.T) { c := cli.NewContext(nil, set, nil) labels := []string{"documentation", "enhancement"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.labels = []string{"documentation", "enhancement"} // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index dfe691fdb..e0aa9c512 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -21,7 +21,7 @@ import ( func TestNative_Parse_Metadata_Bytes(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -49,7 +49,7 @@ func TestNative_Parse_Metadata_Bytes(t *testing.T) { func TestNative_Parse_Metadata_File(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -79,7 +79,7 @@ func TestNative_Parse_Metadata_File(t *testing.T) { func TestNative_Parse_Metadata_Invalid(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) // run test got, _, err := client.Parse(nil, "", new(yaml.Template)) @@ -95,7 +95,7 @@ func TestNative_Parse_Metadata_Invalid(t *testing.T) { func TestNative_Parse_Metadata_Path(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -118,7 +118,7 @@ func TestNative_Parse_Metadata_Path(t *testing.T) { func TestNative_Parse_Metadata_Reader(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -146,7 +146,7 @@ func TestNative_Parse_Metadata_Reader(t *testing.T) { func TestNative_Parse_Metadata_String(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -174,7 +174,7 @@ func TestNative_Parse_Metadata_String(t *testing.T) { func TestNative_Parse_Parameters(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -221,7 +221,7 @@ func TestNative_Parse_Parameters(t *testing.T) { func TestNative_Parse_StagesPipeline(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -350,7 +350,7 @@ func TestNative_Parse_StagesPipeline(t *testing.T) { func TestNative_Parse_StepsPipeline(t *testing.T) { // setup types tBool := true - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -452,7 +452,7 @@ func TestNative_Parse_StepsPipeline(t *testing.T) { func TestNative_Parse_Secrets(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -522,7 +522,7 @@ func TestNative_Parse_Secrets(t *testing.T) { func TestNative_Parse_Stages(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -598,7 +598,7 @@ func TestNative_Parse_Stages(t *testing.T) { func TestNative_Parse_Steps(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 308023e77..2e948e27a 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -87,7 +87,7 @@ func TestNative_ScriptStages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -313,7 +313,7 @@ func TestNative_ScriptSteps(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/substitute_test.go b/compiler/native/substitute_test.go index 459bb2896..809899e1e 100644 --- a/compiler/native/substitute_test.go +++ b/compiler/native/substitute_test.go @@ -112,7 +112,7 @@ func Test_client_SubstituteStages(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -236,7 +236,7 @@ func Test_client_SubstituteSteps(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index e8d596be3..0c1d424e6 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -220,7 +220,7 @@ func TestNative_TransformStages(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } @@ -439,7 +439,7 @@ func TestNative_TransformSteps(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 85896b4a1..d0f23d4fc 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -21,7 +21,7 @@ func TestNative_Validate_NoVersion(t *testing.T) { p := &yaml.Build{} // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -42,7 +42,7 @@ func TestNative_Validate_NoStagesOrSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -85,7 +85,7 @@ func TestNative_Validate_StagesAndSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -122,7 +122,7 @@ func TestNative_Validate_Services(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -159,7 +159,7 @@ func TestNative_Validate_Services_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -196,7 +196,7 @@ func TestNative_Validate_Services_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -231,7 +231,7 @@ func TestNative_Validate_Stages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -265,7 +265,7 @@ func TestNative_Validate_Stages_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -299,7 +299,7 @@ func TestNative_Validate_Stages_NoStepName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -333,7 +333,7 @@ func TestNative_Validate_Stages_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -367,7 +367,7 @@ func TestNative_Validate_Stages_NoCommands(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -403,7 +403,7 @@ func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -433,7 +433,7 @@ func TestNative_Validate_Steps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -461,7 +461,7 @@ func TestNative_Validate_Steps_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -490,7 +490,7 @@ func TestNative_Validate_Steps_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -519,7 +519,7 @@ func TestNative_Validate_Steps_NoCommands(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -556,7 +556,7 @@ func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -595,7 +595,7 @@ func TestNative_Validate_MultiReportAs(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/database/settings/table.go b/database/settings/table.go index 282f6fcca..e20639615 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -14,14 +14,14 @@ const ( CREATE TABLE IF NOT EXISTS settings ( - id SERIAL PRIMARY KEY, - compiler JSON DEFAULT NULL, - queue JSON DEFAULT NULL, - repo_allowlist VARCHAR(1000), - schedule_allowlist VARCHAR(1000), - created_at INTEGER, - updated_at INTEGER, - updated_by VARCHAR(250) + id SERIAL PRIMARY KEY, + compiler JSON DEFAULT NULL, + queue JSON DEFAULT NULL, + repo_allowlist VARCHAR(1000), + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by VARCHAR(250) ); ` @@ -34,7 +34,7 @@ settings ( compiler TEXT, queue TEXT, repo_allowlist VARCHAR(1000), - schedule_allowlist VARCHAR(1000), + schedule_allowlist VARCHAR(1000), created_at INTEGER, updated_at INTEGER, updated_by TEXT diff --git a/database/types/settings.go b/database/types/settings.go index a8f2e0035..4a7e67f39 100644 --- a/database/types/settings.go +++ b/database/types/settings.go @@ -22,7 +22,6 @@ var ( ) type ( - // Platform is the database representation of platform settings. Platform struct { ID sql.NullInt64 `sql:"id"` diff --git a/mock/server/server.go b/mock/server/server.go index 0861da6a6..9527cbc75 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -31,6 +31,7 @@ func FakeHandler() http.Handler { e.PUT("api/v1/admin/clean", cleanResoures) e.GET("/api/v1/admin/settings", getSettings) e.PUT("/api/v1/admin/settings", updateSettings) + e.DELETE("/api/v1/admin/settings", restoreSettings) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/mock/server/settings.go b/mock/server/settings.go index 56ed13cdc..07e5c9419 100644 --- a/mock/server/settings.go +++ b/mock/server/settings.go @@ -61,6 +61,31 @@ const ( "updated_at": 1, "updated_by": "octocat" }` + + // RestoreSettingsResp represents a JSON return for restoring the settings record to the defaults. + RestoreSettingsResp = ` + { + "id": 1, + "compiler": { + "clone_image": "target/vela-git:latest", + "template_depth": 5, + "starlark_exec_limit": 123 + }, + "queue": { + "routes": [ + "vela", + "large" + ] + }, + "repo_allowlist": [], + "schedule_allowlist": [ + "octocat/hello-world", + "octocat/*" + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" + }` ) // getSettings has a param :settings returns mock JSON for a http GET. @@ -82,3 +107,13 @@ func updateSettings(c *gin.Context) { c.JSON(http.StatusOK, body) } + +// restoreSettings returns mock JSON for a http DELETE. +func restoreSettings(c *gin.Context) { + data := []byte(RestoreSettingsResp) + + var body settings.Platform + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} diff --git a/queue/queue.go b/queue/queue.go index fe2946f1b..181ed5899 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -6,10 +6,32 @@ import ( "fmt" "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" "github.com/go-vela/types/constants" ) +// FromCLIContext helper function to setup the queue from the CLI arguments. +func FromCLIContext(c *cli.Context) (Service, error) { + logrus.Debug("Creating queue client from CLI configuration") + + // queue configuration + _setup := &Setup{ + Driver: c.String("queue.driver"), + Address: c.String("queue.addr"), + Cluster: c.Bool("queue.cluster"), + Routes: c.StringSlice("queue.routes"), + Timeout: c.Duration("queue.pop.timeout"), + PrivateKey: c.String("queue.private-key"), + PublicKey: c.String("queue.public-key"), + } + + // setup the queue + // + // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New + return New(_setup) +} + // New creates and returns a Vela service capable of // integrating with the configured queue environment. // Currently, the following queues are supported: diff --git a/router/admin.go b/router/admin.go index 2abd3b612..9c664a9d7 100644 --- a/router/admin.go +++ b/router/admin.go @@ -26,7 +26,7 @@ import ( // POST /api/v1/admin/workers/:worker/register. // GET /api/v1/admin/settings // PUT /api/v1/admin/settings -// DELETE /api/v1/admin/settings. +// DELETE /api/v1/admin/settings. func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) @@ -67,5 +67,6 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin settings endpoints _admin.GET("/settings", admin.GetSettings) _admin.PUT("/settings", admin.UpdateSettings) + _admin.DELETE("/settings", admin.RestoreSettings) } // end of admin endpoints } diff --git a/router/middleware/cli.go b/router/middleware/cli.go new file mode 100644 index 000000000..a029ecac2 --- /dev/null +++ b/router/middleware/cli.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + cliMiddleware "github.com/go-vela/server/router/middleware/cli" + "github.com/urfave/cli/v2" +) + +// CLI is a middleware function that attaches the urface cli client +// to the context of every http.Request. +func CLI(cliCtx *cli.Context) gin.HandlerFunc { + return func(c *gin.Context) { + cliMiddleware.ToContext(c, cliCtx) + + c.Next() + } +} diff --git a/router/middleware/cli/context.go b/router/middleware/cli/context.go new file mode 100644 index 000000000..13952ddc2 --- /dev/null +++ b/router/middleware/cli/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "context" + + "github.com/urfave/cli/v2" +) + +const key = "cli" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the cli context associated with this context. +func FromContext(c context.Context) *cli.Context { + value := c.Value(key) + if value == nil { + return nil + } + + s, ok := value.(*cli.Context) + if !ok { + return nil + } + + return s +} + +// ToContext adds the cli context to this context if it supports +// the Setter interface. +func ToContext(c Setter, s *cli.Context) { + c.Set(key, s) +} diff --git a/router/middleware/cli/context_test.go b/router/middleware/cli/context_test.go new file mode 100644 index 000000000..4269f3c06 --- /dev/null +++ b/router/middleware/cli/context_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" +) + +func TestSettings_FromContext(t *testing.T) { + // setup types + want := &cli.Context{} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestSettings_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_ToContext(t *testing.T) { + // setup types + want := &cli.Context{} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/cli/doc.go b/router/middleware/cli/doc.go new file mode 100644 index 000000000..3bdc404c6 --- /dev/null +++ b/router/middleware/cli/doc.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package cli provides the ability for inserting +// Vela cli resources into or extracting Vela cli +// resources from the middleware chain for the API. +// +// Usage: +// +// import "github.com/go-vela/server/router/middleware/cli" +package cli diff --git a/router/middleware/cli_test.go b/router/middleware/cli_test.go new file mode 100644 index 000000000..d94e68c75 --- /dev/null +++ b/router/middleware/cli_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" +) + +func TestMiddleware_CLI(t *testing.T) { + // setup types + want := &cli.Context{ + App: &cli.App{ + Name: "foo", + }, + } + + got := &cli.Context{} + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(CLI(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("cli").(*cli.Context) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("CLI returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("CLI is %v, want %v", got, want) + } +} diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index 813abb98c..ed625d749 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -21,10 +21,10 @@ import ( func TestMiddleware_CompilerNative(t *testing.T) { // setup types var got compiler.Engine - got, _ = native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + got, _ = native.FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) wantCloneImage := "target/vela-git" - want, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want, _ := native.FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want.SetCloneImage(wantCloneImage) // setup context diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index 489025859..d7af22303 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -285,7 +285,7 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) { set := flag.NewFlagSet("test", 0) set.String("clone-image", "target/vela-git:latest", "doc") - comp, err := native.New(cli.NewContext(nil, set, nil)) + comp, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) if err != nil { t.Errorf("unable to create compiler: %v", err) } diff --git a/router/middleware/settings_test.go b/router/middleware/settings_test.go new file mode 100644 index 000000000..1330b2382 --- /dev/null +++ b/router/middleware/settings_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types/settings" +) + +func TestMiddleware_Settings(t *testing.T) { + // setup types + want := settings.PlatformMockEmpty() + want.SetCloneImage("target/vela-git") + + got := settings.PlatformMockEmpty() + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(Settings(&want)) + engine.GET("/health", func(c *gin.Context) { + got = *c.Value("settings").(*settings.Platform) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Settings returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Settings is %v, want %v", got, want) + } +} diff --git a/router/middleware/worker_test.go b/router/middleware/worker_test.go index 7717d03c5..5f01dc8a4 100644 --- a/router/middleware/worker_test.go +++ b/router/middleware/worker_test.go @@ -37,10 +37,10 @@ func TestMiddleware_Worker(t *testing.T) { engine.ServeHTTP(context.Writer, context.Request) if resp.Code != http.StatusOK { - t.Errorf("Secret returned %v, want %v", resp.Code, http.StatusOK) + t.Errorf("Worker returned %v, want %v", resp.Code, http.StatusOK) } if !reflect.DeepEqual(got, want) { - t.Errorf("Secret is %v, want %v", got, want) + t.Errorf("Worker is %v, want %v", got, want) } } From 297014f096109e339f9835e2c9a5e0bc6c9df282 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 8 May 2024 16:08:02 -0500 Subject: [PATCH 49/65] chore: linter --- api/types/settings/platform.go | 4 ++-- cmd/vela-server/server.go | 4 ++-- router/middleware/cli.go | 3 ++- router/middleware/settings.go | 1 + router/middleware/settings_test.go | 1 + 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index 2be662803..2668461fa 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -230,12 +230,12 @@ func (s *Platform) SetUpdatedBy(v string) { // Update takes another settings record and updates the internal fields, intended // to be used when the refreshing settings record shared across the server. -func (s *Platform) Update(s_ *Platform) { +func (s *Platform) Update(newSettings *Platform) { if s == nil { return } - if s_ == nil { + if newSettings == nil { return } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index e38b69f43..4930961a6 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -244,7 +244,7 @@ func server(c *cli.Context) error { for { time.Sleep(interval) - s_, err := database.GetSettings(context.Background()) + newSettings, err := database.GetSettings(context.Background()) if err != nil { logrus.WithError(err).Warn("unable to refresh platform settings") @@ -252,7 +252,7 @@ func server(c *cli.Context) error { } // update the internal fields for the shared settings record - s.Update(s_) + s.Update(newSettings) } }) diff --git a/router/middleware/cli.go b/router/middleware/cli.go index a029ecac2..a733b991c 100644 --- a/router/middleware/cli.go +++ b/router/middleware/cli.go @@ -4,8 +4,9 @@ package middleware import ( "github.com/gin-gonic/gin" - cliMiddleware "github.com/go-vela/server/router/middleware/cli" "github.com/urfave/cli/v2" + + cliMiddleware "github.com/go-vela/server/router/middleware/cli" ) // CLI is a middleware function that attaches the urface cli client diff --git a/router/middleware/settings.go b/router/middleware/settings.go index dc6074baf..1e985bf73 100644 --- a/router/middleware/settings.go +++ b/router/middleware/settings.go @@ -4,6 +4,7 @@ package middleware import ( "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types/settings" sMiddleware "github.com/go-vela/server/router/middleware/settings" ) diff --git a/router/middleware/settings_test.go b/router/middleware/settings_test.go index 1330b2382..d0b5d2b58 100644 --- a/router/middleware/settings_test.go +++ b/router/middleware/settings_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types/settings" ) From c53ddbe0808b1dd368ec21f91d365b422e8945a9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 8 May 2024 16:10:28 -0500 Subject: [PATCH 50/65] chore: typo --- api/types/settings/platform.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index 2668461fa..ed59b009c 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -239,10 +239,10 @@ func (s *Platform) Update(newSettings *Platform) { return } - s.SetCompiler(s_.GetCompiler()) - s.SetQueue(s_.GetQueue()) - s.SetRepoAllowlist(s_.GetRepoAllowlist()) - s.SetScheduleAllowlist(s_.GetScheduleAllowlist()) + s.SetCompiler(newSettings.GetCompiler()) + s.SetQueue(newSettings.GetQueue()) + s.SetRepoAllowlist(newSettings.GetRepoAllowlist()) + s.SetScheduleAllowlist(newSettings.GetScheduleAllowlist()) } // String implements the Stringer interface for the Platform type. From 4110ac2bc40c48243776bc05fdc0f84960394c42 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 8 May 2024 16:28:02 -0500 Subject: [PATCH 51/65] chore: typos and missing test --- mock/server/settings_test.go | 18 ++++++++++++++++++ router/admin.go | 2 +- router/middleware/cli.go | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go index 7061d4409..d827cc756 100644 --- a/mock/server/settings_test.go +++ b/mock/server/settings_test.go @@ -45,3 +45,21 @@ func TestSettings_UpdateResp(t *testing.T) { } } } + +func TestSettings_RestoreResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(RestoreSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("RestoreSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} diff --git a/router/admin.go b/router/admin.go index 9c664a9d7..22f83973e 100644 --- a/router/admin.go +++ b/router/admin.go @@ -23,7 +23,7 @@ import ( // PUT /api/v1/admin/service // PUT /api/v1/admin/step // PUT /api/v1/admin/user -// POST /api/v1/admin/workers/:worker/register. +// POST /api/v1/admin/workers/:worker/register // GET /api/v1/admin/settings // PUT /api/v1/admin/settings // DELETE /api/v1/admin/settings. diff --git a/router/middleware/cli.go b/router/middleware/cli.go index a733b991c..acc1972eb 100644 --- a/router/middleware/cli.go +++ b/router/middleware/cli.go @@ -9,7 +9,7 @@ import ( cliMiddleware "github.com/go-vela/server/router/middleware/cli" ) -// CLI is a middleware function that attaches the urface cli client +// CLI is a middleware function that attaches the cli client // to the context of every http.Request. func CLI(cliCtx *cli.Context) gin.HandlerFunc { return func(c *gin.Context) { From 409a267e07f51b0d3a3b7e7e9394a59af58673db Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 9 May 2024 14:15:17 -0500 Subject: [PATCH 52/65] fix: typos, swagger, whitespace, 404s --- api/admin/settings.go | 65 ++++++++++++++++++++++++++++++-------- database/settings/table.go | 12 +++---- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 472b288d2..a6b44ca7f 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -31,13 +31,12 @@ import ( // - ApiKeyAuth: [] // responses: // '200': -// description: Successfully retrieved settings from the database +// description: Successfully retrieved settings +// type: json // schema: -// type: array -// items: -// "$ref": "#/definitions/Platform" -// '500': -// description: Unable to retrieve settings from the database +// "$ref": "#/definitions/Platform" +// '404': +// description: Unable to retrieve settings // schema: // "$ref": "#/definitions/Error" @@ -49,12 +48,22 @@ func GetSettings(c *gin.Context) { logrus.Info("Admin: reading settings") + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + c.JSON(http.StatusOK, s) } // swagger:operation PUT /api/v1/admin/settings admin UpdateSettings // -// Update the settings singleton in the database. +// Update the platform settings singleton in the database. // // --- // produces: @@ -70,15 +79,20 @@ func GetSettings(c *gin.Context) { // - ApiKeyAuth: [] // responses: // '200': -// description: Successfully updated the settings in the database +// description: Successfully updated platform settings in the database +// type: json // schema: // "$ref": "#/definitions/Platform" +// '400': +// description: Unable to update settings — bad request +// schema: +// "$ref": "#/definitions/Error" // '404': -// description: Unable to update the settings in the database +// description: Unable to retrieve platform settings to update // schema: // "$ref": "#/definitions/Error" -// '501': -// description: Unable to update the settings in the database +// '500': +// description: Unable to update platform settings in the database // schema: // "$ref": "#/definitions/Error" @@ -92,6 +106,16 @@ func UpdateSettings(c *gin.Context) { logrus.Info("Admin: updating settings") + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + // capture body from API request input := new(settings.Platform) @@ -159,10 +183,13 @@ func UpdateSettings(c *gin.Context) { // responses: // '200': // description: Successfully restored default settings in the database +// type: json +// schema: +// "$ref": "#/definitions/Platform" +// '404': +// description: Unable to retrieve settings to restore // schema: -// type: array -// items: -// "$ref": "#/definitions/Platform" +// "$ref": "#/definitions/Error" // '500': // description: Unable to restore settings in the database // schema: @@ -179,6 +206,16 @@ func RestoreSettings(c *gin.Context) { logrus.Info("Admin: restoring settings") + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + compiler, err := native.FromCLIContext(cliCtx) if err != nil { retErr := fmt.Errorf("unable to restore settings: %w", err) diff --git a/database/settings/table.go b/database/settings/table.go index e20639615..a7268bc76 100644 --- a/database/settings/table.go +++ b/database/settings/table.go @@ -14,14 +14,14 @@ const ( CREATE TABLE IF NOT EXISTS settings ( - id SERIAL PRIMARY KEY, - compiler JSON DEFAULT NULL, - queue JSON DEFAULT NULL, + id SERIAL PRIMARY KEY, + compiler JSON DEFAULT NULL, + queue JSON DEFAULT NULL, repo_allowlist VARCHAR(1000), schedule_allowlist VARCHAR(1000), - created_at INTEGER, - updated_at INTEGER, - updated_by VARCHAR(250) + created_at INTEGER, + updated_at INTEGER, + updated_by VARCHAR(250) ); ` From 81fa5dd4fe7919f196c3332896f092c043326096 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 9 May 2024 14:20:55 -0500 Subject: [PATCH 53/65] fix: consistent variable naming and whitespace --- api/types/settings/platform.go | 142 ++++++++++++++-------------- api/types/settings/platform_test.go | 4 +- database/types/settings.go | 86 ++++++++--------- database/types/settings_test.go | 4 +- queue/redis/redis.go | 2 + 5 files changed, 120 insertions(+), 118 deletions(-) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index ed59b009c..f72698b66 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -6,7 +6,7 @@ import ( "fmt" ) -// Platform is the API representation of platform settings. +// Platform is the API representation of platform settingps. // // swagger:model Platform type Platform struct { @@ -24,231 +24,231 @@ type Platform struct { // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetID() int64 { +func (ps *Platform) GetID() int64 { // return zero value if Platform type or ID field is nil - if s == nil || s.ID == nil { + if ps == nil || ps.ID == nil { return 0 } - return *s.ID + return *ps.ID } // GetCompiler returns the Compiler field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetCompiler() Compiler { +func (ps *Platform) GetCompiler() Compiler { // return zero value if Platform type or Compiler field is nil - if s == nil || s.Compiler == nil { + if ps == nil || ps.Compiler == nil { return Compiler{} } - return *s.Compiler + return *ps.Compiler } // GetQueue returns the Queue field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetQueue() Queue { +func (ps *Platform) GetQueue() Queue { // return zero value if Platform type or Queue field is nil - if s == nil || s.Queue == nil { + if ps == nil || ps.Queue == nil { return Queue{} } - return *s.Queue + return *ps.Queue } // GetRepoAllowlist returns the RepoAllowlist field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetRepoAllowlist() []string { +func (ps *Platform) GetRepoAllowlist() []string { // return zero value if Platform type or RepoAllowlist field is nil - if s == nil || s.RepoAllowlist == nil { + if ps == nil || ps.RepoAllowlist == nil { return []string{} } - return *s.RepoAllowlist + return *ps.RepoAllowlist } // GetScheduleAllowlist returns the ScheduleAllowlist field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetScheduleAllowlist() []string { +func (ps *Platform) GetScheduleAllowlist() []string { // return zero value if Platform type or ScheduleAllowlist field is nil - if s == nil || s.ScheduleAllowlist == nil { + if ps == nil || ps.ScheduleAllowlist == nil { return []string{} } - return *s.ScheduleAllowlist + return *ps.ScheduleAllowlist } // GetCreatedAt returns the CreatedAt field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetCreatedAt() int64 { +func (ps *Platform) GetCreatedAt() int64 { // return zero value if Platform type or CreatedAt field is nil - if s == nil || s.CreatedAt == nil { + if ps == nil || ps.CreatedAt == nil { return 0 } - return *s.CreatedAt + return *ps.CreatedAt } // GetUpdatedAt returns the UpdatedAt field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetUpdatedAt() int64 { +func (ps *Platform) GetUpdatedAt() int64 { // return zero value if Platform type or UpdatedAt field is nil - if s == nil || s.UpdatedAt == nil { + if ps == nil || ps.UpdatedAt == nil { return 0 } - return *s.UpdatedAt + return *ps.UpdatedAt } // GetUpdatedBy returns the UpdatedBy field. // // When the provided Platform type is nil, or the field within // the type is nil, it returns the zero value for the field. -func (s *Platform) GetUpdatedBy() string { +func (ps *Platform) GetUpdatedBy() string { // return zero value if Platform type or UpdatedBy field is nil - if s == nil || s.UpdatedBy == nil { + if ps == nil || ps.UpdatedBy == nil { return "" } - return *s.UpdatedBy + return *ps.UpdatedBy } // SetID sets the ID field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetID(v int64) { +func (ps *Platform) SetID(v int64) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.ID = &v + ps.ID = &v } // SetCompiler sets the Compiler field. // // When the provided Compiler type is nil, it // will set nothing and immediately return. -func (s *Platform) SetCompiler(cs Compiler) { +func (ps *Platform) SetCompiler(cs Compiler) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.Compiler = &cs + ps.Compiler = &cs } // SetQueue sets the Queue field. // // When the provided Queue type is nil, it // will set nothing and immediately return. -func (s *Platform) SetQueue(qs Queue) { +func (ps *Platform) SetQueue(qs Queue) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.Queue = &qs + ps.Queue = &qs } // SetRepoAllowlist sets the RepoAllowlist field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetRepoAllowlist(v []string) { +func (ps *Platform) SetRepoAllowlist(v []string) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.RepoAllowlist = &v + ps.RepoAllowlist = &v } // SetScheduleAllowlist sets the RepoAllowlist field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetScheduleAllowlist(v []string) { +func (ps *Platform) SetScheduleAllowlist(v []string) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.ScheduleAllowlist = &v + ps.ScheduleAllowlist = &v } // SetCreatedAt sets the CreatedAt field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetCreatedAt(v int64) { +func (ps *Platform) SetCreatedAt(v int64) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.CreatedAt = &v + ps.CreatedAt = &v } // SetUpdatedAt sets the UpdatedAt field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetUpdatedAt(v int64) { +func (ps *Platform) SetUpdatedAt(v int64) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.UpdatedAt = &v + ps.UpdatedAt = &v } // SetUpdatedBy sets the UpdatedBy field. // // When the provided Platform type is nil, it // will set nothing and immediately return. -func (s *Platform) SetUpdatedBy(v string) { +func (ps *Platform) SetUpdatedBy(v string) { // return if Platform type is nil - if s == nil { + if ps == nil { return } - s.UpdatedBy = &v + ps.UpdatedBy = &v } // Update takes another settings record and updates the internal fields, intended // to be used when the refreshing settings record shared across the server. -func (s *Platform) Update(newSettings *Platform) { - if s == nil { +func (ps *Platform) Update(newSettingps *Platform) { + if ps == nil { return } - if newSettings == nil { + if newSettingps == nil { return } - s.SetCompiler(newSettings.GetCompiler()) - s.SetQueue(newSettings.GetQueue()) - s.SetRepoAllowlist(newSettings.GetRepoAllowlist()) - s.SetScheduleAllowlist(newSettings.GetScheduleAllowlist()) + ps.SetCompiler(newSettingps.GetCompiler()) + ps.SetQueue(newSettingps.GetQueue()) + ps.SetRepoAllowlist(newSettingps.GetRepoAllowlist()) + ps.SetScheduleAllowlist(newSettingps.GetScheduleAllowlist()) } // String implements the Stringer interface for the Platform type. -func (s *Platform) String() string { - cs := s.GetCompiler() - qs := s.GetQueue() +func (ps *Platform) String() string { + cs := ps.GetCompiler() + qs := ps.GetQueue() return fmt.Sprintf(`{ ID: %d, @@ -260,26 +260,26 @@ func (s *Platform) String() string { UpdatedAt: %d, UpdatedBy: %s, }`, - s.GetID(), + ps.GetID(), cs.String(), qs.String(), - s.GetRepoAllowlist(), - s.GetScheduleAllowlist(), - s.GetCreatedAt(), - s.GetUpdatedAt(), - s.GetUpdatedBy(), + ps.GetRepoAllowlist(), + ps.GetScheduleAllowlist(), + ps.GetCreatedAt(), + ps.GetUpdatedAt(), + ps.GetUpdatedBy(), ) } // PlatformMockEmpty returns an empty Platform type. func PlatformMockEmpty() Platform { - s := Platform{} + ps := Platform{} - s.SetCompiler(CompilerMockEmpty()) - s.SetQueue(QueueMockEmpty()) + ps.SetCompiler(CompilerMockEmpty()) + ps.SetQueue(QueueMockEmpty()) - s.SetRepoAllowlist([]string{}) - s.SetScheduleAllowlist([]string{}) + ps.SetRepoAllowlist([]string{}) + ps.SetScheduleAllowlist([]string{}) - return s + return ps } diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go index b13f4947e..56c9cec7b 100644 --- a/api/types/settings/platform_test.go +++ b/api/types/settings/platform_test.go @@ -48,7 +48,7 @@ func TestTypes_Platform_Getters(t *testing.T) { func TestTypes_Platform_Setters(t *testing.T) { // setup types - var s *Platform + var ps *Platform // setup tests tests := []struct { @@ -60,7 +60,7 @@ func TestTypes_Platform_Setters(t *testing.T) { want: testPlatformSettings(), }, { - platform: s, + platform: ps, want: new(Platform), }, } diff --git a/database/types/settings.go b/database/types/settings.go index 4a7e67f39..325e070ed 100644 --- a/database/types/settings.go +++ b/database/types/settings.go @@ -91,104 +91,104 @@ func (r *Queue) Scan(value interface{}) error { // When a field within the Settings type is the zero // value for the field, the valid flag is set to // false causing it to be NULL in the database. -func (s *Platform) Nullify() *Platform { - if s == nil { +func (ps *Platform) Nullify() *Platform { + if ps == nil { return nil } // check if the ID field should be false - if s.ID.Int64 == 0 { - s.ID.Valid = false + if ps.ID.Int64 == 0 { + ps.ID.Valid = false } // check if the CloneImage field should be false - if len(s.CloneImage.String) == 0 { - s.CloneImage.Valid = false + if len(ps.CloneImage.String) == 0 { + ps.CloneImage.Valid = false } // check if the CreatedAt field should be false - if s.CreatedAt.Int64 < 0 { - s.CreatedAt.Valid = false + if ps.CreatedAt.Int64 < 0 { + ps.CreatedAt.Valid = false } // check if the UpdatedAt field should be false - if s.UpdatedAt.Int64 < 0 { - s.UpdatedAt.Valid = false + if ps.UpdatedAt.Int64 < 0 { + ps.UpdatedAt.Valid = false } - return s + return ps } // ToAPI converts the Settings type // to an API Settings type. -func (s *Platform) ToAPI() *settings.Platform { - ss := new(settings.Platform) - ss.SetID(s.ID.Int64) +func (ps *Platform) ToAPI() *settings.Platform { + psApi := new(settings.Platform) + psApi.SetID(ps.ID.Int64) - ss.SetRepoAllowlist(s.RepoAllowlist) - ss.SetScheduleAllowlist(s.ScheduleAllowlist) + psApi.SetRepoAllowlist(ps.RepoAllowlist) + psApi.SetScheduleAllowlist(ps.ScheduleAllowlist) - ss.Compiler = &settings.Compiler{} - ss.SetCloneImage(s.CloneImage.String) - ss.SetTemplateDepth(int(s.TemplateDepth.Int64)) - ss.SetStarlarkExecLimit(uint64(s.StarlarkExecLimit.Int64)) + psApi.Compiler = &settings.Compiler{} + psApi.SetCloneImage(ps.CloneImage.String) + psApi.SetTemplateDepth(int(ps.TemplateDepth.Int64)) + psApi.SetStarlarkExecLimit(uint64(ps.StarlarkExecLimit.Int64)) - ss.Queue = &settings.Queue{} - ss.SetRoutes(s.Routes) + psApi.Queue = &settings.Queue{} + psApi.SetRoutes(ps.Routes) - ss.SetCreatedAt(s.CreatedAt.Int64) - ss.SetUpdatedAt(s.UpdatedAt.Int64) - ss.SetUpdatedBy(s.UpdatedBy.String) + psApi.SetCreatedAt(ps.CreatedAt.Int64) + psApi.SetUpdatedAt(ps.UpdatedAt.Int64) + psApi.SetUpdatedBy(ps.UpdatedBy.String) - return ss + return psApi } // Validate verifies the necessary fields for // the Settings type are populated correctly. -func (s *Platform) Validate() error { +func (ps *Platform) Validate() error { // verify the CloneImage field is populated - if len(s.CloneImage.String) == 0 { + if len(ps.CloneImage.String) == 0 { return ErrEmptyCloneImage } // verify compiler settings are within limits - if s.TemplateDepth.Int64 <= 0 { - return fmt.Errorf("template depth must be greater than zero, got: %d", s.TemplateDepth.Int64) + if ps.TemplateDepth.Int64 <= 0 { + return fmt.Errorf("template depth must be greater than zero, got: %d", ps.TemplateDepth.Int64) } - if s.StarlarkExecLimit.Int64 <= 0 { - return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", s.StarlarkExecLimit.Int64) + if ps.StarlarkExecLimit.Int64 <= 0 { + return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", ps.StarlarkExecLimit.Int64) } // ensure that all Settings string fields // that can be returned as JSON are sanitized // to avoid unsafe HTML content - s.CloneImage = sql.NullString{String: util.Sanitize(s.CloneImage.String), Valid: s.CloneImage.Valid} + ps.CloneImage = sql.NullString{String: util.Sanitize(ps.CloneImage.String), Valid: ps.CloneImage.Valid} // ensure that all Queue.Routes are sanitized // to avoid unsafe HTML content - for i, v := range s.Routes { - s.Routes[i] = util.Sanitize(v) + for i, v := range ps.Routes { + ps.Routes[i] = util.Sanitize(v) } // ensure that all RepoAllowlist are sanitized // to avoid unsafe HTML content - for i, v := range s.RepoAllowlist { - s.RepoAllowlist[i] = util.Sanitize(v) + for i, v := range ps.RepoAllowlist { + ps.RepoAllowlist[i] = util.Sanitize(v) } // ensure that all ScheduleAllowlist are sanitized // to avoid unsafe HTML content - for i, v := range s.ScheduleAllowlist { - s.ScheduleAllowlist[i] = util.Sanitize(v) + for i, v := range ps.ScheduleAllowlist { + ps.ScheduleAllowlist[i] = util.Sanitize(v) } - if s.CreatedAt.Int64 < 0 { - return fmt.Errorf("created_at must be greater than zero, got: %d", s.CreatedAt.Int64) + if ps.CreatedAt.Int64 < 0 { + return fmt.Errorf("created_at must be greater than zero, got: %d", ps.CreatedAt.Int64) } - if s.UpdatedAt.Int64 < 0 { - return fmt.Errorf("updated_at must be greater than zero, got: %d", s.UpdatedAt.Int64) + if ps.UpdatedAt.Int64 < 0 { + return fmt.Errorf("updated_at must be greater than zero, got: %d", ps.UpdatedAt.Int64) } return nil diff --git a/database/types/settings_test.go b/database/types/settings_test.go index c926411ac..973e5d88d 100644 --- a/database/types/settings_test.go +++ b/database/types/settings_test.go @@ -12,7 +12,7 @@ import ( func TestTypes_Platform_Nullify(t *testing.T) { // setup types - var s *Platform + var ps *Platform want := &Platform{ ID: sql.NullInt64{Int64: 0, Valid: false}, @@ -28,7 +28,7 @@ func TestTypes_Platform_Nullify(t *testing.T) { want: testPlatform(), }, { - repo: s, + repo: ps, want: nil, }, { diff --git a/queue/redis/redis.go b/queue/redis/redis.go index 28bd1ed2a..baaeb4bfb 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -32,7 +32,9 @@ type client struct { config *config Redis *redis.Client Options *redis.Options + settings.Queue + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } From 17a42d9c4ef267203df1ed9f1b74bda35a24959d Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 9 May 2024 14:24:54 -0500 Subject: [PATCH 54/65] chore(queue): rename channels to routes --- queue/redis/driver_test.go | 2 +- queue/redis/length.go | 4 ++-- queue/redis/length_test.go | 18 +++++++++--------- queue/redis/opts.go | 16 ++++++++-------- queue/redis/opts_test.go | 28 ++++++++++++++-------------- queue/redis/ping_test.go | 4 ++-- queue/redis/pop.go | 12 ++++++------ queue/redis/pop_test.go | 18 +++++++++--------- queue/redis/redis.go | 4 ++-- queue/redis/redis_test.go | 2 +- queue/redis/route.go | 2 +- queue/setup.go | 2 +- 12 files changed, 56 insertions(+), 56 deletions(-) diff --git a/queue/redis/driver_test.go b/queue/redis/driver_test.go index 8ed67a3a5..52894f1ba 100644 --- a/queue/redis/driver_test.go +++ b/queue/redis/driver_test.go @@ -28,7 +28,7 @@ func TestRedis_Driver(t *testing.T) { _service, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/length.go b/queue/redis/length.go index d6bc79226..7f5ed9b0b 100644 --- a/queue/redis/length.go +++ b/queue/redis/length.go @@ -6,9 +6,9 @@ import ( "context" ) -// Length tallies all items present in the configured channels in the queue. +// Length tallies all items present in the configured routes in the queue. func (c *client) Length(ctx context.Context) (int64, error) { - c.Logger.Tracef("reading length of all configured channels in queue") + c.Logger.Tracef("reading length of all configured routes in queue") total := int64(0) diff --git a/queue/redis/length_test.go b/queue/redis/length_test.go index 545f9f54a..9656a8c20 100644 --- a/queue/redis/length_test.go +++ b/queue/redis/length_test.go @@ -33,26 +33,26 @@ func TestRedis_Length(t *testing.T) { // setup tests tests := []struct { - channels []string - want int64 + routes []string + want int64 }{ { - channels: []string{"vela"}, - want: 1, + routes: []string{"vela"}, + want: 1, }, { - channels: []string{"vela", "vela:second", "vela:third"}, - want: 4, + routes: []string{"vela", "vela:second", "vela:third"}, + want: 4, }, { - channels: []string{"vela", "vela:second", "phony"}, - want: 6, + routes: []string{"vela", "vela:second", "phony"}, + want: 6, }, } // run tests for _, test := range tests { - for _, channel := range test.channels { + for _, channel := range test.routes { err := _redis.Push(context.Background(), channel, bytes) if err != nil { t.Errorf("unable to push item to queue: %v", err) diff --git a/queue/redis/opts.go b/queue/redis/opts.go index 81a9509f4..f58dab06a 100644 --- a/queue/redis/opts.go +++ b/queue/redis/opts.go @@ -29,18 +29,18 @@ func WithAddress(address string) ClientOpt { } } -// WithChannels sets the channels in the queue client for Redis. -func WithChannels(channels ...string) ClientOpt { +// WithRoutes sets the routes in the queue client for Redis. +func WithRoutes(routes ...string) ClientOpt { return func(c *client) error { - c.Logger.Trace("configuring channels in redis queue client") + c.Logger.Trace("configuring routes in redis queue client") - // check if the channels provided are empty - if len(channels) == 0 { - return fmt.Errorf("no Redis queue channels provided") + // check if the routes provided are empty + if len(routes) == 0 { + return fmt.Errorf("no Redis queue routes provided") } - // set the queue channels in the redis client - c.SetRoutes(channels) + // set the queue routes in the redis client + c.SetRoutes(routes) return nil } diff --git a/queue/redis/opts_test.go b/queue/redis/opts_test.go index 70b8b4e75..6e729ce52 100644 --- a/queue/redis/opts_test.go +++ b/queue/redis/opts_test.go @@ -64,7 +64,7 @@ func TestRedis_ClientOpt_WithAddress(t *testing.T) { } } -func TestRedis_ClientOpt_WithChannels(t *testing.T) { +func TestRedis_ClientOpt_WithRoutes(t *testing.T) { // setup tests // create a local fake redis instance // @@ -76,19 +76,19 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { defer _redis.Close() tests := []struct { - failure bool - channels []string - want []string + failure bool + routes []string + want []string }{ { - failure: false, - channels: []string{"foo", "bar"}, - want: []string{"foo", "bar"}, + failure: false, + routes: []string{"foo", "bar"}, + want: []string{"foo", "bar"}, }, { - failure: true, - channels: []string{}, - want: []string{}, + failure: true, + routes: []string{}, + want: []string{}, }, } @@ -96,23 +96,23 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { for _, test := range tests { _service, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels(test.channels...), + WithRoutes(test.routes...), ) if test.failure { if err == nil { - t.Errorf("WithChannels should have returned err") + t.Errorf("WithRoutes should have returned err") } continue } if err != nil { - t.Errorf("WithChannels returned err: %v", err) + t.Errorf("WithRoutes returned err: %v", err) } if !reflect.DeepEqual(_service.GetRoutes(), test.want) { - t.Errorf("WithChannels is %v, want %v", _service.GetRoutes(), test.want) + t.Errorf("WithRoutes is %v, want %v", _service.GetRoutes(), test.want) } } } diff --git a/queue/redis/ping_test.go b/queue/redis/ping_test.go index d069370f2..39fde9272 100644 --- a/queue/redis/ping_test.go +++ b/queue/redis/ping_test.go @@ -22,7 +22,7 @@ func TestRedis_Ping_Good(t *testing.T) { // setup redis mock goodRedis, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) @@ -49,7 +49,7 @@ func TestRedis_Ping_Bad(t *testing.T) { // setup redis mock badRedis, _ := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/pop.go b/queue/redis/pop.go index 867bc104f..7c5658029 100644 --- a/queue/redis/pop.go +++ b/queue/redis/pop.go @@ -14,23 +14,23 @@ import ( ) // Pop grabs an item from the specified channel off the queue. -func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) { +func (c *client) Pop(ctx context.Context, inRoutes []string) (*models.Item, error) { c.Logger.Tracef("popping item from queue %s", c.GetRoutes()) - // define channels to pop from - var channels []string + // define routes to pop from + var routes []string // if routes were supplied, use those if len(routes) > 0 { - channels = routes + routes = inRoutes } else { - channels = c.GetRoutes() + routes = c.GetRoutes() } // build a redis queue command to pop an item from queue // // https://pkg.go.dev/github.com/go-redis/redis?tab=doc#Client.BLPop - popCmd := c.Redis.BLPop(ctx, c.config.Timeout, channels...) + popCmd := c.Redis.BLPop(ctx, c.config.Timeout, routes...) // blocking call to pop item from queue // diff --git a/queue/redis/pop_test.go b/queue/redis/pop_test.go index 6ab0722a1..190839f80 100644 --- a/queue/redis/pop_test.go +++ b/queue/redis/pop_test.go @@ -76,10 +76,10 @@ func TestRedis_Pop(t *testing.T) { // setup tests tests := []struct { - failure bool - redis *client - want *models.Item - channels []string + failure bool + redis *client + want *models.Item + routes []string }{ { failure: false, @@ -87,10 +87,10 @@ func TestRedis_Pop(t *testing.T) { want: _item, }, { - failure: false, - redis: _redis, - want: _item, - channels: []string{"custom"}, + failure: false, + redis: _redis, + want: _item, + routes: []string{"custom"}, }, { failure: false, @@ -106,7 +106,7 @@ func TestRedis_Pop(t *testing.T) { // run tests for _, test := range tests { - got, err := test.redis.Pop(context.Background(), test.channels) + got, err := test.redis.Pop(context.Background(), test.routes) if test.failure { if err == nil { diff --git a/queue/redis/redis.go b/queue/redis/redis.go index baaeb4bfb..88b962c00 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -177,7 +177,7 @@ func pingQueue(c *client) error { // This function is intended for running tests only. // //nolint:revive // ignore returning unexported client -func NewTest(signingPrivateKey, signingPublicKey string, channels ...string) (*client, error) { +func NewTest(signingPrivateKey, signingPublicKey string, routes ...string) (*client, error) { // create a local fake redis instance // // https://pkg.go.dev/github.com/alicebob/miniredis/v2#Run @@ -188,7 +188,7 @@ func NewTest(signingPrivateKey, signingPublicKey string, channels ...string) (*c return New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels(channels...), + WithRoutes(routes...), WithCluster(false), WithPrivateKey(signingPrivateKey), WithPublicKey(signingPublicKey), diff --git a/queue/redis/redis_test.go b/queue/redis/redis_test.go index 474d57de9..b08fda760 100644 --- a/queue/redis/redis_test.go +++ b/queue/redis/redis_test.go @@ -125,7 +125,7 @@ func TestRedis_New(t *testing.T) { for _, test := range tests { _, err := New( WithAddress(test.address), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/route.go b/queue/redis/route.go index e11f8f60f..e4fc0ae05 100644 --- a/queue/redis/route.go +++ b/queue/redis/route.go @@ -13,7 +13,7 @@ import ( // Route decides which route a build gets placed within the queue. func (c *client) Route(w *pipeline.Worker) (string, error) { - c.Logger.Tracef("deciding route from queue channels %s", c.GetRoutes()) + c.Logger.Tracef("deciding route from queue routes %s", c.GetRoutes()) // create buffer to store route buf := bytes.Buffer{} diff --git a/queue/setup.go b/queue/setup.go index cdac822ea..c554c9238 100644 --- a/queue/setup.go +++ b/queue/setup.go @@ -45,7 +45,7 @@ func (s *Setup) Redis() (Service, error) { // https://pkg.go.dev/github.com/go-vela/server/queue/redis?tab=doc#New return redis.New( redis.WithAddress(s.Address), - redis.WithChannels(s.Routes...), + redis.WithRoutes(s.Routes...), redis.WithCluster(s.Cluster), redis.WithTimeout(s.Timeout), redis.WithPrivateKey(s.PrivateKey), From eb3417bcf0d3be5e326395df5358a0b36c596f19 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 9 May 2024 14:25:37 -0500 Subject: [PATCH 55/65] fix: lint naming --- database/types/settings.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/database/types/settings.go b/database/types/settings.go index 325e070ed..f17bac54b 100644 --- a/database/types/settings.go +++ b/database/types/settings.go @@ -122,25 +122,25 @@ func (ps *Platform) Nullify() *Platform { // ToAPI converts the Settings type // to an API Settings type. func (ps *Platform) ToAPI() *settings.Platform { - psApi := new(settings.Platform) - psApi.SetID(ps.ID.Int64) + psAPI := new(settings.Platform) + psAPI.SetID(ps.ID.Int64) - psApi.SetRepoAllowlist(ps.RepoAllowlist) - psApi.SetScheduleAllowlist(ps.ScheduleAllowlist) + psAPI.SetRepoAllowlist(ps.RepoAllowlist) + psAPI.SetScheduleAllowlist(ps.ScheduleAllowlist) - psApi.Compiler = &settings.Compiler{} - psApi.SetCloneImage(ps.CloneImage.String) - psApi.SetTemplateDepth(int(ps.TemplateDepth.Int64)) - psApi.SetStarlarkExecLimit(uint64(ps.StarlarkExecLimit.Int64)) + psAPI.Compiler = &settings.Compiler{} + psAPI.SetCloneImage(ps.CloneImage.String) + psAPI.SetTemplateDepth(int(ps.TemplateDepth.Int64)) + psAPI.SetStarlarkExecLimit(uint64(ps.StarlarkExecLimit.Int64)) - psApi.Queue = &settings.Queue{} - psApi.SetRoutes(ps.Routes) + psAPI.Queue = &settings.Queue{} + psAPI.SetRoutes(ps.Routes) - psApi.SetCreatedAt(ps.CreatedAt.Int64) - psApi.SetUpdatedAt(ps.UpdatedAt.Int64) - psApi.SetUpdatedBy(ps.UpdatedBy.String) + psAPI.SetCreatedAt(ps.CreatedAt.Int64) + psAPI.SetUpdatedAt(ps.UpdatedAt.Int64) + psAPI.SetUpdatedBy(ps.UpdatedBy.String) - return psApi + return psAPI } // Validate verifies the necessary fields for From a56ea849c987e8139f88e8b26bffb94899f1be9d Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 9 May 2024 14:30:36 -0500 Subject: [PATCH 56/65] enhance: check record not found error instead of nil --- cmd/vela-server/server.go | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 4930961a6..bae9ec326 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -4,6 +4,7 @@ package main import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -16,6 +17,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" + "gorm.io/gorm" "k8s.io/apimachinery/pkg/util/wait" "github.com/go-vela/server/api/types/settings" @@ -107,42 +109,40 @@ func server(c *cli.Context) error { time.Sleep(jitter) - s, err := database.GetSettings(context.Background()) - if s == nil || err != nil { - // only log errors - if s == nil { - logrus.Error("unable to get platform settings") - } + ps, err := database.GetSettings(context.Background()) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } - if err != nil { - logrus.WithError(err).Error("unable to get platform settings") - } + // platform settings record does not exist + if err != nil { + logrus.Info("creating initial platform settings") // create initial settings record - s = new(settings.Platform) + ps = new(settings.Platform) // singleton record ID should always be 1 - s.SetID(1) + ps.SetID(1) - s.SetCreatedAt(time.Now().UTC().Unix()) - s.SetUpdatedAt(time.Now().UTC().Unix()) - s.SetUpdatedBy("vela-server") + ps.SetCreatedAt(time.Now().UTC().Unix()) + ps.SetUpdatedAt(time.Now().UTC().Unix()) + ps.SetUpdatedBy("vela-server") // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() - s.SetCompiler(compilerSettings) + ps.SetCompiler(compilerSettings) queueSettings := queue.GetSettings() - s.SetQueue(queueSettings) + ps.SetQueue(queueSettings) // set repos permitted to be added - s.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) + ps.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) // set repos permitted to use schedules - s.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) + ps.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) // create the settings record in the database - _, err = database.CreateSettings(context.Background(), s) + _, err = database.CreateSettings(context.Background(), ps) if err != nil { return err } @@ -150,12 +150,12 @@ func server(c *cli.Context) error { // update any internal settings, this occurs in middleware // to keep settings refreshed for each request - queue.SetSettings(s) - compiler.SetSettings(s) + queue.SetSettings(ps) + compiler.SetSettings(ps) router := router.Load( middleware.CLI(c), - middleware.Settings(s), + middleware.Settings(ps), middleware.Compiler(compiler), middleware.Database(database), middleware.Logger(logrus.StandardLogger(), time.RFC3339), @@ -252,7 +252,7 @@ func server(c *cli.Context) error { } // update the internal fields for the shared settings record - s.Update(newSettings) + ps.Update(newSettings) } }) @@ -298,10 +298,10 @@ func server(c *cli.Context) error { time.Sleep(jitter) // update internal settings updated through refresh - compiler.SetSettings(s) - queue.SetSettings(s) + compiler.SetSettings(ps) + queue.SetSettings(ps) - err = processSchedules(ctx, start, s, compiler, database, metadata, queue, scm) + err = processSchedules(ctx, start, ps, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { From 197e42ff5919269bd98ec8fb21e148d028131cbc Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 10:23:47 -0500 Subject: [PATCH 57/65] enhance: add docker image validation --- api/admin/settings.go | 15 ++++++- compiler/native/native.go | 12 +++++- go.mod | 2 + go.sum | 4 ++ internal/image/doc.go | 9 +++++ internal/image/image.go | 35 +++++++++++++++++ internal/image/image_test.go | 76 ++++++++++++++++++++++++++++++++++++ 7 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 internal/image/doc.go create mode 100644 internal/image/image.go create mode 100644 internal/image/image_test.go diff --git a/api/admin/settings.go b/api/admin/settings.go index a6b44ca7f..4241b56de 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/image" "github.com/go-vela/server/queue" cliMiddleware "github.com/go-vela/server/router/middleware/cli" sMiddleware "github.com/go-vela/server/router/middleware/settings" @@ -130,7 +131,19 @@ func UpdateSettings(c *gin.Context) { if input.Compiler != nil { if input.CloneImage != nil { - s.SetCloneImage(*input.CloneImage) + // validate clone image + cloneImage := *input.CloneImage + + _, err = image.ParseWithError(cloneImage) + if err != nil { + retErr := fmt.Errorf("invalid clone image %s: %v", cloneImage, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + s.SetCloneImage(cloneImage) } if input.TemplateDepth != nil { diff --git a/compiler/native/native.go b/compiler/native/native.go index 16b4f54ee..d84be4ba4 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -3,6 +3,7 @@ package native import ( + "fmt" "time" "github.com/sirupsen/logrus" @@ -14,6 +15,7 @@ import ( "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" "github.com/go-vela/server/internal" + "github.com/go-vela/server/internal/image" ) type ModificationConfig struct { @@ -70,8 +72,16 @@ func FromCLIContext(ctx *cli.Context) (*client, error) { c.Compiler = settings.Compiler{} + cloneImage := ctx.String("clone-image") + + // validate clone image + _, err = image.ParseWithError(cloneImage) + if err != nil { + return nil, fmt.Errorf("invalid clone image %s: %v", cloneImage, err) + } + // set the clone image to use for the injected clone step - c.SetCloneImage(ctx.String("clone-image")) + c.SetCloneImage(cloneImage) // set the template depth to use for nested templates c.SetTemplateDepth(ctx.Int("max-template-depth")) diff --git a/go.mod b/go.mod index aefa64a29..4993597bb 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/alicebob/miniredis/v2 v2.32.1 github.com/aws/aws-sdk-go v1.51.0 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 + github.com/distribution/reference v0.6.0 github.com/drone/envsubst v1.0.3 github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.9.1 @@ -99,6 +100,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/go.sum b/go.sum index 3b30dd8ed..75bac3763 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -213,6 +215,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/image/doc.go b/internal/image/doc.go new file mode 100644 index 000000000..62ae9f1a2 --- /dev/null +++ b/internal/image/doc.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package image provides the ability for Vela to manage +// and manipulate a images. +// +// Usage: +// +// import "github.com/go-vela/server/internal/image" +package image diff --git a/internal/image/image.go b/internal/image/image.go new file mode 100644 index 000000000..daa5b37bc --- /dev/null +++ b/internal/image/image.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "github.com/distribution/reference" +) + +// ParseWithError digests the provided image into a +// fully qualified canonical reference. If an error +// occurs, it will return the last digested form of +// the image. +func ParseWithError(_image string) (string, error) { + // parse the image provided into a + // named, fully qualified reference + // + // https://pkg.go.dev/github.com/distribution/reference#ParseAnyReference + _reference, err := reference.ParseAnyReference(_image) + if err != nil { + return _image, err + } + + // ensure we have the canonical form of the named reference + // + // https://pkg.go.dev/github.com/distribution/reference#ParseNamed + _canonical, err := reference.ParseNamed(_reference.String()) + if err != nil { + return _reference.String(), err + } + + // ensure the canonical reference has a tag + // + // https://pkg.go.dev/github.com/distribution/reference#TagNameOnly + return reference.TagNameOnly(_canonical).String(), nil +} diff --git a/internal/image/image_test.go b/internal/image/image_test.go new file mode 100644 index 000000000..365fad2ff --- /dev/null +++ b/internal/image/image_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "strings" + "testing" +) + +func TestImage_ParseWithError(t *testing.T) { + // setup tests + tests := []struct { + name string + failure bool + image string + want string + }{ + { + name: "image only", + failure: false, + image: "golang", + want: "docker.io/library/golang:latest", + }, + { + name: "image and tag", + failure: false, + image: "golang:latest", + want: "docker.io/library/golang:latest", + }, + { + name: "image and tag", + failure: false, + image: "golang:1.14", + want: "docker.io/library/golang:1.14", + }, + { + name: "fails with bad image", + failure: true, + image: "!@#$%^&*()", + want: "!@#$%^&*()", + }, + { + name: "fails with image sha", + failure: true, + image: "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + want: "sha256:1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := ParseWithError(test.image) + + if test.failure { + if err == nil { + t.Errorf("ParseWithError should have returned err") + } + + if !strings.EqualFold(got, test.want) { + t.Errorf("ParseWithError is %s want %s", got, test.want) + } + + return // continue to next test + } + + if err != nil { + t.Errorf("ParseWithError returned err: %v", err) + } + + if !strings.EqualFold(got, test.want) { + t.Errorf("ParseWithError is %s want %s", got, test.want) + } + }) + } +} From 1f8ef15d2ba4c5c808d2a82e0474afc241779d64 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 10:25:09 -0500 Subject: [PATCH 58/65] fix: typo --- internal/image/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/image/doc.go b/internal/image/doc.go index 62ae9f1a2..66d473d15 100644 --- a/internal/image/doc.go +++ b/internal/image/doc.go @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Package image provides the ability for Vela to manage -// and manipulate a images. +// and manipulate images. // // Usage: // From da472a4875766d20c74540958b42ab4ae0a40935 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 10:26:05 -0500 Subject: [PATCH 59/65] fix: lint fmt --- api/admin/settings.go | 2 +- compiler/native/native.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 4241b56de..4fbe43a03 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -136,7 +136,7 @@ func UpdateSettings(c *gin.Context) { _, err = image.ParseWithError(cloneImage) if err != nil { - retErr := fmt.Errorf("invalid clone image %s: %v", cloneImage, err) + retErr := fmt.Errorf("invalid clone image %s: %w", cloneImage, err) util.HandleError(c, http.StatusBadRequest, retErr) diff --git a/compiler/native/native.go b/compiler/native/native.go index d84be4ba4..93d93c5ea 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -77,7 +77,7 @@ func FromCLIContext(ctx *cli.Context) (*client, error) { // validate clone image _, err = image.ParseWithError(cloneImage) if err != nil { - return nil, fmt.Errorf("invalid clone image %s: %v", cloneImage, err) + return nil, fmt.Errorf("invalid clone image %s: %w", cloneImage, err) } // set the clone image to use for the injected clone step From 3a627efa06871c0926c195c59d7425a921acdc6b Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 10:51:04 -0500 Subject: [PATCH 60/65] fix: test case flags --- router/middleware/compiler_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index ed625d749..702ea85cf 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -20,13 +20,21 @@ import ( func TestMiddleware_CompilerNative(t *testing.T) { // setup types - var got compiler.Engine - got, _ = native.FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + defaultCloneImage := "target/vela-git" + wantCloneImage := "target/vela-git:latest" + + fs := flag.NewFlagSet("", flag.ExitOnError) + fs.String("clone-image", "", "") + fs.Set("clone-image", defaultCloneImage) + + cliCtx := cli.NewContext(nil, fs, nil) - wantCloneImage := "target/vela-git" - want, _ := native.FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want, _ := native.FromCLIContext(cliCtx) want.SetCloneImage(wantCloneImage) + var got compiler.Engine + got, _ = native.FromCLIContext(cliCtx) + // setup context gin.SetMode(gin.TestMode) From f12002929986bb8c9b9471ec452628df512c7279 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 10:54:23 -0500 Subject: [PATCH 61/65] fix: test case lint --- router/middleware/compiler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index 702ea85cf..f20bddac1 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -25,7 +25,7 @@ func TestMiddleware_CompilerNative(t *testing.T) { fs := flag.NewFlagSet("", flag.ExitOnError) fs.String("clone-image", "", "") - fs.Set("clone-image", defaultCloneImage) + _ = fs.Set("clone-image", defaultCloneImage) cliCtx := cli.NewContext(nil, fs, nil) From a43546e83ddd47c1da398c5e40967eea3d4cde9f Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 11:43:15 -0500 Subject: [PATCH 62/65] fix: clone image compiler test --- compiler/native/compile_test.go | 8 ++++++++ compiler/native/environment_test.go | 4 ++++ compiler/native/expand_test.go | 7 +++++++ compiler/native/initialize_test.go | 2 ++ compiler/native/native_test.go | 17 +++++++++++++++++ compiler/native/script_test.go | 2 ++ compiler/native/substitute_test.go | 2 ++ compiler/native/transform_test.go | 2 ++ compiler/native/validate_test.go | 18 ++++++++++++++++++ router/middleware/compiler_test.go | 11 ++++------- 10 files changed, 66 insertions(+), 7 deletions(-) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 59cee9bc8..15f2ab8f3 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -1370,6 +1370,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -1828,6 +1829,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { func TestNative_Compile_NoStepsorStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) name := "foo" author := "author" @@ -1844,6 +1846,10 @@ func TestNative_Compile_NoStepsorStages(t *testing.T) { t.Errorf("Creating compiler returned err: %v", err) } + // todo: this needs to be fixed in compiler validation + // this is a dirty hack to make this test pass + compiler.SetCloneImage("") + compiler.repo = &api.Repo{Name: &author} compiler.build = &api.Build{Author: &name, Number: &number} @@ -1860,6 +1866,7 @@ func TestNative_Compile_NoStepsorStages(t *testing.T) { func TestNative_Compile_StepsandStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) name := "foo" author := "author" @@ -3016,6 +3023,7 @@ func Test_CompileLite(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index bb3f9d8da..9c55ed41d 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -20,6 +20,7 @@ import ( func TestNative_EnvironmentStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -78,6 +79,7 @@ func TestNative_EnvironmentStages(t *testing.T) { func TestNative_EnvironmentSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ @@ -253,6 +255,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { func TestNative_EnvironmentServices(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ @@ -398,6 +401,7 @@ func TestNative_EnvironmentServices(t *testing.T) { func TestNative_EnvironmentSecrets(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 171c80d30..49e6b51b7 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -44,6 +44,7 @@ func TestNative_ExpandStages(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -226,6 +227,7 @@ func TestNative_ExpandSteps(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testRepo := new(api.Repo) @@ -404,6 +406,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -670,6 +673,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -760,6 +764,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) @@ -928,6 +933,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) @@ -1014,6 +1020,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) diff --git a/compiler/native/initialize_test.go b/compiler/native/initialize_test.go index fb2a6548b..ea196216a 100644 --- a/compiler/native/initialize_test.go +++ b/compiler/native/initialize_test.go @@ -15,6 +15,7 @@ import ( func TestNative_InitStage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -79,6 +80,7 @@ func TestNative_InitStage(t *testing.T) { func TestNative_InitStep(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" diff --git a/compiler/native/native_test.go b/compiler/native/native_test.go index 3c1eb8075..6c5b6d2be 100644 --- a/compiler/native/native_test.go +++ b/compiler/native/native_test.go @@ -18,12 +18,14 @@ import ( func TestNative_New(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") want := &client{ Github: public, Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test got, err := FromCLIContext(c) @@ -45,6 +47,7 @@ func TestNative_New_PrivateGithub(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") private, _ := github.New(url, token) @@ -54,6 +57,7 @@ func TestNative_New_PrivateGithub(t *testing.T) { UsePrivateGithub: true, Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test got, err := FromCLIContext(c) @@ -75,6 +79,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") private, _ := github.New(url, token) @@ -84,6 +89,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { UsePrivateGithub: true, Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test got, err := FromCLIContext(c) @@ -100,6 +106,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { func TestNative_DuplicateStripBuild(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) @@ -123,6 +130,7 @@ func TestNative_DuplicateStripBuild(t *testing.T) { func TestNative_WithBuild(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) @@ -145,6 +153,7 @@ func TestNative_WithBuild(t *testing.T) { func TestNative_WithFiles(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) f := []string{"foo"} @@ -166,6 +175,7 @@ func TestNative_WithFiles(t *testing.T) { func TestNative_WithComment(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) comment := "ok to test" @@ -186,6 +196,7 @@ func TestNative_WithComment(t *testing.T) { func TestNative_WithLocal(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) local := true @@ -206,6 +217,7 @@ func TestNative_WithLocal(t *testing.T) { func TestNative_WithLocalTemplates(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) localTemplates := []string{"example:tmpl.yml", "exmpl:template.yml"} @@ -226,6 +238,7 @@ func TestNative_WithLocalTemplates(t *testing.T) { func TestNative_WithMetadata(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -270,6 +283,7 @@ func TestNative_WithPrivateGitHub(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) private, _ := github.New(url, token) @@ -291,6 +305,7 @@ func TestNative_WithPrivateGitHub(t *testing.T) { func TestNative_WithRepo(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) @@ -313,6 +328,7 @@ func TestNative_WithRepo(t *testing.T) { func TestNative_WithUser(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) @@ -335,6 +351,7 @@ func TestNative_WithUser(t *testing.T) { func TestNative_WithLabels(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) labels := []string{"documentation", "enhancement"} diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 2e948e27a..0f03e6865 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -16,6 +16,7 @@ import ( func TestNative_ScriptStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) baseEnv := environment(nil, nil, nil, nil) @@ -105,6 +106,7 @@ func TestNative_ScriptStages(t *testing.T) { func TestNative_ScriptSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) emptyEnv := environment(nil, nil, nil, nil) diff --git a/compiler/native/substitute_test.go b/compiler/native/substitute_test.go index 809899e1e..0be8eac03 100644 --- a/compiler/native/substitute_test.go +++ b/compiler/native/substitute_test.go @@ -19,6 +19,7 @@ func Test_client_SubstituteStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tests := []struct { @@ -137,6 +138,7 @@ func Test_client_SubstituteSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tests := []struct { diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index 0c1d424e6..bcd318103 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -17,6 +17,7 @@ import ( func TestNative_TransformStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -257,6 +258,7 @@ func TestNative_TransformStages(t *testing.T) { func TestNative_TransformSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index d0f23d4fc..226c75aa9 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -16,6 +16,7 @@ import ( func TestNative_Validate_NoVersion(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{} @@ -35,6 +36,7 @@ func TestNative_Validate_NoVersion(t *testing.T) { func TestNative_Validate_NoStagesOrSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{ @@ -56,6 +58,7 @@ func TestNative_Validate_NoStagesOrSteps(t *testing.T) { func TestNative_Validate_StagesAndSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -99,6 +102,7 @@ func TestNative_Validate_StagesAndSteps(t *testing.T) { func TestNative_Validate_Services(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -136,6 +140,7 @@ func TestNative_Validate_Services(t *testing.T) { func TestNative_Validate_Services_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -173,6 +178,7 @@ func TestNative_Validate_Services_NoName(t *testing.T) { func TestNative_Validate_Services_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -210,6 +216,7 @@ func TestNative_Validate_Services_NoImage(t *testing.T) { func TestNative_Validate_Stages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -245,6 +252,7 @@ func TestNative_Validate_Stages(t *testing.T) { func TestNative_Validate_Stages_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -279,6 +287,7 @@ func TestNative_Validate_Stages_NoName(t *testing.T) { func TestNative_Validate_Stages_NoStepName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -313,6 +322,7 @@ func TestNative_Validate_Stages_NoStepName(t *testing.T) { func TestNative_Validate_Stages_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -347,6 +357,7 @@ func TestNative_Validate_Stages_NoImage(t *testing.T) { func TestNative_Validate_Stages_NoCommands(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -381,6 +392,7 @@ func TestNative_Validate_Stages_NoCommands(t *testing.T) { func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -417,6 +429,7 @@ func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { func TestNative_Validate_Steps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -447,6 +460,7 @@ func TestNative_Validate_Steps(t *testing.T) { func TestNative_Validate_Steps_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{ @@ -475,6 +489,7 @@ func TestNative_Validate_Steps_NoName(t *testing.T) { func TestNative_Validate_Steps_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -504,6 +519,7 @@ func TestNative_Validate_Steps_NoImage(t *testing.T) { func TestNative_Validate_Steps_NoCommands(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -533,6 +549,7 @@ func TestNative_Validate_Steps_NoCommands(t *testing.T) { func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -571,6 +588,7 @@ func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { func TestNative_Validate_MultiReportAs(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index f20bddac1..d018b06a9 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -23,17 +23,14 @@ func TestMiddleware_CompilerNative(t *testing.T) { defaultCloneImage := "target/vela-git" wantCloneImage := "target/vela-git:latest" - fs := flag.NewFlagSet("", flag.ExitOnError) - fs.String("clone-image", "", "") - _ = fs.Set("clone-image", defaultCloneImage) + set := flag.NewFlagSet("", flag.ExitOnError) + set.String("clone-image", defaultCloneImage, "doc") - cliCtx := cli.NewContext(nil, fs, nil) - - want, _ := native.FromCLIContext(cliCtx) + want, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) want.SetCloneImage(wantCloneImage) var got compiler.Engine - got, _ = native.FromCLIContext(cliCtx) + got, _ = native.FromCLIContext(cli.NewContext(nil, set, nil)) // setup context gin.SetMode(gin.TestMode) From 4cad3e66a5fe1c6bd5d5b145a9dc295b624547ea Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 11:47:26 -0500 Subject: [PATCH 63/65] fix: dont update shared pointer --- api/admin/settings.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 4fbe43a03..5b0a147a9 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -117,6 +117,10 @@ func UpdateSettings(c *gin.Context) { return } + // duplicate settings to not alter the shared pointer + _s := new(settings.Platform) + _s.Update(s) + // capture body from API request input := new(settings.Platform) @@ -143,36 +147,36 @@ func UpdateSettings(c *gin.Context) { return } - s.SetCloneImage(cloneImage) + _s.SetCloneImage(cloneImage) } if input.TemplateDepth != nil { - s.SetTemplateDepth(*input.TemplateDepth) + _s.SetTemplateDepth(*input.TemplateDepth) } if input.StarlarkExecLimit != nil { - s.SetStarlarkExecLimit(*input.StarlarkExecLimit) + _s.SetStarlarkExecLimit(*input.StarlarkExecLimit) } } if input.Queue != nil { if input.Queue.Routes != nil { - s.SetRoutes(input.GetRoutes()) + _s.SetRoutes(input.GetRoutes()) } } if input.RepoAllowlist != nil { - s.SetRepoAllowlist(input.GetRepoAllowlist()) + _s.SetRepoAllowlist(input.GetRepoAllowlist()) } if input.ScheduleAllowlist != nil { - s.SetScheduleAllowlist(input.GetScheduleAllowlist()) + _s.SetScheduleAllowlist(input.GetScheduleAllowlist()) } - s.SetUpdatedBy(u.GetName()) + _s.SetUpdatedBy(u.GetName()) // send API call to update the settings - s, err = database.FromContext(c).UpdateSettings(ctx, s) + _s, err = database.FromContext(c).UpdateSettings(ctx, _s) if err != nil { retErr := fmt.Errorf("unable to update settings: %w", err) @@ -181,7 +185,7 @@ func UpdateSettings(c *gin.Context) { return } - c.JSON(http.StatusOK, s) + c.JSON(http.StatusOK, _s) } // swagger:operation DELETE /api/v1/admin/settings admin RestoreSettings From b6e2672cd7430e3fea8f5613ba42054fa8590952 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 11:50:14 -0500 Subject: [PATCH 64/65] fix: set ID --- api/admin/settings.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/admin/settings.go b/api/admin/settings.go index 5b0a147a9..0b067372b 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -121,6 +121,9 @@ func UpdateSettings(c *gin.Context) { _s := new(settings.Platform) _s.Update(s) + // ensure we update the singleton record + _s.SetID(1) + // capture body from API request input := new(settings.Platform) From f54aa675717e92cf173d40f4422246fbbfcae801 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 10 May 2024 12:09:49 -0500 Subject: [PATCH 65/65] fix: test merge --- compiler/native/expand_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index daaba9250..2f036f974 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -897,7 +897,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment, Templates: yaml.TemplateSlice{test.tmpls["chain"]}}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment, Templates: yaml.TemplateSlice{test.tmpls["chain"]}}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) }