diff --git a/.github/workflows/test_sast.yml b/.github/workflows/test_sast.yml index dacd396b..2ed4eeba 100644 --- a/.github/workflows/test_sast.yml +++ b/.github/workflows/test_sast.yml @@ -1,6 +1,7 @@ name: Test - SAST on: + push: pull_request: permissions: diff --git a/.github/workflows/test_unit.yml b/.github/workflows/test_unit.yml index 43ad7b77..8197f5aa 100644 --- a/.github/workflows/test_unit.yml +++ b/.github/workflows/test_unit.yml @@ -23,4 +23,4 @@ jobs: - name: Test run: | go mod download - go test --race --coverprofile cover.out -v ./... \ No newline at end of file + go test --race --count=1 --coverprofile cover.out -v ./... diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 76b30808..d850a9b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: hooks: - id: go-mod-tidy-repo - id: go-test-repo-mod - args: [ -race ] + args: [ -race, -count=1 ] - id: go-vet-repo-mod - id: go-fumpt-repo args: [ -l, -w ] diff --git a/.vscode/settings.json b/.vscode/settings.json index 867e79be..d35d0a6b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "go.testFlags": [ "-race", - "-cover" + "-cover", + "-count=1" ] } \ No newline at end of file diff --git a/README.md b/README.md index be19162c..e3ce5606 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,17 @@ - [About this component](#about-this-component) - [Installation](#installation) - - [Binary](#binary) - - [Container Image](#container-image) - - [Helm](#helm) + - [Binary](#binary) + - [Container Image](#container-image) + - [Helm](#helm) - [Usage](#usage) - - [Container Image](#container-image-1) + - [Container Image](#container-image-1) - [Configuration](#configuration) - - [Startup](#startup) - - [Loader](#loader) - - [Runtime](#runtime) - - [Check: Health](#check-health) + - [Startup](#startup) + - [Loader](#loader) + - [Runtime](#runtime) + - [TargetManager](#targetmanager) + - [Check: Health](#check-health) - [Health Metrics](#health-metrics) - [Check: Latency](#check-latency) - [Latency Metrics](#latency-metrics) @@ -38,7 +39,8 @@ The `sparrow` performs several checks to monitor the health of the infrastructur The following checks are available: 1. Health check - `health`: The `sparrow` is able to perform an HTTP-based (HTTP/1.1) health check to the provided - endpoints. The `sparrow` will expose its own health check endpoint as well. + endpoints. + The `sparrow` will expose its own health check endpoint as well. 2. Latency check - `latency`: The `sparrow` is able to communicate with other `sparrow` instances to calculate the time a request takes to the target and back. The check is http (HTTP/1.1) based as well. @@ -168,6 +170,35 @@ checks: enabled: true ``` +### Target Manager + +The `sparrow` is able to manage the targets for the checks and register the `sparrow` as target on a (remote) backend. +This is done via a `TargetManager` interface, which can be configured on startup. The available configuration options +are listed below and can be set in a startup YAML configuration file (per default `tmconfig.yaml` in the current +directory). + +| Type | Description | Default | +|--------------------------------------|--------------------------------------------------------------------------------------|----------------------| +| `targetManager.checkInterval` | The interval in seconds to check for new targets. | `300` | +| `targetManager.unhealthyThreshold` | The threshold in seconds to mark a target as unhealthy and remove it from the state. | `600` | +| `targetManager.registrationInterval` | The interval in seconds to register the current sparrow at the targets backend. | `300` | +| `targetManager.gitlab.token` | The token to authenticate against the gitlab instance. | `""` | +| `targetManager.gitlab.baseUrl` | The base URL of the gitlab instance. | `https://gitlab.com` | +| `targetManager.gitlab.projectId` | The project ID of the gitlab project to use as a remote state backend. | `""` | + +Currently, only one target manager exists: the Gitlab target manager. It uses a gitlab project as the remote state +backend. The various `sparrow` instances will +register themselves as targets in the project. The `sparrow` instances will also check the project for new targets and +add them to the local state. The registration is done by committing a "state" file in the main branch of the repository, +which is named after the DNS name of the `sparrow`. The state file contains the following information: + +```json +{ + "url": "https://", + "lastSeen": "2021-09-30T12:00:00Z" +} +``` + ### Check: Health Available configuration options: diff --git a/cmd/gen-docs.go b/cmd/gen-docs.go index bbb1bcdf..82ce54e7 100644 --- a/cmd/gen-docs.go +++ b/cmd/gen-docs.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/cobra/doc" ) -// NewCmdRun creates a new gen-docs command +// NewCmdGenDocs creates a new gen-docs command func NewCmdGenDocs(rootCmd *cobra.Command) *cobra.Command { var docPath string diff --git a/cmd/run.go b/cmd/run.go index aa330eea..8a185959 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -48,6 +48,7 @@ func NewCmdRun() *cobra.Command { LoaderHttpRetryCount: "loaderHttpRetryCount", LoaderHttpRetryDelay: "loaderHttpRetryDelay", LoaderFilePath: "loaderFilePath", + TargetManagerConfig: "tmconfig", } cmd := &cobra.Command{ @@ -67,6 +68,7 @@ func NewCmdRun() *cobra.Command { cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryCount, defaultHttpRetryCount, "http loader: Amount of retries trying to load the configuration") cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryDelay, defaultHttpRetryDelay, "http loader: The initial delay between retries in seconds") cmd.PersistentFlags().String(flagMapping.LoaderFilePath, "config.yaml", "file loader: The path to the file to read the runtime config from") + cmd.PersistentFlags().String(flagMapping.TargetManagerConfig, "tmconfig.yaml", "target manager: The path to the file to read the target manager config from") _ = viper.BindPFlag(flagMapping.ApiAddress, cmd.PersistentFlags().Lookup(flagMapping.ApiAddress)) _ = viper.BindPFlag(flagMapping.LoaderType, cmd.PersistentFlags().Lookup(flagMapping.LoaderType)) @@ -77,6 +79,7 @@ func NewCmdRun() *cobra.Command { _ = viper.BindPFlag(flagMapping.LoaderHttpRetryCount, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryCount)) _ = viper.BindPFlag(flagMapping.LoaderHttpRetryDelay, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryDelay)) _ = viper.BindPFlag(flagMapping.LoaderFilePath, cmd.PersistentFlags().Lookup(flagMapping.LoaderFilePath)) + _ = viper.BindPFlag(flagMapping.TargetManagerConfig, cmd.PersistentFlags().Lookup(flagMapping.TargetManagerConfig)) return cmd } @@ -88,6 +91,7 @@ func run(fm *config.RunFlagsNameMapping) func(cmd *cobra.Command, args []string) ctx := logger.IntoContext(context.Background(), log) cfg := config.NewConfig() + cfg.SetTargetManagerConfig(config.NewTargetManagerConfig(viper.GetString(fm.TargetManagerConfig))) cfg.SetApiAddress(viper.GetString(fm.ApiAddress)) diff --git a/docs/sparrow_run.md b/docs/sparrow_run.md index 52ac1f90..80b12f51 100644 --- a/docs/sparrow_run.md +++ b/docs/sparrow_run.md @@ -23,6 +23,7 @@ sparrow run [flags] --loaderHttpUrl string http loader: The url where to get the remote configuration --loaderInterval int defines the interval the loader reloads the configuration in seconds (default 300) -l, --loaderType string defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader (default "http") + --tmconfig string target manager: The path to the file to read the target manager config from (default "tmconfig.yaml") ``` ### Options inherited from parent commands diff --git a/go.mod b/go.mod index 42af17d3..07a8f93f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/getkin/kin-openapi v0.120.0 github.com/go-chi/chi/v5 v5.0.10 + github.com/go-test/deep v1.0.8 github.com/jarcoal/httpmock v1.3.1 github.com/mitchellh/mapstructure v1.5.0 github.com/spf13/cobra v1.8.0 diff --git a/pkg/checks/checks.go b/pkg/checks/checks.go index 4853d85e..6957dece 100644 --- a/pkg/checks/checks.go +++ b/pkg/checks/checks.go @@ -77,6 +77,13 @@ type Result struct { Err string `json:"error"` } +// GlobalTarget includes the basic information regarding +// other Sparrow instances, which this Sparrow can communicate with. +type GlobalTarget struct { + Url string `json:"url"` + LastSeen time.Time `json:"lastSeen"` +} + type ResultDTO struct { Name string Result *Result diff --git a/pkg/config/config.go b/pkg/config/config.go index 50f9fb15..4f57b84b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,15 +19,32 @@ package config import ( + "os" "time" + "gopkg.in/yaml.v3" + "github.com/caas-team/sparrow/internal/helper" ) +type GitlabTargetManagerConfig struct { + BaseURL string `yaml:"baseUrl"` + Token string `yaml:"token"` + ProjectID int `yaml:"projectId"` +} + +type TargetManagerConfig struct { + CheckInterval time.Duration `yaml:"checkInterval"` + RegistrationInterval time.Duration `yaml:"registrationInterval"` + UnhealthyThreshold time.Duration `yaml:"unhealthyThreshold"` + Gitlab GitlabTargetManagerConfig `yaml:"gitlab"` +} + type Config struct { - Checks map[string]any - Loader LoaderConfig - Api ApiConfig + Checks map[string]any + Loader LoaderConfig + Api ApiConfig + TargetManager TargetManagerConfig } // ApiConfig is the configuration for the data API @@ -56,6 +73,23 @@ type FileLoaderConfig struct { path string } +// NewTargetManagerConfig creates a new TargetManagerConfig +// from the passed file +func NewTargetManagerConfig(path string) TargetManagerConfig { + var res TargetManagerConfig + f, err := os.ReadFile(path) //#nosec G304 + if err != nil { + panic("failed to read config file " + err.Error()) + } + + err = yaml.Unmarshal(f, &res) + if err != nil { + panic("failed to parse config file: " + err.Error()) + } + + return res +} + // NewConfig creates a new Config func NewConfig() *Config { return &Config{ @@ -108,3 +142,8 @@ func (c *Config) SetLoaderHttpRetryCount(retryCount int) { func (c *Config) SetLoaderHttpRetryDelay(retryDelay int) { c.Loader.http.retryCfg.Delay = time.Duration(retryDelay) * time.Second } + +// SetTargetManagerConfig sets the target manager config +func (c *Config) SetTargetManagerConfig(config TargetManagerConfig) { + c.TargetManager = config +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 00000000..3a453611 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "testing" + "time" + + "github.com/go-test/deep" +) + +func Test_NewTargetManagerConfig_Gitlab(t *testing.T) { + got := NewTargetManagerConfig("testdata/tmconfig.yaml") + want := TargetManagerConfig{ + CheckInterval: 300 * time.Second, + RegistrationInterval: 600 * time.Second, + UnhealthyThreshold: 900 * time.Second, + Gitlab: GitlabTargetManagerConfig{ + BaseURL: "https://gitlab.devops.telekom.de", + ProjectID: 666, + Token: "gitlab-token", + }, + } + + if diff := deep.Equal(got, want); diff != nil { + t.Error(diff) + } +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go index 45eea6ac..098adc1b 100644 --- a/pkg/config/flags.go +++ b/pkg/config/flags.go @@ -29,4 +29,6 @@ type RunFlagsNameMapping struct { LoaderHttpRetryCount string LoaderHttpRetryDelay string LoaderFilePath string + + TargetManagerConfig string } diff --git a/pkg/config/http.go b/pkg/config/http.go index afc18a8e..2de62d82 100644 --- a/pkg/config/http.go +++ b/pkg/config/http.go @@ -42,7 +42,7 @@ func NewHttpLoader(cfg *Config, cCfgChecks chan<- map[string]any) *HttpLoader { } } -// GetRuntimeConfig gets the runtime configuration +// Run gets the runtime configuration // from the http remote endpoint. // The config is will be loaded periodically defined by the // loader interval configuration. A failed request will be retried defined diff --git a/pkg/config/testdata/tmconfig.yaml b/pkg/config/testdata/tmconfig.yaml new file mode 100644 index 00000000..9f615528 --- /dev/null +++ b/pkg/config/testdata/tmconfig.yaml @@ -0,0 +1,7 @@ +checkInterval: 300s +registrationInterval: 600s +unhealthyThreshold: 900s +gitlab: + token: gitlab-token + baseUrl: https://gitlab.devops.telekom.de + projectId: 666 diff --git a/pkg/sparrow/gitlab/gitlab.go b/pkg/sparrow/gitlab/gitlab.go new file mode 100644 index 00000000..e65f5c29 --- /dev/null +++ b/pkg/sparrow/gitlab/gitlab.go @@ -0,0 +1,319 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package gitlab + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/caas-team/sparrow/internal/logger" + "github.com/caas-team/sparrow/pkg/checks" +) + +// Gitlab handles interaction with a gitlab repository containing +// the global targets for the Sparrow instance +type Gitlab interface { + FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) + PutFile(ctx context.Context, file File) error + PostFile(ctx context.Context, file File) error +} + +// Client implements Gitlab +type Client struct { + // the base URL of the gitlab instance + baseUrl string + // the ID of the project containing the global targets + projectID int + // the token used to authenticate with the gitlab instance + token string + client *http.Client +} + +func New(baseURL, token string, pid int) Gitlab { + return &Client{ + baseUrl: baseURL, + token: token, + projectID: pid, + client: &http.Client{}, + } +} + +// FetchFiles fetches the files from the global targets repository from the configured gitlab repository +func (g *Client) FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) { + log := logger.FromContext(ctx) + fl, err := g.fetchFileList(ctx) + if err != nil { + log.Error("Failed to fetch files", "error", err) + return nil, err + } + + var result []checks.GlobalTarget + for _, f := range fl { + gl, err := g.fetchFile(ctx, f) + if err != nil { + log.Error("Failed fetching files", "error", err) + return nil, err + } + result = append(result, gl) + } + log.Info("Successfully fetched all target files", "files", len(result)) + return result, nil +} + +// fetchFile fetches the file from the global targets repository from the configured gitlab repository +func (g *Client) fetchFile(ctx context.Context, f string) (checks.GlobalTarget, error) { + log := logger.FromContext(ctx).With("file", f) + var res checks.GlobalTarget + // URL encode the name + n := url.PathEscape(f) + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s/raw?ref=main", g.baseUrl, g.projectID, n), + http.NoBody, + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return res, err + } + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) //nolint:bodyclose // closed in defer + if err != nil { + log.Error("Failed to fetch file", "error", err) + return res, err + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Error("Failed to close response body", "error", err) + } + }(resp.Body) + + if resp.StatusCode != http.StatusOK { + log.Error("Failed to fetch file", "status", resp.Status) + return res, fmt.Errorf("request failed, status is %s", resp.Status) + } + + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + log.Error("Failed to decode file after fetching", "error", err) + return res, err + } + + log.Debug("Successfully fetched file") + return res, nil +} + +// fetchFileList fetches the filenames from the global targets repository from the configured gitlab repository, +// so they may be fetched individually +func (g *Client) fetchFileList(ctx context.Context) ([]string, error) { + log := logger.FromContext(ctx) + log.Debug("Fetching file list from gitlab") + type file struct { + Name string `json:"name"` + } + + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, + fmt.Sprintf("%s/api/v4/projects/%d/repository/tree?ref=main", g.baseUrl, g.projectID), + http.NoBody, + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return nil, err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) //nolint:bodyclose // closed in defer + if err != nil { + log.Error("Failed to fetch file list", "error", err) + return nil, err + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Error("Failed to close response body", "error", err) + } + }(resp.Body) + + if resp.StatusCode != http.StatusOK { + log.Error("Failed to fetch file list", "status", resp.Status) + return nil, fmt.Errorf("request failed, status is %s", resp.Status) + } + + var fl []file + err = json.NewDecoder(resp.Body).Decode(&fl) + if err != nil { + log.Error("Failed to decode file list", "error", err) + return nil, err + } + + var result []string + for _, f := range fl { + result = append(result, f.Name) + } + + log.Debug("Successfully fetched file list", "files", len(result)) + return result, nil +} + +// PutFile commits the current instance to the configured gitlab repository +// as a global target for other sparrow instances to discover +func (g *Client) PutFile(ctx context.Context, body File) error { //nolint: dupl,gocritic // no need to refactor yet + log := logger.FromContext(ctx) + log.Debug("Registering sparrow instance to gitlab") + + // chose method based on whether the registration has already happened + n := url.PathEscape(body.fileName) + b, err := body.Bytes() + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPut, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), + bytes.NewBuffer(b), + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) //nolint:bodyclose // closed in defer + if err != nil { + log.Error("Failed to push registration file", "error", err) + return err + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Error("Failed to close response body", "error", err) + } + }(resp.Body) + + if resp.StatusCode != http.StatusOK { + log.Error("Failed to push registration file", "status", resp.Status) + return fmt.Errorf("request failed, status is %s", resp.Status) + } + + return nil +} + +// PostFile commits the current instance to the configured gitlab repository +// as a global target for other sparrow instances to discover +func (g *Client) PostFile(ctx context.Context, body File) error { //nolint:dupl,gocritic // no need to refactor yet + log := logger.FromContext(ctx) + log.Debug("Posting registration file to gitlab") + + // chose method based on whether the registration has already happened + n := url.PathEscape(body.fileName) + b, err := body.Bytes() + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, + fmt.Sprintf("%s/api/v4/projects/%d/repository/files/%s", g.baseUrl, g.projectID, n), + bytes.NewBuffer(b), + ) + if err != nil { + log.Error("Failed to create request", "error", err) + return err + } + + req.Header.Add("PRIVATE-TOKEN", g.token) + req.Header.Add("Content-Type", "application/json") + + resp, err := g.client.Do(req) //nolint:bodyclose // closed in defer + if err != nil { + log.Error("Failed to post file", "error", err) + return err + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Error("Failed to close response body", "error", err) + } + }(resp.Body) + + if resp.StatusCode != http.StatusCreated { + log.Error("Failed to post file", "status", resp.Status) + return fmt.Errorf("request failed, status is %s", resp.Status) + } + + return nil +} + +// File represents a File manipulation operation via the Gitlab API +type File struct { + Branch string `json:"branch"` + AuthorEmail string `json:"author_email"` + AuthorName string `json:"author_name"` + Content checks.GlobalTarget `json:"content"` + CommitMessage string `json:"commit_message"` + fileName string +} + +// Bytes returns the File as a byte array. The Content +// is base64 encoded for Gitlab API compatibility. +func (g *File) Bytes() ([]byte, error) { + content, err := json.Marshal(g.Content) + if err != nil { + return nil, err + } + + // base64 encode the content + enc := base64.NewEncoder(base64.StdEncoding, bytes.NewBuffer(content)) + _, err = enc.Write(content) + _ = enc.Close() + + if err != nil { + return nil, err + } + return json.Marshal(map[string]string{ + "branch": g.Branch, + "author_email": g.AuthorEmail, + "author_name": g.AuthorName, + "content": string(content), + "commit_message": g.CommitMessage, + }) +} + +// SetFileName sets the filename of the File +func (g *File) SetFileName(name string) { + g.fileName = name +} diff --git a/pkg/sparrow/gitlab/gitlab_test.go b/pkg/sparrow/gitlab/gitlab_test.go new file mode 100644 index 00000000..21925d81 --- /dev/null +++ b/pkg/sparrow/gitlab/gitlab_test.go @@ -0,0 +1,442 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package gitlab + +import ( + "context" + "fmt" + "net/http" + "reflect" + "testing" + "time" + + "github.com/caas-team/sparrow/pkg/checks" + "github.com/jarcoal/httpmock" +) + +func Test_gitlab_fetchFileList(t *testing.T) { + type file struct { + Name string `json:"name"` + } + tests := []struct { + name string + want []string + wantErr bool + mockBody []file + mockCode int + }{ + { + name: "success - 0 targets", + want: nil, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{}, + }, + { + name: "success - 1 target", + want: []string{ + "test", + }, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{ + { + Name: "test", + }, + }, + }, + { + name: "success - 2 targets", + want: []string{ + "test", + "test2", + }, + wantErr: false, + mockCode: http.StatusOK, + mockBody: []file{ + { + Name: "test", + }, + { + Name: "test2", + }, + }, + }, + { + name: "failure - API error", + want: nil, + wantErr: true, + mockCode: http.StatusInternalServerError, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.mockBody) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp) + + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + got, err := g.fetchFileList(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("FetchFiles() got = %v, want %v", got, tt.want) + } + }) + } +} + +// The filelist and url are the same, so we HTTP responders can +// be created without much hassle +func Test_gitlab_FetchFiles(t *testing.T) { + type file struct { + Name string `json:"name"` + } + + tests := []struct { + name string + want []checks.GlobalTarget + fileList []file + wantErr bool + mockCode int + }{ + { + name: "success - 0 targets", + want: nil, + wantErr: false, + mockCode: http.StatusOK, + }, + { + name: "success - 1 target", + want: []checks.GlobalTarget{ + { + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fileList: []file{ + { + Name: "test", + }, + }, + wantErr: false, + mockCode: http.StatusOK, + }, + { + name: "success - 2 targets", + want: []checks.GlobalTarget{ + { + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + { + Url: "test2", + LastSeen: time.Date(2021, 2, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fileList: []file{ + { + Name: "test", + }, + { + Name: "test2", + }, + }, + wantErr: false, + mockCode: http.StatusOK, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // setup mock responses + for i, target := range tt.want { + resp, err := httpmock.NewJsonResponder(tt.mockCode, target) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i].Name), resp) + } + + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.fileList) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp) + + got, err := g.FetchFiles(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("FetchFiles() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("FetchFiles() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_gitlab_fetchFiles_error_cases(t *testing.T) { + type file struct { + Name string `json:"name"` + } + type mockResponses struct { + response checks.GlobalTarget + err bool + } + + tests := []struct { + name string + mockResponses []mockResponses + fileList []file + }{ + { + name: "failure - direct API error", + mockResponses: []mockResponses{ + { + err: true, + }, + }, + fileList: []file{ + { + Name: "test", + }, + }, + }, + { + name: "failure - API error after one successful request", + mockResponses: []mockResponses{ + { + response: checks.GlobalTarget{ + Url: "test", + LastSeen: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), + }, + err: false, + }, + { + response: checks.GlobalTarget{}, + err: true, + }, + }, + fileList: []file{ + {Name: "test"}, + {Name: "test2-will-fail"}, + }, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, target := range tt.mockResponses { + if target.err { + errResp := httpmock.NewStringResponder(http.StatusInternalServerError, "") + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i].Name), errResp) + continue + } + resp, err := httpmock.NewJsonResponder(http.StatusOK, target) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s/raw?ref=main", tt.fileList[i].Name), resp) + } + + _, err := g.FetchFiles(context.Background()) + if err == nil { + t.Fatalf("Expected error but got none.") + } + }) + } +} + +func TestClient_PutFile(t *testing.T) { //nolint:dupl // no need to refactor yet + now := time.Now() + tests := []struct { + name string + file File + mockCode int + wantErr bool + }{ + { + name: "success", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusOK, + }, + { + name: "failure - API error", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusInternalServerError, + wantErr: true, + }, + { + name: "failure - empty file", + wantErr: true, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr { + resp := httpmock.NewStringResponder(tt.mockCode, "") + httpmock.RegisterResponder("PUT", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } else { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.file) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("PUT", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } + + if err := g.PutFile(context.Background(), tt.file); (err != nil) != tt.wantErr { + t.Fatalf("PutFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestClient_PostFile(t *testing.T) { //nolint:dupl // no need to refactor yet + now := time.Now() + tests := []struct { + name string + file File + mockCode int + wantErr bool + }{ + { + name: "success", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusCreated, + }, + { + name: "failure - API error", + file: File{ + Branch: "main", + AuthorEmail: "test@sparrow", + AuthorName: "sparrpw", + Content: checks.GlobalTarget{ + Url: "https://test.de", + LastSeen: now, + }, + CommitMessage: "test-commit", + fileName: "test.de.json", + }, + mockCode: http.StatusInternalServerError, + wantErr: true, + }, + { + name: "failure - empty file", + wantErr: true, + }, + } + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + g := &Client{ + baseUrl: "http://test", + projectID: 1, + token: "test", + client: http.DefaultClient, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr { + resp := httpmock.NewStringResponder(tt.mockCode, "") + httpmock.RegisterResponder("POST", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } else { + resp, err := httpmock.NewJsonResponder(tt.mockCode, tt.file) + if err != nil { + t.Fatalf("error creating mock response: %v", err) + } + httpmock.RegisterResponder("POST", fmt.Sprintf("http://test/api/v4/projects/1/repository/files/%s", tt.file.fileName), resp) + } + + if err := g.PostFile(context.Background(), tt.file); (err != nil) != tt.wantErr { + t.Fatalf("PostFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/sparrow/gitlab/test/mockclient.go b/pkg/sparrow/gitlab/test/mockclient.go new file mode 100644 index 00000000..87f27c9f --- /dev/null +++ b/pkg/sparrow/gitlab/test/mockclient.go @@ -0,0 +1,74 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package gitlabmock + +import ( + "context" + + "github.com/caas-team/sparrow/internal/logger" + "github.com/caas-team/sparrow/pkg/checks" + "github.com/caas-team/sparrow/pkg/sparrow/gitlab" +) + +type MockClient struct { + targets []checks.GlobalTarget + fetchFilesErr error + putFileErr error + postFileErr error +} + +func (m *MockClient) PutFile(ctx context.Context, _ gitlab.File) error { //nolint: gocritic // irrelevant + log := logger.FromContext(ctx) + log.Info("MockPutFile called", "err", m.putFileErr) + return m.putFileErr +} + +func (m *MockClient) PostFile(ctx context.Context, _ gitlab.File) error { //nolint: gocritic // irrelevant + log := logger.FromContext(ctx) + log.Info("MockPostFile called", "err", m.postFileErr) + return m.postFileErr +} + +func (m *MockClient) FetchFiles(ctx context.Context) ([]checks.GlobalTarget, error) { + log := logger.FromContext(ctx) + log.Info("MockFetchFiles called", "targets", len(m.targets), "err", m.fetchFilesErr) + return m.targets, m.fetchFilesErr +} + +// SetFetchFilesErr sets the error returned by FetchFiles +func (m *MockClient) SetFetchFilesErr(err error) { + m.fetchFilesErr = err +} + +// SetPutFileErr sets the error returned by PutFile +func (m *MockClient) SetPutFileErr(err error) { + m.putFileErr = err +} + +// SetPostFileErr sets the error returned by PostFile +func (m *MockClient) SetPostFileErr(err error) { + m.postFileErr = err +} + +// New creates a new MockClient to mock Gitlab interaction +func New(targets []checks.GlobalTarget) *MockClient { + return &MockClient{ + targets: targets, + } +} diff --git a/pkg/sparrow/run.go b/pkg/sparrow/run.go index f9fc31ba..41925bb7 100644 --- a/pkg/sparrow/run.go +++ b/pkg/sparrow/run.go @@ -23,6 +23,8 @@ import ( "fmt" "net/http" + targets "github.com/caas-team/sparrow/pkg/sparrow/targets" + "github.com/caas-team/sparrow/internal/logger" "github.com/caas-team/sparrow/pkg/api" "github.com/caas-team/sparrow/pkg/checks" @@ -32,8 +34,6 @@ import ( ) type Sparrow struct { - // TODO refactor this struct to be less convoluted - // split up responsibilities more clearly db db.DB // the existing checks checks map[string]checks.Check @@ -47,6 +47,7 @@ type Sparrow struct { cfg *config.Config loader config.Loader cCfgChecks chan map[string]any + targets targets.TargetManager routingTree *api.RoutingTree router chi.Router @@ -67,6 +68,10 @@ func New(cfg *config.Config) *Sparrow { router: chi.NewRouter(), } + // Set the target manager + gm := targets.NewGitlabManager("sparrow-with-cfg-file", cfg.TargetManager) + sparrow.targets = gm + sparrow.loader = config.NewLoader(cfg, sparrow.cCfgChecks) sparrow.db = db.NewInMemory() return sparrow @@ -77,8 +82,9 @@ func (s *Sparrow) Run(ctx context.Context) error { ctx, cancel := logger.NewContextWithLogger(ctx, "sparrow") defer cancel() - // Start the runtime configuration loader go s.loader.Run(ctx) + go s.targets.Reconcile(ctx) + // Start the api go func() { err := s.api(ctx) if err != nil { diff --git a/pkg/sparrow/targets/gitlab.go b/pkg/sparrow/targets/gitlab.go new file mode 100644 index 00000000..666bf0f0 --- /dev/null +++ b/pkg/sparrow/targets/gitlab.go @@ -0,0 +1,211 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package targets + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/caas-team/sparrow/pkg/config" + + "github.com/caas-team/sparrow/pkg/checks" + "github.com/caas-team/sparrow/pkg/sparrow/gitlab" + + "github.com/caas-team/sparrow/internal/logger" +) + +var _ TargetManager = &gitlabTargetManager{} + +// gitlabTargetManager implements TargetManager +type gitlabTargetManager struct { + targets []checks.GlobalTarget + mu sync.RWMutex + done chan struct{} + gitlab gitlab.Gitlab + // the DNS name used for self-registration + name string + // the interval for the target reconciliation process + checkInterval time.Duration + // the amount of time a target can be + // unhealthy before it is removed from the global target list + unhealthyThreshold time.Duration + // how often the instance should register itself as a global target + registrationInterval time.Duration + // whether the instance has already registered itself as a global target + registered bool +} + +// NewGitlabManager creates a new gitlabTargetManager +func NewGitlabManager(name string, gtmConfig config.TargetManagerConfig) *gitlabTargetManager { + return &gitlabTargetManager{ + gitlab: gitlab.New(gtmConfig.Gitlab.BaseURL, gtmConfig.Gitlab.Token, gtmConfig.Gitlab.ProjectID), + name: name, + checkInterval: gtmConfig.CheckInterval, + registrationInterval: gtmConfig.RegistrationInterval, + unhealthyThreshold: gtmConfig.UnhealthyThreshold, + mu: sync.RWMutex{}, + done: make(chan struct{}, 1), + } +} + +// Reconcile reconciles the targets of the gitlabTargetManager. +// The global targets are parsed from a gitlab repository. +// +// The global targets are evaluated for healthiness and +// unhealthy gitlabTargetManager are removed. +func (t *gitlabTargetManager) Reconcile(ctx context.Context) { + log := logger.FromContext(ctx) + log.Info("Starting global gitlabTargetManager reconciler") + + checkTimer := time.NewTimer(t.checkInterval) + registrationTimer := time.NewTimer(t.registrationInterval) + + defer checkTimer.Stop() + defer registrationTimer.Stop() + + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); err != nil { + log.Error("Context canceled", "error", err) + err = t.Shutdown(ctx) + if err != nil { + log.Error("Failed to shutdown gracefully", "error", err) + return + } + } + case <-t.done: + log.Info("Ending Reconcile routine") + return + case <-checkTimer.C: + err := t.refreshTargets(ctx) + if err != nil { + log.Error("Failed to get global targets", "error", err) + } + checkTimer.Reset(t.checkInterval) + case <-registrationTimer.C: + err := t.updateRegistration(ctx) + if err != nil { + log.Error("Failed to register self as global target", "error", err) + } + registrationTimer.Reset(t.registrationInterval) + } + } +} + +// GetTargets returns the current targets of the gitlabTargetManager +func (t *gitlabTargetManager) GetTargets() []checks.GlobalTarget { + t.mu.RLock() + defer t.mu.RUnlock() + return t.targets +} + +// Shutdown shuts down the gitlabTargetManager and deletes the file containing +// the sparrow's registration from Gitlab +func (t *gitlabTargetManager) Shutdown(ctx context.Context) error { + t.mu.Lock() + defer t.mu.Unlock() + log := logger.FromContext(ctx) + log.Info("Shutting down global gitlabTargetManager") + t.registered = false + + select { + case t.done <- struct{}{}: + log.Debug("Stopping reconcile routine") + default: + } + + return nil +} + +// updateRegistration registers the current instance as a global target +func (t *gitlabTargetManager) updateRegistration(ctx context.Context) error { + log := logger.FromContext(ctx) + log.Debug("Updating registration") + + t.mu.Lock() + defer t.mu.Unlock() + f := gitlab.File{ + Branch: "main", + AuthorEmail: fmt.Sprintf("%s@sparrow", t.name), + AuthorName: t.name, + Content: checks.GlobalTarget{Url: fmt.Sprintf("https://%s", t.name), LastSeen: time.Now().UTC()}, + } + f.SetFileName(fmt.Sprintf("%s.json", t.name)) + + if t.Registered() { + f.CommitMessage = "Updated registration" + err := t.gitlab.PutFile(ctx, f) + if err != nil { + log.Error("Failed to update registration", "error", err) + return err + } + log.Debug("Successfully updated registration") + return nil + } + + f.CommitMessage = "Initial registration" + err := t.gitlab.PostFile(ctx, f) + if err != nil { + log.Error("Failed to register global gitlabTargetManager", "error", err) + return err + } + + log.Debug("Successfully registered") + t.registered = true + return nil +} + +// refreshTargets updates the targets of the gitlabTargetManager +// with the latest available healthy targets +func (t *gitlabTargetManager) refreshTargets(ctx context.Context) error { + log := logger.FromContext(ctx) + t.mu.Lock() + defer t.mu.Unlock() + var healthyTargets []checks.GlobalTarget + targets, err := t.gitlab.FetchFiles(ctx) + if err != nil { + log.Error("Failed to update global targets", "error", err) + return err + } + + // filter unhealthy targets - this may be removed in the future + for _, target := range targets { + if !t.Registered() && target.Url == fmt.Sprintf("https://%s", t.name) { + log.Debug("Found self as global target", "lastSeenMin", time.Since(target.LastSeen).Minutes()) + t.registered = true + } + if time.Now().Add(-t.unhealthyThreshold).After(target.LastSeen) { + log.Debug("Skipping unhealthy target", "target", target) + continue + } + healthyTargets = append(healthyTargets, target) + } + + t.targets = healthyTargets + log.Debug("Updated global targets", "targets", len(t.targets)) + return nil +} + +// Registered returns whether the instance is registered as a global target +func (t *gitlabTargetManager) Registered() bool { + return t.registered +} diff --git a/pkg/sparrow/targets/gitlab_test.go b/pkg/sparrow/targets/gitlab_test.go new file mode 100644 index 00000000..695644df --- /dev/null +++ b/pkg/sparrow/targets/gitlab_test.go @@ -0,0 +1,399 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package targets + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/caas-team/sparrow/pkg/checks" + gitlabmock "github.com/caas-team/sparrow/pkg/sparrow/gitlab/test" +) + +func Test_gitlabTargetManager_refreshTargets(t *testing.T) { + now := time.Now() + tooOld := now.Add(-time.Hour * 2) + + tests := []struct { + name string + mockTargets []checks.GlobalTarget + expectedHealthy []checks.GlobalTarget + expectedRegisteredAfter bool + wantErr error + }{ + { + name: "success with 0 targets", + mockTargets: []checks.GlobalTarget{}, + expectedHealthy: []checks.GlobalTarget{}, + }, + { + name: "success with 1 healthy target", + mockTargets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + }, + expectedHealthy: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + }, + expectedRegisteredAfter: true, + }, + { + name: "success with 1 unhealthy target", + mockTargets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: tooOld, + }, + }, + expectedRegisteredAfter: true, + }, + { + name: "success with 1 healthy and 1 unhealthy targets", + mockTargets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + { + Url: "https://test2", + LastSeen: tooOld, + }, + }, + expectedHealthy: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + }, + expectedRegisteredAfter: true, + }, + { + name: "failure getting targets", + mockTargets: nil, + expectedHealthy: nil, + wantErr: fmt.Errorf("failed to fetch files"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gitlab := gitlabmock.New(tt.mockTargets) + if tt.wantErr != nil { + gitlab.SetFetchFilesErr(tt.wantErr) + } + gtm := &gitlabTargetManager{ + targets: nil, + gitlab: gitlab, + name: "test", + unhealthyThreshold: time.Hour, + } + if err := gtm.refreshTargets(context.Background()); (err != nil) != (tt.wantErr != nil) { + t.Fatalf("refreshTargets() error = %v, wantErr %v", err, tt.wantErr) + } + + if gtm.Registered() != tt.expectedRegisteredAfter { + t.Fatalf("expected registered to be %v, got %v", tt.expectedRegisteredAfter, gtm.Registered()) + } + }) + } +} + +func Test_gitlabTargetManager_GetTargets(t *testing.T) { + now := time.Now() + tests := []struct { + name string + targets []checks.GlobalTarget + want []checks.GlobalTarget + }{ + { + name: "success with 0 targets", + targets: nil, + want: nil, + }, + { + name: "success with 1 target", + targets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + }, + want: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + }, + }, + { + name: "success with 2 targets", + targets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + { + Url: "https://test2", + LastSeen: now, + }, + }, + want: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: now, + }, + { + Url: "https://test2", + LastSeen: now, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gtm := &gitlabTargetManager{ + targets: tt.targets, + } + got := gtm.GetTargets() + + if len(got) != len(tt.want) { + t.Fatalf("GetTargets() got = %v, want %v", got, tt.want) + } + + for i := range got { + if got[i].Url != tt.want[i].Url { + t.Fatalf("GetTargets() got = %v, want %v", got, tt.want) + } + if !got[i].LastSeen.Equal(tt.want[i].LastSeen) { + t.Fatalf("GetTargets() got = %v, want %v", got, tt.want) + } + } + }) + } +} + +func Test_gitlabTargetManager_updateRegistration(t *testing.T) { + tests := []struct { + name string + registered bool + wantPostError bool + wantPutError bool + }{ + { + name: "success - first registration", + }, + { + name: "success - update registration", + registered: true, + }, + { + name: "failure - failed to register", + wantPostError: true, + }, + { + name: "failure - failed to update registration", + registered: true, + wantPutError: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + glmock := gitlabmock.New(nil) + if tt.wantPostError { + glmock.SetPostFileErr(fmt.Errorf("failed to register")) + } + if tt.wantPutError { + glmock.SetPutFileErr(fmt.Errorf("failed to update registration")) + } + gtm := &gitlabTargetManager{ + gitlab: glmock, + registered: tt.registered, + } + wantErr := tt.wantPutError || tt.wantPostError + if err := gtm.updateRegistration(context.Background()); (err != nil) != wantErr { + t.Fatalf("updateRegistration() error = %v, wantErr %v", err, wantErr) + } + }) + } +} + +// Test_gitlabTargetManager_Reconcile_success tests that the Reconcile method +// will register the target if it is not registered yet and update the +// registration if it is already registered +func Test_gitlabTargetManager_Reconcile_success(t *testing.T) { + tests := []struct { + name string + registered bool + wantPostError bool + wantPutError bool + }{ + { + name: "success - first registration", + }, + { + name: "success - update registration", + registered: true, + }, + } + + glmock := gitlabmock.New( + []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: time.Now(), + }, + }, + ) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gtm := mockGitlabTargetManager(glmock, "test") + ctx := context.Background() + go func() { + gtm.Reconcile(ctx) + }() + + time.Sleep(time.Millisecond * 300) + if gtm.GetTargets()[0].Url != "https://test" { + t.Fatalf("Reconcile() did not receive the correct target") + } + if !gtm.Registered() { + t.Fatalf("Reconcile() did not register") + } + + err := gtm.Shutdown(ctx) + if err != nil { + t.Fatalf("Reconcile() failed to shutdown") + } + }) + } +} + +// Test_gitlabTargetManager_Reconcile_failure tests that the Reconcile method +// will handle API failures gracefully +func Test_gitlabTargetManager_Reconcile_failure(t *testing.T) { + tests := []struct { + name string + registered bool + targets []checks.GlobalTarget + postErr error + putError error + }{ + { + name: "failure - failed to register", + postErr: errors.New("failed to register"), + }, + { + name: "failure - failed to update registration", + registered: true, + putError: errors.New("failed to update registration"), + targets: []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: time.Now(), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + glmock := gitlabmock.New(tt.targets) + + gtm := mockGitlabTargetManager(glmock, "test") + glmock.SetPostFileErr(tt.postErr) + glmock.SetPutFileErr(tt.putError) + + ctx := context.Background() + go func() { + gtm.Reconcile(ctx) + }() + + time.Sleep(time.Millisecond * 300) + + gtm.mu.Lock() + if tt.postErr != nil && gtm.Registered() { + t.Fatalf("Reconcile() should not have registered") + } + + if tt.putError != nil && !gtm.Registered() { + t.Fatalf("Reconcile() should still be registered") + } + gtm.mu.Unlock() + + err := gtm.Shutdown(ctx) + if err != nil { + t.Fatalf("Reconcile() failed to shutdown") + } + }) + } +} + +// Test_gitlabTargetManager_Reconcile_Context_Canceled tests that the Reconcile +// method will shutdown gracefully when the context is canceled. +func Test_gitlabTargetManager_Reconcile_Context_Canceled(t *testing.T) { + glmock := gitlabmock.New( + []checks.GlobalTarget{ + { + Url: "https://test", + LastSeen: time.Now(), + }, + }, + ) + + gtm := mockGitlabTargetManager(glmock, "test") + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + gtm.Reconcile(ctx) + }() + + time.Sleep(time.Millisecond * 250) + cancel() + time.Sleep(time.Millisecond * 250) + + gtm.mu.Lock() + // instance shouldn't be registered anymore + if gtm.Registered() { + t.Fatalf("Reconcile() should not be registered") + } + gtm.mu.Unlock() +} + +func mockGitlabTargetManager(g *gitlabmock.MockClient, name string) *gitlabTargetManager { + return &gitlabTargetManager{ + targets: nil, + mu: sync.RWMutex{}, + done: make(chan struct{}, 1), + gitlab: g, + name: name, + checkInterval: 100 * time.Millisecond, + unhealthyThreshold: 1 * time.Second, + registrationInterval: 150 * time.Millisecond, + registered: false, + } +} diff --git a/pkg/sparrow/targets/targetmanager.go b/pkg/sparrow/targets/targetmanager.go new file mode 100644 index 00000000..f9a3eacb --- /dev/null +++ b/pkg/sparrow/targets/targetmanager.go @@ -0,0 +1,38 @@ +// sparrow +// (C) 2023, Deutsche Telekom IT GmbH +// +// Deutsche Telekom IT GmbH and all other contributors / +// copyright owners license this file to you under the Apache +// License, Version 2.0 (the "License"); you may not use this +// file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package targets + +import ( + "context" + + "github.com/caas-team/sparrow/pkg/checks" +) + +// TargetManager handles the management of globalTargets for +// a Sparrow instance +type TargetManager interface { + // Reconcile fetches the global targets from the configured + // endpoint and updates the local state + Reconcile(ctx context.Context) + // GetTargets returns the current global targets + GetTargets() []checks.GlobalTarget + // Shutdown shuts down the target manager + // and unregisters the instance as a global target + Shutdown(ctx context.Context) error +}