From 44192fe13a9c7b1862c52df7b83df2f3b45a209b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 May 2020 11:53:38 +0200 Subject: [PATCH 1/5] download from snapshot repo --- .../pkg/agent/application/stream.go | 2 +- .../download/localremote/downloader.go | 22 ++++- .../artifact/download/localremote/verifier.go | 22 ++++- .../artifact/download/snapshot/downloader.go | 95 +++++++++++++++++++ .../artifact/download/snapshot/verifier.go | 21 ++++ 5 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go create mode 100644 x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go diff --git a/x-pack/elastic-agent/pkg/agent/application/stream.go b/x-pack/elastic-agent/pkg/agent/application/stream.go index 5d8880a9e076..55ed49656d4b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/stream.go +++ b/x-pack/elastic-agent/pkg/agent/application/stream.go @@ -77,7 +77,7 @@ func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config return nil, err } - fetcher := downloader.NewDownloader(operatorConfig.DownloadConfig) + fetcher := downloader.NewDownloader(log, operatorConfig.DownloadConfig) installer, err := install.NewInstaller(operatorConfig.DownloadConfig) if err != nil { return nil, errors.New(err, "initiating installer") diff --git a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go index 8757cb2abbce..a1661fb0b61a 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go @@ -10,10 +10,28 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/composed" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/fs" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/snapshot" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) // NewDownloader creates a downloader which first checks local directory // and then fallbacks to remote if configured. -func NewDownloader(config *artifact.Config, downloaders ...download.Downloader) download.Downloader { - return composed.NewDownloader(fs.NewDownloader(config), http.NewDownloader(config)) +func NewDownloader(log *logger.Logger, config *artifact.Config) download.Downloader { + downloaders := make([]download.Downloader, 0, 3) + downloaders = append(downloaders, fs.NewDownloader(config)) + + // try snapshot repo before official + log.Errorf("IS SNAPSHOT", release.Snapshot()) + if release.Snapshot() { + snapDownloader, err := snapshot.NewDownloader(config) + if err != nil { + log.Error(err) + } else { + downloaders = append(downloaders, snapDownloader) + } + } + + downloaders = append(downloaders, http.NewDownloader(config)) + return composed.NewDownloader(downloaders...) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go index 08cb61cb7c3e..34863270679b 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go @@ -10,19 +10,37 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/composed" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/fs" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/snapshot" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) // NewVerifier creates a downloader which first checks local directory // and then fallbacks to remote if configured. -func NewVerifier(config *artifact.Config, downloaders ...download.Downloader) (download.Verifier, error) { +func NewVerifier(log *logger.Logger, config *artifact.Config) (download.Verifier, error) { + verifiers := make([]download.Verifier, 0, 3) + fsVer, err := fs.NewVerifier(config) if err != nil { return nil, err } + verifiers = append(verifiers, fsVer) + + // try snapshot repo before official + if release.Snapshot() { + snapshotVerifier, err := snapshot.NewVerifier(config) + if err != nil { + log.Error(err) + } else { + verifiers = append(verifiers, snapshotVerifier) + } + } + remoteVer, err := http.NewVerifier(config) if err != nil { return nil, err } + verifiers = append(verifiers, remoteVer) - return composed.NewVerifier(fsVer, remoteVer), nil + return composed.NewVerifier(verifiers...), nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go new file mode 100644 index 000000000000..10c8cf38994d --- /dev/null +++ b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go @@ -0,0 +1,95 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package snapshot + +import ( + "encoding/json" + "fmt" + gohttp "net/http" + "strings" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +) + +// NewDownloader creates a downloader which first checks local directory +// and then fallbacks to remote if configured. +func NewDownloader(config *artifact.Config) (download.Downloader, error) { + cfg, err := snapshotConfig(config) + if err != nil { + return nil, err + } + return http.NewDownloader(cfg), nil +} + +func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { + snapshotUri, err := snapshotUri() + if err != nil { + return nil, fmt.Errorf("failed to detect remote snapshot repo, proceeding with configured: %v", err) + } + + fmt.Println(">>>>>> SNAPSHOT URI", snapshotUri) + + return &artifact.Config{ + OperatingSystem: config.OperatingSystem, + Architecture: config.Architecture, + BeatsSourceURI: snapshotUri, + TargetDirectory: config.TargetDirectory, + Timeout: config.Timeout, + PgpFile: config.PgpFile, + InstallPath: config.InstallPath, + DropPath: config.DropPath, + }, nil +} + +func snapshotUri() (string, error) { + artifactsUri := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", release.Version()) + resp, err := gohttp.Get(artifactsUri) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body := struct { + Packages map[string]interface{} `json:"packages"` + }{} + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&body); err != nil { + return "", err + } + + if len(body.Packages) == 0 { + return "", fmt.Errorf("no packages found in snapshot repo") + } + + for k, pkg := range body.Packages { + pkgMap, ok := pkg.(map[string]interface{}) + if !ok { + return "", fmt.Errorf("content of '%s' is not a map", k) + } + + uriVal, found := pkgMap["url"] + if !found { + return "", fmt.Errorf("item '%s' does not contain url", k) + } + + uri, ok := uriVal.(string) + if !ok { + return "", fmt.Errorf("uri is not a string") + } + + index := strings.Index(uri, "/elastic-agent/") + if index == -1 { + return "", fmt.Errorf("not an agent uri: '%s'", uri) + } + + return uri[:index], nil + } + + return "", fmt.Errorf("uri not detected") +} diff --git a/x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go new file mode 100644 index 000000000000..91626a6b55b0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package snapshot + +import ( + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/http" +) + +// NewVerifier creates a downloader which first checks local directory +// and then fallbacks to remote if configured. +func NewVerifier(config *artifact.Config, downloaders ...download.Downloader) (download.Verifier, error) { + cfg, err := snapshotConfig(config) + if err != nil { + return nil, err + } + return http.NewVerifier(cfg) +} From a74a4bbd05f94c0fac42c9cf3cb31e70d56b7d04 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 May 2020 12:16:02 +0200 Subject: [PATCH 2/5] lint --- .../pkg/artifact/download/snapshot/downloader.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go index 10c8cf38994d..c9a43d2a48d0 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go @@ -27,13 +27,11 @@ func NewDownloader(config *artifact.Config) (download.Downloader, error) { } func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { - snapshotUri, err := snapshotUri() + snapshotUri, err := snapshotURI() if err != nil { return nil, fmt.Errorf("failed to detect remote snapshot repo, proceeding with configured: %v", err) } - fmt.Println(">>>>>> SNAPSHOT URI", snapshotUri) - return &artifact.Config{ OperatingSystem: config.OperatingSystem, Architecture: config.Architecture, @@ -46,9 +44,9 @@ func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { }, nil } -func snapshotUri() (string, error) { - artifactsUri := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", release.Version()) - resp, err := gohttp.Get(artifactsUri) +func snapshotURI() (string, error) { + artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s-SNAPSHOT/elastic-agent", release.Version()) + resp, err := gohttp.Get(artifactsURI) if err != nil { return "", err } From 43ec89dc0ed4f075d08159181e90ad3acd2730b2 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 May 2020 12:16:56 +0200 Subject: [PATCH 3/5] removed debug message --- .../pkg/artifact/download/localremote/downloader.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go index a1661fb0b61a..6448af25aca0 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/localremote/downloader.go @@ -22,7 +22,6 @@ func NewDownloader(log *logger.Logger, config *artifact.Config) download.Downloa downloaders = append(downloaders, fs.NewDownloader(config)) // try snapshot repo before official - log.Errorf("IS SNAPSHOT", release.Snapshot()) if release.Snapshot() { snapDownloader, err := snapshot.NewDownloader(config) if err != nil { From c4f3366cd48346ac86604613b89679ae3404def1 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 May 2020 16:06:56 +0200 Subject: [PATCH 4/5] lint --- .../pkg/artifact/download/snapshot/downloader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go index c9a43d2a48d0..022cda0ffffc 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go @@ -27,7 +27,7 @@ func NewDownloader(config *artifact.Config) (download.Downloader, error) { } func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { - snapshotUri, err := snapshotURI() + snapshotURI, err := snapshotURI() if err != nil { return nil, fmt.Errorf("failed to detect remote snapshot repo, proceeding with configured: %v", err) } @@ -35,7 +35,7 @@ func snapshotConfig(config *artifact.Config) (*artifact.Config, error) { return &artifact.Config{ OperatingSystem: config.OperatingSystem, Architecture: config.Architecture, - BeatsSourceURI: snapshotUri, + BeatsSourceURI: snapshotURI, TargetDirectory: config.TargetDirectory, Timeout: config.Timeout, PgpFile: config.PgpFile, From a18abeecc0ec59334af1bf258ac1b821816772a5 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 5 Jun 2020 13:30:05 +0200 Subject: [PATCH 5/5] conflicts --- x-pack/elastic-agent/pkg/agent/application/stream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/stream.go b/x-pack/elastic-agent/pkg/agent/application/stream.go index 05c9787571dc..1b9fda77cc95 100644 --- a/x-pack/elastic-agent/pkg/agent/application/stream.go +++ b/x-pack/elastic-agent/pkg/agent/application/stream.go @@ -78,7 +78,7 @@ func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config } fetcher := downloader.NewDownloader(log, operatorConfig.DownloadConfig) - verifier, err := downloader.NewVerifier(operatorConfig.DownloadConfig) + verifier, err := downloader.NewVerifier(log, operatorConfig.DownloadConfig) if err != nil { return nil, errors.New(err, "initiating verifier") }