From 5e86214c1702a6fea756ab278ecc40661020b8fe Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Wed, 26 Feb 2025 10:06:11 +0100 Subject: [PATCH] feat: add release markers --- publisher/config/config.go | 8 + publisher/go.mod | 5 +- publisher/go.sum | 6 + publisher/publisher.go | 30 +- publisher/release/marker.go | 55 +++ publisher/release/marker_aws.go | 204 +++++++++ publisher/release/marker_aws_test.go | 644 +++++++++++++++++++++++++++ publisher/upload/upload.go | 14 +- publisher/upload/upload_test.go | 195 +++++++- 9 files changed, 1150 insertions(+), 11 deletions(-) create mode 100644 publisher/release/marker.go create mode 100644 publisher/release/marker_aws.go create mode 100644 publisher/release/marker_aws_test.go diff --git a/publisher/config/config.go b/publisher/config/config.go index 5b238e3..7836aa3 100644 --- a/publisher/config/config.go +++ b/publisher/config/config.go @@ -35,6 +35,8 @@ type Config struct { ArtifactsDestFolder string // s3 mounted folder ArtifactsSrcFolder string AptlyFolder string + SchemaURL string + Schema string UploadSchemaFilePath string GpgPassphrase string GpgKeyRing string @@ -42,6 +44,7 @@ type Config struct { AwsRoleARN string // locking properties (candidate for factoring) AwsLockBucket string + AwsBucket string AwsTags string LockGroup string DisableLock bool @@ -85,6 +88,8 @@ func LoadConfig() (Config, error) { viper.BindEnv("artifacts_src_folder") viper.BindEnv("aptly_folder") viper.BindEnv("upload_schema_file_path") + viper.BindEnv("schema_url") + viper.BindEnv("schema") viper.BindEnv("dest_prefix") viper.BindEnv("gpg_passphrase") viper.BindEnv("gpg_key_ring") @@ -132,10 +137,13 @@ func LoadConfig() (Config, error) { ArtifactsSrcFolder: viper.GetString("artifacts_src_folder"), AptlyFolder: aptlyF, UploadSchemaFilePath: viper.GetString("upload_schema_file_path"), + SchemaURL: viper.GetString("schema_url"), + Schema: viper.GetString("schema"), GpgPassphrase: viper.GetString("gpg_passphrase"), GpgKeyRing: viper.GetString("gpg_key_ring"), LockGroup: lockGroup, AwsLockBucket: viper.GetString("aws_s3_lock_bucket_name"), + AwsBucket: viper.GetString("aws_s3_bucket_name"), AwsRoleARN: viper.GetString("aws_role_arn"), AwsRegion: viper.GetString("aws_region"), AwsTags: viper.GetString("aws_tags"), diff --git a/publisher/go.mod b/publisher/go.mod index b85d766..5df09d9 100644 --- a/publisher/go.mod +++ b/publisher/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/aws-sdk-go v1.37.11 github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.10.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -22,9 +22,10 @@ require ( github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/publisher/go.sum b/publisher/go.sum index cdffdbd..344d146 100644 --- a/publisher/go.sum +++ b/publisher/go.sum @@ -291,11 +291,15 @@ github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8q github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= @@ -467,6 +471,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/publisher/publisher.go b/publisher/publisher.go index d1b98c2..e0d8b4c 100644 --- a/publisher/publisher.go +++ b/publisher/publisher.go @@ -4,13 +4,14 @@ package main import ( "fmt" - "log" - "net/http" - "github.com/newrelic/infrastructure-publish-action/publisher/config" "github.com/newrelic/infrastructure-publish-action/publisher/download" "github.com/newrelic/infrastructure-publish-action/publisher/lock" + "github.com/newrelic/infrastructure-publish-action/publisher/release" "github.com/newrelic/infrastructure-publish-action/publisher/upload" + "log" + "net/http" + "strings" ) const ( @@ -42,6 +43,11 @@ func main() { l.Fatal("loading config: " + err.Error()) } + releaseMarker, err := newReleaseMarker(conf) + if err != nil { + l.Fatal("creating release marker: " + err.Error()) + } + var bucketLock lock.BucketLock if conf.DisableLock { bucketLock = lock.NewNoop() @@ -105,9 +111,25 @@ func main() { conf.ArtifactsSrcFolder = conf.LocalPackagesPath } - err = upload.UploadArtifacts(conf, uploadSchemas, bucketLock) + err = upload.UploadArtifacts(conf, uploadSchemas, bucketLock, releaseMarker) if err != nil { l.Fatal(err) } l.Println("🎉 upload phase complete") } + +func newReleaseMarker(conf config.Config) (release.Marker, error) { + // We'll leave the release marker file in the root of the repository + // i.e. + // repo = /infrastructure_agent/linux/apt/ + // release marker = /infrastructure_agent/releases.json + repoRootDir := strings.Split(strings.TrimPrefix(conf.DestPrefix, "/"), "/")[0] + markerS3Conf := release.S3Config{ + Bucket: conf.AwsBucket, + RoleARN: conf.AwsRoleARN, + Region: conf.AwsRegion, + Directory: repoRootDir, + } + + return release.NewMarkerAWS(markerS3Conf, l.Printf) +} diff --git a/publisher/release/marker.go b/publisher/release/marker.go new file mode 100644 index 0000000..5e00b5f --- /dev/null +++ b/publisher/release/marker.go @@ -0,0 +1,55 @@ +package release + +import ( + "encoding/json" + "time" +) + +// Mark represents a release mark. It will contain the name of the release (appName, tag...) +// and the start and end of a release +// When the release has been started, the end will be zero +type Mark struct { + AppName string `json:"app_name"` + Tag string `json:"tag"` + RunID string `json:"run_id"` + Start CustomTime `json:"start"` + End CustomTime `json:"end"` + RepoName string `json:"repo_name"` + Schema string `json:"schema"` + SchemaURl string `json:"schema_url"` +} + +// Marker abstracts the persistence of the start and end of a release +type Marker interface { + Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) + End(mark Mark) error +} + +// CustomTime is a wrapper around time.Time that +// allows to marshal and unmarshal time.Time in a custom format +type CustomTime struct { + time.Time +} + +const ctLayout = time.RFC3339 + +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + t, err := time.Parse(ctLayout, s) + if err != nil { + return err + } + ct.Time = t + return nil +} + +func (ct *CustomTime) MarshalJSON() ([]byte, error) { + return json.Marshal(ct.Time.Format(ctLayout)) +} + +func (ct *CustomTime) Equals(t CustomTime) bool { + return ct.Truncate(time.Second).Equal(t.Truncate(time.Second)) +} diff --git a/publisher/release/marker_aws.go b/publisher/release/marker_aws.go new file mode 100644 index 0000000..806d031 --- /dev/null +++ b/publisher/release/marker_aws.go @@ -0,0 +1,204 @@ +package release + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "time" +) + +type Logf func(format string, args ...interface{}) + +const markerName = "releases.json" + +var ErrLastMarkerEnded = errors.New("last marker is already ended") +var ErrNoStartedMarkersFound = errors.New("no started markers found") +var ErrNoStartedMarkerFoundForApp = errors.New("no started marker found for app") +var ErrCannotWriteMarkerFile = errors.New("cannot write marker file") +var ErrNotStartedMark = errors.New("not started mark") + +// S3Config markerAWS lock config DTO. +type S3Config struct { + Directory string + Bucket string + RoleARN string + Region string +} + +// S3Client aws client interface for testing +type S3Client interface { + PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +type TimeProvider interface { + Now() time.Time +} + +type RealTimeProvider struct{} + +func (RealTimeProvider) Now() time.Time { + return time.Now() +} + +type markerAWS struct { + client S3Client + conf S3Config + timeProvider TimeProvider + logfn Logf +} + +// NewMarkerAWS creates a new marker using AWS S3 +// it returns an interface on purpose, so this way +// we can have markerAWS unexported and force the +// usage of the constructor +func NewMarkerAWS(s3Config S3Config, logfn Logf) (Marker, error) { + sess, err := session.NewSession() + if err != nil { + return nil, err + } + + creds := stscreds.NewCredentials(sess, s3Config.RoleARN, func(p *stscreds.AssumeRoleProvider) {}) + awsCfg := aws.Config{ + Credentials: creds, + Region: aws.String(s3Config.Region), + } + + return &markerAWS{ + client: s3.New(sess, &awsCfg), + conf: s3Config, + timeProvider: RealTimeProvider{}, + logfn: logfn, + }, nil +} + +// Start will: +// load all the markers from the file +// append a new started marker +// write the markers back to the file +func (s *markerAWS) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) { + s.logfn("[marker] starting %s", appName) + markers, err := s.readMarkers() + if err != nil { + if !isNoSuchKeyError(err) { + return Mark{}, err + } + } + + mark := Mark{ + AppName: appName, + Tag: tag, + RunID: runID, + RepoName: repoName, + Schema: schema, + SchemaURl: schemaURL, + Start: CustomTime{s.now()}, + } + + markers = append(markers, mark) + if err = s.writeMarkers(markers); err != nil { + return mark, err + } + + return mark, nil +} + +// End will: +// load all the markers from the file +// find the last started marker +// append the end time to the last started marker +// write the markers back to the file +func (s *markerAWS) End(mark Mark) error { + s.logfn("[marker] ending %s", mark.AppName) + if mark.Start.IsZero() { + return ErrNotStartedMark + } + + markers, err := s.readMarkers() + if err != nil { + return err + } + //ensure the latest marker is the one being ended + if len(markers) == 0 { + return ErrNoStartedMarkersFound + } + + lastMarker := markers[len(markers)-1] + if !lastMarker.End.IsZero() { + return ErrLastMarkerEnded + } + + if lastMarker.AppName != mark.AppName || !lastMarker.Start.Equals(mark.Start) { + return fmt.Errorf("%w started:%s appName:%s", ErrNoStartedMarkerFoundForApp, mark.Start, mark.AppName) + } + + lastMarker.End = CustomTime{s.now()} + markers[len(markers)-1] = lastMarker + + err = s.writeMarkers(markers) + if err != nil { + return fmt.Errorf("%w: %w", ErrCannotWriteMarkerFile, err) + } + + return nil +} + +func (s *markerAWS) writeMarkers(markers []Mark) error { + markersBytes, err := json.MarshalIndent(markers, "", " ") + if err != nil { + return fmt.Errorf("cannot encode marker file: %w", err) + } + + s.logfn("[marker] writing bucket:%s key:%s", s.conf.Bucket, s.markerPath()) + _, err = s.client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(s.markerPath()), + Body: aws.ReadSeekCloser(bytes.NewReader(markersBytes)), + }) + if err != nil { + return fmt.Errorf("cannot write marker file: %w", err) + } + + return nil +} + +func (s *markerAWS) readMarkers() ([]Mark, error) { + objOutput, err := s.client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(s.markerPath()), + }) + if err != nil { + return nil, fmt.Errorf("cannot read marker file: %w", err) + } + + var markers []Mark + err = json.NewDecoder(objOutput.Body).Decode(&markers) + if err != nil { + return nil, fmt.Errorf("cannot decode marker file: %w", err) + } + + return markers, nil +} + +func (s *markerAWS) markerPath() string { + return s.conf.Directory + "/" + markerName +} + +func (s *markerAWS) now() time.Time { + return s.timeProvider.Now().UTC() +} + +// from github.com/aws/aws-sdk-go/aws/awserr/error.go +func isNoSuchKeyError(err error) bool { + var awsErr awserr.Error + if errors.As(err, &awsErr) { + return awsErr.Code() == "NoSuchKey" + } + return false +} diff --git a/publisher/release/marker_aws_test.go b/publisher/release/marker_aws_test.go new file mode 100644 index 0000000..a2bce0c --- /dev/null +++ b/publisher/release/marker_aws_test.go @@ -0,0 +1,644 @@ +package release + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io" + "testing" + "time" + + "github.com/stretchr/testify/mock" +) + +var nolog = func(format string, args ...interface{}) {} + +func Test_Start(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "app2", "tag": "v1.1", "run_id": "run2", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo2", "schema": "schema2", "schema_url": "url2"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the new marker + startTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(startTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"app2","tag":"v1.1","run_id":"run2","start":"2023-01-02T00:00:00Z","end":"2023-01-02T01:00:00Z","repo_name":"repo2","schema":"schema2","schema_url":"url2"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2025-03-04T11:12:13Z","end":"0001-01-01T00:00:00Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + s3ClientMock.ShouldPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + &s3.PutObjectOutput{}, + ) + + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + marker, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + require.NoError(t, err) + require.Equal(t, appName, marker.AppName) + require.Equal(t, tag, marker.Tag) + require.Equal(t, runID, marker.RunID) + require.Equal(t, repoName, marker.RepoName) + require.Equal(t, schema, marker.Schema) + require.Equal(t, schemaURL, marker.SchemaURl) +} + +func Test_StartErrorReadingMarkers(t *testing.T) { + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + timeProviderMock := &TimeProviderMock{} + s3ClientMock := &S3ClientMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + var someError = errors.New("error reading markers") + s3ClientMock.ShouldReturnErrorOnGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + someError) + + markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} + _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + assert.ErrorIs(t, err, someError) +} + +func Test_StartErrorWritingMarkers(t *testing.T) { + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + timeProviderMock := &TimeProviderMock{} + s3ClientMock := &S3ClientMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "app2", "tag": "v1.1", "run_id": "run2", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo2", "schema": "schema2", "schema_url": "url2"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the new marker + startTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(startTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"app2","tag":"v1.1","run_id":"run2","start":"2023-01-02T00:00:00Z","end":"2023-01-02T01:00:00Z","repo_name":"repo2","schema":"schema2","schema_url":"url2"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2025-03-04T11:12:13Z","end":"0001-01-01T00:00:00Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + var someError = errors.New("error writing markers") + s3ClientMock.ShouldReturnErrorOnPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + someError) + + markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} + _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + assert.ErrorIs(t, err, someError) +} +func Test_End(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2023-01-02T00:00:00Z","end":"2025-03-04T11:12:13Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + s3ClientMock.ShouldPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + &s3.PutObjectOutput{}, + ) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + require.NoError(t, err) +} + +func Test_End_ErrorOnWriting(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2023-01-02T00:00:00Z","end":"2025-03-04T11:12:13Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + var someError = errors.New("error writing markers") + s3ClientMock.ShouldReturnErrorOnPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + someError) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, someError) +} + +func Test_End_ErrorIfNoMarkerFound(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := `[]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkersFound) +} +func Test_End_ErrorOnReadingMarkers(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + var someError = errors.New("error reading markers") + s3ClientMock.ShouldReturnErrorOnGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + someError) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, someError) +} + +func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrLastMarkerEnded) +} + +func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "another-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) +} + +func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T11:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) +} + +func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T11:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Time{} + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNotStartedMark) +} + +/////////////////////////////////////////////////////////////// +// S3 client Mock +/////////////////////////////////////////////////////////////// + +type S3ClientMock struct { + mock.Mock +} + +func (m *S3ClientMock) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + args := m.Called(input) + + return args.Get(0).(*s3.PutObjectOutput), args.Error(1) +} + +func (m *S3ClientMock) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + args := m.Called(input) + + return args.Get(0).(*s3.GetObjectOutput), args.Error(1) +} + +func (m *S3ClientMock) ShouldPutObject(input *s3.PutObjectInput, output *s3.PutObjectOutput) { + m. + On("PutObject", input). + Once(). + Return(output, nil) +} + +func (m *S3ClientMock) ShouldReturnErrorOnPutObject(input *s3.PutObjectInput, err error) { + m. + On("PutObject", input). + Once(). + Return(&s3.PutObjectOutput{}, err) +} + +func (m *S3ClientMock) ShouldGetObject(input *s3.GetObjectInput, output *s3.GetObjectOutput) { + m. + On("GetObject", input). + Once(). + Return(output, nil) +} + +func (m *S3ClientMock) ShouldReturnErrorOnGetObject(input *s3.GetObjectInput, err error) { + m. + On("GetObject", input). + Once(). + Return(&s3.GetObjectOutput{}, err) +} + +/////////////////////////////////////////////////////////////// +// Time provider Mock +/////////////////////////////////////////////////////////////// + +type TimeProviderMock struct { + mock.Mock +} + +func (m *TimeProviderMock) Now() time.Time { + args := m.Called() + + return args.Get(0).(time.Time) +} + +func (m *TimeProviderMock) ShouldProvideNow(now time.Time) { + m. + On("Now"). + Once(). + Return(now) +} + +func mustPrettify(jsonStr string) string { + var jsonObj []Mark + err := json.Unmarshal([]byte(jsonStr), &jsonObj) + if err != nil { + panic(err) + } + + prettyJSON, err := json.MarshalIndent(jsonObj, "", " ") + if err != nil { + panic(err) + } + + return string(prettyJSON) +} diff --git a/publisher/upload/upload.go b/publisher/upload/upload.go index eb109d5..d9669e9 100644 --- a/publisher/upload/upload.go +++ b/publisher/upload/upload.go @@ -2,6 +2,7 @@ package upload import ( "fmt" + "github.com/newrelic/infrastructure-publish-action/publisher/release" "io/ioutil" "net/http" "net/url" @@ -64,11 +65,22 @@ func uploadArtifact(conf config.Config, schema config.UploadArtifactSchema, uplo return nil } -func UploadArtifacts(conf config.Config, schema config.UploadArtifactSchemas, bucketLock lock.BucketLock) (err error) { +func UploadArtifacts(conf config.Config, schema config.UploadArtifactSchemas, bucketLock lock.BucketLock, releaseMarker release.Marker) (err error) { if err = bucketLock.Lock(); err != nil { return } + // Write the release marker + mark, err := releaseMarker.Start(conf.AppName, conf.Tag, conf.RunID, conf.RepoName, conf.Schema, conf.SchemaURL) + if err != nil { + return fmt.Errorf("cannot start release marker: %w", err) + } + defer func() { + markerErr := releaseMarker.End(mark) + if markerErr != nil { + utils.Logger.Printf("ERROR: cannot end release marker %v", markerErr) + } + errRelease := bucketLock.Release() if err == nil { err = errRelease diff --git a/publisher/upload/upload_test.go b/publisher/upload/upload_test.go index fa16b52..05584fa 100644 --- a/publisher/upload/upload_test.go +++ b/publisher/upload/upload_test.go @@ -1,6 +1,9 @@ package upload import ( + "errors" + "github.com/newrelic/infrastructure-publish-action/publisher/release" + "github.com/stretchr/testify/mock" "io/ioutil" "os" "path" @@ -164,7 +167,13 @@ func TestUploadArtifacts(t *testing.T) { err := writeDummyFile(path.Join(src, dummyFile)) assert.NoError(t, err) } - err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory()) + + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) assert.NoError(t, err) for _, expectedFile := range artifact.expectedFiles { @@ -176,6 +185,131 @@ func TestUploadArtifacts(t *testing.T) { } +func TestUploadArtifactsShouldFailIfMarkerCannotBeStarted(t *testing.T) { + var testArtifacts = []struct { + name string + schema []config.UploadArtifactSchema + dummyFiles []string + expectedFiles []string + }{ + { + name: "AppName, arch and app version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{arch}-{version}.txt", []string{"amd64", "386"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + {"{app_name}-{arch}-{version}.txt", nil, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + }, + dummyFiles: []string{"nri-foobar-amd64-2.0.0.txt", "nri-foobar-386-2.0.0.txt"}, + expectedFiles: []string{"amd64/nri-foobar/nri-foobar-amd64-2.0.0.txt", "386/nri-foobar/nri-foobar-386-2.0.0.txt"}, + }, + } + + cfg := config.Config{ + Version: "2.0.0", + UploadSchemaFilePath: "", + AppName: "nri-foobar", + } + + for _, artifact := range testArtifacts { + t.Run(artifact.name, func(t *testing.T) { + markerErr := errors.New("some error") + marker := &MarkerMock{} + marker.ShouldFailOnStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, markerErr) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) + assert.ErrorIs(t, err, markerErr) + }) + } +} + +func TestUploadArtifactsShouldNotFailIfMarkerCannotBeEnded(t *testing.T) { + var testArtifacts = []struct { + name string + schema []config.UploadArtifactSchema + dummyFiles []string + expectedFiles []string + }{ + { + name: "AppName, arch and app version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{arch}-{version}.txt", []string{"amd64", "386"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + {"{app_name}-{arch}-{version}.txt", nil, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + }, + dummyFiles: []string{"nri-foobar-amd64-2.0.0.txt", "nri-foobar-386-2.0.0.txt"}, + expectedFiles: []string{"amd64/nri-foobar/nri-foobar-amd64-2.0.0.txt", "386/nri-foobar/nri-foobar-386-2.0.0.txt"}, + }, + { + name: "AppName, arch, app version and os version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{version}-1.amazonlinux-{os_version}.{arch}.rpm.sum", []string{"x86_64"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{os_version}/{src}", + OsVersion: []string{"2", "2022"}, + }, + }}, + }, + dummyFiles: []string{"nri-foobar-2.0.0-1.amazonlinux-2.x86_64.rpm.sum", "nri-foobar-2.0.0-1.amazonlinux-2022.x86_64.rpm.sum"}, + expectedFiles: []string{"x86_64/nri-foobar/2/nri-foobar-2.0.0-1.amazonlinux-2.x86_64.rpm.sum", "x86_64/nri-foobar/2022/nri-foobar-2.0.0-1.amazonlinux-2022.x86_64.rpm.sum"}, + }, + } + + dest, err := ioutil.TempDir("", "") + assert.NoError(t, err) + src, err := ioutil.TempDir("", "") + assert.NoError(t, err) + + cfg := config.Config{ + Version: "2.0.0", + ArtifactsDestFolder: dest, + ArtifactsSrcFolder: src, + UploadSchemaFilePath: "", + AppName: "nri-foobar", + } + + for _, artifact := range testArtifacts { + t.Run(artifact.name, func(t *testing.T) { + for _, dummyFile := range artifact.dummyFiles { + err := writeDummyFile(path.Join(src, dummyFile)) + assert.NoError(t, err) + } + + markerErr := errors.New("some error") + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldFailOnEnd(mark, markerErr) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) + assert.NoError(t, err) + + for _, expectedFile := range artifact.expectedFiles { + _, err = os.Stat(path.Join(dest, expectedFile)) + assert.NoError(t, err) + } + }) + } +} + func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { schema := []config.UploadArtifactSchema{ {"{app_name}-{arch}-{version}.txt", []string{"amd64"}, []config.Upload{ @@ -212,13 +346,18 @@ func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { l := lock.NewInMemory() go func() { <-ready - err1 = UploadArtifacts(cfg, schema, l) + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + err1 = UploadArtifacts(cfg, schema, l, marker) wg.Done() }() go func() { <-ready time.Sleep(1 * time.Millisecond) - err2 = UploadArtifacts(cfg, schema, l) + marker := &MarkerMock{} + err2 = UploadArtifacts(cfg, schema, l, marker) wg.Done() }() @@ -279,7 +418,11 @@ func TestUploadArtifacts_errorsIfAnyArchFails(t *testing.T) { err = writeDummyFile(path.Join(src, "nri-foobar-386-2.0.0.txt")) assert.NoError(t, err) - err = UploadArtifacts(cfg, tc.schema, lock.NewNoop()) + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + err = UploadArtifacts(cfg, tc.schema, lock.NewNoop(), marker) if tc.expectsError { assert.Error(t, err) } else { @@ -309,3 +452,47 @@ func Test_generateRepoFileContent(t *testing.T) { assert.Equal(t, expectedContent, repoFileContent) } + +type MarkerMock struct { + mock.Mock +} + +func (m *MarkerMock) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (release.Mark, error) { + args := m.Called(appName, tag, runID, repoName, schema, schemaURL) + + return args.Get(0).(release.Mark), args.Error(1) +} + +func (m *MarkerMock) End(mark release.Mark) error { + args := m.Called(mark) + + return args.Error(0) +} + +func (m *MarkerMock) ShouldStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, mark release.Mark) { + m. + On("Start", appName, tag, runID, repoName, schema, schemaURL). + Once(). + Return(mark, nil) +} + +func (m *MarkerMock) ShouldFailOnStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, err error) { + m. + On("Start", appName, tag, runID, repoName, schema, schemaURL). + Once(). + Return(release.Mark{}, err) +} + +func (m *MarkerMock) ShouldEnd(mark release.Mark) { + m. + On("End", mark). + Once(). + Return(nil) +} + +func (m *MarkerMock) ShouldFailOnEnd(mark release.Mark, err error) { + m. + On("End", mark). + Once(). + Return(err) +}