From 5e86214c1702a6fea756ab278ecc40661020b8fe Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Wed, 26 Feb 2025 10:06:11 +0100 Subject: [PATCH 1/5] feat: add release markers --- publisher/config/config.go | 8 + publisher/go.mod | 5 +- publisher/go.sum | 6 + publisher/publisher.go | 30 +- publisher/release/marker.go | 55 +++ publisher/release/marker_aws.go | 204 +++++++++ publisher/release/marker_aws_test.go | 644 +++++++++++++++++++++++++++ publisher/upload/upload.go | 14 +- publisher/upload/upload_test.go | 195 +++++++- 9 files changed, 1150 insertions(+), 11 deletions(-) create mode 100644 publisher/release/marker.go create mode 100644 publisher/release/marker_aws.go create mode 100644 publisher/release/marker_aws_test.go diff --git a/publisher/config/config.go b/publisher/config/config.go index 5b238e3..7836aa3 100644 --- a/publisher/config/config.go +++ b/publisher/config/config.go @@ -35,6 +35,8 @@ type Config struct { ArtifactsDestFolder string // s3 mounted folder ArtifactsSrcFolder string AptlyFolder string + SchemaURL string + Schema string UploadSchemaFilePath string GpgPassphrase string GpgKeyRing string @@ -42,6 +44,7 @@ type Config struct { AwsRoleARN string // locking properties (candidate for factoring) AwsLockBucket string + AwsBucket string AwsTags string LockGroup string DisableLock bool @@ -85,6 +88,8 @@ func LoadConfig() (Config, error) { viper.BindEnv("artifacts_src_folder") viper.BindEnv("aptly_folder") viper.BindEnv("upload_schema_file_path") + viper.BindEnv("schema_url") + viper.BindEnv("schema") viper.BindEnv("dest_prefix") viper.BindEnv("gpg_passphrase") viper.BindEnv("gpg_key_ring") @@ -132,10 +137,13 @@ func LoadConfig() (Config, error) { ArtifactsSrcFolder: viper.GetString("artifacts_src_folder"), AptlyFolder: aptlyF, UploadSchemaFilePath: viper.GetString("upload_schema_file_path"), + SchemaURL: viper.GetString("schema_url"), + Schema: viper.GetString("schema"), GpgPassphrase: viper.GetString("gpg_passphrase"), GpgKeyRing: viper.GetString("gpg_key_ring"), LockGroup: lockGroup, AwsLockBucket: viper.GetString("aws_s3_lock_bucket_name"), + AwsBucket: viper.GetString("aws_s3_bucket_name"), AwsRoleARN: viper.GetString("aws_role_arn"), AwsRegion: viper.GetString("aws_region"), AwsTags: viper.GetString("aws_tags"), diff --git a/publisher/go.mod b/publisher/go.mod index b85d766..5df09d9 100644 --- a/publisher/go.mod +++ b/publisher/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/aws-sdk-go v1.37.11 github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.10.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -22,9 +22,10 @@ require ( github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/publisher/go.sum b/publisher/go.sum index cdffdbd..344d146 100644 --- a/publisher/go.sum +++ b/publisher/go.sum @@ -291,11 +291,15 @@ github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8q github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= @@ -467,6 +471,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/publisher/publisher.go b/publisher/publisher.go index d1b98c2..e0d8b4c 100644 --- a/publisher/publisher.go +++ b/publisher/publisher.go @@ -4,13 +4,14 @@ package main import ( "fmt" - "log" - "net/http" - "github.com/newrelic/infrastructure-publish-action/publisher/config" "github.com/newrelic/infrastructure-publish-action/publisher/download" "github.com/newrelic/infrastructure-publish-action/publisher/lock" + "github.com/newrelic/infrastructure-publish-action/publisher/release" "github.com/newrelic/infrastructure-publish-action/publisher/upload" + "log" + "net/http" + "strings" ) const ( @@ -42,6 +43,11 @@ func main() { l.Fatal("loading config: " + err.Error()) } + releaseMarker, err := newReleaseMarker(conf) + if err != nil { + l.Fatal("creating release marker: " + err.Error()) + } + var bucketLock lock.BucketLock if conf.DisableLock { bucketLock = lock.NewNoop() @@ -105,9 +111,25 @@ func main() { conf.ArtifactsSrcFolder = conf.LocalPackagesPath } - err = upload.UploadArtifacts(conf, uploadSchemas, bucketLock) + err = upload.UploadArtifacts(conf, uploadSchemas, bucketLock, releaseMarker) if err != nil { l.Fatal(err) } l.Println("🎉 upload phase complete") } + +func newReleaseMarker(conf config.Config) (release.Marker, error) { + // We'll leave the release marker file in the root of the repository + // i.e. + // repo = /infrastructure_agent/linux/apt/ + // release marker = /infrastructure_agent/releases.json + repoRootDir := strings.Split(strings.TrimPrefix(conf.DestPrefix, "/"), "/")[0] + markerS3Conf := release.S3Config{ + Bucket: conf.AwsBucket, + RoleARN: conf.AwsRoleARN, + Region: conf.AwsRegion, + Directory: repoRootDir, + } + + return release.NewMarkerAWS(markerS3Conf, l.Printf) +} diff --git a/publisher/release/marker.go b/publisher/release/marker.go new file mode 100644 index 0000000..5e00b5f --- /dev/null +++ b/publisher/release/marker.go @@ -0,0 +1,55 @@ +package release + +import ( + "encoding/json" + "time" +) + +// Mark represents a release mark. It will contain the name of the release (appName, tag...) +// and the start and end of a release +// When the release has been started, the end will be zero +type Mark struct { + AppName string `json:"app_name"` + Tag string `json:"tag"` + RunID string `json:"run_id"` + Start CustomTime `json:"start"` + End CustomTime `json:"end"` + RepoName string `json:"repo_name"` + Schema string `json:"schema"` + SchemaURl string `json:"schema_url"` +} + +// Marker abstracts the persistence of the start and end of a release +type Marker interface { + Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) + End(mark Mark) error +} + +// CustomTime is a wrapper around time.Time that +// allows to marshal and unmarshal time.Time in a custom format +type CustomTime struct { + time.Time +} + +const ctLayout = time.RFC3339 + +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + t, err := time.Parse(ctLayout, s) + if err != nil { + return err + } + ct.Time = t + return nil +} + +func (ct *CustomTime) MarshalJSON() ([]byte, error) { + return json.Marshal(ct.Time.Format(ctLayout)) +} + +func (ct *CustomTime) Equals(t CustomTime) bool { + return ct.Truncate(time.Second).Equal(t.Truncate(time.Second)) +} diff --git a/publisher/release/marker_aws.go b/publisher/release/marker_aws.go new file mode 100644 index 0000000..806d031 --- /dev/null +++ b/publisher/release/marker_aws.go @@ -0,0 +1,204 @@ +package release + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "time" +) + +type Logf func(format string, args ...interface{}) + +const markerName = "releases.json" + +var ErrLastMarkerEnded = errors.New("last marker is already ended") +var ErrNoStartedMarkersFound = errors.New("no started markers found") +var ErrNoStartedMarkerFoundForApp = errors.New("no started marker found for app") +var ErrCannotWriteMarkerFile = errors.New("cannot write marker file") +var ErrNotStartedMark = errors.New("not started mark") + +// S3Config markerAWS lock config DTO. +type S3Config struct { + Directory string + Bucket string + RoleARN string + Region string +} + +// S3Client aws client interface for testing +type S3Client interface { + PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +type TimeProvider interface { + Now() time.Time +} + +type RealTimeProvider struct{} + +func (RealTimeProvider) Now() time.Time { + return time.Now() +} + +type markerAWS struct { + client S3Client + conf S3Config + timeProvider TimeProvider + logfn Logf +} + +// NewMarkerAWS creates a new marker using AWS S3 +// it returns an interface on purpose, so this way +// we can have markerAWS unexported and force the +// usage of the constructor +func NewMarkerAWS(s3Config S3Config, logfn Logf) (Marker, error) { + sess, err := session.NewSession() + if err != nil { + return nil, err + } + + creds := stscreds.NewCredentials(sess, s3Config.RoleARN, func(p *stscreds.AssumeRoleProvider) {}) + awsCfg := aws.Config{ + Credentials: creds, + Region: aws.String(s3Config.Region), + } + + return &markerAWS{ + client: s3.New(sess, &awsCfg), + conf: s3Config, + timeProvider: RealTimeProvider{}, + logfn: logfn, + }, nil +} + +// Start will: +// load all the markers from the file +// append a new started marker +// write the markers back to the file +func (s *markerAWS) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) { + s.logfn("[marker] starting %s", appName) + markers, err := s.readMarkers() + if err != nil { + if !isNoSuchKeyError(err) { + return Mark{}, err + } + } + + mark := Mark{ + AppName: appName, + Tag: tag, + RunID: runID, + RepoName: repoName, + Schema: schema, + SchemaURl: schemaURL, + Start: CustomTime{s.now()}, + } + + markers = append(markers, mark) + if err = s.writeMarkers(markers); err != nil { + return mark, err + } + + return mark, nil +} + +// End will: +// load all the markers from the file +// find the last started marker +// append the end time to the last started marker +// write the markers back to the file +func (s *markerAWS) End(mark Mark) error { + s.logfn("[marker] ending %s", mark.AppName) + if mark.Start.IsZero() { + return ErrNotStartedMark + } + + markers, err := s.readMarkers() + if err != nil { + return err + } + //ensure the latest marker is the one being ended + if len(markers) == 0 { + return ErrNoStartedMarkersFound + } + + lastMarker := markers[len(markers)-1] + if !lastMarker.End.IsZero() { + return ErrLastMarkerEnded + } + + if lastMarker.AppName != mark.AppName || !lastMarker.Start.Equals(mark.Start) { + return fmt.Errorf("%w started:%s appName:%s", ErrNoStartedMarkerFoundForApp, mark.Start, mark.AppName) + } + + lastMarker.End = CustomTime{s.now()} + markers[len(markers)-1] = lastMarker + + err = s.writeMarkers(markers) + if err != nil { + return fmt.Errorf("%w: %w", ErrCannotWriteMarkerFile, err) + } + + return nil +} + +func (s *markerAWS) writeMarkers(markers []Mark) error { + markersBytes, err := json.MarshalIndent(markers, "", " ") + if err != nil { + return fmt.Errorf("cannot encode marker file: %w", err) + } + + s.logfn("[marker] writing bucket:%s key:%s", s.conf.Bucket, s.markerPath()) + _, err = s.client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(s.markerPath()), + Body: aws.ReadSeekCloser(bytes.NewReader(markersBytes)), + }) + if err != nil { + return fmt.Errorf("cannot write marker file: %w", err) + } + + return nil +} + +func (s *markerAWS) readMarkers() ([]Mark, error) { + objOutput, err := s.client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(s.markerPath()), + }) + if err != nil { + return nil, fmt.Errorf("cannot read marker file: %w", err) + } + + var markers []Mark + err = json.NewDecoder(objOutput.Body).Decode(&markers) + if err != nil { + return nil, fmt.Errorf("cannot decode marker file: %w", err) + } + + return markers, nil +} + +func (s *markerAWS) markerPath() string { + return s.conf.Directory + "/" + markerName +} + +func (s *markerAWS) now() time.Time { + return s.timeProvider.Now().UTC() +} + +// from github.com/aws/aws-sdk-go/aws/awserr/error.go +func isNoSuchKeyError(err error) bool { + var awsErr awserr.Error + if errors.As(err, &awsErr) { + return awsErr.Code() == "NoSuchKey" + } + return false +} diff --git a/publisher/release/marker_aws_test.go b/publisher/release/marker_aws_test.go new file mode 100644 index 0000000..a2bce0c --- /dev/null +++ b/publisher/release/marker_aws_test.go @@ -0,0 +1,644 @@ +package release + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io" + "testing" + "time" + + "github.com/stretchr/testify/mock" +) + +var nolog = func(format string, args ...interface{}) {} + +func Test_Start(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "app2", "tag": "v1.1", "run_id": "run2", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo2", "schema": "schema2", "schema_url": "url2"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the new marker + startTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(startTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"app2","tag":"v1.1","run_id":"run2","start":"2023-01-02T00:00:00Z","end":"2023-01-02T01:00:00Z","repo_name":"repo2","schema":"schema2","schema_url":"url2"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2025-03-04T11:12:13Z","end":"0001-01-01T00:00:00Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + s3ClientMock.ShouldPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + &s3.PutObjectOutput{}, + ) + + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + marker, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + require.NoError(t, err) + require.Equal(t, appName, marker.AppName) + require.Equal(t, tag, marker.Tag) + require.Equal(t, runID, marker.RunID) + require.Equal(t, repoName, marker.RepoName) + require.Equal(t, schema, marker.Schema) + require.Equal(t, schemaURL, marker.SchemaURl) +} + +func Test_StartErrorReadingMarkers(t *testing.T) { + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + timeProviderMock := &TimeProviderMock{} + s3ClientMock := &S3ClientMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + var someError = errors.New("error reading markers") + s3ClientMock.ShouldReturnErrorOnGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + someError) + + markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} + _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + assert.ErrorIs(t, err, someError) +} + +func Test_StartErrorWritingMarkers(t *testing.T) { + appName := "my-app" + tag := "v1.2" + runID := "run3" + repoName := "repo3" + schema := "schema3" + schemaURL := "url3" + timeProviderMock := &TimeProviderMock{} + s3ClientMock := &S3ClientMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "app2", "tag": "v1.1", "run_id": "run2", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo2", "schema": "schema2", "schema_url": "url2"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the new marker + startTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(startTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"app2","tag":"v1.1","run_id":"run2","start":"2023-01-02T00:00:00Z","end":"2023-01-02T01:00:00Z","repo_name":"repo2","schema":"schema2","schema_url":"url2"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2025-03-04T11:12:13Z","end":"0001-01-01T00:00:00Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + var someError = errors.New("error writing markers") + s3ClientMock.ShouldReturnErrorOnPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + someError) + + markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} + _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + assert.ErrorIs(t, err, someError) +} +func Test_End(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2023-01-02T00:00:00Z","end":"2025-03-04T11:12:13Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + s3ClientMock.ShouldPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + &s3.PutObjectOutput{}, + ) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + require.NoError(t, err) +} + +func Test_End_ErrorOnWriting(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + expectedMarkers := mustPrettify(`[ + {"app_name":"app1","tag":"v1.0","run_id":"run1","start":"2023-01-01T00:00:00Z","end":"2023-01-01T01:00:00Z","repo_name":"repo1","schema":"schema1","schema_url":"url1"}, + {"app_name":"my-app","tag":"v1.2","run_id":"run3","start":"2023-01-02T00:00:00Z","end":"2025-03-04T11:12:13Z","repo_name":"repo3","schema":"schema3","schema_url":"url3"} + ]`) + + putBody := aws.ReadSeekCloser(bytes.NewReader([]byte(expectedMarkers))) + var someError = errors.New("error writing markers") + s3ClientMock.ShouldReturnErrorOnPutObject( + &s3.PutObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName)), Body: putBody}, + someError) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, someError) +} + +func Test_End_ErrorIfNoMarkerFound(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := `[]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkersFound) +} +func Test_End_ErrorOnReadingMarkers(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + var someError = errors.New("error reading markers") + s3ClientMock.ShouldReturnErrorOnGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + someError) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, someError) +} + +func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "2023-01-02T01:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "schema_url", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrLastMarkerEnded) +} + +func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T00:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "another-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) +} + +func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T11:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) +} + +func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { + s3ClientMock := &S3ClientMock{} + timeProviderMock := &TimeProviderMock{} + + s3Config := S3Config{ + Bucket: "bucket", + RoleARN: "role", + Region: "region", + Directory: "directory", + } + + // It should read existing markers + existingMarkers := ` + [ + {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, + {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T11:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} + ]` + + reader := bytes.NewReader([]byte(existingMarkers)) + readCloser := io.NopCloser(reader) + + s3ClientMock.ShouldGetObject( + &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, + &s3.GetObjectOutput{Body: readCloser}) + + // It should get current time for the end of the started marker + endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) + timeProviderMock.ShouldProvideNow(endTime) + + startTime := time.Time{} + mark := Mark{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURl: "url3", + Start: CustomTime{startTime}, + } + + markerS3 := &markerAWS{ + client: s3ClientMock, + conf: s3Config, + timeProvider: timeProviderMock, + logfn: nolog, + } + err := markerS3.End(mark) + assert.ErrorIs(t, err, ErrNotStartedMark) +} + +/////////////////////////////////////////////////////////////// +// S3 client Mock +/////////////////////////////////////////////////////////////// + +type S3ClientMock struct { + mock.Mock +} + +func (m *S3ClientMock) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + args := m.Called(input) + + return args.Get(0).(*s3.PutObjectOutput), args.Error(1) +} + +func (m *S3ClientMock) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + args := m.Called(input) + + return args.Get(0).(*s3.GetObjectOutput), args.Error(1) +} + +func (m *S3ClientMock) ShouldPutObject(input *s3.PutObjectInput, output *s3.PutObjectOutput) { + m. + On("PutObject", input). + Once(). + Return(output, nil) +} + +func (m *S3ClientMock) ShouldReturnErrorOnPutObject(input *s3.PutObjectInput, err error) { + m. + On("PutObject", input). + Once(). + Return(&s3.PutObjectOutput{}, err) +} + +func (m *S3ClientMock) ShouldGetObject(input *s3.GetObjectInput, output *s3.GetObjectOutput) { + m. + On("GetObject", input). + Once(). + Return(output, nil) +} + +func (m *S3ClientMock) ShouldReturnErrorOnGetObject(input *s3.GetObjectInput, err error) { + m. + On("GetObject", input). + Once(). + Return(&s3.GetObjectOutput{}, err) +} + +/////////////////////////////////////////////////////////////// +// Time provider Mock +/////////////////////////////////////////////////////////////// + +type TimeProviderMock struct { + mock.Mock +} + +func (m *TimeProviderMock) Now() time.Time { + args := m.Called() + + return args.Get(0).(time.Time) +} + +func (m *TimeProviderMock) ShouldProvideNow(now time.Time) { + m. + On("Now"). + Once(). + Return(now) +} + +func mustPrettify(jsonStr string) string { + var jsonObj []Mark + err := json.Unmarshal([]byte(jsonStr), &jsonObj) + if err != nil { + panic(err) + } + + prettyJSON, err := json.MarshalIndent(jsonObj, "", " ") + if err != nil { + panic(err) + } + + return string(prettyJSON) +} diff --git a/publisher/upload/upload.go b/publisher/upload/upload.go index eb109d5..d9669e9 100644 --- a/publisher/upload/upload.go +++ b/publisher/upload/upload.go @@ -2,6 +2,7 @@ package upload import ( "fmt" + "github.com/newrelic/infrastructure-publish-action/publisher/release" "io/ioutil" "net/http" "net/url" @@ -64,11 +65,22 @@ func uploadArtifact(conf config.Config, schema config.UploadArtifactSchema, uplo return nil } -func UploadArtifacts(conf config.Config, schema config.UploadArtifactSchemas, bucketLock lock.BucketLock) (err error) { +func UploadArtifacts(conf config.Config, schema config.UploadArtifactSchemas, bucketLock lock.BucketLock, releaseMarker release.Marker) (err error) { if err = bucketLock.Lock(); err != nil { return } + // Write the release marker + mark, err := releaseMarker.Start(conf.AppName, conf.Tag, conf.RunID, conf.RepoName, conf.Schema, conf.SchemaURL) + if err != nil { + return fmt.Errorf("cannot start release marker: %w", err) + } + defer func() { + markerErr := releaseMarker.End(mark) + if markerErr != nil { + utils.Logger.Printf("ERROR: cannot end release marker %v", markerErr) + } + errRelease := bucketLock.Release() if err == nil { err = errRelease diff --git a/publisher/upload/upload_test.go b/publisher/upload/upload_test.go index fa16b52..05584fa 100644 --- a/publisher/upload/upload_test.go +++ b/publisher/upload/upload_test.go @@ -1,6 +1,9 @@ package upload import ( + "errors" + "github.com/newrelic/infrastructure-publish-action/publisher/release" + "github.com/stretchr/testify/mock" "io/ioutil" "os" "path" @@ -164,7 +167,13 @@ func TestUploadArtifacts(t *testing.T) { err := writeDummyFile(path.Join(src, dummyFile)) assert.NoError(t, err) } - err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory()) + + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) assert.NoError(t, err) for _, expectedFile := range artifact.expectedFiles { @@ -176,6 +185,131 @@ func TestUploadArtifacts(t *testing.T) { } +func TestUploadArtifactsShouldFailIfMarkerCannotBeStarted(t *testing.T) { + var testArtifacts = []struct { + name string + schema []config.UploadArtifactSchema + dummyFiles []string + expectedFiles []string + }{ + { + name: "AppName, arch and app version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{arch}-{version}.txt", []string{"amd64", "386"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + {"{app_name}-{arch}-{version}.txt", nil, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + }, + dummyFiles: []string{"nri-foobar-amd64-2.0.0.txt", "nri-foobar-386-2.0.0.txt"}, + expectedFiles: []string{"amd64/nri-foobar/nri-foobar-amd64-2.0.0.txt", "386/nri-foobar/nri-foobar-386-2.0.0.txt"}, + }, + } + + cfg := config.Config{ + Version: "2.0.0", + UploadSchemaFilePath: "", + AppName: "nri-foobar", + } + + for _, artifact := range testArtifacts { + t.Run(artifact.name, func(t *testing.T) { + markerErr := errors.New("some error") + marker := &MarkerMock{} + marker.ShouldFailOnStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, markerErr) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) + assert.ErrorIs(t, err, markerErr) + }) + } +} + +func TestUploadArtifactsShouldNotFailIfMarkerCannotBeEnded(t *testing.T) { + var testArtifacts = []struct { + name string + schema []config.UploadArtifactSchema + dummyFiles []string + expectedFiles []string + }{ + { + name: "AppName, arch and app version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{arch}-{version}.txt", []string{"amd64", "386"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + {"{app_name}-{arch}-{version}.txt", nil, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{src}", + }, + }}, + }, + dummyFiles: []string{"nri-foobar-amd64-2.0.0.txt", "nri-foobar-386-2.0.0.txt"}, + expectedFiles: []string{"amd64/nri-foobar/nri-foobar-amd64-2.0.0.txt", "386/nri-foobar/nri-foobar-386-2.0.0.txt"}, + }, + { + name: "AppName, arch, app version and os version expansion", + schema: []config.UploadArtifactSchema{ + {"{app_name}-{version}-1.amazonlinux-{os_version}.{arch}.rpm.sum", []string{"x86_64"}, []config.Upload{ + { + Type: "file", + Dest: "{arch}/{app_name}/{os_version}/{src}", + OsVersion: []string{"2", "2022"}, + }, + }}, + }, + dummyFiles: []string{"nri-foobar-2.0.0-1.amazonlinux-2.x86_64.rpm.sum", "nri-foobar-2.0.0-1.amazonlinux-2022.x86_64.rpm.sum"}, + expectedFiles: []string{"x86_64/nri-foobar/2/nri-foobar-2.0.0-1.amazonlinux-2.x86_64.rpm.sum", "x86_64/nri-foobar/2022/nri-foobar-2.0.0-1.amazonlinux-2022.x86_64.rpm.sum"}, + }, + } + + dest, err := ioutil.TempDir("", "") + assert.NoError(t, err) + src, err := ioutil.TempDir("", "") + assert.NoError(t, err) + + cfg := config.Config{ + Version: "2.0.0", + ArtifactsDestFolder: dest, + ArtifactsSrcFolder: src, + UploadSchemaFilePath: "", + AppName: "nri-foobar", + } + + for _, artifact := range testArtifacts { + t.Run(artifact.name, func(t *testing.T) { + for _, dummyFile := range artifact.dummyFiles { + err := writeDummyFile(path.Join(src, dummyFile)) + assert.NoError(t, err) + } + + markerErr := errors.New("some error") + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldFailOnEnd(mark, markerErr) + + err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) + assert.NoError(t, err) + + for _, expectedFile := range artifact.expectedFiles { + _, err = os.Stat(path.Join(dest, expectedFile)) + assert.NoError(t, err) + } + }) + } +} + func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { schema := []config.UploadArtifactSchema{ {"{app_name}-{arch}-{version}.txt", []string{"amd64"}, []config.Upload{ @@ -212,13 +346,18 @@ func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { l := lock.NewInMemory() go func() { <-ready - err1 = UploadArtifacts(cfg, schema, l) + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + err1 = UploadArtifacts(cfg, schema, l, marker) wg.Done() }() go func() { <-ready time.Sleep(1 * time.Millisecond) - err2 = UploadArtifacts(cfg, schema, l) + marker := &MarkerMock{} + err2 = UploadArtifacts(cfg, schema, l, marker) wg.Done() }() @@ -279,7 +418,11 @@ func TestUploadArtifacts_errorsIfAnyArchFails(t *testing.T) { err = writeDummyFile(path.Join(src, "nri-foobar-386-2.0.0.txt")) assert.NoError(t, err) - err = UploadArtifacts(cfg, tc.schema, lock.NewNoop()) + marker := &MarkerMock{} + mark := release.Mark{} + marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldEnd(mark) + err = UploadArtifacts(cfg, tc.schema, lock.NewNoop(), marker) if tc.expectsError { assert.Error(t, err) } else { @@ -309,3 +452,47 @@ func Test_generateRepoFileContent(t *testing.T) { assert.Equal(t, expectedContent, repoFileContent) } + +type MarkerMock struct { + mock.Mock +} + +func (m *MarkerMock) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (release.Mark, error) { + args := m.Called(appName, tag, runID, repoName, schema, schemaURL) + + return args.Get(0).(release.Mark), args.Error(1) +} + +func (m *MarkerMock) End(mark release.Mark) error { + args := m.Called(mark) + + return args.Error(0) +} + +func (m *MarkerMock) ShouldStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, mark release.Mark) { + m. + On("Start", appName, tag, runID, repoName, schema, schemaURL). + Once(). + Return(mark, nil) +} + +func (m *MarkerMock) ShouldFailOnStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, err error) { + m. + On("Start", appName, tag, runID, repoName, schema, schemaURL). + Once(). + Return(release.Mark{}, err) +} + +func (m *MarkerMock) ShouldEnd(mark release.Mark) { + m. + On("End", mark). + Once(). + Return(nil) +} + +func (m *MarkerMock) ShouldFailOnEnd(mark release.Mark, err error) { + m. + On("End", mark). + Once(). + Return(err) +} From 5dc1f5315e76a306e74bce9d3055f05eec1bc7f6 Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Thu, 27 Feb 2025 14:38:15 +0100 Subject: [PATCH 2/5] release info --- README.md | 20 +++ publisher/release/marker.go | 22 ++-- publisher/release/marker_aws.go | 13 +- publisher/release/marker_aws_test.go | 174 +++++++++++++++------------ publisher/upload/upload.go | 11 +- publisher/upload/upload_test.go | 61 ++++++++-- 6 files changed, 195 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e1fc7b6..d8e308f 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,26 @@ The locking mechanism is based on the existance of a file in a defined `aws_s3_l NOTE: Currently in case of an interrupted job, the lock file will not be removed and it would be necessary to remove it manually from the bucket. +## Release Markers + +In order to track the releases made with the Publish Action, the action appends information about each execution in a file in the S3 bucket. +The file is persisted in the root of the repository (`/infrastructure-agent/`) and contains the following information: + +```json +[ + { + "app_name": "newrelic-infra", + "tag": "1.7.0", + "run_id": "13549016185", + "start": "2025-02-26T16:57:33Z", + "end": "2025-02-26T16:58:47Z", + "repo_name": "newrelic/infrastructure-agent", + "schema": "custom", + "schema_url": "https://raw.githubusercontent.com/newrelic/infrastructure-agent/test_publish_action_markers/build/upload-schema-linux-deb.yml" + } +] +``` + ## Support If you need assistance with New Relic products, you are in good hands with several support diagnostic tools and support channels. diff --git a/publisher/release/marker.go b/publisher/release/marker.go index 5e00b5f..b6944ff 100644 --- a/publisher/release/marker.go +++ b/publisher/release/marker.go @@ -5,23 +5,27 @@ import ( "time" ) +type ReleaseInfo struct { + AppName string `json:"app_name"` + Tag string `json:"tag"` + RunID string `json:"run_id"` + RepoName string `json:"repo_name"` + Schema string `json:"schema"` + SchemaURL string `json:"schema_url"` +} + // Mark represents a release mark. It will contain the name of the release (appName, tag...) // and the start and end of a release // When the release has been started, the end will be zero type Mark struct { - AppName string `json:"app_name"` - Tag string `json:"tag"` - RunID string `json:"run_id"` - Start CustomTime `json:"start"` - End CustomTime `json:"end"` - RepoName string `json:"repo_name"` - Schema string `json:"schema"` - SchemaURl string `json:"schema_url"` + ReleaseInfo + Start CustomTime `json:"start"` + End CustomTime `json:"end"` } // Marker abstracts the persistence of the start and end of a release type Marker interface { - Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) + Start(releaseInfo ReleaseInfo) (Mark, error) End(mark Mark) error } diff --git a/publisher/release/marker_aws.go b/publisher/release/marker_aws.go index 806d031..e8a995d 100644 --- a/publisher/release/marker_aws.go +++ b/publisher/release/marker_aws.go @@ -82,8 +82,8 @@ func NewMarkerAWS(s3Config S3Config, logfn Logf) (Marker, error) { // load all the markers from the file // append a new started marker // write the markers back to the file -func (s *markerAWS) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (Mark, error) { - s.logfn("[marker] starting %s", appName) +func (s *markerAWS) Start(releaseInfo ReleaseInfo) (Mark, error) { + s.logfn("[marker] starting %s", releaseInfo.AppName) markers, err := s.readMarkers() if err != nil { if !isNoSuchKeyError(err) { @@ -92,13 +92,8 @@ func (s *markerAWS) Start(appName string, tag string, runID string, repoName str } mark := Mark{ - AppName: appName, - Tag: tag, - RunID: runID, - RepoName: repoName, - Schema: schema, - SchemaURl: schemaURL, - Start: CustomTime{s.now()}, + ReleaseInfo: releaseInfo, + Start: CustomTime{s.now()}, } markers = append(markers, mark) diff --git a/publisher/release/marker_aws_test.go b/publisher/release/marker_aws_test.go index a2bce0c..1e9762b 100644 --- a/publisher/release/marker_aws_test.go +++ b/publisher/release/marker_aws_test.go @@ -59,35 +59,35 @@ func Test_Start(t *testing.T) { &s3.PutObjectOutput{}, ) - appName := "my-app" - tag := "v1.2" - runID := "run3" - repoName := "repo3" - schema := "schema3" - schemaURL := "url3" markerS3 := &markerAWS{ client: s3ClientMock, conf: s3Config, timeProvider: timeProviderMock, logfn: nolog, } - marker, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + + releaseInfo := ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + } + mark, err := markerS3.Start(releaseInfo) require.NoError(t, err) - require.Equal(t, appName, marker.AppName) - require.Equal(t, tag, marker.Tag) - require.Equal(t, runID, marker.RunID) - require.Equal(t, repoName, marker.RepoName) - require.Equal(t, schema, marker.Schema) - require.Equal(t, schemaURL, marker.SchemaURl) + require.Equal(t, releaseInfo, mark.ReleaseInfo) } func Test_StartErrorReadingMarkers(t *testing.T) { - appName := "my-app" - tag := "v1.2" - runID := "run3" - repoName := "repo3" - schema := "schema3" - schemaURL := "url3" + releaseInfo := ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + } timeProviderMock := &TimeProviderMock{} s3ClientMock := &S3ClientMock{} @@ -103,17 +103,19 @@ func Test_StartErrorReadingMarkers(t *testing.T) { someError) markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} - _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + _, err := markerS3.Start(releaseInfo) assert.ErrorIs(t, err, someError) } func Test_StartErrorWritingMarkers(t *testing.T) { - appName := "my-app" - tag := "v1.2" - runID := "run3" - repoName := "repo3" - schema := "schema3" - schemaURL := "url3" + releaseInfo := ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + } timeProviderMock := &TimeProviderMock{} s3ClientMock := &S3ClientMock{} @@ -154,7 +156,7 @@ func Test_StartErrorWritingMarkers(t *testing.T) { someError) markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} - _, err := markerS3.Start(appName, tag, runID, repoName, schema, schemaURL) + _, err := markerS3.Start(releaseInfo) assert.ErrorIs(t, err, someError) } func Test_End(t *testing.T) { @@ -196,15 +198,18 @@ func Test_End(t *testing.T) { &s3.PutObjectOutput{}, ) - startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) - mark := Mark{ + releaseInfo := ReleaseInfo{ AppName: "my-app", Tag: "v1.2", RunID: "run3", RepoName: "repo3", Schema: "schema3", - SchemaURl: "schema_url", - Start: CustomTime{startTime}, + SchemaURL: "url3", + } + startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) + mark := Mark{ + ReleaseInfo: releaseInfo, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -215,6 +220,7 @@ func Test_End(t *testing.T) { } err := markerS3.End(mark) require.NoError(t, err) + require.Equal(t, releaseInfo, mark.ReleaseInfo) } func Test_End_ErrorOnWriting(t *testing.T) { @@ -257,14 +263,18 @@ func Test_End_ErrorOnWriting(t *testing.T) { someError) startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) - mark := Mark{ + + releaseInfo := ReleaseInfo{ AppName: "my-app", Tag: "v1.2", RunID: "run3", RepoName: "repo3", Schema: "schema3", - SchemaURl: "schema_url", - Start: CustomTime{startTime}, + SchemaURL: "url3", + } + mark := Mark{ + ReleaseInfo: releaseInfo, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -304,13 +314,15 @@ func Test_End_ErrorIfNoMarkerFound(t *testing.T) { startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ - AppName: "my-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "schema_url", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -340,13 +352,15 @@ func Test_End_ErrorOnReadingMarkers(t *testing.T) { startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ - AppName: "my-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "schema_url", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -390,13 +404,15 @@ func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ - AppName: "my-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "schema_url", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -440,13 +456,15 @@ func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ - AppName: "another-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "url3", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "another-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -490,13 +508,15 @@ func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ - AppName: "my-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "url3", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ @@ -540,13 +560,15 @@ func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { startTime := time.Time{} mark := Mark{ - AppName: "my-app", - Tag: "v1.2", - RunID: "run3", - RepoName: "repo3", - Schema: "schema3", - SchemaURl: "url3", - Start: CustomTime{startTime}, + ReleaseInfo: ReleaseInfo{ + AppName: "my-app", + Tag: "v1.2", + RunID: "run3", + RepoName: "repo3", + Schema: "schema3", + SchemaURL: "url3", + }, + Start: CustomTime{startTime}, } markerS3 := &markerAWS{ diff --git a/publisher/upload/upload.go b/publisher/upload/upload.go index d9669e9..078baaa 100644 --- a/publisher/upload/upload.go +++ b/publisher/upload/upload.go @@ -70,7 +70,16 @@ func UploadArtifacts(conf config.Config, schema config.UploadArtifactSchemas, bu return } // Write the release marker - mark, err := releaseMarker.Start(conf.AppName, conf.Tag, conf.RunID, conf.RepoName, conf.Schema, conf.SchemaURL) + mark, err := releaseMarker.Start( + release.ReleaseInfo{ + AppName: conf.AppName, + Tag: conf.Tag, + RunID: conf.RunID, + RepoName: conf.RepoName, + Schema: conf.Schema, + SchemaURL: conf.SchemaURL, + }, + ) if err != nil { return fmt.Errorf("cannot start release marker: %w", err) } diff --git a/publisher/upload/upload_test.go b/publisher/upload/upload_test.go index 05584fa..96b1a69 100644 --- a/publisher/upload/upload_test.go +++ b/publisher/upload/upload_test.go @@ -169,8 +169,16 @@ func TestUploadArtifacts(t *testing.T) { } marker := &MarkerMock{} + releaseInfo := release.ReleaseInfo{ + AppName: cfg.AppName, + Tag: cfg.Tag, + RunID: cfg.RunID, + RepoName: cfg.RepoName, + Schema: cfg.Schema, + SchemaURL: cfg.SchemaURL, + } mark := release.Mark{} - marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldStart(releaseInfo, mark) marker.ShouldEnd(mark) err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) @@ -223,7 +231,14 @@ func TestUploadArtifactsShouldFailIfMarkerCannotBeStarted(t *testing.T) { t.Run(artifact.name, func(t *testing.T) { markerErr := errors.New("some error") marker := &MarkerMock{} - marker.ShouldFailOnStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, markerErr) + marker.ShouldFailOnStart(release.ReleaseInfo{ + AppName: cfg.AppName, + Tag: cfg.Tag, + RunID: cfg.RunID, + RepoName: cfg.RepoName, + Schema: cfg.Schema, + SchemaURL: cfg.SchemaURL, + }, markerErr) err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) assert.ErrorIs(t, err, markerErr) @@ -295,8 +310,16 @@ func TestUploadArtifactsShouldNotFailIfMarkerCannotBeEnded(t *testing.T) { markerErr := errors.New("some error") marker := &MarkerMock{} + releaseInfo := release.ReleaseInfo{ + AppName: cfg.AppName, + Tag: cfg.Tag, + RunID: cfg.RunID, + RepoName: cfg.RepoName, + Schema: cfg.Schema, + SchemaURL: cfg.SchemaURL, + } mark := release.Mark{} - marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldStart(releaseInfo, mark) marker.ShouldFailOnEnd(mark, markerErr) err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) @@ -347,8 +370,16 @@ func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { go func() { <-ready marker := &MarkerMock{} + releaseInfo := release.ReleaseInfo{ + AppName: cfg.AppName, + Tag: cfg.Tag, + RunID: cfg.RunID, + RepoName: cfg.RepoName, + Schema: cfg.Schema, + SchemaURL: cfg.SchemaURL, + } mark := release.Mark{} - marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldStart(releaseInfo, mark) marker.ShouldEnd(mark) err1 = UploadArtifacts(cfg, schema, l, marker) wg.Done() @@ -419,8 +450,16 @@ func TestUploadArtifacts_errorsIfAnyArchFails(t *testing.T) { assert.NoError(t, err) marker := &MarkerMock{} + releaseInfo := release.ReleaseInfo{ + AppName: cfg.AppName, + Tag: cfg.Tag, + RunID: cfg.RunID, + RepoName: cfg.RepoName, + Schema: cfg.Schema, + SchemaURL: cfg.SchemaURL, + } mark := release.Mark{} - marker.ShouldStart(cfg.AppName, cfg.Tag, cfg.RunID, cfg.RepoName, cfg.Schema, cfg.SchemaURL, mark) + marker.ShouldStart(releaseInfo, mark) marker.ShouldEnd(mark) err = UploadArtifacts(cfg, tc.schema, lock.NewNoop(), marker) if tc.expectsError { @@ -457,8 +496,8 @@ type MarkerMock struct { mock.Mock } -func (m *MarkerMock) Start(appName string, tag string, runID string, repoName string, schema string, schemaURL string) (release.Mark, error) { - args := m.Called(appName, tag, runID, repoName, schema, schemaURL) +func (m *MarkerMock) Start(releaseInfo release.ReleaseInfo) (release.Mark, error) { + args := m.Called(releaseInfo) return args.Get(0).(release.Mark), args.Error(1) } @@ -469,16 +508,16 @@ func (m *MarkerMock) End(mark release.Mark) error { return args.Error(0) } -func (m *MarkerMock) ShouldStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, mark release.Mark) { +func (m *MarkerMock) ShouldStart(releaseInfo release.ReleaseInfo, mark release.Mark) { m. - On("Start", appName, tag, runID, repoName, schema, schemaURL). + On("Start", releaseInfo). Once(). Return(mark, nil) } -func (m *MarkerMock) ShouldFailOnStart(appName string, tag string, runID string, repoName string, schema string, schemaURL string, err error) { +func (m *MarkerMock) ShouldFailOnStart(releaseInfo release.ReleaseInfo, err error) { m. - On("Start", appName, tag, runID, repoName, schema, schemaURL). + On("Start", releaseInfo). Once(). Return(release.Mark{}, err) } From 1b5c84e283d87e12d9d9705718a9aefbf0055172 Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Fri, 28 Feb 2025 14:59:27 +0100 Subject: [PATCH 3/5] add assertions for mocks expectations --- publisher/release/marker_aws_test.go | 45 +++++++--------------------- publisher/upload/upload_test.go | 6 ++++ 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/publisher/release/marker_aws_test.go b/publisher/release/marker_aws_test.go index 1e9762b..ef43711 100644 --- a/publisher/release/marker_aws_test.go +++ b/publisher/release/marker_aws_test.go @@ -77,6 +77,7 @@ func Test_Start(t *testing.T) { mark, err := markerS3.Start(releaseInfo) require.NoError(t, err) require.Equal(t, releaseInfo, mark.ReleaseInfo) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_StartErrorReadingMarkers(t *testing.T) { @@ -105,6 +106,7 @@ func Test_StartErrorReadingMarkers(t *testing.T) { markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} _, err := markerS3.Start(releaseInfo) assert.ErrorIs(t, err, someError) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_StartErrorWritingMarkers(t *testing.T) { @@ -158,6 +160,7 @@ func Test_StartErrorWritingMarkers(t *testing.T) { markerS3 := &markerAWS{timeProvider: timeProviderMock, conf: s3Config, client: s3ClientMock, logfn: nolog} _, err := markerS3.Start(releaseInfo) assert.ErrorIs(t, err, someError) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_End(t *testing.T) { s3ClientMock := &S3ClientMock{} @@ -221,6 +224,7 @@ func Test_End(t *testing.T) { err := markerS3.End(mark) require.NoError(t, err) require.Equal(t, releaseInfo, mark.ReleaseInfo) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_End_ErrorOnWriting(t *testing.T) { @@ -285,6 +289,7 @@ func Test_End_ErrorOnWriting(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, someError) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_End_ErrorIfNoMarkerFound(t *testing.T) { @@ -308,10 +313,6 @@ func Test_End_ErrorIfNoMarkerFound(t *testing.T) { &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, &s3.GetObjectOutput{Body: readCloser}) - // It should get current time for the end of the started marker - endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) - timeProviderMock.ShouldProvideNow(endTime) - startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ ReleaseInfo: ReleaseInfo{ @@ -333,6 +334,7 @@ func Test_End_ErrorIfNoMarkerFound(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, ErrNoStartedMarkersFound) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_End_ErrorOnReadingMarkers(t *testing.T) { s3ClientMock := &S3ClientMock{} @@ -371,6 +373,7 @@ func Test_End_ErrorOnReadingMarkers(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, someError) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { @@ -398,10 +401,6 @@ func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, &s3.GetObjectOutput{Body: readCloser}) - // It should get current time for the end of the started marker - endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) - timeProviderMock.ShouldProvideNow(endTime) - startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ ReleaseInfo: ReleaseInfo{ @@ -423,6 +422,7 @@ func Test_EndFailsIfLatestMarkerIsEnded(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, ErrLastMarkerEnded) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { @@ -450,10 +450,6 @@ func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, &s3.GetObjectOutput{Body: readCloser}) - // It should get current time for the end of the started marker - endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) - timeProviderMock.ShouldProvideNow(endTime) - startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ ReleaseInfo: ReleaseInfo{ @@ -475,6 +471,7 @@ func Test_EndFailsIfMarkerForAppNotFound(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { @@ -502,10 +499,6 @@ func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, &s3.GetObjectOutput{Body: readCloser}) - // It should get current time for the end of the started marker - endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) - timeProviderMock.ShouldProvideNow(endTime) - startTime := time.Date(2023, 1, 2, 00, 00, 00, 0, time.UTC) mark := Mark{ ReleaseInfo: ReleaseInfo{ @@ -527,6 +520,7 @@ func Test_EndFailsIfMarkerStartTimeIsDifferent(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, ErrNoStartedMarkerFoundForApp) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { @@ -540,24 +534,6 @@ func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { Directory: "directory", } - // It should read existing markers - existingMarkers := ` - [ - {"app_name": "app1", "tag": "v1.0", "run_id": "run1", "start": "2023-01-01T00:00:00Z", "end": "2023-01-01T01:00:00Z", "repo_name": "repo1", "schema": "schema1", "schema_url": "url1"}, - {"app_name": "my-app", "tag": "v1.2", "run_id": "run3", "start": "2023-01-02T11:00:00Z", "end": "0001-01-01T00:00:00Z", "repo_name": "repo3", "schema": "schema3", "schema_url": "url3"} - ]` - - reader := bytes.NewReader([]byte(existingMarkers)) - readCloser := io.NopCloser(reader) - - s3ClientMock.ShouldGetObject( - &s3.GetObjectInput{Bucket: &s3Config.Bucket, Key: aws.String(fmt.Sprintf("%s/%s", s3Config.Directory, markerName))}, - &s3.GetObjectOutput{Body: readCloser}) - - // It should get current time for the end of the started marker - endTime := time.Date(2025, 3, 4, 11, 12, 13, 0, time.UTC) - timeProviderMock.ShouldProvideNow(endTime) - startTime := time.Time{} mark := Mark{ ReleaseInfo: ReleaseInfo{ @@ -579,6 +555,7 @@ func Test_EndFailsIfMarkerStartIsZero(t *testing.T) { } err := markerS3.End(mark) assert.ErrorIs(t, err, ErrNotStartedMark) + mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } /////////////////////////////////////////////////////////////// diff --git a/publisher/upload/upload_test.go b/publisher/upload/upload_test.go index 96b1a69..06dfb16 100644 --- a/publisher/upload/upload_test.go +++ b/publisher/upload/upload_test.go @@ -188,6 +188,7 @@ func TestUploadArtifacts(t *testing.T) { _, err = os.Stat(path.Join(dest, expectedFile)) assert.NoError(t, err) } + mock.AssertExpectationsForObjects(t, marker) }) } @@ -242,6 +243,7 @@ func TestUploadArtifactsShouldFailIfMarkerCannotBeStarted(t *testing.T) { err := UploadArtifacts(cfg, artifact.schema, lock.NewInMemory(), marker) assert.ErrorIs(t, err, markerErr) + mock.AssertExpectationsForObjects(t, marker) }) } } @@ -329,6 +331,7 @@ func TestUploadArtifactsShouldNotFailIfMarkerCannotBeEnded(t *testing.T) { _, err = os.Stat(path.Join(dest, expectedFile)) assert.NoError(t, err) } + mock.AssertExpectationsForObjects(t, marker) }) } } @@ -382,6 +385,7 @@ func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { marker.ShouldStart(releaseInfo, mark) marker.ShouldEnd(mark) err1 = UploadArtifacts(cfg, schema, l, marker) + mock.AssertExpectationsForObjects(t, marker) wg.Done() }() go func() { @@ -389,6 +393,7 @@ func TestUploadArtifacts_cantBeRunInParallel(t *testing.T) { time.Sleep(1 * time.Millisecond) marker := &MarkerMock{} err2 = UploadArtifacts(cfg, schema, l, marker) + mock.AssertExpectationsForObjects(t, marker) wg.Done() }() @@ -467,6 +472,7 @@ func TestUploadArtifacts_errorsIfAnyArchFails(t *testing.T) { } else { assert.NoError(t, err) } + mock.AssertExpectationsForObjects(t, marker) }) } } From 27048d7f762efc973a945519ab82a79e96621c3a Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Mon, 3 Mar 2025 11:13:56 +0100 Subject: [PATCH 4/5] add comment for isNoSuchKeyError --- publisher/release/marker_aws.go | 1 + 1 file changed, 1 insertion(+) diff --git a/publisher/release/marker_aws.go b/publisher/release/marker_aws.go index e8a995d..2dcc6bd 100644 --- a/publisher/release/marker_aws.go +++ b/publisher/release/marker_aws.go @@ -86,6 +86,7 @@ func (s *markerAWS) Start(releaseInfo ReleaseInfo) (Mark, error) { s.logfn("[marker] starting %s", releaseInfo.AppName) markers, err := s.readMarkers() if err != nil { + // Do not return error if the file does not exist. The first time we will create a new one. if !isNoSuchKeyError(err) { return Mark{}, err } From fb041c3e231f29ef7c3171ad212e10ff420e0911 Mon Sep 17 00:00:00 2001 From: Ruben Ruiz de Gauna Date: Mon, 3 Mar 2025 15:25:58 +0100 Subject: [PATCH 5/5] new lines missing --- publisher/release/marker_aws_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publisher/release/marker_aws_test.go b/publisher/release/marker_aws_test.go index ef43711..53601ad 100644 --- a/publisher/release/marker_aws_test.go +++ b/publisher/release/marker_aws_test.go @@ -162,6 +162,7 @@ func Test_StartErrorWritingMarkers(t *testing.T) { assert.ErrorIs(t, err, someError) mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } + func Test_End(t *testing.T) { s3ClientMock := &S3ClientMock{} timeProviderMock := &TimeProviderMock{} @@ -336,6 +337,7 @@ func Test_End_ErrorIfNoMarkerFound(t *testing.T) { assert.ErrorIs(t, err, ErrNoStartedMarkersFound) mock.AssertExpectationsForObjects(t, s3ClientMock, timeProviderMock) } + func Test_End_ErrorOnReadingMarkers(t *testing.T) { s3ClientMock := &S3ClientMock{} timeProviderMock := &TimeProviderMock{}