Skip to content

Commit

Permalink
feat(storage): Add new OLM conditions (#2512)
Browse files Browse the repository at this point in the history 
Support CustomTime and NoncurrentTime OLM options. Includes
a new metadata field for CustomTime.

Fixes #2208
Fixes #2517
  • Loading branch information
@tritone
tritone authored Aug 26, 2020
1 parent fe7d8f6 commit 363003c
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 35 deletions.
53 changes: 46 additions & 7 deletions storage/bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,8 @@ type RetentionPolicy struct {
}

const (
// RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule.
// RFC3339 timestamp with only the date segment, used for CreatedBefore,
// CustomTimeBefore, and NoncurrentTimeBefore in LifecycleRule.
rfc3339Date = "2006-01-02"

// DeleteAction is a lifecycle action that deletes a live and/or archived
Expand Down Expand Up @@ -455,6 +456,21 @@ type LifecycleCondition struct {
// the specified date in UTC.
CreatedBefore time.Time

// CustomTimeBefore is the CustomTime metadata field of the object. This
// condition is satisfied when an object's CustomTime timestamp is before
// midnight of the specified date in UTC.
//
// This condition can only be satisfied if CustomTime has been set.
CustomTimeBefore time.Time

// DaysSinceCustomTime is the days elapsed since the CustomTime date of the
// object. This condition can only be satisfied if CustomTime has been set.
DaysSinceCustomTime int64

// DaysSinceNoncurrentTime is the days elapsed since the noncurrent timestamp
// of the object. This condition is relevant only for versioned objects.
DaysSinceNoncurrentTime int64

// Liveness specifies the object's liveness. Relevant only for versioned objects
Liveness Liveness

Expand All @@ -464,6 +480,13 @@ type LifecycleCondition struct {
// Values include "STANDARD", "NEARLINE", "COLDLINE" and "ARCHIVE".
MatchesStorageClasses []string

// NoncurrentTimeBefore is the noncurrent timestamp of the object. This
// condition is satisfied when an object's noncurrent timestamp is before
// midnight of the specified date in UTC.
//
// This condition is relevant only for versioned objects.
NoncurrentTimeBefore time.Time

// NumNewerVersions is the condition matching objects with a number of newer versions.
//
// If the value is N, this condition is satisfied when there are at least N
Expand Down Expand Up @@ -946,9 +969,11 @@ func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
StorageClass: r.Action.StorageClass,
},
Condition: &raw.BucketLifecycleRuleCondition{
Age: r.Condition.AgeInDays,
MatchesStorageClass: r.Condition.MatchesStorageClasses,
NumNewerVersions: r.Condition.NumNewerVersions,
Age: r.Condition.AgeInDays,
DaysSinceCustomTime: r.Condition.DaysSinceCustomTime,
DaysSinceNoncurrentTime: r.Condition.DaysSinceNoncurrentTime,
MatchesStorageClass: r.Condition.MatchesStorageClasses,
NumNewerVersions: r.Condition.NumNewerVersions,
},
}

Expand All @@ -964,6 +989,12 @@ func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
if !r.Condition.CreatedBefore.IsZero() {
rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date)
}
if !r.Condition.CustomTimeBefore.IsZero() {
rr.Condition.CustomTimeBefore = r.Condition.CustomTimeBefore.Format(rfc3339Date)
}
if !r.Condition.NoncurrentTimeBefore.IsZero() {
rr.Condition.NoncurrentTimeBefore = r.Condition.NoncurrentTimeBefore.Format(rfc3339Date)
}
rl.Rule = append(rl.Rule, rr)
}
return &rl
Expand All @@ -981,9 +1012,11 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
StorageClass: rr.Action.StorageClass,
},
Condition: LifecycleCondition{
AgeInDays: rr.Condition.Age,
MatchesStorageClasses: rr.Condition.MatchesStorageClass,
NumNewerVersions: rr.Condition.NumNewerVersions,
AgeInDays: rr.Condition.Age,
DaysSinceCustomTime: rr.Condition.DaysSinceCustomTime,
DaysSinceNoncurrentTime: rr.Condition.DaysSinceNoncurrentTime,
MatchesStorageClasses: rr.Condition.MatchesStorageClass,
NumNewerVersions: rr.Condition.NumNewerVersions,
},
}

Expand All @@ -998,6 +1031,12 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
if rr.Condition.CreatedBefore != "" {
r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
}
if rr.Condition.CustomTimeBefore != "" {
r.Condition.CustomTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.CustomTimeBefore)
}
if rr.Condition.NoncurrentTimeBefore != "" {
r.Condition.NoncurrentTimeBefore, _ = time.Parse(rfc3339Date, rr.Condition.NoncurrentTimeBefore)
}
l.Rules = append(l.Rules, r)
}
return l
Expand Down
70 changes: 47 additions & 23 deletions storage/bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,25 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
},
}, {
Action: LifecycleAction{
Type: DeleteAction,
Type: SetStorageClassAction,
StorageClass: "ARCHIVE",
},
Condition: LifecycleCondition{
AgeInDays: 30,
CustomTimeBefore: time.Date(2020, 1, 2, 3, 0, 0, 0, time.UTC),
DaysSinceCustomTime: 100,
Liveness: Live,
CreatedBefore: time.Date(2017, 1, 2, 3, 4, 5, 6, time.UTC),
MatchesStorageClasses: []string{"NEARLINE"},
NumNewerVersions: 10,
MatchesStorageClasses: []string{"STANDARD"},
},
}, {
Action: LifecycleAction{
Type: DeleteAction,
},
Condition: LifecycleCondition{
DaysSinceNoncurrentTime: 30,
Liveness: Live,
NoncurrentTimeBefore: time.Date(2017, 1, 2, 3, 4, 5, 6, time.UTC),
MatchesStorageClasses: []string{"NEARLINE"},
NumNewerVersions: 10,
},
}, {
Action: LifecycleAction{
Expand Down Expand Up @@ -137,25 +148,38 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
MatchesStorageClass: []string{"STANDARD"},
NumNewerVersions: 3,
},
}, {
Action: &raw.BucketLifecycleRuleAction{
Type: DeleteAction,
},
Condition: &raw.BucketLifecycleRuleCondition{
Age: 30,
IsLive: googleapi.Bool(true),
CreatedBefore: "2017-01-02",
MatchesStorageClass: []string{"NEARLINE"},
NumNewerVersions: 10,
},
}, {
Action: &raw.BucketLifecycleRuleAction{
Type: DeleteAction,
},
Condition: &raw.BucketLifecycleRuleCondition{
IsLive: googleapi.Bool(false),
},
{
Action: &raw.BucketLifecycleRuleAction{
StorageClass: "ARCHIVE",
Type: SetStorageClassAction,
},
Condition: &raw.BucketLifecycleRuleCondition{
IsLive: googleapi.Bool(true),
CustomTimeBefore: "2020-01-02",
DaysSinceCustomTime: 100,
MatchesStorageClass: []string{"STANDARD"},
},
},
}},
{
Action: &raw.BucketLifecycleRuleAction{
Type: DeleteAction,
},
Condition: &raw.BucketLifecycleRuleCondition{
DaysSinceNoncurrentTime: 30,
IsLive: googleapi.Bool(true),
NoncurrentTimeBefore: "2017-01-02",
MatchesStorageClass: []string{"NEARLINE"},
NumNewerVersions: 10,
},
}, {
Action: &raw.BucketLifecycleRuleAction{
Type: DeleteAction,
},
Condition: &raw.BucketLifecycleRuleCondition{
IsLive: googleapi.Bool(false),
},
}},
},
}
if msg := testutil.Diff(got, want); msg != "" {
Expand Down
45 changes: 40 additions & 5 deletions storage/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,14 +265,25 @@ func TestIntegration_BucketMethods(t *testing.T) {
},
}, {
Action: LifecycleAction{
Type: DeleteAction,
Type: SetStorageClassAction,
StorageClass: "ARCHIVE",
},
Condition: LifecycleCondition{
AgeInDays: 30,
CustomTimeBefore: time.Date(2020, 1, 2, 3, 0, 0, 0, time.UTC),
DaysSinceCustomTime: 20,
Liveness: Live,
CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
MatchesStorageClasses: []string{"NEARLINE"},
NumNewerVersions: 10,
MatchesStorageClasses: []string{"STANDARD"},
},
}, {
Action: LifecycleAction{
Type: DeleteAction,
},
Condition: LifecycleCondition{
DaysSinceNoncurrentTime: 30,
Liveness: Live,
NoncurrentTimeBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
MatchesStorageClasses: []string{"NEARLINE"},
NumNewerVersions: 10,
},
}},
},
Expand Down Expand Up @@ -2643,6 +2654,30 @@ func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) {
}
}

func TestIntegration_CustomTime(t *testing.T) {
ctx := context.Background()
client := testConfig(ctx, t)
defer client.Close()
h := testHelper{t}

// Create object with CustomTime.
bkt := client.Bucket(bucketName)
obj := bkt.Object("custom-time-obj")
w := obj.NewWriter(ctx)
ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC)
w.ObjectAttrs.CustomTime = ct
h.mustWrite(w, randomContents())

// Validate that CustomTime has been set
attrs, err := obj.Attrs(ctx)
if err != nil {
t.Fatalf("failed to get object attrs: %v", err)
}
if got := attrs.CustomTime; got != ct {
t.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct)
}
}

func TestIntegration_UpdateRetentionPolicy(t *testing.T) {
ctx := context.Background()
client := testConfig(ctx, t)
Expand Down
15 changes: 15 additions & 0 deletions storage/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1051,6 +1051,10 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object {
if !o.RetentionExpirationTime.IsZero() {
ret = o.RetentionExpirationTime.Format(time.RFC3339)
}
var ct string
if !o.CustomTime.IsZero() {
ct = o.CustomTime.Format(time.RFC3339)
}
return &raw.Object{
Bucket: bucket,
Name: o.Name,
Expand All @@ -1065,6 +1069,7 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object {
StorageClass: o.StorageClass,
Acl: toRawObjectACL(o.ACL),
Metadata: o.Metadata,
CustomTime: ct,
}
}

Expand Down Expand Up @@ -1203,6 +1208,14 @@ type ObjectAttrs struct {
// Etag is the HTTP/1.1 Entity tag for the object.
// This field is read-only.
Etag string

// A user-specified timestamp which can be applied to an object. This is
// typically set in order to use the CustomTimeBefore and DaysSinceCustomTime
// LifecycleConditions to manage object lifecycles.
//
// CustomTime cannot be removed once set on an object. It can be updated to a
// later value but not to an earlier one.
CustomTime time.Time
}

// convertTime converts a time in RFC3339 format to time.Time.
Expand Down Expand Up @@ -1256,6 +1269,7 @@ func newObject(o *raw.Object) *ObjectAttrs {
Deleted: convertTime(o.TimeDeleted),
Updated: convertTime(o.Updated),
Etag: o.Etag,
CustomTime: convertTime(o.CustomTime),
}
}

Expand Down Expand Up @@ -1344,6 +1358,7 @@ var attrToFieldMap = map[string]string{
"Deleted": "timeDeleted",
"Updated": "updated",
"Etag": "etag",
"CustomTime": "customTime",
}

// SetAttrSelection makes the query populate only specific attributes of
Expand Down
2 changes: 2 additions & 0 deletions storage/storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1116,6 +1116,7 @@ func TestRawObjectToObjectAttrs(t *testing.T) {
Bucket: "Test",
ContentLanguage: "en-us",
ContentType: "video/mpeg",
CustomTime: "2020-08-25T19:33:36Z",
EventBasedHold: false,
Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
Generation: 7,
Expand All @@ -1132,6 +1133,7 @@ func TestRawObjectToObjectAttrs(t *testing.T) {
Created: time.Date(2019, 3, 31, 19, 32, 10, 0, time.UTC),
ContentLanguage: "en-us",
ContentType: "video/mpeg",
CustomTime: time.Date(2020, 8, 25, 19, 33, 36, 0, time.UTC),
Deleted: time.Date(2019, 3, 31, 19, 33, 39, 0, time.UTC),
EventBasedHold: false,
Etag: "Zkyw9ACJZUvcYmlFaKGChzhmtnE/dt1zHSfweiWpwzdGsqXwuJZqiD0",
Expand Down

0 comments on commit 363003c

Please sign in to comment.