From 5dda155f81c7dc5016a0dd94b9a0967e10bc6c33 Mon Sep 17 00:00:00 2001 From: poornas Date: Wed, 20 Nov 2019 16:17:07 -0800 Subject: [PATCH] mirror object lock configuration to dest bucket (#2964) if bucket does not exist on the target side. --- cmd/client-fs.go | 11 +++++++++++ cmd/client-s3.go | 29 +++++++++++++++++++++++++++++ cmd/client.go | 2 ++ cmd/mirror-main.go | 22 +++++++++++++++++++++- go.mod | 2 +- go.sum | 2 ++ 6 files changed, 66 insertions(+), 2 deletions(-) diff --git a/cmd/client-fs.go b/cmd/client-fs.go index 608a8508ec..c2587124fc 100644 --- a/cmd/client-fs.go +++ b/cmd/client-fs.go @@ -37,6 +37,7 @@ import ( "github.com/minio/mc/pkg/hookreader" "github.com/minio/mc/pkg/ioutils" "github.com/minio/mc/pkg/probe" + minio "github.com/minio/minio-go/v6" "github.com/minio/minio-go/v6/pkg/encrypt" ) @@ -999,6 +1000,16 @@ func (f *fsClient) MakeBucket(region string, ignoreExisting, withLock bool) *pro return nil } +// Set object lock configuration of bucket. +func (f *fsClient) SetObjectLockConfig(mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit) *probe.Error { + return probe.NewError(APINotImplemented{API: "SetObjectLockConfig", APIType: "filesystem"}) +} + +// Get object lock configuration of bucket. +func (f *fsClient) GetObjectLockConfig() (mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, perr *probe.Error) { + return nil, nil, nil, probe.NewError(APINotImplemented{API: "GetObjectLockConfig", APIType: "filesystem"}) +} + // GetAccessRules - unsupported API func (f *fsClient) GetAccessRules() (map[string]string, *probe.Error) { return map[string]string{}, probe.NewError(APINotImplemented{ diff --git a/cmd/client-s3.go b/cmd/client-s3.go index 2e12e6f59c..7c4fe42261 100644 --- a/cmd/client-s3.go +++ b/cmd/client-s3.go @@ -72,6 +72,10 @@ const ( fileHeaderType = "fileheader" commentCharType = "commentchar" typeJSONType = "type" + // AmzObjectLockMode sets object lock mode + AmzObjectLockMode = "X-Amz-Object-Lock-Mode" + // AmzObjectLockRetainUntilDate sets object lock retain until date + AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" ) // cseHeaders is list of client side encryption headers @@ -81,6 +85,8 @@ var cseHeaders = []string{ "X-Amz-Meta-X-Amz-Matdesc", } +var timeSentinel = time.Unix(0, 0).UTC() + // newFactory encloses New function with client cache. func newFactory() func(config *Config) (Client, *probe.Error) { clientCache := make(map[uint32]*minio.Client) @@ -799,6 +805,23 @@ func (c *s3Client) Put(ctx context.Context, reader io.Reader, size int64, metada if ok { delete(metadata, "X-Amz-Storage-Class") } + + lockModeStr, ok := metadata[AmzObjectLockMode] + lockMode := minio.RetentionMode("") + if ok { + lockMode = minio.RetentionMode(lockModeStr) + delete(metadata, AmzObjectLockMode) + } + + retainUntilDateStr, ok := metadata[AmzObjectLockRetainUntilDate] + retainUntilDate := timeSentinel + if ok { + delete(metadata, AmzObjectLockRetainUntilDate) + if t, e := time.Parse(time.RFC3339, retainUntilDateStr); e == nil { + retainUntilDate = t.UTC() + } + } + if bucket == "" { return 0, probe.NewError(BucketNameEmpty{}) } @@ -814,6 +837,12 @@ func (c *s3Client) Put(ctx context.Context, reader io.Reader, size int64, metada StorageClass: strings.ToUpper(storageClass), ServerSideEncryption: sse, } + if retainUntilDate != timeSentinel { + opts.RetainUntilDate = &retainUntilDate + } + if lockModeStr != "" { + opts.Mode = &lockMode + } n, e := c.api.PutObjectWithContext(ctx, bucket, object, reader, size, opts) if e != nil { errResponse := minio.ToErrorResponse(e) diff --git a/cmd/client.go b/cmd/client.go index 4841953f0e..c320bac32c 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -50,6 +50,8 @@ type Client interface { // Bucket operations MakeBucket(region string, ignoreExisting, withLock bool) *probe.Error + SetObjectLockConfig(mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit) *probe.Error + GetObjectLockConfig() (mode *minio.RetentionMode, validity *uint, unit *minio.ValidityUnit, perr *probe.Error) // Access policy operations. GetAccess() (access string, policyJSON string, error *probe.Error) diff --git a/cmd/mirror-main.go b/cmd/mirror-main.go index 715c33d179..dab820b3c6 100644 --- a/cmd/mirror-main.go +++ b/cmd/mirror-main.go @@ -714,11 +714,24 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr newDstClt, _ := newClient(newTgtURL) if d.Diff == differInFirst { + withLock := false + mode, validity, unit, err := newSrcClt.GetObjectLockConfig() + if err == nil { + withLock = true + } // Bucket only exists in the source, create the same bucket in the destination - if err := newDstClt.MakeBucket(ctx.String("region"), false, false); err != nil { + if err := newDstClt.MakeBucket(ctx.String("region"), false, withLock); err != nil { errorIf(err, "Cannot created bucket in `"+newTgtURL+"`.") continue } + // object lock configuration set on bucket + if mode != nil { + err := newDstClt.SetObjectLockConfig(mode, validity, unit) + if err != nil { + errorIf(err, "Cannot set object lock config in `"+newTgtURL+"`.") + continue + } + } // Copy policy rules from source to dest if flag is activated // and all buckets are mirrored. if ctx.Bool("a") { @@ -743,6 +756,13 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr errorIf(err, "Cannot copy bucket policies to `"+dstClt.GetURL().String()+"`.") } } + // object lock configuration set on bucket + if srcMode, srcValidity, srcUnit, srcErr := srcClt.GetObjectLockConfig(); srcMode != nil && srcErr == nil { + err := dstClt.SetObjectLockConfig(srcMode, srcValidity, srcUnit) + if err != nil { + errorIf(err, "Cannot set object lock config in `"+dstClt.GetURL().String()+"`.") + } + } } if !mirrorAllBuckets && mj.isWatch { diff --git a/go.mod b/go.mod index 22d0774e14..460e308317 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/mattn/go-runewidth v0.0.5 // indirect github.com/minio/cli v1.22.0 github.com/minio/minio v0.0.0-20191119214813-7cdb67680e72 - github.com/minio/minio-go/v6 v6.0.41 + github.com/minio/minio-go/v6 v6.0.42 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/profile v1.3.0 diff --git a/go.sum b/go.sum index 97685e2caa..10e5b05e0f 100644 --- a/go.sum +++ b/go.sum @@ -469,6 +469,8 @@ github.com/minio/minio-go/v6 v6.0.39 h1:9qmKCTBpQpMdGlDAbs3mbb4mmL45/lwRUvHL1VLh github.com/minio/minio-go/v6 v6.0.39/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/minio-go/v6 v6.0.41 h1:ybV6itOYLsjCpp4+QjE4gokic/xhJU7D7wRzxqPHTio= github.com/minio/minio-go/v6 v6.0.41/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.42 h1:aIAm+bMIOWCr634eZQdWGxelaVXA8/y2tOBrG9wV8+Y= +github.com/minio/minio-go/v6 v6.0.42/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679 h1:OMKaN/82sBHUZPvjYNBFituHExa1OGY63eACDGtetKs= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679/go.mod h1:J+goXSuzlte5imWMqb6cUWC/tbYYysUHctwmKXomYzM= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=