From 332290276c0722d4fe400f975cf4e0dca34ace87 Mon Sep 17 00:00:00 2001 From: yahavi Date: Fri, 9 Feb 2024 16:48:17 +0200 Subject: [PATCH] Support multipart upload --- artifactory/cli.go | 31 ++++++++++++++++++---------- artifactory/cli_test.go | 36 +++++++++++++++++++++++++++++++-- go.mod | 4 ++-- go.sum | 8 ++++---- utils/cliutils/cli_consts.go | 5 +++++ utils/cliutils/commandsflags.go | 28 +++++++++++++++++-------- 6 files changed, 85 insertions(+), 27 deletions(-) diff --git a/artifactory/cli.go b/artifactory/cli.go index ef9ce9081..38abbde55 100644 --- a/artifactory/cli.go +++ b/artifactory/cli.go @@ -3,11 +3,12 @@ package artifactory import ( "errors" "fmt" - "github.com/jfrog/jfrog-cli/utils/accesstoken" "os" "strconv" "strings" + "github.com/jfrog/jfrog-cli/utils/accesstoken" + "github.com/jfrog/gofrog/version" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferinstall" "github.com/jfrog/jfrog-cli/docs/artifactory/transferplugininstall" @@ -922,16 +923,16 @@ func GetCommands() []cli.Command { }) } -func getSplitCount(c *cli.Context) (splitCount int, err error) { - splitCount = cliutils.DownloadSplitCount +func getSplitCount(c *cli.Context, defaultSplitCount, defaultMaxSplitCount int) (splitCount int, err error) { + splitCount = defaultSplitCount err = nil if c.String("split-count") != "" { splitCount, err = strconv.Atoi(c.String("split-count")) if err != nil { err = errors.New("The '--split-count' option should have a numeric value. " + cliutils.GetDocumentationMessage()) } - if splitCount > cliutils.DownloadMaxSplitCount { - err = errors.New("The '--split-count' option value is limited to a maximum of " + strconv.Itoa(cliutils.DownloadMaxSplitCount) + ".") + if splitCount > defaultMaxSplitCount { + err = errors.New("The '--split-count' option value is limited to a maximum of " + strconv.Itoa(defaultMaxSplitCount) + ".") } if splitCount < 0 { err = errors.New("the '--split-count' option cannot have a negative value") @@ -940,10 +941,10 @@ func getSplitCount(c *cli.Context) (splitCount int, err error) { return } -func getMinSplit(c *cli.Context) (minSplitSize int64, err error) { - minSplitSize = cliutils.DownloadMinSplitKb - if c.String("min-split") != "" { - minSplitSize, err = strconv.ParseInt(c.String("min-split"), 10, 64) +func getMinSplit(c *cli.Context, defaultMinSplit int64) (minSplitSize int64, err error) { + minSplitSize = defaultMinSplit + if c.String(cliutils.MinSplit) != "" { + minSplitSize, err = strconv.ParseInt(c.String(cliutils.MinSplit), 10, 64) if err != nil { err = errors.New("The '--min-split' option should have a numeric value. " + cliutils.GetDocumentationMessage()) return 0, err @@ -2611,11 +2612,11 @@ func createDefaultDownloadSpec(c *cli.Context) (*spec.SpecFiles, error) { func createDownloadConfiguration(c *cli.Context) (downloadConfiguration *utils.DownloadConfiguration, err error) { downloadConfiguration = new(utils.DownloadConfiguration) - downloadConfiguration.MinSplitSize, err = getMinSplit(c) + downloadConfiguration.MinSplitSize, err = getMinSplit(c, cliutils.DownloadMinSplitKb) if err != nil { return nil, err } - downloadConfiguration.SplitCount, err = getSplitCount(c) + downloadConfiguration.SplitCount, err = getSplitCount(c, cliutils.DownloadSplitCount, cliutils.DownloadMaxSplitCount) if err != nil { return nil, err } @@ -2689,6 +2690,14 @@ func fixWinPathsForDownloadCmd(uploadSpec *spec.SpecFiles, c *cli.Context) { func createUploadConfiguration(c *cli.Context) (uploadConfiguration *utils.UploadConfiguration, err error) { uploadConfiguration = new(utils.UploadConfiguration) + uploadConfiguration.MinSplitSizeMB, err = getMinSplit(c, cliutils.UploadMinSplitMb) + if err != nil { + return nil, err + } + uploadConfiguration.SplitCount, err = getSplitCount(c, cliutils.UploadSplitCount, cliutils.UploadMaxSplitCount) + if err != nil { + return nil, err + } uploadConfiguration.Threads, err = cliutils.GetThreadsCount(c) if err != nil { return nil, err diff --git a/artifactory/cli_test.go b/artifactory/cli_test.go index e951be591..ead08ebb7 100644 --- a/artifactory/cli_test.go +++ b/artifactory/cli_test.go @@ -2,12 +2,15 @@ package artifactory import ( "bytes" + "path/filepath" + "testing" + + commonCliUtils "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" "github.com/jfrog/jfrog-cli-core/v2/common/spec" + "github.com/jfrog/jfrog-cli/utils/cliutils" "github.com/jfrog/jfrog-cli/utils/tests" "github.com/stretchr/testify/assert" "github.com/urfave/cli" - "path/filepath" - "testing" ) func TestPrepareSearchDownloadDeleteCommands(t *testing.T) { @@ -119,3 +122,32 @@ func assertGenericCommand(t *testing.T, err error, buffer *bytes.Buffer, expectE func getSpecPath(spec string) string { return filepath.Join("..", "testdata", "filespecs", spec) } + +var createUploadConfigurationCases = []struct { + name string + flags []string + expectedMinSplit int64 + expectedSplitCount int + expectedThreads int + expectedDeb string +}{ + {"empty", []string{}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, ""}, + {"min-split", []string{"min-split=101"}, 101, cliutils.UploadSplitCount, commonCliUtils.Threads, ""}, + {"split-count", []string{"split-count=6"}, cliutils.UploadMinSplitMb, 6, commonCliUtils.Threads, ""}, + {"threads", []string{"threads=6"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, 6, ""}, + {"deb", []string{"deb=jammy/main/i386"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "jammy/main/i386"}, +} + +func TestCreateUploadConfiguration(t *testing.T) { + for _, testCase := range createUploadConfigurationCases { + t.Run(testCase.name, func(t *testing.T) { + context, _ := tests.CreateContext(t, testCase.flags, []string{}) + uploadConfiguration, err := createUploadConfiguration(context) + assert.NoError(t, err) + assert.Equal(t, testCase.expectedMinSplit, uploadConfiguration.MinSplitSizeMB) + assert.Equal(t, testCase.expectedSplitCount, uploadConfiguration.SplitCount) + assert.Equal(t, testCase.expectedThreads, uploadConfiguration.Threads) + assert.Equal(t, testCase.expectedDeb, uploadConfiguration.Deb) + }) + } +} diff --git a/go.mod b/go.mod index e0eb08eaf..276a47c72 100644 --- a/go.mod +++ b/go.mod @@ -131,9 +131,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9 +replace github.com/jfrog/jfrog-cli-core/v2 => github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0 -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df +replace github.com/jfrog/jfrog-client-go => github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057 // replace github.com/jfrog/jfrog-cli-security => github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac diff --git a/go.sum b/go.sum index 48a904b0d..d8ab3d5cd 100644 --- a/go.sum +++ b/go.sum @@ -128,12 +128,8 @@ github.com/jfrog/gofrog v1.5.1 h1:2AXL8hHu1jJFMIoCqTp2OyRUfEqEp4nC7J8fwn6KtwE= github.com/jfrog/gofrog v1.5.1/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9 h1:rBtOFiF7YRdnC8dzMHVXUc/9d0y+W0iZ7iHQb9ur2qQ= -github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9/go.mod h1:+eraSKhahQf7tj09+g3rAA2Z+XPnZGfMc0y8uUDecZw= github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac h1:tNn3TQXaIJZ9Fu5jiVB9lWpJAKkEGWNjz/6WzHhHePI= github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac/go.mod h1:X4rz1639L8vWKJgpLxpO3ddkIW7KaCaQjbwani7FPf4= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df h1:7yQ7m0N3AQNqGnymRP+Fka6FZCEYr9GzkTbVkUcxwtQ= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU= github.com/jszwec/csvutil v1.9.0 h1:iTmq9G1P0e+AUq/MkFg6tetJ+1BH3fOX8Xi0RAcwiGc= github.com/jszwec/csvutil v1.9.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -285,6 +281,10 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0 h1:D6BdBwvSA+auq4yV6ABFCdQUccg8Sns3luMv5GYuTGc= +github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0/go.mod h1:NUjgJRs1hC8DhdoTxahbYdNh56ga56HnEC2YNW6TZpU= +github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057 h1:ok2hthLuAAzx9UCvhLCtNnp8JZfaU7ozk3vNqg4cTqE= +github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/utils/cliutils/cli_consts.go b/utils/cliutils/cli_consts.go index a142f81f3..9a2d5abf9 100644 --- a/utils/cliutils/cli_consts.go +++ b/utils/cliutils/cli_consts.go @@ -24,6 +24,11 @@ const ( DownloadSplitCount = 3 DownloadMaxSplitCount = 15 + // Upload + UploadMinSplitMb = 200 + UploadSplitCount = 5 + UploadMaxSplitCount = 100 + // Common Retries = 3 RetryWaitMilliSecs = 0 diff --git a/utils/cliutils/commandsflags.go b/utils/cliutils/commandsflags.go index 9946e24cf..e7fbc7c10 100644 --- a/utils/cliutils/commandsflags.go +++ b/utils/cliutils/commandsflags.go @@ -206,6 +206,8 @@ const ( fromRt = "from-rt" transitive = "transitive" Status = "status" + MinSplit = "min-split" + SplitCount = "split-count" // Config flags interactive = "interactive" @@ -223,6 +225,8 @@ const ( uploadTargetProps = uploadPrefix + targetProps uploadSyncDeletes = uploadPrefix + syncDeletes uploadArchive = uploadPrefix + archive + uploadMinSplit = uploadPrefix + MinSplit + uploadSplitCount = uploadPrefix + SplitCount deb = "deb" symlinks = "symlinks" uploadAnt = uploadPrefix + antFlag @@ -235,8 +239,8 @@ const ( downloadProps = downloadPrefix + props downloadExcludeProps = downloadPrefix + excludeProps downloadSyncDeletes = downloadPrefix + syncDeletes - minSplit = "min-split" - splitCount = "split-count" + downloadMinSplit = downloadPrefix + MinSplit + downloadSplitCount = downloadPrefix + SplitCount validateSymlinks = "validate-symlinks" skipChecksum = "skip-checksum" @@ -804,6 +808,14 @@ var flagsMap = map[string]cli.Flag{ Name: archive, Usage: "[Optional] Set to \"zip\" to deploy the files to Artifactory in a ZIP archive.` `", }, + uploadMinSplit: cli.StringFlag{ + Name: MinSplit, + Usage: "[Default: " + strconv.Itoa(UploadMinSplitMb) + "] Minimum file size in MB to split into ranges when uploading. Require Artifactory with S3 storage.` `", + }, + uploadSplitCount: cli.StringFlag{ + Name: SplitCount, + Usage: "[Default: " + strconv.Itoa(UploadSplitCount) + "] Number of parts to split a file when uploading. Set to 0 for no splits.` `", + }, syncDeletesQuiet: cli.BoolFlag{ Name: quiet, Usage: "[Default: $CI] Set to true to skip the sync-deletes confirmation message.` `", @@ -816,8 +828,8 @@ var flagsMap = map[string]cli.Flag{ Name: flat, Usage: "[Default: false] Set to true if you do not wish to have the Artifactory repository path structure created locally for your downloaded files.` `", }, - minSplit: cli.StringFlag{ - Name: minSplit, + downloadMinSplit: cli.StringFlag{ + Name: MinSplit, Value: "", Usage: "[Default: " + strconv.Itoa(DownloadMinSplitKb) + "] Minimum file size in KB to split into ranges when downloading. Set to -1 for no splits.` `", }, @@ -825,8 +837,8 @@ var flagsMap = map[string]cli.Flag{ Name: skipChecksum, Usage: "[Default: false] Set to true to skip checksum verification when downloading.` `", }, - splitCount: cli.StringFlag{ - Name: splitCount, + downloadSplitCount: cli.StringFlag{ + Name: SplitCount, Value: "", Usage: "[Default: " + strconv.Itoa(DownloadSplitCount) + "] Number of parts to split a file when downloading. Set to 0 for no splits.` `", }, @@ -1737,12 +1749,12 @@ var commandFlags = map[string][]string{ ClientCertKeyPath, specFlag, specVars, buildName, buildNumber, module, uploadExclusions, deb, uploadRecursive, uploadFlat, uploadRegexp, retries, retryWaitTime, dryRun, uploadExplode, symlinks, includeDirs, failNoOp, threads, uploadSyncDeletes, syncDeletesQuiet, InsecureTls, detailedSummary, Project, - uploadAnt, uploadArchive, + uploadAnt, uploadArchive, uploadMinSplit, uploadSplitCount, }, Download: { url, user, password, accessToken, sshPassphrase, sshKeyPath, serverId, ClientCertPath, ClientCertKeyPath, specFlag, specVars, buildName, buildNumber, module, exclusions, sortBy, - sortOrder, limit, offset, downloadRecursive, downloadFlat, build, includeDeps, excludeArtifacts, minSplit, splitCount, + sortOrder, limit, offset, downloadRecursive, downloadFlat, build, includeDeps, excludeArtifacts, downloadMinSplit, downloadSplitCount, retries, retryWaitTime, dryRun, downloadExplode, bypassArchiveInspection, validateSymlinks, bundle, publicGpgKey, includeDirs, downloadProps, downloadExcludeProps, failNoOp, threads, archiveEntries, downloadSyncDeletes, syncDeletesQuiet, InsecureTls, detailedSummary, Project, skipChecksum,