Support multipart upload

artifactory/cli.go

@@ -3,11 +3,12 @@ package artifactory
 import (
 	"errors"
 	"fmt"
-	"github.com/jfrog/jfrog-cli/utils/accesstoken"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/jfrog/jfrog-cli/utils/accesstoken"
+
 	"github.com/jfrog/gofrog/version"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferinstall"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/transferplugininstall"
@@ -922,16 +923,16 @@ func GetCommands() []cli.Command {
 	})
 }
 
-func getSplitCount(c *cli.Context) (splitCount int, err error) {
-	splitCount = cliutils.DownloadSplitCount
+func getSplitCount(c *cli.Context, defaultSplitCount, defaultMaxSplitCount int) (splitCount int, err error) {
+	splitCount = defaultSplitCount
 	err = nil
 	if c.String("split-count") != "" {
 		splitCount, err = strconv.Atoi(c.String("split-count"))
 		if err != nil {
 			err = errors.New("The '--split-count' option should have a numeric value. " + cliutils.GetDocumentationMessage())
 		}
-		if splitCount > cliutils.DownloadMaxSplitCount {
-			err = errors.New("The '--split-count' option value is limited to a maximum of " + strconv.Itoa(cliutils.DownloadMaxSplitCount) + ".")
+		if splitCount > defaultMaxSplitCount {
+			err = errors.New("The '--split-count' option value is limited to a maximum of " + strconv.Itoa(defaultMaxSplitCount) + ".")
 		}
 		if splitCount < 0 {
 			err = errors.New("the '--split-count' option cannot have a negative value")
@@ -940,10 +941,10 @@ func getSplitCount(c *cli.Context) (splitCount int, err error) {
 	return
 }
 
-func getMinSplit(c *cli.Context) (minSplitSize int64, err error) {
-	minSplitSize = cliutils.DownloadMinSplitKb
-	if c.String("min-split") != "" {
-		minSplitSize, err = strconv.ParseInt(c.String("min-split"), 10, 64)
+func getMinSplit(c *cli.Context, defaultMinSplit int64) (minSplitSize int64, err error) {
+	minSplitSize = defaultMinSplit
+	if c.String(cliutils.MinSplit) != "" {
+		minSplitSize, err = strconv.ParseInt(c.String(cliutils.MinSplit), 10, 64)
 		if err != nil {
 			err = errors.New("The '--min-split' option should have a numeric value. " + cliutils.GetDocumentationMessage())
 			return 0, err
@@ -2611,11 +2612,11 @@ func createDefaultDownloadSpec(c *cli.Context) (*spec.SpecFiles, error) {
 
 func createDownloadConfiguration(c *cli.Context) (downloadConfiguration *utils.DownloadConfiguration, err error) {
 	downloadConfiguration = new(utils.DownloadConfiguration)
-	downloadConfiguration.MinSplitSize, err = getMinSplit(c)
+	downloadConfiguration.MinSplitSize, err = getMinSplit(c, cliutils.DownloadMinSplitKb)
 	if err != nil {
 		return nil, err
 	}
-	downloadConfiguration.SplitCount, err = getSplitCount(c)
+	downloadConfiguration.SplitCount, err = getSplitCount(c, cliutils.DownloadSplitCount, cliutils.DownloadMaxSplitCount)
 	if err != nil {
 		return nil, err
 	}
@@ -2689,6 +2690,14 @@ func fixWinPathsForDownloadCmd(uploadSpec *spec.SpecFiles, c *cli.Context) {
 
 func createUploadConfiguration(c *cli.Context) (uploadConfiguration *utils.UploadConfiguration, err error) {
 	uploadConfiguration = new(utils.UploadConfiguration)
+	uploadConfiguration.MinSplitSizeMB, err = getMinSplit(c, cliutils.UploadMinSplitMb)
+	if err != nil {
+		return nil, err
+	}
+	uploadConfiguration.SplitCount, err = getSplitCount(c, cliutils.UploadSplitCount, cliutils.UploadMaxSplitCount)
+	if err != nil {
+		return nil, err
+	}
 	uploadConfiguration.Threads, err = cliutils.GetThreadsCount(c)
 	if err != nil {
 		return nil, err

artifactory/cli_test.go

@@ -2,12 +2,15 @@ package artifactory
 
 import (
 	"bytes"
+	"path/filepath"
+	"testing"
+
+	commonCliUtils "github.com/jfrog/jfrog-cli-core/v2/common/cliutils"
 	"github.com/jfrog/jfrog-cli-core/v2/common/spec"
+	"github.com/jfrog/jfrog-cli/utils/cliutils"
 	"github.com/jfrog/jfrog-cli/utils/tests"
 	"github.com/stretchr/testify/assert"
 	"github.com/urfave/cli"
-	"path/filepath"
-	"testing"
 )
 
 func TestPrepareSearchDownloadDeleteCommands(t *testing.T) {
@@ -119,3 +122,32 @@ func assertGenericCommand(t *testing.T, err error, buffer *bytes.Buffer, expectE
 func getSpecPath(spec string) string {
 	return filepath.Join("..", "testdata", "filespecs", spec)
 }
+
+var createUploadConfigurationCases = []struct {
+	name               string
+	flags              []string
+	expectedMinSplit   int64
+	expectedSplitCount int
+	expectedThreads    int
+	expectedDeb        string
+}{
+	{"empty", []string{}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, ""},
+	{"min-split", []string{"min-split=101"}, 101, cliutils.UploadSplitCount, commonCliUtils.Threads, ""},
+	{"split-count", []string{"split-count=6"}, cliutils.UploadMinSplitMb, 6, commonCliUtils.Threads, ""},
+	{"threads", []string{"threads=6"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, 6, ""},
+	{"deb", []string{"deb=jammy/main/i386"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "jammy/main/i386"},
+}
+
+func TestCreateUploadConfiguration(t *testing.T) {
+	for _, testCase := range createUploadConfigurationCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			context, _ := tests.CreateContext(t, testCase.flags, []string{})
+			uploadConfiguration, err := createUploadConfiguration(context)
+			assert.NoError(t, err)
+			assert.Equal(t, testCase.expectedMinSplit, uploadConfiguration.MinSplitSizeMB)
+			assert.Equal(t, testCase.expectedSplitCount, uploadConfiguration.SplitCount)
+			assert.Equal(t, testCase.expectedThreads, uploadConfiguration.Threads)
+			assert.Equal(t, testCase.expectedDeb, uploadConfiguration.Deb)
+		})
+	}
+}

go.mod

@@ -131,9 +131,9 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0
 
-replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df
+replace github.com/jfrog/jfrog-client-go => github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057
 
 // replace github.com/jfrog/jfrog-cli-security => github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac
 

go.sum

@@ -128,12 +128,8 @@ github.com/jfrog/gofrog v1.5.1 h1:2AXL8hHu1jJFMIoCqTp2OyRUfEqEp4nC7J8fwn6KtwE=
 github.com/jfrog/gofrog v1.5.1/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9 h1:rBtOFiF7YRdnC8dzMHVXUc/9d0y+W0iZ7iHQb9ur2qQ=
-github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240204105703-557f7a17d7f9/go.mod h1:+eraSKhahQf7tj09+g3rAA2Z+XPnZGfMc0y8uUDecZw=
 github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac h1:tNn3TQXaIJZ9Fu5jiVB9lWpJAKkEGWNjz/6WzHhHePI=
 github.com/jfrog/jfrog-cli-security v0.0.0-20240122124933-edf9cb4ca3ac/go.mod h1:X4rz1639L8vWKJgpLxpO3ddkIW7KaCaQjbwani7FPf4=
-github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df h1:7yQ7m0N3AQNqGnymRP+Fka6FZCEYr9GzkTbVkUcxwtQ=
-github.com/jfrog/jfrog-client-go v1.28.1-0.20240204105421-dd3f7041f3df/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU=
 github.com/jszwec/csvutil v1.9.0 h1:iTmq9G1P0e+AUq/MkFg6tetJ+1BH3fOX8Xi0RAcwiGc=
 github.com/jszwec/csvutil v1.9.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
@@ -285,6 +281,10 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
+github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0 h1:D6BdBwvSA+auq4yV6ABFCdQUccg8Sns3luMv5GYuTGc=
+github.com/yahavi/jfrog-cli-core/v2 v2.0.0-20240213073338-398e494844b0/go.mod h1:NUjgJRs1hC8DhdoTxahbYdNh56ga56HnEC2YNW6TZpU=
+github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057 h1:ok2hthLuAAzx9UCvhLCtNnp8JZfaU7ozk3vNqg4cTqE=
+github.com/yahavi/jfrog-client-go v0.1.2-0.20240213070734-494a414e3057/go.mod h1:y1WF6eiZ7V2DortiwjpMEicEH6NIJH+hOXI5QI2W3NU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

utils/cliutils/cli_consts.go

@@ -24,6 +24,11 @@ const (
 	DownloadSplitCount    = 3
 	DownloadMaxSplitCount = 15
 
+	// Upload
+	UploadMinSplitMb    = 200
+	UploadSplitCount    = 5
+	UploadMaxSplitCount = 100
+
 	// Common
 	Retries                       = 3
 	RetryWaitMilliSecs            = 0

utils/cliutils/commandsflags.go

@@ -206,6 +206,8 @@ const (
 	fromRt                  = "from-rt"
 	transitive              = "transitive"
 	Status                  = "status"
+	MinSplit                = "min-split"
+	SplitCount              = "split-count"
 
 	// Config flags
 	interactive   = "interactive"
@@ -223,6 +225,8 @@ const (
 	uploadTargetProps = uploadPrefix + targetProps
 	uploadSyncDeletes = uploadPrefix + syncDeletes
 	uploadArchive     = uploadPrefix + archive
+	uploadMinSplit    = uploadPrefix + MinSplit
+	uploadSplitCount  = uploadPrefix + SplitCount
 	deb               = "deb"
 	symlinks          = "symlinks"
 	uploadAnt         = uploadPrefix + antFlag
@@ -235,8 +239,8 @@ const (
 	downloadProps        = downloadPrefix + props
 	downloadExcludeProps = downloadPrefix + excludeProps
 	downloadSyncDeletes  = downloadPrefix + syncDeletes
-	minSplit             = "min-split"
-	splitCount           = "split-count"
+	downloadMinSplit     = downloadPrefix + MinSplit
+	downloadSplitCount   = downloadPrefix + SplitCount
 	validateSymlinks     = "validate-symlinks"
 	skipChecksum         = "skip-checksum"
 
@@ -804,6 +808,14 @@ var flagsMap = map[string]cli.Flag{
 		Name:  archive,
 		Usage: "[Optional] Set to \"zip\" to deploy the files to Artifactory in a ZIP archive.` `",
 	},
+	uploadMinSplit: cli.StringFlag{
+		Name:  MinSplit,
+		Usage: "[Default: " + strconv.Itoa(UploadMinSplitMb) + "] Minimum file size in MB to split into ranges when uploading. Require Artifactory with S3 storage.` `",
+	},
+	uploadSplitCount: cli.StringFlag{
+		Name:  SplitCount,
+		Usage: "[Default: " + strconv.Itoa(UploadSplitCount) + "] Number of parts to split a file when uploading. Set to 0 for no splits.` `",
+	},
 	syncDeletesQuiet: cli.BoolFlag{
 		Name:  quiet,
 		Usage: "[Default: $CI] Set to true to skip the sync-deletes confirmation message.` `",
@@ -816,17 +828,17 @@ var flagsMap = map[string]cli.Flag{
 		Name:  flat,
 		Usage: "[Default: false] Set to true if you do not wish to have the Artifactory repository path structure created locally for your downloaded files.` `",
 	},
-	minSplit: cli.StringFlag{
-		Name:  minSplit,
+	downloadMinSplit: cli.StringFlag{
+		Name:  MinSplit,
 		Value: "",
 		Usage: "[Default: " + strconv.Itoa(DownloadMinSplitKb) + "] Minimum file size in KB to split into ranges when downloading. Set to -1 for no splits.` `",
 	},
 	skipChecksum: cli.BoolFlag{
 		Name:  skipChecksum,
 		Usage: "[Default: false] Set to true to skip checksum verification when downloading.` `",
 	},
-	splitCount: cli.StringFlag{
-		Name:  splitCount,
+	downloadSplitCount: cli.StringFlag{
+		Name:  SplitCount,
 		Value: "",
 		Usage: "[Default: " + strconv.Itoa(DownloadSplitCount) + "] Number of parts to split a file when downloading. Set to 0 for no splits.` `",
 	},
@@ -1737,12 +1749,12 @@ var commandFlags = map[string][]string{
 		ClientCertKeyPath, specFlag, specVars, buildName, buildNumber, module, uploadExclusions, deb,
 		uploadRecursive, uploadFlat, uploadRegexp, retries, retryWaitTime, dryRun, uploadExplode, symlinks, includeDirs,
 		failNoOp, threads, uploadSyncDeletes, syncDeletesQuiet, InsecureTls, detailedSummary, Project,
-		uploadAnt, uploadArchive,
+		uploadAnt, uploadArchive, uploadMinSplit, uploadSplitCount,
 	},
 	Download: {
 		url, user, password, accessToken, sshPassphrase, sshKeyPath, serverId, ClientCertPath,
 		ClientCertKeyPath, specFlag, specVars, buildName, buildNumber, module, exclusions, sortBy,
-		sortOrder, limit, offset, downloadRecursive, downloadFlat, build, includeDeps, excludeArtifacts, minSplit, splitCount,
+		sortOrder, limit, offset, downloadRecursive, downloadFlat, build, includeDeps, excludeArtifacts, downloadMinSplit, downloadSplitCount,
 		retries, retryWaitTime, dryRun, downloadExplode, bypassArchiveInspection, validateSymlinks, bundle, publicGpgKey, includeDirs,
 		downloadProps, downloadExcludeProps, failNoOp, threads, archiveEntries, downloadSyncDeletes, syncDeletesQuiet, InsecureTls, detailedSummary, Project,
 		skipChecksum,