From 3cea121c58c7ea95f27283fef2ca9373502d738b Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Thu, 22 Feb 2024 15:53:41 +0200 Subject: [PATCH 01/15] Artifactory Multi-Part Upload support (#900) --- .github/workflows/tests.yml | 4 +- README.md | 24 +- artifactory/manager.go | 2 + artifactory/services/download.go | 16 +- artifactory/services/fspatterns/utils.go | 8 +- artifactory/services/upload.go | 129 ++++-- artifactory/services/utils/multipartupload.go | 436 ++++++++++++++++++ .../services/utils/multipartupload_test.go | 367 +++++++++++++++ artifactory/services/utils/storageutils.go | 7 + go.mod | 2 +- go.sum | 4 +- http/httpclient/client.go | 5 +- tests/artifactorymultipartupload_test.go | 89 ++++ tests/jfrogclient_test.go | 2 +- tests/utils_test.go | 9 +- utils/io/fileutils/files.go | 4 +- utils/io/progress.go | 6 +- utils/retryexecutor.go | 2 +- 18 files changed, 1028 insertions(+), 88 deletions(-) create mode 100644 artifactory/services/utils/multipartupload.go create mode 100644 artifactory/services/utils/multipartupload_test.go create mode 100644 tests/artifactorymultipartupload_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d2f95187e..dc9fb5f2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,13 +83,13 @@ jobs: - name: artifactory tests run: go test -v github.com/jfrog/jfrog-client-go/tests --timeout 0 --test.${{ matrix.suite }} --ci.runId=${{ runner.os }}-${{ matrix.suite }} - JFrog-Client-Go-Ds-Xr-Tests: + JFrog-Client-Go-Ds-Xr-MPU-Tests: needs: Pretest name: ${{ matrix.suite }} ${{ matrix.os }} strategy: fail-fast: false matrix: - suite: [ distribution, xray ] + suite: [ distribution, xray, mpu ] os: [ ubuntu, windows, macos ] runs-on: ${{ matrix.os }}-latest steps: diff --git a/README.md b/README.md index 4e761e7e3..feac593cc 100644 --- a/README.md +++ b/README.md @@ -254,14 +254,15 @@ content of this repository is deleted. #### Test Types -| Type | Description | Prerequisites | -| -------------------- | ------------------ | ----------------------------- | -| `-test.artifactory` | Artifactory tests | Artifactory Pro | -| `-test.distribution` | Distribution tests | Artifactory with Distribution | -| `-test.xray` | Xray tests | Artifactory with Xray | -| `-test.pipelines` | Pipelines tests | JFrog Pipelines | -| `-test.access` | Access tests | Artifactory Pro | -| `-test.repositories` | Access tests | Artifactory Pro | +| Type | Description | Prerequisites | +| -------------------- | ---------------------- | ------------------------------- | +| `-test.artifactory` | Artifactory tests | Artifactory Pro | +| `-test.distribution` | Distribution tests | Artifactory with Distribution | +| `-test.xray` | Xray tests | Artifactory with Xray | +| `-test.pipelines` | Pipelines tests | JFrog Pipelines | +| `-test.access` | Access tests | Artifactory Pro | +| `-test.repositories` | Repositories tests | Artifactory Pro | +| `-test.mpu` | Multipart upload tests | Artifactory Pro with S3 storage | #### Connection Details @@ -400,6 +401,12 @@ params.Symlink = false params.Exclusions = "(.*)a.zip" // Retries default value: 3 params.Retries = 5 +// The maximum number of parts that can be concurrently uploaded per file during a multi-part upload. Set to 0 to disable multi-part upload. +// SplitCount default value: 5 +params.SplitCount = 10 +// The minimum file size in MiB required to attempt a multi-part upload. +// MinSplitSize default value: 200 +params.MinSplitSize = 100 // The min file size in bytes for "checksum deploy". // "Checksum deploy" is the action of calculating the file checksum locally, before // the upload, and skipping the actual file transfer if the file already @@ -2230,7 +2237,6 @@ xscVersion, err := scanService.IsXscEnabled() multiScanId, err := scanService.SendScanGitInfoContext(details) ``` - ## Pipelines APIs ### Creating Pipelines Service Manager diff --git a/artifactory/manager.go b/artifactory/manager.go index 8827abd28..9d9e627c3 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -321,6 +321,8 @@ func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadSer uploadService.ArtDetails = sm.config.GetServiceDetails() uploadService.DryRun = sm.config.IsDryRun() uploadService.Progress = sm.progress + httpClientDetails := uploadService.ArtDetails.CreateHttpClientDetails() + uploadService.MultipartUpload = utils.NewMultipartUpload(sm.client, &httpClientDetails, uploadService.ArtDetails.GetUrl()) return uploadService } diff --git a/artifactory/services/download.go b/artifactory/services/download.go index 8445e76ec..61bf3242c 100644 --- a/artifactory/services/download.go +++ b/artifactory/services/download.go @@ -440,19 +440,11 @@ func createLocalSymlink(localPath, localFileName, symlinkArtifact string, symlin if !fileutils.IsPathExists(symlinkArtifact, false) { return errorutils.CheckErrorf("symlink validation failed, target doesn't exist: " + symlinkArtifact) } - file, err := os.Open(symlinkArtifact) - if err = errorutils.CheckError(err); err != nil { - return err - } - defer func() { - err = errors.Join(err, errorutils.CheckError(file.Close())) - }() - checksumInfo, err := biutils.CalcChecksums(file, biutils.SHA1) - if err = errorutils.CheckError(err); err != nil { - return err + var checksums map[biutils.Algorithm]string + if checksums, err = biutils.GetFileChecksums(symlinkArtifact, biutils.SHA1); err != nil { + return errorutils.CheckError(err) } - sha1 := checksumInfo[biutils.SHA1] - if sha1 != symlinkContentChecksum { + if checksums[biutils.SHA1] != symlinkContentChecksum { return errorutils.CheckErrorf("symlink validation failed for target: " + symlinkArtifact) } } diff --git a/artifactory/services/fspatterns/utils.go b/artifactory/services/fspatterns/utils.go index 5cfc3d578..01427d4d0 100644 --- a/artifactory/services/fspatterns/utils.go +++ b/artifactory/services/fspatterns/utils.go @@ -178,15 +178,15 @@ func GetRootPath(pattern, target, archiveTarget string, patternType utils.Patter // When handling symlink we want to simulate the creation of empty file func CreateSymlinkFileDetails() (*fileutils.FileDetails, error) { - checksumInfo, err := biutils.CalcChecksums(bytes.NewBuffer([]byte(fileutils.SymlinkFileContent))) + checksums, err := biutils.CalcChecksums(bytes.NewBuffer([]byte(fileutils.SymlinkFileContent))) if err != nil { return nil, errorutils.CheckError(err) } details := new(fileutils.FileDetails) - details.Checksum.Md5 = checksumInfo[biutils.MD5] - details.Checksum.Sha1 = checksumInfo[biutils.SHA1] - details.Checksum.Sha256 = checksumInfo[biutils.SHA256] + details.Checksum.Md5 = checksums[biutils.MD5] + details.Checksum.Sha1 = checksums[biutils.SHA1] + details.Checksum.Sha256 = checksums[biutils.SHA256] details.Size = int64(0) return details, nil } diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 6ac6a81c7..0fbd9a077 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -30,14 +30,24 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) +const ( + // 10 KiB + DefaultMinChecksumDeploy = utils.SizeKib * 10 + // The default minimum file size for attempting multi-part upload + defaultUploadMinSplit = utils.SizeMiB * 200 + // The default maximum number of parts that can be concurrently uploaded per file during a multi-part upload + defaultUploadSplitCount = 5 +) + type UploadService struct { - client *jfroghttpclient.JfrogHttpClient - Progress ioutils.ProgressMgr - ArtDetails auth.ServiceDetails - DryRun bool - Threads int - saveSummary bool - resultsManager *resultsManager + client *jfroghttpclient.JfrogHttpClient + Progress ioutils.ProgressMgr + ArtDetails auth.ServiceDetails + MultipartUpload *utils.MultipartUpload + DryRun bool + Threads int + saveSummary bool + resultsManager *resultsManager } const JfrogCliUploadEmptyArchiveEnv = "JFROG_CLI_UPLOAD_EMPTY_ARCHIVE" @@ -179,19 +189,12 @@ func createProperties(artifact clientutils.Artifact, uploadParams UploadParams) } // If Symlink target exists -> get SHA1 if isn't a directory } else if !fileInfo.IsDir() { - file, err := os.Open(artifact.LocalPath) - if err != nil { - return nil, errorutils.CheckError(err) - } - defer func() { - err = errors.Join(err, errorutils.CheckError(file.Close())) - }() - checksumInfo, err := biutils.CalcChecksums(file, biutils.SHA1) + var checksums map[biutils.Algorithm]string + checksums, err := biutils.GetFileChecksums(artifact.LocalPath, biutils.SHA1) if err != nil { return nil, errorutils.CheckError(err) } - sha1 := checksumInfo[biutils.SHA1] - artifactProps.AddProperty(utils.SymlinkSha1, sha1) + artifactProps.AddProperty(utils.SymlinkSha1, checksums[biutils.SHA1]) } artifactProps.AddProperty(utils.ArtifactorySymlink, artifactSymlink) } @@ -486,7 +489,7 @@ func (us *UploadService) uploadFile(artifact UploadData, uploadParams UploadPara if uploadParams.IsSymlink() && fileutils.IsFileSymlink(fileInfo) { resp, details, body, err = us.uploadSymlink(targetPathWithProps, logMsgPrefix, httpClientsDetails, uploadParams) } else { - resp, details, body, checksumDeployed, err = us.doUpload(artifact.Artifact.LocalPath, targetPathWithProps, logMsgPrefix, httpClientsDetails, fileInfo, uploadParams) + resp, details, body, checksumDeployed, err = us.doUpload(artifact, targetPathWithProps, logMsgPrefix, httpClientsDetails, fileInfo, uploadParams) } if err != nil { return nil, false, err @@ -503,6 +506,21 @@ func (us *UploadService) shouldTryChecksumDeploy(fileSize int64, uploadParams Up return uploadParams.ChecksumsCalcEnabled && fileSize >= uploadParams.MinChecksumDeploy && !uploadParams.IsExplodeArchive() } +func (us *UploadService) shouldDoMultipartUpload(fileSize int64, uploadParams UploadParams) (bool, error) { + if uploadParams.SplitCount == 0 || fileSize < uploadParams.MinSplitSize { + return false, nil + } + if fileSize > utils.MaxMultipartUploadFileSize { + log.Debug(fmt.Sprintf("Max file size for multipart upload exceeded: %d>%d", fileSize, utils.MaxMultipartUploadFileSize)) + return false, nil + } + if uploadParams.IsExplodeArchive() { + // Explode archives is not supported in multipart uploads + return false, nil + } + return us.MultipartUpload.IsSupported(us.ArtDetails) +} + // Reads a file from a Reader that is given from a function (getReaderFunc) and uploads it to the specified target path. // getReaderFunc is called only if checksum deploy was successful. // Returns true if the file was successfully uploaded. @@ -514,7 +532,7 @@ func (us *UploadService) uploadFileFromReader(getReaderFunc func() (io.Reader, e httpClientsDetails := us.ArtDetails.CreateHttpClientDetails() if !us.DryRun { if us.shouldTryChecksumDeploy(details.Size, uploadParams) { - resp, body, e = us.tryChecksumDeploy(details, targetUrlWithProps, httpClientsDetails, us.client) + resp, body, e = us.doChecksumDeploy(details, targetUrlWithProps, httpClientsDetails, us.client) if e != nil { return false, e } @@ -570,37 +588,50 @@ func (us *UploadService) uploadSymlink(targetPath, logMsgPrefix string, httpClie return } -func (us *UploadService) doUpload(localPath, targetUrlWithProps, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails, fileInfo os.FileInfo, uploadParams UploadParams) (*http.Response, *fileutils.FileDetails, []byte, bool, error) { - var details *fileutils.FileDetails - var checksumDeployed bool - var resp *http.Response - var body []byte - var err error - addExplodeHeader(&httpClientsDetails, uploadParams.IsExplodeArchive()) - if !us.DryRun { - if us.shouldTryChecksumDeploy(fileInfo.Size(), uploadParams) { - details, err = fileutils.GetFileDetails(localPath, uploadParams.ChecksumsCalcEnabled) - if err != nil { - return resp, details, body, checksumDeployed, err - } - resp, body, err = us.tryChecksumDeploy(details, targetUrlWithProps, httpClientsDetails, us.client) - if err != nil { - return resp, details, body, checksumDeployed, err - } - checksumDeployed = isSuccessfulUploadStatusCode(resp.StatusCode) +func (us *UploadService) doUpload(artifact UploadData, targetUrlWithProps, logMsgPrefix string, httpClientsDetails httputils.HttpClientDetails, fileInfo os.FileInfo, uploadParams UploadParams) ( + resp *http.Response, details *fileutils.FileDetails, body []byte, checksumDeployed bool, err error) { + // Get local file details + details, err = fileutils.GetFileDetails(artifact.Artifact.LocalPath, uploadParams.ChecksumsCalcEnabled) + if err != nil { + return + } + + // Return if dry run + if us.DryRun { + return + } + + // Try checksum deploy + if us.shouldTryChecksumDeploy(fileInfo.Size(), uploadParams) { + resp, body, err = us.doChecksumDeploy(details, targetUrlWithProps, httpClientsDetails, us.client) + if err != nil { + return resp, details, body, checksumDeployed, err } - if !checksumDeployed { - resp, body, err = utils.UploadFile(localPath, targetUrlWithProps, logMsgPrefix, &us.ArtDetails, details, - httpClientsDetails, us.client, uploadParams.ChecksumsCalcEnabled, us.Progress) - if err != nil { - return resp, details, body, checksumDeployed, err - } + if isSuccessfulUploadStatusCode(resp.StatusCode) { + checksumDeployed = true + return } } - if details == nil { - details, err = fileutils.GetFileDetails(localPath, uploadParams.ChecksumsCalcEnabled) + + // Try multipart upload + var shouldTryMultipart bool + if shouldTryMultipart, err = us.shouldDoMultipartUpload(fileInfo.Size(), uploadParams); err != nil { + return + } + if shouldTryMultipart { + if err = us.MultipartUpload.UploadFileConcurrently(artifact.Artifact.LocalPath, artifact.Artifact.TargetPath, fileInfo.Size(), details.Checksum.Sha1, us.Progress, uploadParams.SplitCount); err != nil { + return + } + // Once the file is uploaded to the storage, we finalize the multipart upload by performing a checksum deployment to save the file in Artifactory. + resp, body, err = us.doChecksumDeploy(details, targetUrlWithProps, httpClientsDetails, us.client) + return } - return resp, details, body, checksumDeployed, err + + // Do regular upload + addExplodeHeader(&httpClientsDetails, uploadParams.IsExplodeArchive()) + resp, body, err = utils.UploadFile(artifact.Artifact.LocalPath, targetUrlWithProps, logMsgPrefix, &us.ArtDetails, details, + httpClientsDetails, us.client, uploadParams.ChecksumsCalcEnabled, us.Progress) + return } func (us *UploadService) doUploadFromReader(fileReader io.Reader, targetUrlWithProps string, httpClientsDetails httputils.HttpClientDetails, uploadParams UploadParams, details *fileutils.FileDetails) (*http.Response, *fileutils.FileDetails, []byte, error) { @@ -643,7 +674,7 @@ func addExplodeHeader(httpClientsDetails *httputils.HttpClientDetails, isExplode } } -func (us *UploadService) tryChecksumDeploy(details *fileutils.FileDetails, targetPath string, httpClientsDetails httputils.HttpClientDetails, +func (us *UploadService) doChecksumDeploy(details *fileutils.FileDetails, targetPath string, httpClientsDetails httputils.HttpClientDetails, client *jfroghttpclient.JfrogHttpClient) (resp *http.Response, body []byte, err error) { requestClientDetails := httpClientsDetails.Clone() utils.AddHeader("X-Checksum-Deploy", "true", &requestClientDetails.Headers) @@ -676,6 +707,8 @@ type UploadParams struct { Flat bool AddVcsProps bool MinChecksumDeploy int64 + MinSplitSize int64 + SplitCount int ChecksumsCalcEnabled bool Archive string // When using the 'archive' option for upload, we can control the target path inside the uploaded archive using placeholders. This operation determines the TargetPathInArchive value. @@ -683,7 +716,7 @@ type UploadParams struct { } func NewUploadParams() UploadParams { - return UploadParams{CommonParams: &utils.CommonParams{}, MinChecksumDeploy: 10240, ChecksumsCalcEnabled: true} + return UploadParams{CommonParams: &utils.CommonParams{}, MinChecksumDeploy: DefaultMinChecksumDeploy, ChecksumsCalcEnabled: true, MinSplitSize: defaultUploadMinSplit, SplitCount: defaultUploadSplitCount} } func DeepCopyUploadParams(params *UploadParams) UploadParams { diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go new file mode 100644 index 000000000..5ac06b3e3 --- /dev/null +++ b/artifactory/services/utils/multipartupload.go @@ -0,0 +1,436 @@ +package utils + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + biUtils "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/parallel" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + ioutils "github.com/jfrog/jfrog-client-go/utils/io" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" +) + +type supportedStatus int +type completionStatus string + +const ( + // TODO - Update version + minArtifactoryVersion = "8.0.0" + + // Supported status + // Multipart upload support is not yet determined + undetermined supportedStatus = iota + // Multipart upload is supported + multipartSupported + // Multipart upload is not supported + multipartNotSupported + + // Completion status + parts completionStatus = "PARTS" + queued completionStatus = "QUEUED" + processing completionStatus = "PROCESSING" + finished completionStatus = "FINISHED" + retryableError completionStatus = "RETRYABLE_ERROR" + nonRetryableError completionStatus = "NON_RETRYABLE_ERROR" + aborted completionStatus = "ABORTED" + + // API constants + uploadsApi = "/api/v1/uploads/" + artifactoryNodeIdHeader = "X-Artifactory-Node-Id" + + // Sizes and limits constants + MaxMultipartUploadFileSize = SizeTiB * 5 + uploadPartSize int64 = SizeMiB * 20 + + // Retries and polling constants + retriesInterval = time.Second * 5 + // A week of retries + maxPollingRetries = time.Hour * 168 / retriesInterval + mergingLoggingInterval = time.Minute +) + +var ( + errTooManyAttempts = errors.New("too many upload attempts failed") + supportedMutex sync.Mutex +) + +type MultipartUpload struct { + client *jfroghttpclient.JfrogHttpClient + httpClientsDetails *httputils.HttpClientDetails + artifactoryUrl string + supportedStatus supportedStatus +} + +func NewMultipartUpload(client *jfroghttpclient.JfrogHttpClient, httpClientsDetails *httputils.HttpClientDetails, artifactoryUrl string) *MultipartUpload { + return &MultipartUpload{client, httpClientsDetails, strings.TrimSuffix(artifactoryUrl, "/"), undetermined} +} + +func (mu *MultipartUpload) IsSupported(serviceDetails auth.ServiceDetails) (supported bool, err error) { + supportedMutex.Lock() + defer supportedMutex.Unlock() + if mu.supportedStatus != undetermined { + // If the supported status was determined earlier, return true if multipart upload is supported or false if not + return mu.supportedStatus == multipartSupported, nil + } + + artifactoryVersion, err := serviceDetails.GetVersion() + if err != nil { + return + } + + if versionErr := utils.ValidateMinimumVersion(utils.Artifactory, artifactoryVersion, minArtifactoryVersion); versionErr != nil { + log.Debug("Multipart upload is not supported in versions below " + minArtifactoryVersion + ". Proceeding with regular upload...") + mu.supportedStatus = multipartNotSupported + return + } + + url := fmt.Sprintf("%s%sconfig", mu.artifactoryUrl, uploadsApi) + resp, body, _, err := mu.client.SendGet(url, true, mu.httpClientsDetails) + if err != nil { + return + } + log.Debug("Artifactory response:", string(body), resp.Status) + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + + var getConfigResponse getConfigResponse + err = errorutils.CheckError(json.Unmarshal(body, &getConfigResponse)) + if getConfigResponse.Supported { + mu.supportedStatus = multipartSupported + } else { + mu.supportedStatus = multipartNotSupported + } + return getConfigResponse.Supported, err +} + +type getConfigResponse struct { + Supported bool `json:"supported,omitempty"` +} + +func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, fileSize int64, sha1 string, progress ioutils.ProgressMgr, splitCount int) (err error) { + repoAndPath := strings.SplitN(targetPath, "/", 2) + repoKey := repoAndPath[0] + repoPath := repoAndPath[1] + logMsgPrefix := fmt.Sprintf("[Multipart upload %s] ", repoPath) + + token, err := mu.createMultipartUpload(repoKey, repoPath, calculatePartSize(fileSize, 0)) + if err != nil { + return + } + + multipartUploadClient := &httputils.HttpClientDetails{ + AccessToken: token, + Transport: mu.httpClientsDetails.Transport, + DialTimeout: mu.httpClientsDetails.DialTimeout, + OverallRequestTimeout: mu.httpClientsDetails.OverallRequestTimeout, + } + + var progressReader ioutils.Progress + if progress != nil { + progressReader = progress.NewProgressReader(fileSize, "Multipart upload", targetPath) + defer progress.RemoveProgress(progressReader.GetId()) + } + + defer func() { + if err == nil { + log.Info(logMsgPrefix + "Upload completed successfully!") + } else { + err = errors.Join(err, mu.abort(logMsgPrefix, multipartUploadClient)) + } + }() + + if err = mu.uploadPartsConcurrently(logMsgPrefix, fileSize, splitCount, localPath, progressReader, multipartUploadClient); err != nil { + return + } + + if sha1 == "" { + var checksums map[biUtils.Algorithm]string + if checksums, err = biUtils.GetFileChecksums(localPath); errorutils.CheckError(err) != nil { + return + } + sha1 = checksums[biUtils.SHA1] + } + + if progress != nil { + progressReader = progress.SetMergingState(progressReader.GetId(), false) + } + + log.Info(logMsgPrefix + "Starting parts merge...") + // The total number of attempts is determined by the number of retries + 1 + return mu.completeAndPollForStatus(logMsgPrefix, uint(mu.client.GetHttpClient().GetRetries())+1, sha1, multipartUploadClient, progressReader) +} + +func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize int64, splitCount int, localPath string, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { + numberOfParts := calculateNumberOfParts(fileSize) + log.Info(fmt.Sprintf("%sSplitting file to %d parts, using %d working threads for uploading...", logMsgPrefix, numberOfParts, splitCount)) + producerConsumer := parallel.NewRunner(splitCount, uint(numberOfParts), false) + + wg := new(sync.WaitGroup) + wg.Add(int(numberOfParts)) + attemptsAllowed := new(atomic.Uint64) + attemptsAllowed.Add(uint64(numberOfParts) * uint64(mu.client.GetHttpClient().GetRetries())) + go func() { + for i := 0; i < int(numberOfParts); i++ { + if err = mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, int64(i), progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { + return + } + } + }() + go func() { + defer producerConsumer.Done() + wg.Wait() + }() + producerConsumer.Run() + if attemptsAllowed.Load() == 0 { + return errorutils.CheckError(errTooManyAttempts) + } + return +} + +func (mu *MultipartUpload) produceUploadTask(producerConsumer parallel.Runner, logMsgPrefix, localPath string, fileSize, numberOfParts, partId int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails, attemptsAllowed *atomic.Uint64, wg *sync.WaitGroup) (retErr error) { + _, retErr = producerConsumer.AddTaskWithError(func(int) error { + uploadErr := mu.uploadPart(logMsgPrefix, localPath, fileSize, partId, progressReader, multipartUploadClient) + if uploadErr == nil { + log.Info(fmt.Sprintf("%sCompleted uploading part %d/%d", logMsgPrefix, partId+1, numberOfParts)) + wg.Done() + } + return uploadErr + }, func(uploadErr error) { + if attemptsAllowed.Load() == 0 { + wg.Done() + return + } + log.Warn(fmt.Sprintf("%sPart %d/%d - %s", logMsgPrefix, partId+1, numberOfParts, uploadErr.Error())) + attemptsAllowed.Add(^uint64(0)) + + // Sleep before trying again + time.Sleep(retriesInterval) + if err := mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, partId, progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { + retErr = err + } + }) + return +} + +func (mu *MultipartUpload) uploadPart(logMsgPrefix, localPath string, fileSize, partId int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { + file, err := os.Open(localPath) + if err != nil { + return errorutils.CheckError(err) + } + defer func() { + err = errors.Join(err, errorutils.CheckError(file.Close())) + }() + if _, err = file.Seek(partId*uploadPartSize, io.SeekStart); err != nil { + return errorutils.CheckError(err) + } + partSize := calculatePartSize(fileSize, partId) + + limitReader := io.LimitReader(file, partSize) + limitReader = bufio.NewReader(limitReader) + if progressReader != nil { + limitReader = progressReader.ActionWithProgress(limitReader) + } + + urlPart, err := mu.generateUrlPart(logMsgPrefix, partId, multipartUploadClient) + if err != nil { + return + } + + resp, body, err := mu.client.GetHttpClient().UploadFileFromReader(limitReader, urlPart, httputils.HttpClientDetails{}, partSize) + if err != nil { + return + } + log.Debug("Artifactory response:", string(body), resp.Status) + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) +} + +func (mu *MultipartUpload) createMultipartUpload(repoKey, repoPath string, partSize int64) (token string, err error) { + url := fmt.Sprintf("%s%screate?repoKey=%s&repoPath=%s&partSizeMB=%d", mu.artifactoryUrl, uploadsApi, repoKey, repoPath, partSize/SizeMiB) + resp, body, err := mu.client.SendPost(url, []byte{}, mu.httpClientsDetails) + if err != nil { + return + } + // We don't log the response body because it includes credentials + + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + + var createMultipartUploadResponse createMultipartUploadResponse + err = json.Unmarshal(body, &createMultipartUploadResponse) + return createMultipartUploadResponse.Token, err +} + +type createMultipartUploadResponse struct { + Token string `json:"token,omitempty"` +} + +func (mu *MultipartUpload) generateUrlPart(logMsgPrefix string, partNumber int64, multipartUploadClient *httputils.HttpClientDetails) (partUrl string, err error) { + url := fmt.Sprintf("%s%surlPart?partNumber=%d", mu.artifactoryUrl, uploadsApi, partNumber+1) + resp, body, err := mu.client.GetHttpClient().SendPost(url, []byte{}, *multipartUploadClient, logMsgPrefix) + if err != nil { + return "", err + } + // We don't log the response body because it includes credentials + + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + var urlPartResponse urlPartResponse + err = json.Unmarshal(body, &urlPartResponse) + return urlPartResponse.Url, errorutils.CheckError(err) +} + +type urlPartResponse struct { + Url string `json:"url,omitempty"` +} + +func (mu *MultipartUpload) completeAndPollForStatus(logMsgPrefix string, completionAttemptsLeft uint, sha1 string, multipartUploadClient *httputils.HttpClientDetails, progressReader ioutils.Progress) (err error) { + nodeId, err := mu.completeMultipartUpload(logMsgPrefix, sha1, multipartUploadClient) + if err != nil { + return + } + + err = mu.pollCompletionStatus(logMsgPrefix, completionAttemptsLeft, sha1, nodeId, multipartUploadClient, progressReader) + return +} + +func (mu *MultipartUpload) pollCompletionStatus(logMsgPrefix string, completionAttemptsLeft uint, sha1, nodeId string, multipartUploadClient *httputils.HttpClientDetails, progressReader ioutils.Progress) error { + multipartUploadClientWithNodeId := multipartUploadClient.Clone() + multipartUploadClientWithNodeId.Headers = map[string]string{artifactoryNodeIdHeader: nodeId} + + lastMergeLog := time.Now() + pollingExecutor := &utils.RetryExecutor{ + MaxRetries: int(maxPollingRetries), + RetriesIntervalMilliSecs: int(retriesInterval.Milliseconds()), + LogMsgPrefix: logMsgPrefix, + ExecutionHandler: func() (shouldRetry bool, err error) { + // Get completion status + status, err := mu.status(logMsgPrefix, multipartUploadClientWithNodeId) + if err != nil { + return false, err + } + + // Parse status + shouldRetry, shouldRerunComplete, err := parseMultipartUploadStatus(status) + if err != nil { + return false, err + } + + // Rerun complete if needed + if shouldRerunComplete { + if completionAttemptsLeft == 0 { + return false, errorutils.CheckErrorf("multipart upload failed after %d attempts", mu.client.GetHttpClient().GetRetries()) + } + err = mu.completeAndPollForStatus(logMsgPrefix, completionAttemptsLeft-1, sha1, multipartUploadClient, progressReader) + } + + // Log status + if status.Progress != nil { + if progressReader != nil { + progressReader.SetProgress(int64(*status.Progress)) + } + if time.Since(lastMergeLog) > mergingLoggingInterval { + log.Info(fmt.Sprintf("%sMerging progress: %d%%", logMsgPrefix, *status.Progress)) + lastMergeLog = time.Now() + } + } + return + }, + } + return pollingExecutor.Execute() +} + +func (mu *MultipartUpload) completeMultipartUpload(logMsgPrefix, sha1 string, multipartUploadClient *httputils.HttpClientDetails) (string, error) { + url := fmt.Sprintf("%s%scomplete?sha1=%s", mu.artifactoryUrl, uploadsApi, sha1) + resp, body, err := mu.client.GetHttpClient().SendPost(url, []byte{}, *multipartUploadClient, logMsgPrefix) + if err != nil { + return "", err + } + log.Debug("Artifactory response:", string(body), resp.Status) + return resp.Header.Get(artifactoryNodeIdHeader), errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) +} + +func (mu *MultipartUpload) status(logMsgPrefix string, multipartUploadClientWithNodeId *httputils.HttpClientDetails) (status statusResponse, err error) { + url := fmt.Sprintf("%s%sstatus", mu.artifactoryUrl, uploadsApi) + resp, body, err := mu.client.GetHttpClient().SendPost(url, []byte{}, *multipartUploadClientWithNodeId, logMsgPrefix) + if err != nil { + return + } + log.Debug("Artifactory response:", string(body), resp.Status) + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + err = errorutils.CheckError(json.Unmarshal(body, &status)) + return +} + +type statusResponse struct { + Status completionStatus `json:"status,omitempty"` + Error string `json:"error,omitempty"` + Progress *int `json:"progress,omitempty"` +} + +func (mu *MultipartUpload) abort(logMsgPrefix string, multipartUploadClient *httputils.HttpClientDetails) (err error) { + log.Info("Aborting multipart upload...") + url := fmt.Sprintf("%s%sabort", mu.artifactoryUrl, uploadsApi) + resp, body, err := mu.client.GetHttpClient().SendPost(url, []byte{}, *multipartUploadClient, logMsgPrefix) + if err != nil { + return + } + log.Debug("Artifactory response:", string(body), resp.Status) + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusNoContent) +} + +// Calculates the part size based on the file size and the part number. +// fileSize - the file size +// partNumber - the current part number +func calculatePartSize(fileSize int64, partNumber int64) int64 { + partOffset := partNumber * uploadPartSize + if partOffset+uploadPartSize > fileSize { + return fileSize - partOffset + } + return uploadPartSize +} + +// Calculates the number of parts based on the file size and the default part size. +// fileSize - the file size +func calculateNumberOfParts(fileSize int64) int64 { + return (fileSize + uploadPartSize - 1) / uploadPartSize +} + +func parseMultipartUploadStatus(status statusResponse) (shouldKeepPolling, shouldRerunComplete bool, err error) { + switch status.Status { + case queued, processing: + // File merging had not yet been completed - keep polling + return true, false, nil + case retryableError: + // Retryable error was received - stop polling and rerun the /complete API again + log.Warn("received error upon multipart upload completion process: '%s', retrying...", status.Error) + return false, true, nil + case finished, aborted: + // Upload finished or aborted + return false, false, nil + case nonRetryableError: + // Fatal error occurred - stop the entire process + return false, false, errorutils.CheckErrorf("received non retryable error upon multipart upload completion process: '%s'", status.Error) + default: + // Unexpected status - stop the entire process + return false, false, errorutils.CheckErrorf("received unexpected status upon multipart upload completion process: '%s', error: '%s'", status.Status, status.Error) + } +} diff --git a/artifactory/services/utils/multipartupload_test.go b/artifactory/services/utils/multipartupload_test.go new file mode 100644 index 000000000..76c45b3b7 --- /dev/null +++ b/artifactory/services/utils/multipartupload_test.go @@ -0,0 +1,367 @@ +package utils + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/jfrog/jfrog-client-go/utils/tests" + "github.com/stretchr/testify/assert" + "golang.org/x/exp/rand" +) + +const ( + localPath = "localPath" + repoKey = "repoKey" + repoPath = "repoPath" + partSize = SizeGiB + partSizeMB = 1024 + partNumber = 2 + splitCount = 3 + token = "token" + partUrl = "http://dummy-url-part" + sha1 = "sha1" + nodeId = "nodeId" +) + +func TestIsSupported(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodGet, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/config", r.URL.Path) + + // Send response 200 OK + w.WriteHeader(http.StatusOK) + response, err := json.Marshal(getConfigResponse{Supported: true}) + assert.NoError(t, err) + _, err = w.Write(response) + assert.NoError(t, err) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Create Artifactory service details + rtDetails := &dummyArtifactoryServiceDetails{version: minArtifactoryVersion} + + // Execute IsSupported + supported, err := multipartUpload.IsSupported(rtDetails) + assert.NoError(t, err) + assert.True(t, supported) +} + +func TestUnsupportedVersion(t *testing.T) { + // Create Artifactory service details with unsupported Artifactory version + rtDetails := &dummyArtifactoryServiceDetails{version: "6.0.0"} + + // Create mock multipart upload with server + client, err := jfroghttpclient.JfrogClientBuilder().Build() + assert.NoError(t, err) + + // Execute IsSupported + supported, err := NewMultipartUpload(client, nil, "").IsSupported(rtDetails) + assert.NoError(t, err) + assert.False(t, supported) +} + +func TestCreateMultipartUpload(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/create", r.URL.Path) + assert.Equal(t, fmt.Sprintf("repoKey=%s&repoPath=%s&partSizeMB=%d", repoKey, repoPath, partSizeMB), r.URL.RawQuery) + + // Send response 200 OK + w.WriteHeader(http.StatusOK) + response, err := json.Marshal(createMultipartUploadResponse{Token: token}) + assert.NoError(t, err) + _, err = w.Write(response) + assert.NoError(t, err) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute CreateMultipartUpload + actualToken, err := multipartUpload.createMultipartUpload(repoKey, repoPath, partSize) + assert.NoError(t, err) + assert.Equal(t, token, actualToken) +} + +func TestUploadPartsConcurrentlyTooManyAttempts(t *testing.T) { + // Create temp file + tempFile, cleanUp := createTempFile(t) + defer cleanUp() + + // Write something to the file + buf := make([]byte, uploadPartSize*3) + _, err := rand.Read(buf) + assert.NoError(t, err) + _, err = tempFile.Write(buf) + assert.NoError(t, err) + + var multipartUpload *MultipartUpload + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + // Generate part URL for upload + case http.MethodPost: + // Check URL + assert.Equal(t, "/api/v1/uploads/urlPart", r.URL.Path) + + // Send response 200 OK + w.WriteHeader(http.StatusOK) + response, unmarshalErr := json.Marshal(urlPartResponse{Url: multipartUpload.artifactoryUrl}) + assert.NoError(t, unmarshalErr) + _, err = w.Write(response) + assert.NoError(t, err) + // Fail the upload to trigger retry + case http.MethodPut: + assert.Equal(t, "/", r.URL.Path) + + // Send response 502 OK + w.WriteHeader(http.StatusBadGateway) + default: + assert.Fail(t, "unexpected method "+r.Method) + } + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp = createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute uploadPartsConcurrently + fileSize := int64(len(buf)) + err = multipartUpload.uploadPartsConcurrently("", fileSize, splitCount, tempFile.Name(), nil, &httputils.HttpClientDetails{}) + assert.ErrorIs(t, err, errTooManyAttempts) +} + +func TestGenerateUrlPart(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/urlPart", r.URL.Path) + assert.Equal(t, fmt.Sprintf("partNumber=%d", partNumber+1), r.URL.RawQuery) + + // Send response 200 OK + w.WriteHeader(http.StatusOK) + response, err := json.Marshal(urlPartResponse{Url: partUrl}) + assert.NoError(t, err) + _, err = w.Write(response) + assert.NoError(t, err) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute GenerateUrlPart + actualPartUrl, err := multipartUpload.generateUrlPart("", partNumber, &httputils.HttpClientDetails{}) + assert.NoError(t, err) + assert.Equal(t, partUrl, actualPartUrl) +} + +func TestCompleteMultipartUpload(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/complete", r.URL.Path) + assert.Equal(t, fmt.Sprintf("sha1=%s", sha1), r.URL.RawQuery) + + // Add the "X-Artifactory-Node-Id" header to the response + w.Header().Add(artifactoryNodeIdHeader, nodeId) + + // Send response 202 Accepted + w.WriteHeader(http.StatusAccepted) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute completeMultipartUpload + actualNodeId, err := multipartUpload.completeMultipartUpload("", sha1, &httputils.HttpClientDetails{}) + assert.NoError(t, err) + assert.Equal(t, nodeId, actualNodeId) +} + +func TestStatus(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/status", r.URL.Path) + + // Check "X-Artifactory-Node-Id" header + assert.Equal(t, nodeId, r.Header.Get(artifactoryNodeIdHeader)) + + // Send response 200 OK + w.WriteHeader(http.StatusOK) + response, err := json.Marshal(statusResponse{Status: finished, Progress: utils.Pointer(100)}) + assert.NoError(t, err) + _, err = w.Write(response) + assert.NoError(t, err) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute status + clientDetails := &httputils.HttpClientDetails{Headers: map[string]string{artifactoryNodeIdHeader: nodeId}} + status, err := multipartUpload.status("", clientDetails) + assert.NoError(t, err) + assert.Equal(t, statusResponse{Status: finished, Progress: utils.Pointer(100)}, status) +} + +func TestAbort(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/abort", r.URL.Path) + + // Send response 204 No Content + w.WriteHeader(http.StatusNoContent) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute status + clientDetails := &httputils.HttpClientDetails{} + err := multipartUpload.abort("", clientDetails) + assert.NoError(t, err) +} + +var calculatePartSizeProvider = []struct { + fileSize int64 + partNumber int64 + expectedPartSize int64 +}{ + {uploadPartSize - 1, 0, uploadPartSize - 1}, + {uploadPartSize, 0, uploadPartSize}, + {uploadPartSize + 1, 0, uploadPartSize}, + + {uploadPartSize*2 - 1, 1, uploadPartSize - 1}, + {uploadPartSize * 2, 1, uploadPartSize}, + {uploadPartSize*2 + 1, 1, uploadPartSize}, +} + +func TestCalculatePartSize(t *testing.T) { + for _, testCase := range calculatePartSizeProvider { + t.Run(fmt.Sprintf("fileSize: %d partNumber: %d", testCase.fileSize, testCase.partNumber), func(t *testing.T) { + assert.Equal(t, testCase.expectedPartSize, calculatePartSize(testCase.fileSize, testCase.partNumber)) + }) + } +} + +var calculateNumberOfPartsProvider = []struct { + fileSize int64 + expectedNumberOfParts int64 +}{ + {0, 0}, + {1, 1}, + {uploadPartSize - 1, 1}, + {uploadPartSize, 1}, + {uploadPartSize + 1, 2}, + + {uploadPartSize*2 - 1, 2}, + {uploadPartSize * 2, 2}, + {uploadPartSize*2 + 1, 3}, +} + +func TestCalculateNumberOfParts(t *testing.T) { + for _, testCase := range calculateNumberOfPartsProvider { + t.Run(fmt.Sprintf("fileSize: %d", testCase.fileSize), func(t *testing.T) { + assert.Equal(t, testCase.expectedNumberOfParts, calculateNumberOfParts(testCase.fileSize)) + }) + } +} + +var parseMultipartUploadStatusProvider = []struct { + status completionStatus + shouldKeepPolling bool + shouldRerunComplete bool + expectedError string +}{ + {queued, true, false, ""}, + {processing, true, false, ""}, + {parts, false, false, "received unexpected status upon multipart upload completion process: 'PARTS', error: 'Some error'"}, + {finished, false, false, ""}, + {aborted, false, false, ""}, + {retryableError, false, true, ""}, + {nonRetryableError, false, false, "received non retryable error upon multipart upload completion process: 'Some error'"}, +} + +func TestParseMultipartUploadStatus(t *testing.T) { + previousLog := tests.RedirectLogOutputToNil() + defer func() { + log.SetLogger(previousLog) + }() + + for _, testCase := range parseMultipartUploadStatusProvider { + t.Run(string(testCase.status), func(t *testing.T) { + + shouldKeepPolling, shouldRerunComplete, err := parseMultipartUploadStatus(statusResponse{Status: testCase.status, Error: "Some error"}) + if testCase.expectedError != "" { + assert.EqualError(t, err, testCase.expectedError) + return + } + assert.NoError(t, err) + assert.Equal(t, testCase.shouldKeepPolling, shouldKeepPolling) + assert.Equal(t, testCase.shouldRerunComplete, shouldRerunComplete) + }) + } +} + +func createTempFile(t *testing.T) (tempFile *os.File, cleanUp func()) { + // Create a temporary file + tempFile, err := fileutils.CreateTempFile() + assert.NoError(t, err) + cleanUp = func() { + assert.NoError(t, tempFile.Close()) + assert.NoError(t, fileutils.RemovePath(localPath)) + } + return +} + +func createMockMultipartUpload(t *testing.T, handler http.Handler) (multipartUpload *MultipartUpload, cleanUp func()) { + ts := httptest.NewServer(handler) + cleanUp = ts.Close + + client, err := jfroghttpclient.JfrogClientBuilder().Build() + assert.NoError(t, err) + multipartUpload = NewMultipartUpload(client, &httputils.HttpClientDetails{}, ts.URL) + return +} + +type dummyArtifactoryServiceDetails struct { + auth.CommonConfigFields + version string +} + +func (dasd *dummyArtifactoryServiceDetails) GetVersion() (string, error) { + return dasd.version, nil +} diff --git a/artifactory/services/utils/storageutils.go b/artifactory/services/utils/storageutils.go index bb5f2504d..6b87946ce 100644 --- a/artifactory/services/utils/storageutils.go +++ b/artifactory/services/utils/storageutils.go @@ -5,6 +5,13 @@ import ( "errors" ) +const ( + SizeKib int64 = 1 << 10 + SizeMiB int64 = 1 << 20 + SizeGiB int64 = 1 << 30 + SizeTiB int64 = 1 << 40 +) + type FolderInfo struct { Uri string `json:"uri,omitempty"` Repo string `json:"repo,omitempty"` diff --git a/go.mod b/go.mod index cdf60162e..706f9d012 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231220102935-c8776c613ad8 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev diff --git a/go.sum b/go.sum index d1a5a201e..234252eba 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= -github.com/jfrog/build-info-go v1.9.23 h1:+TwUIBEJwRvz9skR8xBfY5ti8Vl4Z6iMCkFbkclnEN0= -github.com/jfrog/build-info-go v1.9.23/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= +github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e h1:NzB2yvEojIhP5KIX9SeCqSljZmoiE98hBzXYvvi52D0= +github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8= github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/http/httpclient/client.go b/http/httpclient/client.go index 7c70a4532..658bd7f6e 100644 --- a/http/httpclient/client.go +++ b/http/httpclient/client.go @@ -286,6 +286,7 @@ func (jc *HttpClient) doUploadFile(localPath, url string, httpClientsDetails htt } else { reader = reqContent } + resp, body, err = jc.UploadFileFromReader(reader, url, httpClientsDetails, size) return } @@ -481,7 +482,7 @@ func (jc *HttpClient) DownloadFileConcurrently(flags ConcurrentDownloadFlags, lo var downloadProgressId int if progress != nil { - downloadProgress := progress.NewProgressReader(flags.FileSize, "Downloading", flags.RelativePath) + downloadProgress := progress.NewProgressReader(flags.FileSize, "Multipart download", flags.RelativePath) downloadProgressId = downloadProgress.GetId() // Aborting order matters. mergingProgress depends on the existence of downloadingProgress defer progress.RemoveProgress(downloadProgressId) @@ -511,7 +512,7 @@ func (jc *HttpClient) DownloadFileConcurrently(flags ConcurrentDownloadFlags, lo } } if progress != nil { - progress.SetProgressState(downloadProgressId, "Merging") + progress.SetMergingState(downloadProgressId, true) } err = mergeChunks(chunksPaths, flags) if errorutils.CheckError(err) != nil { diff --git a/tests/artifactorymultipartupload_test.go b/tests/artifactorymultipartupload_test.go new file mode 100644 index 000000000..78a4fe11a --- /dev/null +++ b/tests/artifactorymultipartupload_test.go @@ -0,0 +1,89 @@ +package tests + +import ( + "os" + "path/filepath" + "testing" + + "github.com/jfrog/jfrog-client-go/artifactory/services" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/stretchr/testify/assert" +) + +const ( + bigFileSize = 100 << 20 + propertyKey = "prop-key" + propertyValue = "prop-value" +) + +func initArtifactoryMultipartUploadTest(t *testing.T) { + if !*TestMultipartUpload { + t.Skip("Skipping multipart upload test. To run artifactory test add the '-test.mpu=true' option.") + } + + supported, err := testsUploadService.MultipartUpload.IsSupported(testsUploadService.ArtDetails) + assert.NoError(t, err) + if !supported { + t.Skip("Skipping multipart upload test. Multipart upload test is not supported in the provided Artifactory server.") + } +} + +func TestArtifactoryMultipartUpload(t *testing.T) { + initArtifactoryMultipartUploadTest(t) + t.Run("multipartUpload", multipartUpload) +} + +func multipartUpload(t *testing.T) { + bigFile, cleanup := createBigFile(t) + defer cleanup() + + // Create upload parameters + up := services.NewUploadParams() + props := utils.NewProperties() + props.AddProperty(propertyKey, propertyValue) + up.CommonParams = &utils.CommonParams{Pattern: bigFile.Name(), Target: getRtTargetRepo(), TargetProps: props} + up.Flat = true + up.MinChecksumDeploy = bigFileSize + 1 + up.MinSplitSize = bigFileSize + + // Upload file and verify success + summary, err := testsUploadService.UploadFiles(up) + assert.NoError(t, err) + assert.Equal(t, 1, summary.TotalSucceeded) + assert.Zero(t, summary.TotalFailed) + + // Search for the uploaded file in Artifactory + searchParams := services.NewSearchParams() + searchParams.Pattern = getRtTargetRepo() + reader, err := testsSearchService.Search(searchParams) + defer readerCloseAndAssert(t, reader) + assert.NoError(t, err) + length, err := reader.Length() + assert.NoError(t, err) + assert.Equal(t, 1, length) + + // Ensure existence of the uploaded file and verify properties + for item := new(utils.ResultItem); reader.NextRecord(item) == nil; item = new(utils.ResultItem) { + assert.Equal(t, filepath.Base(bigFile.Name()), item.Name) + assert.Equal(t, propertyValue, item.GetProperty(propertyKey)) + } + readerGetErrorAndAssert(t, reader) + + // Cleanup + artifactoryCleanup(t) +} + +func createBigFile(t *testing.T) (bigFile *os.File, cleanUp func()) { + bigFile, err := fileutils.CreateTempFile() + assert.NoError(t, err) + + cleanUp = func() { + assert.NoError(t, os.Remove(bigFile.Name())) + } + + data := make([]byte, int(bigFileSize)) + _, err = bigFile.Write(data) + assert.NoError(t, err) + return +} diff --git a/tests/jfrogclient_test.go b/tests/jfrogclient_test.go index cdba5cc94..ceda8bd8a 100644 --- a/tests/jfrogclient_test.go +++ b/tests/jfrogclient_test.go @@ -28,7 +28,7 @@ func TestMain(m *testing.M) { func setupIntegrationTests() { flag.Parse() log.SetLogger(log.NewLogger(log.DEBUG, nil)) - if *TestArtifactory || *TestDistribution || *TestXray || *TestRepositories { + if *TestArtifactory || *TestDistribution || *TestXray || *TestRepositories || *TestMultipartUpload { createArtifactoryUploadManager() createArtifactorySearchManager() createArtifactoryDeleteManager() diff --git a/tests/utils_test.go b/tests/utils_test.go index 6f38ea735..7a0ff6829 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -52,6 +52,7 @@ var ( TestPipelines *bool TestAccess *bool TestRepositories *bool + TestMultipartUpload *bool RtUrl *string DistUrl *string XrayUrl *string @@ -147,6 +148,7 @@ func init() { TestPipelines = flag.Bool("test.pipelines", false, "Test pipelines") TestAccess = flag.Bool("test.access", false, "Test access") TestRepositories = flag.Bool("test.repositories", false, "Test repositories in Artifactory") + TestMultipartUpload = flag.Bool("test.mpu", false, "Test Artifactory multipart upload") RtUrl = flag.String("rt.url", "http://localhost:8081/artifactory", "Artifactory url") DistUrl = flag.String("ds.url", "", "Distribution url") XrayUrl = flag.String("xr.url", "", "Xray url") @@ -216,6 +218,8 @@ func createArtifactoryUploadManager() { testsUploadService = services.NewUploadService(client) testsUploadService.ArtDetails = artDetails testsUploadService.Threads = 3 + httpClientDetails := testsUploadService.ArtDetails.CreateHttpClientDetails() + testsUploadService.MultipartUpload = utils.NewMultipartUpload(client, &httpClientDetails, testsUploadService.ArtDetails.GetUrl()) } func createArtifactoryUserManager() { @@ -427,6 +431,7 @@ func createArtifactoryAqlManager() { func createJfrogHttpClient(artDetailsPtr *auth.ServiceDetails) (*jfroghttpclient.JfrogHttpClient, error) { artDetails := *artDetailsPtr return jfroghttpclient.JfrogClientBuilder(). + SetRetries(3). SetClientCertPath(artDetails.GetClientCertPath()). SetClientCertKeyPath(artDetails.GetClientCertKeyPath()). AppendPreRequestInterceptor(artDetails.RunPreRequestFunctions). @@ -667,7 +672,7 @@ func artifactoryCleanup(t *testing.T) { } func createRepo() error { - if !(*TestArtifactory || *TestDistribution || *TestXray || *TestRepositories) { + if !(*TestArtifactory || *TestDistribution || *TestXray || *TestRepositories || *TestMultipartUpload) { return nil } var err error @@ -684,7 +689,7 @@ func createRepo() error { } func teardownIntegrationTests() { - if !(*TestArtifactory || *TestDistribution || *TestXray || *TestRepositories) { + if !(*TestArtifactory || *TestDistribution || *TestXray || *TestRepositories || *TestMultipartUpload) { return } repo := getRtTargetRepoKey() diff --git a/utils/io/fileutils/files.go b/utils/io/fileutils/files.go index 3b64a4b04..119d5b8d3 100644 --- a/utils/io/fileutils/files.go +++ b/utils/io/fileutils/files.go @@ -404,11 +404,11 @@ func GetFileDetailsFromReader(reader io.Reader, includeChecksums bool) (details } func calcChecksumDetailsFromReader(reader io.Reader) (entities.Checksum, error) { - checksumInfo, err := biutils.CalcChecksums(reader) + checksums, err := biutils.CalcChecksums(reader) if err != nil { return entities.Checksum{}, errorutils.CheckError(err) } - return entities.Checksum{Md5: checksumInfo[biutils.MD5], Sha1: checksumInfo[biutils.SHA1], Sha256: checksumInfo[biutils.SHA256]}, nil + return entities.Checksum{Md5: checksums[biutils.MD5], Sha1: checksums[biutils.SHA1], Sha256: checksums[biutils.SHA256]}, nil } type FileDetails struct { diff --git a/utils/io/progress.go b/utils/io/progress.go index e8335ec68..d67a2dd07 100644 --- a/utils/io/progress.go +++ b/utils/io/progress.go @@ -8,8 +8,8 @@ type ProgressMgr interface { // Input: 'total' - file size, 'label' - the title of the operation, 'path' - the path of the file being processed. // Output: progress indicator id NewProgressReader(total int64, label, path string) (progress Progress) - // Changes progress indicator state. - SetProgressState(id int, state string) + // Changes progress indicator state to merging. + SetMergingState(id int, useSpinner bool) (bar Progress) // Returns the requested progress indicator. GetProgress(id int) (progress Progress) // Aborts a progress indicator. Called on both successful and unsuccessful operations. @@ -32,6 +32,8 @@ type ProgressMgr interface { type Progress interface { // Used for updating the progress indicator progress. ActionWithProgress(reader io.Reader) (results io.Reader) + // Used for setting the progress indicator progress. + SetProgress(progress int64) // Aborts a progress indicator. Called on both successful and unsuccessful operations Abort() // Returns the Progress ID diff --git a/utils/retryexecutor.go b/utils/retryexecutor.go index e3b2faeb9..bad3dc16e 100644 --- a/utils/retryexecutor.go +++ b/utils/retryexecutor.go @@ -10,7 +10,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" ) -type ExecutionHandlerFunc func() (bool, error) +type ExecutionHandlerFunc func() (shouldRetry bool, err error) type RetryExecutor struct { // The context From e55c7d7acbee79659edf8b1e3a8a3928b4ecf09a Mon Sep 17 00:00:00 2001 From: Kishan Bhat Date: Thu, 22 Feb 2024 07:56:38 -0800 Subject: [PATCH 02/15] Artifactory Release Lifecycle Management - Support creation by artifacts and sync for distribution (#903) --- README.md | 38 ++++++++++------- lifecycle/manager.go | 26 ++++++++++-- lifecycle/services/create.go | 23 +++++++++++ lifecycle/services/distribute.go | 33 ++++++++++++--- lifecycle/services/status.go | 71 ++++++++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index feac593cc..432330ae4 100644 --- a/README.md +++ b/README.md @@ -2510,22 +2510,30 @@ resp, err := serviceManager.GetReleaseBundlePromotionStatus(rbDetails, projectKe #### Distribute Release Bundle ```go -rules := &distribution.DistributionCommonParams{ -SiteName: "*", -CityName: "*", -CountryCodes: []string{"*"}, +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} +pathMappings := []services.PathMapping{ + { + Pattern: "(*)/(*)", + Target: "{1}/target/{2}", + }, } -params := distribution.NewDistributeReleaseBundleParams("rbName", "rbVersion") -params.DistributionRules = append(params.DistributionRules, rules) -autoCreateRepo := true - -pathMapping := services.PathMapping{ - Pattern: "(*)/(*)", - Target: "{1}/target/{2}", +rules := &distribution.DistributionCommonParams{ + SiteName: "*", + CityName: "*", + CountryCodes: []string{"*"}, +} +dsParams := DistributeReleaseBundleParams{ + Sync: true, + AutoCreateRepo: true, + MaxWaitMinutes: 60, + PathMappings: pathMappings, + DistributionRules: []*dmUtils.DistributionCommonParams{ + rules, + }, } -resp, err := serviceManager.DistributeReleaseBundle(params, autoCreateRepo, pathMapping) +resp, err := serviceManager.DistributeReleaseBundle(rbDetails, dsParams) ``` #### Delete Release Bundle @@ -2543,9 +2551,9 @@ resp, err := serviceManager.DeleteReleaseBundle(rbDetails, queryParams) ```go rules := &distribution.DistributionCommonParams{ -SiteName: "*", -CityName: "*", -CountryCodes: []string{"*"}, + SiteName: "*", + CityName: "*", + CountryCodes: []string{"*"}, } params := distribution.NewDistributeReleaseBundleParams("rbName", "rbVersion") params.DistributionRules = append(params.DistributionRules, rules) diff --git a/lifecycle/manager.go b/lifecycle/manager.go index f8911bb68..da379c0a0 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -36,6 +36,12 @@ func (lcs *LifecycleServicesManager) Client() *jfroghttpclient.JfrogHttpClient { return lcs.client } +func (lcs *LifecycleServicesManager) CreateReleaseBundleFromArtifacts(rbDetails lifecycle.ReleaseBundleDetails, + queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, sourceArtifacts lifecycle.CreateFromArtifacts) error { + rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) + return rbService.CreateFromArtifacts(rbDetails, queryParams, signingKeyName, sourceArtifacts) +} + func (lcs *LifecycleServicesManager) CreateReleaseBundleFromBuilds(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, sourceBuilds lifecycle.CreateFromBuildsSource) error { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) @@ -68,13 +74,25 @@ func (lcs *LifecycleServicesManager) DeleteReleaseBundle(rbDetails lifecycle.Rel return rbService.DeleteReleaseBundle(rbDetails, queryParams) } -func (lcs *LifecycleServicesManager) DistributeReleaseBundle(params distribution.DistributionParams, autoCreateRepo bool, pathMapping lifecycle.PathMapping) error { +func (lcs *LifecycleServicesManager) DistributeReleaseBundle(rbDetails lifecycle.ReleaseBundleDetails, distributeParams lifecycle.DistributeReleaseBundleParams) error { distributeBundleService := lifecycle.NewDistributeReleaseBundleService(lcs.client) distributeBundleService.LcDetails = lcs.config.GetServiceDetails() distributeBundleService.DryRun = lcs.config.IsDryRun() - distributeBundleService.AutoCreateRepo = autoCreateRepo - distributeBundleService.DistributeParams = params - distributeBundleService.PathMapping = pathMapping + + distributeBundleService.DistributeParams = distribution.DistributionParams{ + Name: rbDetails.ReleaseBundleName, + Version: rbDetails.ReleaseBundleVersion, + DistributionRules: distributeParams.DistributionRules, + } + distributeBundleService.AutoCreateRepo = distributeParams.AutoCreateRepo + distributeBundleService.Sync = distributeParams.Sync + distributeBundleService.MaxWaitMinutes = distributeParams.MaxWaitMinutes + + m := &distributeBundleService.Modifications.PathMappings + for _, pathMapping := range distributeParams.PathMappings { + *m = append(*m, + distribution.CreatePathMappingsFromPatternAndTarget(pathMapping.Pattern, pathMapping.Target)...) + } return distributeBundleService.Distribute() } diff --git a/lifecycle/services/create.go b/lifecycle/services/create.go index a095a5e40..e5e44ba6a 100644 --- a/lifecycle/services/create.go +++ b/lifecycle/services/create.go @@ -7,6 +7,7 @@ const ( type sourceType string const ( + artifacts sourceType = "artifacts" builds sourceType = "builds" releaseBundles sourceType = "release_bundles" ) @@ -37,6 +38,19 @@ func (c *createOperation) getSigningKeyName() string { return c.signingKeyName } +func (rbs *ReleaseBundlesService) CreateFromArtifacts(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, sourceArtifacts CreateFromArtifacts) error { + operation := createOperation{ + reqBody: RbCreationBody{ + ReleaseBundleDetails: rbDetails, + SourceType: artifacts, + Source: sourceArtifacts}, + params: params, + signingKeyName: signingKeyName, + } + _, err := rbs.doOperation(&operation) + return err +} + func (rbs *ReleaseBundlesService) CreateFromBuilds(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, sourceBuilds CreateFromBuildsSource) error { operation := createOperation{ reqBody: RbCreationBody{ @@ -69,10 +83,19 @@ type SourceBuildDetails struct { ProjectKey string } +type CreateFromArtifacts struct { + Artifacts []ArtifactSource `json:"artifacts,omitempty"` +} + type CreateFromBuildsSource struct { Builds []BuildSource `json:"builds,omitempty"` } +type ArtifactSource struct { + Path string `json:"path,omitempty"` + Sha256 string `json:"sha256,omitempty"` +} + type BuildSource struct { BuildName string `json:"build_name,omitempty"` BuildNumber string `json:"build_number,omitempty"` diff --git a/lifecycle/services/distribute.go b/lifecycle/services/distribute.go index 447583a28..f6aa40eb3 100644 --- a/lifecycle/services/distribute.go +++ b/lifecycle/services/distribute.go @@ -18,8 +18,18 @@ type DistributeReleaseBundleService struct { LcDetails auth.ServiceDetails DryRun bool AutoCreateRepo bool + Sync bool + MaxWaitMinutes int DistributeParams distribution.DistributionParams - PathMapping + Modifications +} + +type DistributeReleaseBundleParams struct { + Sync bool + AutoCreateRepo bool + MaxWaitMinutes int + DistributionRules []*distribution.DistributionCommonParams + PathMappings []PathMapping } func (dr *DistributeReleaseBundleService) GetHttpClient() *jfroghttpclient.JfrogHttpClient { @@ -34,6 +44,14 @@ func (dr *DistributeReleaseBundleService) IsDryRun() bool { return dr.DryRun } +func (dr *DistributeReleaseBundleService) IsSync() bool { + return dr.Sync +} + +func (dr *DistributeReleaseBundleService) GetMaxWaitMinutes() int { + return dr.MaxWaitMinutes +} + func (dr *DistributeReleaseBundleService) IsAutoCreateRepo() bool { return dr.AutoCreateRepo } @@ -55,16 +73,19 @@ func NewDistributeReleaseBundleService(client *jfroghttpclient.JfrogHttpClient) } func (dr *DistributeReleaseBundleService) Distribute() error { - _, err := distribution.DoDistribute(dr) - return err + trackerId, err := distribution.DoDistribute(dr) + if err != nil || !dr.IsSync() || dr.IsDryRun() { + return err + } + + // Sync distribution + return dr.waitForDistributionOperationCompletion(&dr.DistributeParams, trackerId) } func (dr *DistributeReleaseBundleService) createDistributeBody() ReleaseBundleDistributeBody { return ReleaseBundleDistributeBody{ ReleaseBundleDistributeV1Body: distribution.CreateDistributeV1Body(dr.DistributeParams, dr.DryRun, dr.AutoCreateRepo), - Modifications: Modifications{ - PathMappings: distribution.CreatePathMappingsFromPatternAndTarget(dr.Pattern, dr.Target), - }, + Modifications: dr.Modifications, } } diff --git a/lifecycle/services/status.go b/lifecycle/services/status.go index 35364a79a..346015563 100644 --- a/lifecycle/services/status.go +++ b/lifecycle/services/status.go @@ -2,17 +2,23 @@ package services import ( "encoding/json" + "errors" "fmt" "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/distribution" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" "net/http" "path" "time" + + dsServices "github.com/jfrog/jfrog-client-go/distribution/services" ) const ( statusesApi = "statuses" + trackersApi = "trackers" defaultMaxWait = 60 * time.Minute DefaultSyncSleepInterval = 10 * time.Second ) @@ -68,6 +74,24 @@ func (rbs *ReleaseBundlesService) getReleaseBundleStatus(restApi string, project return } +func (dbs *DistributeReleaseBundleService) getReleaseBundleDistributionStatus(distributeParams *distribution.DistributionParams, trackerId json.Number) (statusResp *dsServices.DistributionStatusResponse, body []byte, err error) { + restApi := path.Join(distributionBaseApi, trackersApi, distributeParams.Name, distributeParams.Version, trackerId.String()) + requestFullUrl, err := utils.BuildUrl(dbs.LcDetails.GetUrl(), restApi, nil) + if err != nil { + return + } + httpClientsDetails := dbs.LcDetails.CreateHttpClientDetails() + resp, body, _, err := dbs.client.SendGet(requestFullUrl, true, &httpClientsDetails) + if err != nil { + return + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + err = errorutils.CheckError(json.Unmarshal(body, &statusResp)) + return +} + func getStatusResponse(respBody []byte) (ReleaseBundleStatusResponse, error) { var rbStatusResponse ReleaseBundleStatusResponse err := json.Unmarshal(respBody, &rbStatusResponse) @@ -103,6 +127,53 @@ func (rbs *ReleaseBundlesService) waitForRbOperationCompletion(restApi, projectK return getStatusResponse(finalRespBody) } +func (dbs *DistributeReleaseBundleService) waitForDistributionOperationCompletion(distributeParams *distribution.DistributionParams, trackerId json.Number) error { + maxWait := time.Duration(dbs.GetMaxWaitMinutes()) * time.Minute + if maxWait.Minutes() < 1 { + maxWait = defaultMaxWait + } + + pollingAction := func() (shouldStop bool, responseBody []byte, err error) { + statusResponse, responseBody, err := dbs.getReleaseBundleDistributionStatus(distributeParams, trackerId) + if err != nil { + return true, nil, err + } + + switch statusResponse.Status { + case dsServices.NotDistributed, dsServices.InProgress: + return false, nil, nil + case dsServices.Failed, dsServices.Completed: + return true, responseBody, nil + default: + return true, nil, errorutils.CheckErrorf("received unexpected status: '%s'", statusResponse.Status) + } + } + pollingExecutor := &httputils.PollingExecutor{ + Timeout: maxWait, + PollingInterval: SyncSleepInterval, + PollingAction: pollingAction, + MsgPrefix: fmt.Sprintf("Sync: Distributing %s/%s...", distributeParams.Name, distributeParams.Version), + } + finalRespBody, err := pollingExecutor.Execute() + if err != nil { + return err + } + + var dsStatusResponse dsServices.DistributionStatusResponse + if err = json.Unmarshal(finalRespBody, &dsStatusResponse); err != nil { + return errorutils.CheckError(err) + } + + if dsStatusResponse.Status != dsServices.Completed { + for _, st := range dsStatusResponse.Sites { + err = errors.Join(err, fmt.Errorf("target %s name:%s error:%s", st.TargetArtifactory.Type, st.TargetArtifactory.Name, st.Error)) + } + return errorutils.CheckError(err) + } + log.Info("Distribution Completed!") + return nil +} + type ReleaseBundleStatusResponse struct { Status RbStatus `json:"status,omitempty"` Messages []Message `json:"messages,omitempty"` From e5fed3788ecaeb613f9c89b285d6b777c1b2d682 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Sun, 25 Feb 2024 17:07:56 +0200 Subject: [PATCH 03/15] Artifactory Release Lifecycle Management - Avoid sending null path mapping (#912) --- go.mod | 2 +- go.sum | 4 ++-- lifecycle/manager.go | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 706f9d012..b92a9412d 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev diff --git a/go.sum b/go.sum index 234252eba..155abb8a2 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= -github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e h1:NzB2yvEojIhP5KIX9SeCqSljZmoiE98hBzXYvvi52D0= -github.com/jfrog/build-info-go v1.8.9-0.20240222124058-bd9687a8666e/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= +github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c h1:M1QiuCYGCYN1IiGyxogrLzfetYGkkhE2pgDh5K4Wo9A= +github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8= github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/lifecycle/manager.go b/lifecycle/manager.go index da379c0a0..b6437f292 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -1,6 +1,7 @@ package lifecycle import ( + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/config" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" lifecycle "github.com/jfrog/jfrog-client-go/lifecycle/services" @@ -89,6 +90,7 @@ func (lcs *LifecycleServicesManager) DistributeReleaseBundle(rbDetails lifecycle distributeBundleService.MaxWaitMinutes = distributeParams.MaxWaitMinutes m := &distributeBundleService.Modifications.PathMappings + *m = []utils.PathMapping{} for _, pathMapping := range distributeParams.PathMappings { *m = append(*m, distribution.CreatePathMappingsFromPatternAndTarget(pathMapping.Pattern, pathMapping.Target)...) From f768c79b4f4120d2db3cb81aca2d7a4d90ff599e Mon Sep 17 00:00:00 2001 From: Kishan Bhat Date: Sun, 25 Feb 2024 23:41:48 -0800 Subject: [PATCH 04/15] Artifactory Release Lifecycle Management - get lifecycle release bundle specification (#910) --- README.md | 7 +++ distribution/services/getstatus.go | 1 + lifecycle/lifecycle_test.go | 70 ++++++++++++++++++++++++++++++ lifecycle/manager.go | 5 +++ lifecycle/services/status.go | 42 +++++++++++++++++- 5 files changed, 124 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 432330ae4..b0adddcd3 100644 --- a/README.md +++ b/README.md @@ -2507,6 +2507,13 @@ projectKey := "default" resp, err := serviceManager.GetReleaseBundlePromotionStatus(rbDetails, projectKey, createdMillis, sync) ``` +#### Get Release Bundle Specification + +```go +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} +resp, err := serviceManager.GetReleaseBundleSpecification(rbDetails) +``` + #### Distribute Release Bundle ```go diff --git a/distribution/services/getstatus.go b/distribution/services/getstatus.go index aeea27944..91350f55c 100644 --- a/distribution/services/getstatus.go +++ b/distribution/services/getstatus.go @@ -105,6 +105,7 @@ type DistributionStatus string const ( NotDistributed DistributionStatus = "Not distributed" InProgress DistributionStatus = "In progress" + InQueue DistributionStatus = "In queue" Completed DistributionStatus = "Completed" Failed DistributionStatus = "Failed" ) diff --git a/lifecycle/lifecycle_test.go b/lifecycle/lifecycle_test.go index 726aeb733..28b8ea99b 100644 --- a/lifecycle/lifecycle_test.go +++ b/lifecycle/lifecycle_test.go @@ -85,6 +85,76 @@ func testGetRBStatus(t *testing.T, test testCase, handlerFunc http.HandlerFunc) assert.Equal(t, test.finalStatus, statusResp.Status) } +func TestGetReleaseBundleSpecArtifactsOnly(t *testing.T) { + mockServer, rbService := createMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/"+lifecycle.GetReleaseBundleSpecificationRestApi(testRb) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{ + "schema_version": "1.0.0", + "service_id": "jfrt@01h0nvs1pwjtzs15x7kbtv1sve", + "created_by": "admin", + "created": "2023-05-18T11:26:02.912Z", + "created_millis": 1684409162912, + "artifacts": [ + { + "path": "catalina/release-notes-1.0.0.txt", + "checksum": "e06f59f5a976c7f4a5406907790bb8cad6148406282f07cd143fd1de64ca169d", + "source_repository_key": "catalina-dev-generic-local", + "package_type": "generic", + "size": 470, + "properties": [ + { + "key": "build.name", + "values": [ + "Catalina-Build" + ] + } + ] + } + ], + "checked_webhooks": [], + "source": { + "aql": "{source-AQL}", + "builds": [ + { + "build_repository": "artifactory-build-info", + "build_name": "Commons-Build", + "build_number": "1.0.1", + "build_started": "2023-04-05T07:00:00.000+0200", + "include_dependencies": false + } + ], + "release_bundles": [ + { + "project_key": "default", + "repository_key": "release-bundles-v2", + "release_bundle_name": "Commons-Bundle", + "release_bundle_version": "1.0.0" + } + ] + } + }`)) + assert.NoError(t, err) + } + }) + defer mockServer.Close() + + specResp, err := rbService.GetReleaseBundleSpecification(testRb) + assert.NoError(t, err) + assert.Equal(t, "admin", specResp.CreatedBy) + assert.Equal(t, "2023-05-18T11:26:02Z", specResp.Created.Format(time.RFC3339)) + assert.Equal(t, 1684409162912, specResp.CreatedMillis) + + assert.Len(t, specResp.Artifacts, 1) + assert.Equal(t, "catalina/release-notes-1.0.0.txt", specResp.Artifacts[0].Path) + assert.Equal(t, "generic", specResp.Artifacts[0].PackageType) + assert.Equal(t, "catalina-dev-generic-local", specResp.Artifacts[0].SourceRepositoryKey) + assert.Equal(t, 470, specResp.Artifacts[0].Size) + assert.Len(t, specResp.Artifacts[0].Properties, 1) + assert.Equal(t, "build.name", specResp.Artifacts[0].Properties[0].Key) + assert.Equal(t, []string{"Catalina-Build"}, specResp.Artifacts[0].Properties[0].Values) +} + func createMockServer(t *testing.T, testHandler http.HandlerFunc) (*httptest.Server, *lifecycle.ReleaseBundlesService) { testServer := httptest.NewServer(testHandler) diff --git a/lifecycle/manager.go b/lifecycle/manager.go index b6437f292..891e3bf81 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -55,6 +55,11 @@ func (lcs *LifecycleServicesManager) CreateReleaseBundleFromBundles(rbDetails li return rbService.CreateFromBundles(rbDetails, queryParams, signingKeyName, sourceReleaseBundles) } +func (lcs *LifecycleServicesManager) GetReleaseBundleSpecification(rbDetails lifecycle.ReleaseBundleDetails) (lifecycle.ReleaseBundleSpecResponse, error) { + rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) + return rbService.GetReleaseBundleSpecification(rbDetails) +} + func (lcs *LifecycleServicesManager) PromoteReleaseBundle(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, promotionParams lifecycle.RbPromotionParams) (lifecycle.RbPromotionResp, error) { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) return rbService.Promote(rbDetails, queryParams, signingKeyName, promotionParams) diff --git a/lifecycle/services/status.go b/lifecycle/services/status.go index 346015563..98d475706 100644 --- a/lifecycle/services/status.go +++ b/lifecycle/services/status.go @@ -17,6 +17,7 @@ import ( ) const ( + recordsApi = "records" statusesApi = "statuses" trackersApi = "trackers" defaultMaxWait = 60 * time.Minute @@ -44,6 +45,10 @@ func GetReleaseBundleCreationStatusRestApi(rbDetails ReleaseBundleDetails) strin return path.Join(releaseBundleBaseApi, statusesApi, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) } +func GetReleaseBundleSpecificationRestApi(rbDetails ReleaseBundleDetails) string { + return path.Join(releaseBundleBaseApi, recordsApi, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) +} + func (rbs *ReleaseBundlesService) GetReleaseBundlePromotionStatus(rbDetails ReleaseBundleDetails, projectKey, createdMillis string, sync bool) (ReleaseBundleStatusResponse, error) { restApi := path.Join(promotionBaseApi, statusesApi, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion, createdMillis) return rbs.getReleaseBundleOperationStatus(restApi, projectKey, sync, "promotion") @@ -74,6 +79,24 @@ func (rbs *ReleaseBundlesService) getReleaseBundleStatus(restApi string, project return } +func (rbs *ReleaseBundlesService) GetReleaseBundleSpecification(rbDetails ReleaseBundleDetails) (specResp ReleaseBundleSpecResponse, err error) { + restApi := GetReleaseBundleSpecificationRestApi(rbDetails) + requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, nil) + if err != nil { + return + } + httpClientsDetails := rbs.GetLifecycleDetails().CreateHttpClientDetails() + resp, body, _, err := rbs.client.SendGet(requestFullUrl, true, &httpClientsDetails) + if err != nil { + return + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + err = errorutils.CheckError(json.Unmarshal(body, &specResp)) + return +} + func (dbs *DistributeReleaseBundleService) getReleaseBundleDistributionStatus(distributeParams *distribution.DistributionParams, trackerId json.Number) (statusResp *dsServices.DistributionStatusResponse, body []byte, err error) { restApi := path.Join(distributionBaseApi, trackersApi, distributeParams.Name, distributeParams.Version, trackerId.String()) requestFullUrl, err := utils.BuildUrl(dbs.LcDetails.GetUrl(), restApi, nil) @@ -140,7 +163,7 @@ func (dbs *DistributeReleaseBundleService) waitForDistributionOperationCompletio } switch statusResponse.Status { - case dsServices.NotDistributed, dsServices.InProgress: + case dsServices.NotDistributed, dsServices.InProgress, dsServices.InQueue: return false, nil, nil case dsServices.Failed, dsServices.Completed: return true, responseBody, nil @@ -179,6 +202,23 @@ type ReleaseBundleStatusResponse struct { Messages []Message `json:"messages,omitempty"` } +type ReleaseBundleSpecResponse struct { + CreatedBy string `json:"created_by,omitempty"` + Created time.Time `json:"created"` + CreatedMillis int `json:"created_millis,omitempty"` + Artifacts []struct { + Path string `json:"path,omitempty"` + Checksum string `json:"checksum,omitempty"` + SourceRepositoryKey string `json:"source_repository_key,omitempty"` + PackageType string `json:"package_type,omitempty"` + Size int `json:"size,omitempty"` + Properties []struct { + Key string `json:"key"` + Values []string `json:"values"` + } `json:"properties,omitempty"` + } `json:"artifacts,omitempty"` +} + type Message struct { Source string `json:"source,omitempty"` Text string `json:"text,omitempty"` From 3414cc0ffcb6d587fa6d8959a2f00b67a71da2f9 Mon Sep 17 00:00:00 2001 From: davidlesfrog <160411306+davidlesfrog@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:12:57 +0200 Subject: [PATCH 05/15] Add file info rest API for artifactory (#914) --- artifactory/emptymanager.go | 5 ++++ artifactory/manager.go | 5 ++++ artifactory/services/storage.go | 31 +++++++++++++++++----- artifactory/services/utils/storageutils.go | 20 ++++++++++++++ tests/artifactorystorage_test.go | 26 ++++++++++++++---- 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/artifactory/emptymanager.go b/artifactory/emptymanager.go index 79ffa8a1e..1e9387150 100644 --- a/artifactory/emptymanager.go +++ b/artifactory/emptymanager.go @@ -101,6 +101,7 @@ type ArtifactoryServicesManager interface { TriggerFederatedRepositoryFullSyncMirror(repoKey string, mirrorUrl string) error Export(params services.ExportParams) error FolderInfo(relativePath string) (*utils.FolderInfo, error) + FileInfo(relativePath string) (*utils.FileInfo, error) FileList(relativePath string, optionalParams utils.FileListParams) (*utils.FileListResponse, error) GetStorageInfo() (*utils.StorageInfo, error) CalculateStorageInfo() error @@ -448,6 +449,10 @@ func (esm *EmptyArtifactoryServicesManager) FolderInfo(string) (*utils.FolderInf panic("Failed: Method is not implemented") } +func (esm *EmptyArtifactoryServicesManager) FileInfo(string) (*utils.FileInfo, error) { + panic("Failed: Method is not implemented") +} + func (esm *EmptyArtifactoryServicesManager) FileList(string, utils.FileListParams) (*utils.FileListResponse, error) { panic("Failed: Method is not implemented") } diff --git a/artifactory/manager.go b/artifactory/manager.go index 9d9e627c3..2d19c9871 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -586,6 +586,11 @@ func (sm *ArtifactoryServicesManagerImp) FolderInfo(relativePath string) (*utils return storageService.FolderInfo(relativePath) } +func (sm *ArtifactoryServicesManagerImp) FileInfo(relativePath string) (*utils.FileInfo, error) { + storageService := services.NewStorageService(sm.config.GetServiceDetails(), sm.client) + return storageService.FileInfo(relativePath) +} + func (sm *ArtifactoryServicesManagerImp) FileList(relativePath string, optionalParams utils.FileListParams) (*utils.FileListResponse, error) { storageService := services.NewStorageService(sm.config.GetServiceDetails(), sm.client) return storageService.FileList(relativePath, optionalParams) diff --git a/artifactory/services/storage.go b/artifactory/services/storage.go index 51c4453c8..f5a9dceda 100644 --- a/artifactory/services/storage.go +++ b/artifactory/services/storage.go @@ -33,16 +33,38 @@ func (s *StorageService) GetJfrogHttpClient() *jfroghttpclient.JfrogHttpClient { return s.client } +func (s *StorageService) FileInfo(relativePath string) (*utils.FileInfo, error) { + body, err := s.getPathInfo(relativePath) + if err != nil { + return nil, err + } + + result := &utils.FileInfo{} + err = json.Unmarshal(body, result) + return result, errorutils.CheckError(err) +} + func (s *StorageService) FolderInfo(relativePath string) (*utils.FolderInfo, error) { + body, err := s.getPathInfo(relativePath) + if err != nil { + return nil, err + } + + result := &utils.FolderInfo{} + err = json.Unmarshal(body, result) + return result, errorutils.CheckError(err) +} + +func (s *StorageService) getPathInfo(relativePath string) ([]byte, error) { client := s.GetJfrogHttpClient() restAPI := path.Join(StorageRestApi, path.Clean(relativePath)) - folderUrl, err := clientutils.BuildUrl(s.GetArtifactoryDetails().GetUrl(), restAPI, make(map[string]string)) + fullUrl, err := clientutils.BuildUrl(s.GetArtifactoryDetails().GetUrl(), restAPI, make(map[string]string)) if err != nil { return nil, err } httpClientsDetails := s.GetArtifactoryDetails().CreateHttpClientDetails() - resp, body, _, err := client.SendGet(folderUrl, true, &httpClientsDetails) + resp, body, _, err := client.SendGet(fullUrl, true, &httpClientsDetails) if err != nil { return nil, err } @@ -50,10 +72,7 @@ func (s *StorageService) FolderInfo(relativePath string) (*utils.FolderInfo, err return nil, err } log.Debug("Artifactory response:", resp.Status) - - result := &utils.FolderInfo{} - err = json.Unmarshal(body, result) - return result, errorutils.CheckError(err) + return body, err } func (s *StorageService) FileList(relativePath string, optionalParams utils.FileListParams) (*utils.FileListResponse, error) { diff --git a/artifactory/services/utils/storageutils.go b/artifactory/services/utils/storageutils.go index 6b87946ce..8d29d1b7d 100644 --- a/artifactory/services/utils/storageutils.go +++ b/artifactory/services/utils/storageutils.go @@ -12,6 +12,26 @@ const ( SizeTiB int64 = 1 << 40 ) +type FileInfo struct { + Uri string `json:"uri,omitempty"` + DownloadUri string `json:"downloadUri,omitempty"` + Repo string `json:"repo,omitempty"` + Path string `json:"path,omitempty"` + RemoteUrl string `json:"remoteUrl,omitempty"` + Created string `json:"created,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + LastModified string `json:"lastModified,omitempty"` + ModifiedBy string `json:"modifiedBy,omitempty"` + LastUpdated string `json:"lastUpdated,omitempty"` + Size string `json:"size,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Checksums struct { + Sha1 string `json:"sha1,omitempty"` + Sha256 string `json:"sha256,omitempty"` + Md5 string `json:"md5,omitempty"` + } `json:"checksums,omitempty"` +} + type FolderInfo struct { Uri string `json:"uri,omitempty"` Repo string `json:"repo,omitempty"` diff --git a/tests/artifactorystorage_test.go b/tests/artifactorystorage_test.go index a9fad1d02..dcce6e9f9 100644 --- a/tests/artifactorystorage_test.go +++ b/tests/artifactorystorage_test.go @@ -2,19 +2,21 @@ package tests import ( "errors" - "github.com/jfrog/jfrog-client-go/artifactory/services" - servicesutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" - "github.com/jfrog/jfrog-client-go/utils" - "github.com/stretchr/testify/assert" "path" "strings" "testing" "time" + + "github.com/jfrog/jfrog-client-go/artifactory/services" + servicesutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/stretchr/testify/assert" ) func TestArtifactoryStorage(t *testing.T) { initArtifactoryTest(t) uploadDummyFile(t) + t.Run("folder info test", folderInfoTest) t.Run("file info test", fileInfoTest) t.Run("file list test", fileListTest) t.Run("storage info test", storageInfoTest) @@ -22,7 +24,7 @@ func TestArtifactoryStorage(t *testing.T) { artifactoryCleanup(t) } -func fileInfoTest(t *testing.T) { +func folderInfoTest(t *testing.T) { info, err := testsStorageService.FolderInfo(getRtTargetRepo() + "test/") if !assert.NoError(t, err) { return @@ -39,6 +41,20 @@ func fileInfoTest(t *testing.T) { assert.False(t, info.Children[0].Folder) } +func fileInfoTest(t *testing.T) { + info, err := testsStorageService.FileInfo(getRtTargetRepo() + "test/a.in") + if !assert.NoError(t, err) { + return + } + assert.Equal(t, utils.AddTrailingSlashIfNeeded(*RtUrl)+path.Join(services.StorageRestApi, getRtTargetRepo()+"test/a.in"), info.Uri) + assert.Equal(t, strings.TrimSuffix(getRtTargetRepo(), "/"), info.Repo) + assert.Equal(t, "/test/a.in", info.Path) + assert.NotEmpty(t, info.Created) + assert.NotEmpty(t, info.CreatedBy) + assert.NotEmpty(t, info.LastModified) + assert.NotEmpty(t, info.LastUpdated) +} + func fileListTest(t *testing.T) { params := servicesutils.NewFileListParams() params.Deep = true From e195efc9b269c8066855aed83796fbb65b619d47 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Thu, 7 Mar 2024 13:26:39 +0200 Subject: [PATCH 06/15] Tweak multipart upload polling (#918) --- artifactory/services/utils/multipartupload.go | 13 ++++--- .../services/utils/multipartupload_test.go | 35 ++++++++++++++++--- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go index 5ac06b3e3..436de4997 100644 --- a/artifactory/services/utils/multipartupload.go +++ b/artifactory/services/utils/multipartupload.go @@ -49,8 +49,8 @@ const ( aborted completionStatus = "ABORTED" // API constants - uploadsApi = "/api/v1/uploads/" - artifactoryNodeIdHeader = "X-Artifactory-Node-Id" + uploadsApi = "/api/v1/uploads/" + routeToHeader = "X-JFrog-Route-To" // Sizes and limits constants MaxMultipartUploadFileSize = SizeTiB * 5 @@ -312,7 +312,7 @@ func (mu *MultipartUpload) completeAndPollForStatus(logMsgPrefix string, complet func (mu *MultipartUpload) pollCompletionStatus(logMsgPrefix string, completionAttemptsLeft uint, sha1, nodeId string, multipartUploadClient *httputils.HttpClientDetails, progressReader ioutils.Progress) error { multipartUploadClientWithNodeId := multipartUploadClient.Clone() - multipartUploadClientWithNodeId.Headers = map[string]string{artifactoryNodeIdHeader: nodeId} + multipartUploadClientWithNodeId.Headers = map[string]string{routeToHeader: nodeId} lastMergeLog := time.Now() pollingExecutor := &utils.RetryExecutor{ @@ -363,12 +363,17 @@ func (mu *MultipartUpload) completeMultipartUpload(logMsgPrefix, sha1 string, mu return "", err } log.Debug("Artifactory response:", string(body), resp.Status) - return resp.Header.Get(artifactoryNodeIdHeader), errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) + return resp.Header.Get(routeToHeader), errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) } func (mu *MultipartUpload) status(logMsgPrefix string, multipartUploadClientWithNodeId *httputils.HttpClientDetails) (status statusResponse, err error) { url := fmt.Sprintf("%s%sstatus", mu.artifactoryUrl, uploadsApi) resp, body, err := mu.client.GetHttpClient().SendPost(url, []byte{}, *multipartUploadClientWithNodeId, logMsgPrefix) + // If the Artifactory node returns a "Service unavailable" error (status 503), attempt to retry the upload completion process on a different node. + if resp != nil && resp.StatusCode == http.StatusServiceUnavailable { + unavailableNodeErr := fmt.Sprintf(logMsgPrefix + fmt.Sprintf("The Artifactory node ID '%s' is unavailable.", multipartUploadClientWithNodeId.Headers[routeToHeader])) + return statusResponse{Status: retryableError, Error: unavailableNodeErr}, nil + } if err != nil { return } diff --git a/artifactory/services/utils/multipartupload_test.go b/artifactory/services/utils/multipartupload_test.go index 76c45b3b7..b467ffe6f 100644 --- a/artifactory/services/utils/multipartupload_test.go +++ b/artifactory/services/utils/multipartupload_test.go @@ -186,8 +186,8 @@ func TestCompleteMultipartUpload(t *testing.T) { assert.Equal(t, "/api/v1/uploads/complete", r.URL.Path) assert.Equal(t, fmt.Sprintf("sha1=%s", sha1), r.URL.RawQuery) - // Add the "X-Artifactory-Node-Id" header to the response - w.Header().Add(artifactoryNodeIdHeader, nodeId) + // Add the "X-JFrog-Route-To" header to the response + w.Header().Add(routeToHeader, nodeId) // Send response 202 Accepted w.WriteHeader(http.StatusAccepted) @@ -211,8 +211,8 @@ func TestStatus(t *testing.T) { // Check URL assert.Equal(t, "/api/v1/uploads/status", r.URL.Path) - // Check "X-Artifactory-Node-Id" header - assert.Equal(t, nodeId, r.Header.Get(artifactoryNodeIdHeader)) + // Check "X-JFrog-Route-To" header + assert.Equal(t, nodeId, r.Header.Get(routeToHeader)) // Send response 200 OK w.WriteHeader(http.StatusOK) @@ -227,12 +227,37 @@ func TestStatus(t *testing.T) { defer cleanUp() // Execute status - clientDetails := &httputils.HttpClientDetails{Headers: map[string]string{artifactoryNodeIdHeader: nodeId}} + clientDetails := &httputils.HttpClientDetails{Headers: map[string]string{routeToHeader: nodeId}} status, err := multipartUpload.status("", clientDetails) assert.NoError(t, err) assert.Equal(t, statusResponse{Status: finished, Progress: utils.Pointer(100)}, status) } +func TestStatusServiceUnavailable(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Check method + assert.Equal(t, http.MethodPost, r.Method) + + // Check URL + assert.Equal(t, "/api/v1/uploads/status", r.URL.Path) + + // Send response 503 Service unavailable + w.WriteHeader(http.StatusServiceUnavailable) + _, err := w.Write([]byte("Service unavailable")) + assert.NoError(t, err) + }) + + // Create mock multipart upload with server + multipartUpload, cleanUp := createMockMultipartUpload(t, handler) + defer cleanUp() + + // Execute status + clientDetails := &httputils.HttpClientDetails{Headers: map[string]string{routeToHeader: nodeId}} + status, err := multipartUpload.status("", clientDetails) + assert.NoError(t, err) + assert.Equal(t, statusResponse{Status: retryableError, Error: "The Artifactory node ID 'nodeId' is unavailable."}, status) +} + func TestAbort(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Check method From f35f6dcde9640fd7c46d3dfe34331a4e0d7bda0c Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Mon, 11 Mar 2024 10:23:58 +0200 Subject: [PATCH 07/15] Update gofrog to 1.6.3 (#919) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b92a9412d..7714a4a4e 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.0 github.com/jfrog/build-info-go v1.9.23 - github.com/jfrog/gofrog v1.6.0 + github.com/jfrog/gofrog v1.6.3 github.com/stretchr/testify v1.8.4 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.19.0 diff --git a/go.sum b/go.sum index 155abb8a2..419a1aedd 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c h1:M1QiuCYGCYN1IiGyxogrLzfetYGkkhE2pgDh5K4Wo9A= github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= -github.com/jfrog/gofrog v1.6.0 h1:jOwb37nHY2PnxePNFJ6e6279Pgkr3di05SbQQw47Mq8= -github.com/jfrog/gofrog v1.6.0/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= +github.com/jfrog/gofrog v1.6.3 h1:F7He0+75HcgCe6SGTSHLFCBDxiE2Ja0tekvvcktW6wc= +github.com/jfrog/gofrog v1.6.3/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= From c76da0f1a7c97a83df7791ad4e035678b05ff10f Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Tue, 12 Mar 2024 11:47:36 +0200 Subject: [PATCH 08/15] Multipart upload - Fix empty nodeId response in completeMultipartUpload (#920) --- artifactory/services/utils/multipartupload.go | 9 +++++---- artifactory/services/utils/multipartupload_test.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go index 436de4997..cb6be37bb 100644 --- a/artifactory/services/utils/multipartupload.go +++ b/artifactory/services/utils/multipartupload.go @@ -49,8 +49,9 @@ const ( aborted completionStatus = "ABORTED" // API constants - uploadsApi = "/api/v1/uploads/" - routeToHeader = "X-JFrog-Route-To" + uploadsApi = "/api/v1/uploads/" + routeToHeader = "X-JFrog-Route-To" + artifactoryNodeId = "X-Artifactory-Node-Id" // Sizes and limits constants MaxMultipartUploadFileSize = SizeTiB * 5 @@ -363,7 +364,7 @@ func (mu *MultipartUpload) completeMultipartUpload(logMsgPrefix, sha1 string, mu return "", err } log.Debug("Artifactory response:", string(body), resp.Status) - return resp.Header.Get(routeToHeader), errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) + return resp.Header.Get(artifactoryNodeId), errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) } func (mu *MultipartUpload) status(logMsgPrefix string, multipartUploadClientWithNodeId *httputils.HttpClientDetails) (status statusResponse, err error) { @@ -426,7 +427,7 @@ func parseMultipartUploadStatus(status statusResponse) (shouldKeepPolling, shoul return true, false, nil case retryableError: // Retryable error was received - stop polling and rerun the /complete API again - log.Warn("received error upon multipart upload completion process: '%s', retrying...", status.Error) + log.Warn(fmt.Printf("received error upon multipart upload completion process: '%s', retrying...", status.Error)) return false, true, nil case finished, aborted: // Upload finished or aborted diff --git a/artifactory/services/utils/multipartupload_test.go b/artifactory/services/utils/multipartupload_test.go index b467ffe6f..6f8fd2cc9 100644 --- a/artifactory/services/utils/multipartupload_test.go +++ b/artifactory/services/utils/multipartupload_test.go @@ -186,8 +186,8 @@ func TestCompleteMultipartUpload(t *testing.T) { assert.Equal(t, "/api/v1/uploads/complete", r.URL.Path) assert.Equal(t, fmt.Sprintf("sha1=%s", sha1), r.URL.RawQuery) - // Add the "X-JFrog-Route-To" header to the response - w.Header().Add(routeToHeader, nodeId) + // Add the "X-Artifactory-Node-Id" header to the response + w.Header().Add(artifactoryNodeId, nodeId) // Send response 202 Accepted w.WriteHeader(http.StatusAccepted) From 393125c38b53df7ce20aa8eb113286bb9aec8850 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Wed, 13 Mar 2024 10:31:26 +0200 Subject: [PATCH 09/15] Mock artifactory dry run deletion success (#922) --- artifactory/services/delete.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/artifactory/services/delete.go b/artifactory/services/delete.go index c71357fa2..9ce3011ab 100644 --- a/artifactory/services/delete.go +++ b/artifactory/services/delete.go @@ -106,6 +106,8 @@ func (ds *DeleteService) createFileHandlerFunc(result *utils.Result) fileDeleteH } log.Info(logMsgPrefix+"Deleting", resultItem.GetItemRelativePath()) if ds.DryRun { + // Mock success count on dry run + result.SuccessCount[threadId]++ return nil } httpClientsDetails := ds.GetArtifactoryDetails().CreateHttpClientDetails() From b652b7daef9b12029aa5ac82cae2f6845418403a Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Thu, 14 Mar 2024 13:59:34 +0200 Subject: [PATCH 10/15] Add file info service to README.md (#916) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index b0adddcd3..50e00f06b 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ - [Deleting a Group](#deleting-a-group) - [Generating Full System Export](#generating-full-system-export) - [Getting Info of a Folder in Artifactory](#getting-info-of-a-folder-in-artifactory) + - [Getting Info of a File in Artifactory](#getting-info-of-a-file-in-artifactory) - [Getting a listing of files and folders within a folder in Artifactory](#getting-a-listing-of-files-and-folders-within-a-folder-in-artifactory) - [Getting Storage Summary Info of Artifactory](#getting-storage-summary-info-of-artifactory) - [Triggering Storage Info Recalculation in Artifactory](#triggering-storage-info-recalculation-in-artifactory) @@ -1353,6 +1354,12 @@ err := serviceManager.Export(params) serviceManager.FolderInfo("repo/path/") ``` +#### Getting Info of a File in Artifactory + +```go +serviceManager.FileInfo("repo/path/file") +``` + #### Getting a listing of files and folders within a folder in Artifactory ```go From e419c2a9e72344dc6005994bf27a70561cb941b6 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Sun, 17 Mar 2024 18:06:15 +0200 Subject: [PATCH 11/15] Artifactory Release Lifecycle Management - Support release bundles deletion (#911) --- README.md | 39 ++++++++++++- distribution/services/distribute.go | 2 +- lifecycle/lifecycle_test.go | 76 +++++++++++++++++++++++++- lifecycle/manager.go | 18 ++++-- lifecycle/services/delete.go | 85 +++++++++++++++++++++++++++-- lifecycle/services/distribute.go | 45 ++++++++++++++- lifecycle/services/promote.go | 80 +++++++++++++++++++++++++++ lifecycle/services/status.go | 1 + utils/distribution/distribute.go | 10 ++-- 9 files changed, 334 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 50e00f06b..1d05f858d 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,10 @@ - [Promoting a Release Bundle](#promoting-a-release-bundle) - [Get Release Bundle Creation Status](#get-release-bundle-creation-status) - [Get Release Bundle Promotion Status](#get-release-bundle-promotion-status) + - [Get Release Bundle Promotions](#get-release-bundle-promotions) - [Distribute Release Bundle](#distribute-release-bundle) - - [Delete Release Bundle](#delete-release-bundle) + - [Delete Release Bundle Version](#delete-release-bundle-version) + - [Delete Release Bundle Version Promotion](#delete-release-bundle-version-promotion) - [Remote Delete Release Bundle](#remote-delete-release-bundle) ## General @@ -2514,6 +2516,24 @@ projectKey := "default" resp, err := serviceManager.GetReleaseBundlePromotionStatus(rbDetails, projectKey, createdMillis, sync) ``` +#### Get Release Bundle Promotions + +```go +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} + +optionalQueryParams := lifecycle.GetPromotionsOptionalQueryParams{ + Include: "MSG", + Offset: 1, + Limit: 10, + FilterBy: "DEV", + OrderBy: "created", + OrderAsc: true, + ProjectKey: "default", +} + +resp, err := serviceManager.GetReleaseBundleVersionPromotions(rbDetails, optionalQueryParams) +``` + #### Get Release Bundle Specification ```go @@ -2550,15 +2570,28 @@ dsParams := DistributeReleaseBundleParams{ resp, err := serviceManager.DistributeReleaseBundle(rbDetails, dsParams) ``` -#### Delete Release Bundle +#### Delete Release Bundle Version + +```go +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} +queryParams := CommonOptionalQueryParams{} +queryParams.ProjectKey = "project" +queryParams.Async = true + +resp, err := serviceManager.DeleteReleaseBundleVersion(rbDetails, queryParams) +``` + +#### Delete Release Bundle Version Promotion ```go rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} + queryParams := CommonOptionalQueryParams{} queryParams.ProjectKey = "project" queryParams.Async = true -resp, err := serviceManager.DeleteReleaseBundle(rbDetails, queryParams) +created := "1708612052952" +resp, err := serviceManager.DeleteReleaseBundleVersionPromotion(rbDetails, queryParams, created) ``` #### Remote Delete Release Bundle diff --git a/distribution/services/distribute.go b/distribution/services/distribute.go index db3db25b3..0c33cf6e1 100644 --- a/distribution/services/distribute.go +++ b/distribution/services/distribute.go @@ -50,7 +50,7 @@ func (dr *DistributeReleaseBundleV1Service) GetRestApi(name, version string) str } func (dr *DistributeReleaseBundleV1Service) GetDistributeBody() any { - return distribution.CreateDistributeV1Body(dr.DistributeParams, dr.DryRun, dr.AutoCreateRepo) + return distribution.CreateDistributeV1Body(dr.DistributeParams.DistributionRules, dr.DryRun, dr.AutoCreateRepo) } func (dr *DistributeReleaseBundleV1Service) GetDistributionParams() distribution.DistributionParams { diff --git a/lifecycle/lifecycle_test.go b/lifecycle/lifecycle_test.go index 28b8ea99b..063d490ef 100644 --- a/lifecycle/lifecycle_test.go +++ b/lifecycle/lifecycle_test.go @@ -166,8 +166,8 @@ func createMockServer(t *testing.T, testHandler http.HandlerFunc) (*httptest.Ser return testServer, lifecycle.NewReleaseBundlesService(rtDetails, client) } -func writeMockStatusResponse(t *testing.T, w http.ResponseWriter, statusResp lifecycle.ReleaseBundleStatusResponse) { - content, err := json.Marshal(statusResp) +func writeMockStatusResponse(t *testing.T, w http.ResponseWriter, resp interface{}) { + content, err := json.Marshal(resp) assert.NoError(t, err) _, err = w.Write(content) assert.NoError(t, err) @@ -183,3 +183,75 @@ func createDefaultHandlerFunc(t *testing.T, status lifecycle.RbStatus) (http.Han } }, &requestNum } + +func TestRemoteDeleteReleaseBundle(t *testing.T) { + lifecycle.SyncSleepInterval = 1 * time.Second + defer func() { lifecycle.SyncSleepInterval = lifecycle.DefaultSyncSleepInterval }() + + requestNum := 0 + handlerFunc := func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/" + lifecycle.GetReleaseBundleDistributionsApi(testRb): + w.WriteHeader(http.StatusOK) + var rbStatus lifecycle.RbStatus + switch requestNum { + case 0: + rbStatus = lifecycle.InProgress + case 1: + rbStatus = lifecycle.InProgress + case 2: + rbStatus = lifecycle.Completed + } + requestNum++ + writeMockStatusResponse(t, w, lifecycle.GetDistributionsResponse{{Status: rbStatus}}) + case "/" + lifecycle.GetRemoteDeleteReleaseBundleApi(testRb): + w.WriteHeader(http.StatusAccepted) + } + } + + mockServer, rbService := createMockServer(t, handlerFunc) + defer mockServer.Close() + + assert.NoError(t, rbService.RemoteDeleteReleaseBundle(testRb, lifecycle.ReleaseBundleRemoteDeleteParams{MaxWaitMinutes: 2})) +} + +func TestGetReleaseBundleVersionPromotions(t *testing.T) { + mockServer, rbService := createMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/"+lifecycle.GetGetReleaseBundleVersionPromotionsApi(testRb) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{ + "promotions": [ + { + "status": "COMPLETED", + "repository_key": "release-bundles-v2", + "release_bundle_name": "bundle-test", + "release_bundle_version": "1.2.3", + "environment": "PROD", + "service_id": "jfrt@012345r6315rxa03z99nec1zns", + "created_by": "admin", + "created": "2024-03-14T15:26:46.637Z", + "created_millis": 1710430006637 + } + ] +}`)) + assert.NoError(t, err) + } + }) + defer mockServer.Close() + + resp, err := rbService.GetReleaseBundleVersionPromotions(testRb, lifecycle.GetPromotionsOptionalQueryParams{}) + assert.NoError(t, err) + if !assert.Len(t, resp.Promotions, 1) { + return + } + promotion := resp.Promotions[0] + assert.Equal(t, lifecycle.Completed, promotion.Status) + assert.Equal(t, "release-bundles-v2", promotion.RepositoryKey) + assert.Equal(t, testRb.ReleaseBundleName, promotion.ReleaseBundleName) + assert.Equal(t, testRb.ReleaseBundleVersion, promotion.ReleaseBundleVersion) + assert.Equal(t, "PROD", promotion.Environment) + assert.Equal(t, "jfrt@012345r6315rxa03z99nec1zns", promotion.ServiceId) + assert.Equal(t, "admin", promotion.CreatedBy) + assert.Equal(t, "2024-03-14T15:26:46.637Z", promotion.Created) + assert.Equal(t, "1710430006637", promotion.CreatedMillis.String()) +} diff --git a/lifecycle/manager.go b/lifecycle/manager.go index 891e3bf81..0dc1a8347 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -75,9 +75,19 @@ func (lcs *LifecycleServicesManager) GetReleaseBundlePromotionStatus(rbDetails l return rbService.GetReleaseBundlePromotionStatus(rbDetails, projectKey, createdMillis, sync) } -func (lcs *LifecycleServicesManager) DeleteReleaseBundle(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams) error { +func (lcs *LifecycleServicesManager) GetReleaseBundleVersionPromotions(rbDetails lifecycle.ReleaseBundleDetails, optionalQueryParams lifecycle.GetPromotionsOptionalQueryParams) (lifecycle.RbPromotionsResponse, error) { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) - return rbService.DeleteReleaseBundle(rbDetails, queryParams) + return rbService.GetReleaseBundleVersionPromotions(rbDetails, optionalQueryParams) +} + +func (lcs *LifecycleServicesManager) DeleteReleaseBundleVersion(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams) error { + rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) + return rbService.DeleteReleaseBundleVersion(rbDetails, queryParams) +} + +func (lcs *LifecycleServicesManager) DeleteReleaseBundleVersionPromotion(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams, created string) error { + rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) + return rbService.DeleteReleaseBundleVersionPromotion(rbDetails, queryParams, created) } func (lcs *LifecycleServicesManager) DistributeReleaseBundle(rbDetails lifecycle.ReleaseBundleDetails, distributeParams lifecycle.DistributeReleaseBundleParams) error { @@ -103,7 +113,7 @@ func (lcs *LifecycleServicesManager) DistributeReleaseBundle(rbDetails lifecycle return distributeBundleService.Distribute() } -func (lcs *LifecycleServicesManager) RemoteDeleteReleaseBundle(params distribution.DistributionParams, dryRun bool) error { +func (lcs *LifecycleServicesManager) RemoteDeleteReleaseBundle(rbDetails lifecycle.ReleaseBundleDetails, params lifecycle.ReleaseBundleRemoteDeleteParams) error { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) - return rbService.RemoteDeleteReleaseBundle(params, dryRun) + return rbService.RemoteDeleteReleaseBundle(rbDetails, params) } diff --git a/lifecycle/services/delete.go b/lifecycle/services/delete.go index 9834be00f..02d18e112 100644 --- a/lifecycle/services/delete.go +++ b/lifecycle/services/delete.go @@ -2,23 +2,36 @@ package services import ( "encoding/json" + "fmt" rtUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/distribution" "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" "net/http" "path" "strconv" + "time" ) const ( remoteDeleteEndpoint = "remote_delete" ) -func (rbs *ReleaseBundlesService) DeleteReleaseBundle(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams) error { +func (rbs *ReleaseBundlesService) DeleteReleaseBundleVersion(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams) error { + restApi := path.Join(releaseBundleBaseApi, records, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) + return rbs.deleteReleaseBundle(params, restApi) +} + +func (rbs *ReleaseBundlesService) DeleteReleaseBundleVersionPromotion(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, createdMillis string) error { + restApi := path.Join(promotionBaseApi, records, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion, createdMillis) + return rbs.deleteReleaseBundle(params, restApi) +} + +func (rbs *ReleaseBundlesService) deleteReleaseBundle(params CommonOptionalQueryParams, restApi string) error { queryParams := getProjectQueryParam(params.ProjectKey) queryParams[async] = strconv.FormatBool(params.Async) - restApi := path.Join(releaseBundleBaseApi, records, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, queryParams) if err != nil { return err @@ -28,17 +41,26 @@ func (rbs *ReleaseBundlesService) DeleteReleaseBundle(rbDetails ReleaseBundleDet if err != nil { return err } + if params.Async { + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) + } return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusNoContent) } -func (rbs *ReleaseBundlesService) RemoteDeleteReleaseBundle(params distribution.DistributionParams, dryRun bool) error { - rbBody := distribution.CreateDistributeV1Body(params, dryRun, false) +func (rbs *ReleaseBundlesService) RemoteDeleteReleaseBundle(rbDetails ReleaseBundleDetails, params ReleaseBundleRemoteDeleteParams) error { + dryRunStr := "" + if params.DryRun { + dryRunStr = "[Dry run] " + } + log.Info(dryRunStr + "Remote Deleting: " + rbDetails.ReleaseBundleName + "/" + rbDetails.ReleaseBundleVersion) + + rbBody := distribution.CreateDistributeV1Body(params.DistributionRules, params.DryRun, false) content, err := json.Marshal(rbBody) if err != nil { return errorutils.CheckError(err) } - restApi := path.Join(distributionBaseApi, remoteDeleteEndpoint, params.Name, params.Version) + restApi := GetRemoteDeleteReleaseBundleApi(rbDetails) requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, nil) if err != nil { return err @@ -51,5 +73,56 @@ func (rbs *ReleaseBundlesService) RemoteDeleteReleaseBundle(params distribution. return err } - return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) + log.Debug("Artifactory response:", resp.Status) + err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted) + if err != nil || params.Async || params.DryRun { + return err + } + + return rbs.waitForRemoteDeletion(rbDetails, params) +} + +func GetRemoteDeleteReleaseBundleApi(rbDetails ReleaseBundleDetails) string { + return path.Join(distributionBaseApi, remoteDeleteEndpoint, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) +} + +func (rbs *ReleaseBundlesService) waitForRemoteDeletion(rbDetails ReleaseBundleDetails, params ReleaseBundleRemoteDeleteParams) error { + maxWaitTime := defaultMaxWait + if params.MaxWaitMinutes > 0 { + maxWaitTime = time.Duration(params.MaxWaitMinutes) * time.Minute + } + + pollingAction := func() (shouldStop bool, responseBody []byte, err error) { + resp, _, err := rbs.getReleaseBundleDistributions(rbDetails, params.ProjectKey) + if err != nil { + return true, nil, err + } + deletionStatus := resp[len(resp)-1].Status + switch deletionStatus { + case InProgress: + return false, nil, nil + case Completed: + return true, nil, nil + case Failed: + return true, nil, errorutils.CheckErrorf("remote deletion failed!") + default: + return true, nil, errorutils.CheckErrorf("unexpected status for remote deletion: %s", deletionStatus) + } + } + pollingExecutor := &httputils.PollingExecutor{ + Timeout: maxWaitTime, + PollingInterval: SyncSleepInterval, + PollingAction: pollingAction, + MsgPrefix: fmt.Sprintf("Performing sync remote deletion of release bundle %s/%s...", rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion), + } + _, err := pollingExecutor.Execute() + return err +} + +type ReleaseBundleRemoteDeleteParams struct { + DistributionRules []*distribution.DistributionCommonParams + DryRun bool + // Max time in minutes to wait for sync distribution to finish. + MaxWaitMinutes int + CommonOptionalQueryParams } diff --git a/lifecycle/services/distribute.go b/lifecycle/services/distribute.go index f6aa40eb3..55a2ba1d3 100644 --- a/lifecycle/services/distribute.go +++ b/lifecycle/services/distribute.go @@ -1,16 +1,22 @@ package services import ( + "encoding/json" "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + clientUtils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/distribution" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "net/http" "path" ) const ( distributionBaseApi = "api/v2/distribution/" distribute = "distribute" + trackers = "trackers" ) type DistributeReleaseBundleService struct { @@ -84,7 +90,7 @@ func (dr *DistributeReleaseBundleService) Distribute() error { func (dr *DistributeReleaseBundleService) createDistributeBody() ReleaseBundleDistributeBody { return ReleaseBundleDistributeBody{ - ReleaseBundleDistributeV1Body: distribution.CreateDistributeV1Body(dr.DistributeParams, dr.DryRun, dr.AutoCreateRepo), + ReleaseBundleDistributeV1Body: distribution.CreateDistributeV1Body(dr.DistributeParams.DistributionRules, dr.DryRun, dr.AutoCreateRepo), Modifications: dr.Modifications, } } @@ -102,3 +108,40 @@ type PathMapping struct { Pattern string Target string } + +func (rbs *ReleaseBundlesService) getReleaseBundleDistributions(rbDetails ReleaseBundleDetails, projectKey string) (distributionsResp GetDistributionsResponse, body []byte, err error) { + restApi := GetReleaseBundleDistributionsApi(rbDetails) + requestFullUrl, err := clientUtils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, getProjectQueryParam(projectKey)) + if err != nil { + return + } + httpClientsDetails := rbs.GetLifecycleDetails().CreateHttpClientDetails() + resp, body, _, err := rbs.client.SendGet(requestFullUrl, true, &httpClientsDetails) + if err != nil { + return + } + log.Debug("Artifactory response:", resp.Status) + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted, http.StatusOK); err != nil { + return + } + err = errorutils.CheckError(json.Unmarshal(body, &distributionsResp)) + return +} + +func GetReleaseBundleDistributionsApi(rbDetails ReleaseBundleDetails) string { + return path.Join(distributionBaseApi, trackers, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) +} + +type GetDistributionsResponse []struct { + FriendlyId json.Number `json:"distribution_tracker_friendly_id"` + Type string `json:"type"` + ReleaseBundleName string `json:"release_bundle_name"` + ReleaseBundleVersion string `json:"release_bundle_version"` + Repository string `json:"storing_repository"` + Status RbStatus `json:"status"` + DistributedBy string `json:"distributed_by"` + Created string `json:"created"` + StartTime string `json:"start_time"` + FinishTime string `json:"finish_time"` + Targets []string `json:"targets"` +} diff --git a/lifecycle/services/promote.go b/lifecycle/services/promote.go index dd66c8c4c..f7ca665b7 100644 --- a/lifecycle/services/promote.go +++ b/lifecycle/services/promote.go @@ -2,8 +2,12 @@ package services import ( "encoding/json" + "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "net/http" "path" + "strconv" ) const ( @@ -54,6 +58,82 @@ func (rbs *ReleaseBundlesService) Promote(rbDetails ReleaseBundleDetails, queryP return promotionResp, errorutils.CheckError(err) } +type RbPromotionsResponse struct { + Promotions []RbPromotion `json:"promotions,omitempty"` +} + +type RbPromotion struct { + Status RbStatus `json:"status,omitempty"` + RepositoryKey string `json:"repository_key,omitempty"` + ReleaseBundleName string `json:"release_bundle_name,omitempty"` + ReleaseBundleVersion string `json:"release_bundle_version,omitempty"` + Environment string `json:"environment,omitempty"` + ServiceId string `json:"service_id,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Created string `json:"created,omitempty"` + CreatedMillis json.Number `json:"created_millis,omitempty"` + Messages []Message `json:"messages,omitempty"` +} + +type GetPromotionsOptionalQueryParams struct { + Include string + Offset int + Limit int + FilterBy string + OrderBy string + OrderAsc bool + ProjectKey string +} + +func buildGetPromotionsQueryParams(optionalQueryParams GetPromotionsOptionalQueryParams) map[string]string { + params := make(map[string]string) + if optionalQueryParams.ProjectKey != "" { + params["project"] = optionalQueryParams.ProjectKey + } + if optionalQueryParams.Include != "" { + params["include"] = optionalQueryParams.Include + } + if optionalQueryParams.Offset > 0 { + params["offset"] = strconv.Itoa(optionalQueryParams.Offset) + } + if optionalQueryParams.Limit > 0 { + params["limit"] = strconv.Itoa(optionalQueryParams.Limit) + } + if optionalQueryParams.FilterBy != "" { + params["filter_by"] = optionalQueryParams.FilterBy + } + if optionalQueryParams.OrderBy != "" { + params["order_by"] = optionalQueryParams.OrderBy + } + if optionalQueryParams.OrderAsc { + params["order_asc"] = strconv.FormatBool(optionalQueryParams.OrderAsc) + } + return params +} + +func (rbs *ReleaseBundlesService) GetReleaseBundleVersionPromotions(rbDetails ReleaseBundleDetails, optionalQueryParams GetPromotionsOptionalQueryParams) (response RbPromotionsResponse, err error) { + restApi := GetGetReleaseBundleVersionPromotionsApi(rbDetails) + requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, buildGetPromotionsQueryParams(optionalQueryParams)) + if err != nil { + return + } + httpClientsDetails := rbs.GetLifecycleDetails().CreateHttpClientDetails() + resp, body, _, err := rbs.client.SendGet(requestFullUrl, true, &httpClientsDetails) + if err != nil { + return + } + log.Debug("Artifactory response:", resp.Status) + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + err = errorutils.CheckError(json.Unmarshal(body, &response)) + return +} + +func GetGetReleaseBundleVersionPromotionsApi(rbDetails ReleaseBundleDetails) string { + return path.Join(promotionBaseApi, records, rbDetails.ReleaseBundleName, rbDetails.ReleaseBundleVersion) +} + type RbPromotionParams struct { Environment string IncludedRepositoryKeys []string diff --git a/lifecycle/services/status.go b/lifecycle/services/status.go index 98d475706..939f0222a 100644 --- a/lifecycle/services/status.go +++ b/lifecycle/services/status.go @@ -31,6 +31,7 @@ type RbStatus string const ( Completed RbStatus = "COMPLETED" Processing RbStatus = "PROCESSING" + InProgress RbStatus = "IN_PROGRESS" Pending RbStatus = "PENDING" Failed RbStatus = "FAILED" Rejected RbStatus = "REJECTED" diff --git a/utils/distribution/distribute.go b/utils/distribution/distribute.go index 47e80475f..dec61c98a 100644 --- a/utils/distribution/distribute.go +++ b/utils/distribution/distribute.go @@ -20,13 +20,13 @@ type DistributeReleaseBundleExecutor interface { GetDistributionParams() DistributionParams } -func CreateDistributeV1Body(distributeParams DistributionParams, dryRun, isAutoCreateRepo bool) ReleaseBundleDistributeV1Body { +func CreateDistributeV1Body(distCommonParams []*DistributionCommonParams, dryRun, isAutoCreateRepo bool) ReleaseBundleDistributeV1Body { var distributionRules []DistributionRulesBody - for _, spec := range distributeParams.DistributionRules { + for i := range distCommonParams { distributionRule := DistributionRulesBody{ - SiteName: spec.GetSiteName(), - CityName: spec.GetCityName(), - CountryCodes: spec.GetCountryCodes(), + SiteName: distCommonParams[i].GetSiteName(), + CityName: distCommonParams[i].GetCityName(), + CountryCodes: distCommonParams[i].GetCountryCodes(), } distributionRules = append(distributionRules, distributionRule) } From 90669dbbcc54cf2bd0af4cced9c1cc70a8932363 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 08:54:24 +0200 Subject: [PATCH 12/15] Artifactory Release Lifecycle Management - Support release bundle distribution by project (#913) --- distribution/manager.go | 2 +- distribution/services/distribute.go | 8 +++- distribution/services/getstatus.go | 52 ++------------------------ lifecycle/manager.go | 7 ++-- lifecycle/services/delete.go | 2 +- lifecycle/services/distribute.go | 12 +++--- lifecycle/services/operation.go | 11 +----- lifecycle/services/status.go | 30 +++++++-------- tests/distribution_test.go | 10 ++--- utils/distribution/distribute.go | 57 +++++++++++++++++++++++++++-- utils/distribution/utils.go | 8 ++++ 11 files changed, 106 insertions(+), 93 deletions(-) diff --git a/distribution/manager.go b/distribution/manager.go index e0548429a..257d407c7 100644 --- a/distribution/manager.go +++ b/distribution/manager.go @@ -79,7 +79,7 @@ func (sm *DistributionServicesManager) DistributeReleaseBundleSync(params distri return distributeBundleService.Distribute() } -func (sm *DistributionServicesManager) GetDistributionStatus(params services.DistributionStatusParams) (*[]services.DistributionStatusResponse, error) { +func (sm *DistributionServicesManager) GetDistributionStatus(params services.DistributionStatusParams) (*[]distribution.DistributionStatusResponse, error) { distributeBundleService := services.NewDistributionStatusService(sm.client) distributeBundleService.DistDetails = sm.config.GetServiceDetails() return distributeBundleService.GetStatus(params) diff --git a/distribution/services/distribute.go b/distribution/services/distribute.go index 0c33cf6e1..e59f11f87 100644 --- a/distribution/services/distribute.go +++ b/distribution/services/distribute.go @@ -57,6 +57,10 @@ func (dr *DistributeReleaseBundleV1Service) GetDistributionParams() distribution return dr.DistributeParams } +func (dr *DistributeReleaseBundleV1Service) GetProjectKey() string { + return "" +} + func NewDistributeReleaseBundleV1Service(client *jfroghttpclient.JfrogHttpClient) *DistributeReleaseBundleV1Service { return &DistributeReleaseBundleV1Service{client: client} } @@ -94,14 +98,14 @@ func (dr *DistributeReleaseBundleV1Service) waitForDistribution(distributeParams if err != nil { return false, errorutils.CheckError(err) } - if (*response)[0].Status == Failed { + if (*response)[0].Status == distribution.Failed { bytes, err := json.Marshal(response) if err != nil { return false, errorutils.CheckError(err) } return false, errorutils.CheckErrorf("Distribution failed: " + clientUtils.IndentJson(bytes)) } - if (*response)[0].Status == Completed { + if (*response)[0].Status == distribution.Completed { log.Info("Distribution Completed!") return false, nil } diff --git a/distribution/services/getstatus.go b/distribution/services/getstatus.go index 91350f55c..d4c87f134 100644 --- a/distribution/services/getstatus.go +++ b/distribution/services/getstatus.go @@ -27,7 +27,7 @@ func (ds *DistributionStatusService) GetDistDetails() auth.ServiceDetails { return ds.DistDetails } -func (ds *DistributionStatusService) GetStatus(distributionStatusParams DistributionStatusParams) (*[]DistributionStatusResponse, error) { +func (ds *DistributionStatusService) GetStatus(distributionStatusParams DistributionStatusParams) (*[]distribution.DistributionStatusResponse, error) { if err := ds.checkParameters(distributionStatusParams); err != nil { return nil, err } @@ -45,7 +45,7 @@ func (ds *DistributionStatusService) checkParameters(distributionStatusParams Di return errorutils.CheckError(err) } -func (ds *DistributionStatusService) execGetStatus(name, version, trackerId string) (*[]DistributionStatusResponse, error) { +func (ds *DistributionStatusService) execGetStatus(name, version, trackerId string) (*[]distribution.DistributionStatusResponse, error) { httpClientsDetails := ds.DistDetails.CreateHttpClientDetails() url := ds.BuildUrlForGetStatus(ds.DistDetails.GetUrl(), name, version, trackerId) @@ -58,7 +58,7 @@ func (ds *DistributionStatusService) execGetStatus(name, version, trackerId stri } log.Debug("Distribution response:", resp.Status) log.Debug(utils.IndentJson(body)) - var distributionStatusResponse []DistributionStatusResponse + var distributionStatusResponse []distribution.DistributionStatusResponse stringBody := string(body) if !strings.HasPrefix(stringBody, "[") { stringBody = "[" + stringBody + "]" @@ -92,49 +92,3 @@ type DistributionStatusParams struct { func NewDistributionStatusParams() DistributionStatusParams { return DistributionStatusParams{} } - -type DistributionType string - -const ( - Distribute DistributionType = "distribute" - DeleteReleaseBundleVersion DistributionType = "delete_release_bundle_version" -) - -type DistributionStatus string - -const ( - NotDistributed DistributionStatus = "Not distributed" - InProgress DistributionStatus = "In progress" - InQueue DistributionStatus = "In queue" - Completed DistributionStatus = "Completed" - Failed DistributionStatus = "Failed" -) - -type DistributionStatusResponse struct { - Id json.Number `json:"distribution_id"` - FriendlyId json.Number `json:"distribution_friendly_id,omitempty"` - Type DistributionType `json:"type,omitempty"` - Name string `json:"release_bundle_name,omitempty"` - Version string `json:"release_bundle_version,omitempty"` - Status DistributionStatus `json:"status,omitempty"` - DistributionRules []distribution.DistributionRulesBody `json:"distribution_rules,omitempty"` - Sites []DistributionSiteStatus `json:"sites,omitempty"` -} - -type DistributionSiteStatus struct { - Status string `json:"status,omitempty"` - Error string `json:"general_error,omitempty"` - TargetArtifactory TargetArtifactory `json:"target_artifactory,omitempty"` - TotalFiles json.Number `json:"total_files,omitempty"` - TotalBytes json.Number `json:"total_bytes,omitempty"` - DistributedBytes json.Number `json:"distributed_bytes,omitempty"` - DistributedFiles json.Number `json:"distributed_files,omitempty"` - FileErrors []string `json:"file_errors,omitempty"` - FilesInProgress []string `json:"files_in_progress,omitempty"` -} - -type TargetArtifactory struct { - ServiceId string `json:"service_id"` - Name string `json:"name"` - Type string `json:"type"` -} diff --git a/lifecycle/manager.go b/lifecycle/manager.go index 0dc1a8347..154cbeb32 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -103,11 +103,12 @@ func (lcs *LifecycleServicesManager) DistributeReleaseBundle(rbDetails lifecycle distributeBundleService.AutoCreateRepo = distributeParams.AutoCreateRepo distributeBundleService.Sync = distributeParams.Sync distributeBundleService.MaxWaitMinutes = distributeParams.MaxWaitMinutes + distributeBundleService.ProjectKey = distributeParams.ProjectKey - m := &distributeBundleService.Modifications.PathMappings - *m = []utils.PathMapping{} + mappings := &distributeBundleService.Modifications.PathMappings + *mappings = []utils.PathMapping{} for _, pathMapping := range distributeParams.PathMappings { - *m = append(*m, + *mappings = append(*mappings, distribution.CreatePathMappingsFromPatternAndTarget(pathMapping.Pattern, pathMapping.Target)...) } return distributeBundleService.Distribute() diff --git a/lifecycle/services/delete.go b/lifecycle/services/delete.go index 02d18e112..054ef13db 100644 --- a/lifecycle/services/delete.go +++ b/lifecycle/services/delete.go @@ -30,7 +30,7 @@ func (rbs *ReleaseBundlesService) DeleteReleaseBundleVersionPromotion(rbDetails } func (rbs *ReleaseBundlesService) deleteReleaseBundle(params CommonOptionalQueryParams, restApi string) error { - queryParams := getProjectQueryParam(params.ProjectKey) + queryParams := distribution.GetProjectQueryParam(params.ProjectKey) queryParams[async] = strconv.FormatBool(params.Async) requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, queryParams) if err != nil { diff --git a/lifecycle/services/distribute.go b/lifecycle/services/distribute.go index 55a2ba1d3..8a97c4717 100644 --- a/lifecycle/services/distribute.go +++ b/lifecycle/services/distribute.go @@ -27,6 +27,7 @@ type DistributeReleaseBundleService struct { Sync bool MaxWaitMinutes int DistributeParams distribution.DistributionParams + ProjectKey string Modifications } @@ -36,6 +37,7 @@ type DistributeReleaseBundleParams struct { MaxWaitMinutes int DistributionRules []*distribution.DistributionCommonParams PathMappings []PathMapping + ProjectKey string } func (dr *DistributeReleaseBundleService) GetHttpClient() *jfroghttpclient.JfrogHttpClient { @@ -58,10 +60,6 @@ func (dr *DistributeReleaseBundleService) GetMaxWaitMinutes() int { return dr.MaxWaitMinutes } -func (dr *DistributeReleaseBundleService) IsAutoCreateRepo() bool { - return dr.AutoCreateRepo -} - func (dr *DistributeReleaseBundleService) GetRestApi(name, version string) string { return path.Join(distributionBaseApi, distribute, name, version) } @@ -74,6 +72,10 @@ func (dr *DistributeReleaseBundleService) GetDistributionParams() distribution.D return dr.DistributeParams } +func (dr *DistributeReleaseBundleService) GetProjectKey() string { + return dr.ProjectKey +} + func NewDistributeReleaseBundleService(client *jfroghttpclient.JfrogHttpClient) *DistributeReleaseBundleService { return &DistributeReleaseBundleService{client: client} } @@ -111,7 +113,7 @@ type PathMapping struct { func (rbs *ReleaseBundlesService) getReleaseBundleDistributions(rbDetails ReleaseBundleDetails, projectKey string) (distributionsResp GetDistributionsResponse, body []byte, err error) { restApi := GetReleaseBundleDistributionsApi(rbDetails) - requestFullUrl, err := clientUtils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, getProjectQueryParam(projectKey)) + requestFullUrl, err := clientUtils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, distribution.GetProjectQueryParam(projectKey)) if err != nil { return } diff --git a/lifecycle/services/operation.go b/lifecycle/services/operation.go index cb97ff35b..8bb095f23 100644 --- a/lifecycle/services/operation.go +++ b/lifecycle/services/operation.go @@ -6,6 +6,7 @@ import ( "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/distribution" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" "net/http" @@ -36,7 +37,7 @@ type ReleaseBundleOperation interface { } func (rbs *ReleaseBundlesService) doOperation(operation ReleaseBundleOperation) ([]byte, error) { - queryParams := getProjectQueryParam(operation.getOperationParams().ProjectKey) + queryParams := distribution.GetProjectQueryParam(operation.getOperationParams().ProjectKey) queryParams[async] = strconv.FormatBool(operation.getOperationParams().Async) requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), operation.getOperationRestApi(), queryParams) if err != nil { @@ -67,14 +68,6 @@ func (rbs *ReleaseBundlesService) doOperation(operation ReleaseBundleOperation) return body, errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) } -func getProjectQueryParam(projectKey string) map[string]string { - queryParams := make(map[string]string) - if projectKey != "" { - queryParams["project"] = projectKey - } - return queryParams -} - type ReleaseBundleDetails struct { ReleaseBundleName string `json:"release_bundle_name,omitempty"` ReleaseBundleVersion string `json:"release_bundle_version,omitempty"` diff --git a/lifecycle/services/status.go b/lifecycle/services/status.go index 939f0222a..a763da5c8 100644 --- a/lifecycle/services/status.go +++ b/lifecycle/services/status.go @@ -12,8 +12,6 @@ import ( "net/http" "path" "time" - - dsServices "github.com/jfrog/jfrog-client-go/distribution/services" ) const ( @@ -64,7 +62,7 @@ func (rbs *ReleaseBundlesService) getReleaseBundleOperationStatus(restApi string } func (rbs *ReleaseBundlesService) getReleaseBundleStatus(restApi string, projectKey string) (statusResp ReleaseBundleStatusResponse, body []byte, err error) { - requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, getProjectQueryParam(projectKey)) + requestFullUrl, err := utils.BuildUrl(rbs.GetLifecycleDetails().GetUrl(), restApi, distribution.GetProjectQueryParam(projectKey)) if err != nil { return } @@ -98,14 +96,14 @@ func (rbs *ReleaseBundlesService) GetReleaseBundleSpecification(rbDetails Releas return } -func (dbs *DistributeReleaseBundleService) getReleaseBundleDistributionStatus(distributeParams *distribution.DistributionParams, trackerId json.Number) (statusResp *dsServices.DistributionStatusResponse, body []byte, err error) { +func (dr *DistributeReleaseBundleService) getReleaseBundleDistributionStatus(distributeParams *distribution.DistributionParams, trackerId json.Number) (statusResp *distribution.DistributionStatusResponse, body []byte, err error) { restApi := path.Join(distributionBaseApi, trackersApi, distributeParams.Name, distributeParams.Version, trackerId.String()) - requestFullUrl, err := utils.BuildUrl(dbs.LcDetails.GetUrl(), restApi, nil) + requestFullUrl, err := utils.BuildUrl(dr.LcDetails.GetUrl(), restApi, distribution.GetProjectQueryParam(dr.GetProjectKey())) if err != nil { return } - httpClientsDetails := dbs.LcDetails.CreateHttpClientDetails() - resp, body, _, err := dbs.client.SendGet(requestFullUrl, true, &httpClientsDetails) + httpClientsDetails := dr.LcDetails.CreateHttpClientDetails() + resp, body, _, err := dr.client.SendGet(requestFullUrl, true, &httpClientsDetails) if err != nil { return } @@ -151,22 +149,22 @@ func (rbs *ReleaseBundlesService) waitForRbOperationCompletion(restApi, projectK return getStatusResponse(finalRespBody) } -func (dbs *DistributeReleaseBundleService) waitForDistributionOperationCompletion(distributeParams *distribution.DistributionParams, trackerId json.Number) error { - maxWait := time.Duration(dbs.GetMaxWaitMinutes()) * time.Minute +func (dr *DistributeReleaseBundleService) waitForDistributionOperationCompletion(distributeParams *distribution.DistributionParams, trackerId json.Number) error { + maxWait := time.Duration(dr.GetMaxWaitMinutes()) * time.Minute if maxWait.Minutes() < 1 { maxWait = defaultMaxWait } pollingAction := func() (shouldStop bool, responseBody []byte, err error) { - statusResponse, responseBody, err := dbs.getReleaseBundleDistributionStatus(distributeParams, trackerId) + statusResponse, responseBody, err := dr.getReleaseBundleDistributionStatus(distributeParams, trackerId) if err != nil { return true, nil, err } switch statusResponse.Status { - case dsServices.NotDistributed, dsServices.InProgress, dsServices.InQueue: + case distribution.NotDistributed, distribution.InProgress, distribution.InQueue: return false, nil, nil - case dsServices.Failed, dsServices.Completed: + case distribution.Failed, distribution.Completed: return true, responseBody, nil default: return true, nil, errorutils.CheckErrorf("received unexpected status: '%s'", statusResponse.Status) @@ -183,14 +181,16 @@ func (dbs *DistributeReleaseBundleService) waitForDistributionOperationCompletio return err } - var dsStatusResponse dsServices.DistributionStatusResponse + var dsStatusResponse distribution.DistributionStatusResponse if err = json.Unmarshal(finalRespBody, &dsStatusResponse); err != nil { return errorutils.CheckError(err) } - if dsStatusResponse.Status != dsServices.Completed { + if dsStatusResponse.Status != distribution.Completed { for _, st := range dsStatusResponse.Sites { - err = errors.Join(err, fmt.Errorf("target %s name:%s error:%s", st.TargetArtifactory.Type, st.TargetArtifactory.Name, st.Error)) + if st.Status != distribution.Completed { + err = errors.Join(err, fmt.Errorf("target %s name:%s error:%s", st.TargetArtifactory.Type, st.TargetArtifactory.Name, st.Error)) + } } return errorutils.CheckError(err) } diff --git a/tests/distribution_test.go b/tests/distribution_test.go index e6c6144f0..f467eddfb 100644 --- a/tests/distribution_test.go +++ b/tests/distribution_test.go @@ -311,7 +311,7 @@ func createSignDistributeDelete(t *testing.T) { // Assert release bundle in "completed" status response, err = testsBundleDistributionStatusService.GetStatus(distributionStatusParams) if assert.NoError(t, err) && assert.NotEmpty(t, *response) { - assert.Equal(t, services.Completed, (*response)[0].Status) + assert.Equal(t, distribution.Completed, (*response)[0].Status) } } @@ -357,7 +357,7 @@ func createSignSyncDistributeDelete(t *testing.T) { } response, err := testsBundleDistributionStatusService.GetStatus(distributionStatusParams) if assert.NoError(t, err) && assert.NotEmpty(t, *response) { - assert.Equal(t, services.Completed, (*response)[0].Status) + assert.Equal(t, distribution.Completed, (*response)[0].Status) } } @@ -548,12 +548,12 @@ func waitForDistribution(t *testing.T, bundleName string) { assert.Len(t, *response, 1) switch (*response)[0].Status { - case services.Completed: + case distribution.Completed: return - case services.Failed: + case distribution.Failed: t.Error("Distribution failed for " + bundleName + "/" + bundleVersion) return - case services.InProgress, services.NotDistributed: + case distribution.InProgress, distribution.NotDistributed: // Wait } t.Log("Waiting for " + bundleName + "/" + bundleVersion + "...") diff --git a/utils/distribution/distribute.go b/utils/distribution/distribute.go index dec61c98a..4b051cb59 100644 --- a/utils/distribution/distribute.go +++ b/utils/distribution/distribute.go @@ -18,6 +18,7 @@ type DistributeReleaseBundleExecutor interface { GetRestApi(name, version string) string GetDistributeBody() any GetDistributionParams() DistributionParams + GetProjectKey() string } func CreateDistributeV1Body(distCommonParams []*DistributionCommonParams, dryRun, isAutoCreateRepo bool) ReleaseBundleDistributeV1Body { @@ -44,7 +45,6 @@ func DoDistribute(dr DistributeReleaseBundleExecutor) (trackerId json.Number, er } func execDistribute(dr DistributeReleaseBundleExecutor, name, version string) (json.Number, error) { - httpClientsDetails := dr.ServiceDetails().CreateHttpClientDetails() content, err := json.Marshal(dr.GetDistributeBody()) if err != nil { return "", errorutils.CheckError(err) @@ -56,9 +56,14 @@ func execDistribute(dr DistributeReleaseBundleExecutor, name, version string) (j } log.Info(dryRunStr + "Distributing: " + name + "/" + version) - url := dr.ServiceDetails().GetUrl() + dr.GetRestApi(name, version) + requestFullUrl, err := clientUtils.BuildUrl(dr.ServiceDetails().GetUrl(), dr.GetRestApi(name, version), GetProjectQueryParam(dr.GetProjectKey())) + if err != nil { + return "", err + } + + httpClientsDetails := dr.ServiceDetails().CreateHttpClientDetails() artifactoryUtils.SetContentType("application/json", &httpClientsDetails.Headers) - resp, body, err := dr.GetHttpClient().SendPost(url, content, &httpClientsDetails) + resp, body, err := dr.GetHttpClient().SendPost(requestFullUrl, content, &httpClientsDetails) if err != nil { return "", err } @@ -104,3 +109,49 @@ type DistributionRulesBody struct { type DistributionResponseBody struct { TrackerId json.Number `json:"id"` } + +type DistributionStatus string + +const ( + NotDistributed DistributionStatus = "Not distributed" + InProgress DistributionStatus = "In progress" + InQueue DistributionStatus = "In queue" + Completed DistributionStatus = "Completed" + Failed DistributionStatus = "Failed" +) + +type DistributionStatusResponse struct { + Id json.Number `json:"distribution_id"` + FriendlyId json.Number `json:"distribution_friendly_id,omitempty"` + Type DistributionType `json:"type,omitempty"` + Name string `json:"release_bundle_name,omitempty"` + Version string `json:"release_bundle_version,omitempty"` + Status DistributionStatus `json:"status,omitempty"` + DistributionRules []DistributionRulesBody `json:"distribution_rules,omitempty"` + Sites []DistributionSiteStatus `json:"sites,omitempty"` +} + +type DistributionType string + +const ( + Distribute DistributionType = "distribute" + DeleteReleaseBundleVersion DistributionType = "delete_release_bundle_version" +) + +type DistributionSiteStatus struct { + Status DistributionStatus `json:"status,omitempty"` + Error string `json:"general_error,omitempty"` + TargetArtifactory TargetArtifactory `json:"target_artifactory,omitempty"` + TotalFiles json.Number `json:"total_files,omitempty"` + TotalBytes json.Number `json:"total_bytes,omitempty"` + DistributedBytes json.Number `json:"distributed_bytes,omitempty"` + DistributedFiles json.Number `json:"distributed_files,omitempty"` + FileErrors []string `json:"file_errors,omitempty"` + FilesInProgress []string `json:"files_in_progress,omitempty"` +} + +type TargetArtifactory struct { + ServiceId string `json:"service_id"` + Name string `json:"name"` + Type string `json:"type"` +} diff --git a/utils/distribution/utils.go b/utils/distribution/utils.go index 066cb7e45..858e28fa8 100644 --- a/utils/distribution/utils.go +++ b/utils/distribution/utils.go @@ -38,3 +38,11 @@ func CreatePathMappings(input, output string) []utils.PathMapping { Output: output, }} } + +func GetProjectQueryParam(projectKey string) map[string]string { + queryParams := make(map[string]string) + if projectKey != "" { + queryParams["project"] = projectKey + } + return queryParams +} From 75c3311549abfb892f4e5e55f49d01dc6e873142 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 13:18:07 +0200 Subject: [PATCH 13/15] Artifactory Release Lifecycle Management - Support release bundle creation by aql (#915) --- README.md | 38 ++++++++++++++ .../services/utils/artifactoryutils.go | 4 +- artifactory/services/utils/searchutil.go | 2 +- lifecycle/manager.go | 6 +++ lifecycle/services/create.go | 50 +++++++++---------- 5 files changed, 70 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 1d05f858d..bd6606e67 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,8 @@ - [Creating Lifecycle Service Config](#creating-lifeCycle-service-config) - [Creating New Lifecycle Service Manager](#creating-new-lifeCycle-service-manager) - [Using Lifecycle Services](#using-lifeCycle-services) + - [Creating a Release Bundle From AQL](#creating-a-release-bundle-from-aql) + - [Creating a Release Bundle From Artifacts](#creating-a-release-bundle-from-artifacts) - [Creating a Release Bundle From Published Builds](#creating-a-release-bundle-from-published-builds) - [Creating a Release Bundle From Release Bundles](#creating-a-release-bundle-from-release-bundles) - [Promoting a Release Bundle](#promoting-a-release-bundle) @@ -2434,6 +2436,42 @@ lifecycleManager, err := lifecycle.New(serviceConfig) ### Using Lifecycle Services +#### Creating a Release Bundle From AQL + +```go +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} +queryParams := CommonOptionalQueryParams{} +queryParams.ProjectKey = "project" +queryParams.Async = true + +// The GPG/RSA key-pair name given in Artifactory. +signingKeyName = "key-pair" + +aqlQuery := `items.find({"repo": "my-repo","path": ".","name": "a2.in"})` +serviceManager.CreateReleaseBundleFromAql(rbDetails, queryParams, signingKeyName, aqlQuery) +``` + +#### Creating a Release Bundle From Artifacts + +```go +rbDetails := ReleaseBundleDetails{"rbName", "rbVersion"} +queryParams := CommonOptionalQueryParams{} +queryParams.ProjectKey = "project" +queryParams.Async = true + +// The GPG/RSA key-pair name given in Artifactory. +signingKeyName = "key-pair" + +artifacts := CreateFromArtifacts{Artifacts: []ArtifactSource{ + { + Path: "repo/path/file", + Sha256: "3e3deb6628658a48cf0d280a2210211f9d977ec2e10a4619b95d5fb85cb10450", + }, +}} + +serviceManager.CreateReleaseBundleFromArtifacts(rbDetails, queryParams, signingKeyName, artifacts) +``` + #### Creating a Release Bundle From Published Builds ```go diff --git a/artifactory/services/utils/artifactoryutils.go b/artifactory/services/utils/artifactoryutils.go index 3006de7b8..7cc293d22 100644 --- a/artifactory/services/utils/artifactoryutils.go +++ b/artifactory/services/utils/artifactoryutils.go @@ -139,7 +139,7 @@ func HasPrefix(paths []string, prefix string) bool { // If no buildNumber provided LATEST will be downloaded. // If buildName or buildNumber contains "/" (slash) it should be escaped by "\" (backslash). // Result examples of parsing: "aaa/123" > "aaa"-"123", "aaa" > "aaa"-"LATEST", "aaa\\/aaa" > "aaa/aaa"-"LATEST", "aaa/12\\/3" > "aaa"-"12/3". -func getBuildNameAndNumberFromBuildIdentifier(buildIdentifier, projectKey string, flags CommonConf) (string, string, error) { +func GetBuildNameAndNumberFromBuildIdentifier(buildIdentifier, projectKey string, flags CommonConf) (string, string, error) { buildName, buildNumber, err := ParseNameAndVersion(buildIdentifier, true) if err != nil { return "", "", err @@ -265,7 +265,7 @@ func filterAqlSearchResultsByBuild(specFile *CommonParams, reader *content.Conte var wg sync.WaitGroup wg.Add(2) // If 'build-number' is missing in spec file, we fetch the latest from artifactory. - buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, specFile.Project, flags) + buildName, buildNumber, err := GetBuildNameAndNumberFromBuildIdentifier(specFile.Build, specFile.Project, flags) if err != nil { return nil, err } diff --git a/artifactory/services/utils/searchutil.go b/artifactory/services/utils/searchutil.go index f4909f157..4e625e9ba 100644 --- a/artifactory/services/utils/searchutil.go +++ b/artifactory/services/utils/searchutil.go @@ -36,7 +36,7 @@ const ( // Use this function when searching by build without pattern or aql. // Collect build artifacts and build dependencies separately, then merge the results into one reader. func SearchBySpecWithBuild(specFile *CommonParams, flags CommonConf) (readerContent *content.ContentReader, err error) { - buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, specFile.Project, flags) + buildName, buildNumber, err := GetBuildNameAndNumberFromBuildIdentifier(specFile.Build, specFile.Project, flags) if err != nil { return nil, err } diff --git a/lifecycle/manager.go b/lifecycle/manager.go index 154cbeb32..e7286b8f4 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -37,6 +37,12 @@ func (lcs *LifecycleServicesManager) Client() *jfroghttpclient.JfrogHttpClient { return lcs.client } +func (lcs *LifecycleServicesManager) CreateReleaseBundleFromAql(rbDetails lifecycle.ReleaseBundleDetails, + queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, aqlQuery string) error { + rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) + return rbService.CreateFromAql(rbDetails, queryParams, signingKeyName, aqlQuery) +} + func (lcs *LifecycleServicesManager) CreateReleaseBundleFromArtifacts(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, sourceArtifacts lifecycle.CreateFromArtifacts) error { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) diff --git a/lifecycle/services/create.go b/lifecycle/services/create.go index e5e44ba6a..8206f148a 100644 --- a/lifecycle/services/create.go +++ b/lifecycle/services/create.go @@ -4,12 +4,13 @@ const ( releaseBundleBaseApi = "api/v2/release_bundle" ) -type sourceType string +type SourceType string const ( - artifacts sourceType = "artifacts" - builds sourceType = "builds" - releaseBundles sourceType = "release_bundles" + Aql SourceType = "aql" + Artifacts SourceType = "artifacts" + Builds SourceType = "builds" + ReleaseBundles SourceType = "release_bundles" ) type createOperation struct { @@ -38,38 +39,29 @@ func (c *createOperation) getSigningKeyName() string { return c.signingKeyName } +func (rbs *ReleaseBundlesService) CreateFromAql(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, aqlQuery string) error { + return rbs.CreateReleaseBundle(rbDetails, params, signingKeyName, Aql, CreateFromAqlSource{Aql: aqlQuery}) +} + func (rbs *ReleaseBundlesService) CreateFromArtifacts(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, sourceArtifacts CreateFromArtifacts) error { - operation := createOperation{ - reqBody: RbCreationBody{ - ReleaseBundleDetails: rbDetails, - SourceType: artifacts, - Source: sourceArtifacts}, - params: params, - signingKeyName: signingKeyName, - } - _, err := rbs.doOperation(&operation) - return err + return rbs.CreateReleaseBundle(rbDetails, params, signingKeyName, Artifacts, sourceArtifacts) } func (rbs *ReleaseBundlesService) CreateFromBuilds(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, sourceBuilds CreateFromBuildsSource) error { - operation := createOperation{ - reqBody: RbCreationBody{ - ReleaseBundleDetails: rbDetails, - SourceType: builds, - Source: sourceBuilds}, - params: params, - signingKeyName: signingKeyName, - } - _, err := rbs.doOperation(&operation) - return err + return rbs.CreateReleaseBundle(rbDetails, params, signingKeyName, Builds, sourceBuilds) } func (rbs *ReleaseBundlesService) CreateFromBundles(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, signingKeyName string, sourceReleaseBundles CreateFromReleaseBundlesSource) error { + return rbs.CreateReleaseBundle(rbDetails, params, signingKeyName, ReleaseBundles, sourceReleaseBundles) +} + +func (rbs *ReleaseBundlesService) CreateReleaseBundle(rbDetails ReleaseBundleDetails, params CommonOptionalQueryParams, + signingKeyName string, rbSourceType SourceType, source interface{}) error { operation := createOperation{ reqBody: RbCreationBody{ ReleaseBundleDetails: rbDetails, - SourceType: releaseBundles, - Source: sourceReleaseBundles}, + SourceType: rbSourceType, + Source: source}, params: params, signingKeyName: signingKeyName, } @@ -77,6 +69,10 @@ func (rbs *ReleaseBundlesService) CreateFromBundles(rbDetails ReleaseBundleDetai return err } +type CreateFromAqlSource struct { + Aql string `json:"aql,omitempty"` +} + type SourceBuildDetails struct { BuildName string BuildNumber string @@ -115,6 +111,6 @@ type ReleaseBundleSource struct { type RbCreationBody struct { ReleaseBundleDetails - SourceType sourceType `json:"source_type,omitempty"` + SourceType SourceType `json:"source_type,omitempty"` Source interface{} `json:"source,omitempty"` } From 6267b7025659128e96fc691bc462ce1879c8bc0c Mon Sep 17 00:00:00 2001 From: idand1741 <130966676+idand1741@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:46:20 +0200 Subject: [PATCH 14/15] Update multipartupload.go for artifactory version (#923) --- artifactory/services/utils/multipartupload.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go index cb6be37bb..b212ea845 100644 --- a/artifactory/services/utils/multipartupload.go +++ b/artifactory/services/utils/multipartupload.go @@ -28,8 +28,7 @@ type supportedStatus int type completionStatus string const ( - // TODO - Update version - minArtifactoryVersion = "8.0.0" + minArtifactoryVersion = "7.82.2" // Supported status // Multipart upload support is not yet determined From e5506e73f1fe2faf8da846606853134fa6c9a51e Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 18 Mar 2024 16:30:48 +0200 Subject: [PATCH 15/15] Promote version to 1.38.0 (#924) --- go.mod | 4 ++-- go.sum | 4 ++-- utils/utils.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 7714a4a4e..fdfbde22d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.0 - github.com/jfrog/build-info-go v1.9.23 + github.com/jfrog/build-info-go v1.9.24 github.com/jfrog/gofrog v1.6.3 github.com/stretchr/testify v1.8.4 github.com/xanzy/ssh-agent v0.3.3 @@ -57,6 +57,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c +// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev diff --git a/go.sum b/go.sum index 419a1aedd..1932dea6e 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= -github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c h1:M1QiuCYGCYN1IiGyxogrLzfetYGkkhE2pgDh5K4Wo9A= -github.com/jfrog/build-info-go v1.8.9-0.20240225113943-096bf22ca54c/go.mod h1:QHcKuesY4MrBVBuEwwBz4uIsX6mwYuMEDV09ng4AvAU= +github.com/jfrog/build-info-go v1.9.24 h1:MjT+4bYecbNQ+dbLczg0lkE5DoLAhdyrF0cRXtnEJqI= +github.com/jfrog/build-info-go v1.9.24/go.mod h1:CaCKqcg3V2W9/ZysE4ZvXZMgsvunclhjrTTQQGp3CzM= github.com/jfrog/gofrog v1.6.3 h1:F7He0+75HcgCe6SGTSHLFCBDxiE2Ja0tekvvcktW6wc= github.com/jfrog/gofrog v1.6.3/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/utils/utils.go b/utils/utils.go index 1d943f81f..ecba3e4d5 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -27,7 +27,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.37.1" + Version = "1.38.0" ) type MinVersionProduct string