From d432990b994e4b7b57f0aeebb84812a358f7e672 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov Date: Wed, 10 Mar 2021 14:12:40 +0300 Subject: [PATCH] [DownloadBuildArtifactsV0] Respect "Parallelization limit" input (#14540) - Add function to resolve "parallelProcessingLimit" value - Changed "configureDownloaderOptions" function to respect "Parallelization limit" input - Remove "timeoutPromise" function since not used in task --- .../download_helper.ts | 67 ++++++++++++++++--- Tasks/DownloadBuildArtifactsV0/main.ts | 22 ++++-- Tasks/DownloadBuildArtifactsV0/task.json | 2 +- Tasks/DownloadBuildArtifactsV0/task.loc.json | 2 +- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/Tasks/DownloadBuildArtifactsV0/download_helper.ts b/Tasks/DownloadBuildArtifactsV0/download_helper.ts index bf88a9add8e8..452b868058c3 100644 --- a/Tasks/DownloadBuildArtifactsV0/download_helper.ts +++ b/Tasks/DownloadBuildArtifactsV0/download_helper.ts @@ -2,15 +2,6 @@ import { debug, loc } from 'azure-pipelines-task-lib/task'; import { ArtifactDownloadTicket, ItemType, TicketState } from 'artifact-engine/Models'; import { getFileSizeInBytes } from './file_helper'; -/** - * Just a Promise wrapper for setTimeout function - * @param {number} interval - timeout interval in milliseconds - */ -export function timeoutPromise(interval: number): Promise<{}> { - debug(`Wait for ${interval} milliseconds`); - return new Promise(resolve => setTimeout(resolve, interval)); -} - /** * This function checks a result of artifact download * @param {Array} downloadTickets @@ -96,3 +87,61 @@ function isItemCorrupted(ticket: ArtifactDownloadTicket): boolean { return isCorrupted; } + +/** + * This function resolves the value for the `parallelProcessingLimit` option of `ArtifactEngine` + * + * Earlier the only way to set parallelProcessingLimit in the task + * was by declaring the `release.artifact.download.parallellimit` variable. + * + * To maintain backward compatibility we will use the following strategy: + * + * Firstly, investigate the `release.artifact.download.parallellimit` variable. + * If everything is okay with this variable, the task will use the value from this variable. + * + * Secondly, investigate the `Parallelization limit` input of the task. + * If everything is okay with the value in the related task's input, the task will use the value from this input. + * + * If validation failed for both cases the function will return the `defaultLimit` for the `parallelProcessingLimit` option. + * + * @param {string} artifactDownloadLimit - value of `release.artifact.download.parallellimit` variable + * @param {string} taskLimit - value of `Parallelization limit` task input + * @param {number} defaultLimit - the default value that will be returned if `artifactDownloadLimit` and `taskLimit` contain invalid values. + * @returns {number} - parallel processing limit + */ +export function resolveParallelProcessingLimit(artifactDownloadLimit: string, taskLimit: string, defaultLimit: number): number { + debug(`Checking value of the "release.artifact.download.parallellimit" variable - ${artifactDownloadLimit}`); + const artifactDownloadParallelLimit: number = Number(artifactDownloadLimit); + if (isParallelProcessingLimitCorrect(artifactDownloadParallelLimit)) { + return artifactDownloadParallelLimit; + } + + debug(`Checking value of the "Parallelization limit" input - ${taskLimit}`); + const taskInputParallelLimit: number = Number(taskLimit); + if (isParallelProcessingLimitCorrect(taskInputParallelLimit)) { + return taskInputParallelLimit; + } + + debug(`The parallelization limit is set to default value - ${defaultLimit}`); + return defaultLimit; +} + +/** + * This function checks the input value for the `parallelProcessingLimit` option of `ArtifactEngine` + * + * The parallel processing limit must be a number greater than 0. + * + * @param {number} limit - value of parallel processing limit + * @returns {boolean} true if parallel processing limit is correct, false otherwise. + */ +function isParallelProcessingLimitCorrect(limit: number): boolean { + const isCorrect: boolean = (!isNaN(limit) && limit > 0); + + if (isCorrect) { + debug(`The value is correct, the parallelization limit is set to ${limit}`); + } else { + debug(`The value is incorrect ${limit}`); + } + + return isCorrect; +} diff --git a/Tasks/DownloadBuildArtifactsV0/main.ts b/Tasks/DownloadBuildArtifactsV0/main.ts index 65dc0ad54b57..a57ec2e9d102 100644 --- a/Tasks/DownloadBuildArtifactsV0/main.ts +++ b/Tasks/DownloadBuildArtifactsV0/main.ts @@ -17,11 +17,14 @@ import { DownloadHandlerContainer } from './DownloadHandlers/DownloadHandlerCont import { DownloadHandlerContainerZip } from './DownloadHandlers/DownloadHandlerContainerZip'; import { DownloadHandlerFilePath } from './DownloadHandlers/DownloadHandlerFilePath'; +import { resolveParallelProcessingLimit } from './download_helper'; + var taskJson = require('./task.json'); tl.setResourcePath(path.join(__dirname, 'task.json')); const area: string = 'DownloadBuildArtifacts'; +const DefaultParallelProcessingLimit: number = 8; function getDefaultProps() { var hostType = (tl.getVariable('SYSTEM.HOSTTYPE') || "").toLowerCase(); @@ -73,7 +76,7 @@ async function main(): Promise { var buildId: number = null; var buildVersionToDownload: string = tl.getInput("buildVersionToDownload", false); var allowPartiallySucceededBuilds: boolean = tl.getBoolInput("allowPartiallySucceededBuilds", false); - var branchName: string = tl.getInput("branchName", false);; + var branchName: string = tl.getInput("branchName", false); var downloadPath: string = tl.getInput("downloadPath", true); var downloadType: string = tl.getInput("downloadType", true); var tagFiltersInput: string = tl.getInput("tags", false); @@ -253,7 +256,7 @@ async function main(): Promise { if (artifacts) { var downloadPromises: Array> = []; artifacts.forEach(async function (artifact, index, artifacts) { - let downloaderOptions = configureDownloaderOptions(); + const downloaderOptions: engine.ArtifactEngineOptions = configureDownloaderOptions(); const config: IBaseHandlerConfig = { artifactInfo: artifact, @@ -372,11 +375,16 @@ function getRetryIntervalInSeconds(retryCount: number): number { } function configureDownloaderOptions(): engine.ArtifactEngineOptions { - var downloaderOptions = new engine.ArtifactEngineOptions(); - downloaderOptions.itemPattern = tl.getInput('itemPattern', false) || "**"; - downloaderOptions.parallelProcessingLimit = +tl.getVariable("release.artifact.download.parallellimit") || 8; - var debugMode = tl.getVariable('System.Debug'); - downloaderOptions.verbose = debugMode ? debugMode.toLowerCase() != 'false' : false; + const downloaderOptions: engine.ArtifactEngineOptions = new engine.ArtifactEngineOptions(); + + const debugMode: string = tl.getVariable('System.Debug'); + downloaderOptions.verbose = debugMode ? debugMode.toLowerCase() !== 'false' : false; + + const artifactDownloadLimit: string = tl.getVariable('release.artifact.download.parallellimit'); + const taskInputParallelLimit: string = tl.getInput('parallelizationLimit', false); + downloaderOptions.parallelProcessingLimit = resolveParallelProcessingLimit(artifactDownloadLimit, taskInputParallelLimit, DefaultParallelProcessingLimit); + + downloaderOptions.itemPattern = tl.getInput('itemPattern', false) || '**'; return downloaderOptions; } diff --git a/Tasks/DownloadBuildArtifactsV0/task.json b/Tasks/DownloadBuildArtifactsV0/task.json index 1f0473e79557..49784e794299 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.json +++ b/Tasks/DownloadBuildArtifactsV0/task.json @@ -9,7 +9,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 183, + "Minor": 184, "Patch": 0 }, "groups": [ diff --git a/Tasks/DownloadBuildArtifactsV0/task.loc.json b/Tasks/DownloadBuildArtifactsV0/task.loc.json index 04261432414a..b945bdc4d164 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.loc.json +++ b/Tasks/DownloadBuildArtifactsV0/task.loc.json @@ -9,7 +9,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 183, + "Minor": 184, "Patch": 0 }, "groups": [