From f1416bb9353ab82aed2439f9c8769f28971b2d8a Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:38:03 -0700 Subject: [PATCH 1/7] Support multiple test proxies --- sdk/test-utils/perfstress/CHANGELOG.md | 4 ++++ sdk/test-utils/perfstress/GettingStarted.md | 12 +++++----- sdk/test-utils/perfstress/README.md | 2 +- sdk/test-utils/perfstress/src/options.ts | 8 +++---- sdk/test-utils/perfstress/src/program.ts | 6 ++--- sdk/test-utils/perfstress/src/tests.ts | 25 ++++++++++++++++----- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/sdk/test-utils/perfstress/CHANGELOG.md b/sdk/test-utils/perfstress/CHANGELOG.md index 53b38c0cd57e..f8e5e8a18df2 100644 --- a/sdk/test-utils/perfstress/CHANGELOG.md +++ b/sdk/test-utils/perfstress/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.0.0 (Unreleased) +### 2021-10-05 + +- Support multiple test proxies + ### 2021-10-01 - Calls runAsync() once before starting recording, to avoid capturing one-time setup like authorization requests. diff --git a/sdk/test-utils/perfstress/GettingStarted.md b/sdk/test-utils/perfstress/GettingStarted.md index fb6ac5495dde..9d47b72f7a33 100644 --- a/sdk/test-utils/perfstress/GettingStarted.md +++ b/sdk/test-utils/perfstress/GettingStarted.md @@ -291,21 +291,21 @@ Run this command Reference: https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy#via-docker-image -To use the proxy-tool in your test pass this option in cli `--test-proxy http://localhost:5000`(Make sure the port is same as what you have used to run the `docker run` command). +To use the proxy-tool in your test pass this option in cli `--test-proxies http://localhost:5000`(Make sure the port is same as what you have used to run the `docker run` command). Sample command(using storage-blob perf tests as example (Core-v1)!) -> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --test-proxy http://localhost:5000 +> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --test-proxies http://localhost:5000 -> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxy http://localhost:5000 +> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxies http://localhost:5000 -> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxy https://localhost:5001 --insecure true +> npm run perf-test:node -- StorageBlobDownloadTest --warmup 2 --duration 7 --iterations 2 --parallel 2 --test-proxies https://localhost:5001 --insecure true Sample command(using data-tables perf tests as example (Core-v2)!) -> npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxy http://localhost:5000 +> npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxies http://localhost:5000 -> npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxy https://localhost:5001 --insecure true +> npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 --test-proxies https://localhost:5001 --insecure true > npm run perf-test:node -- ListComplexEntitiesTest --duration 7 --iterations 2 --parallel 2 diff --git a/sdk/test-utils/perfstress/README.md b/sdk/test-utils/perfstress/README.md index d5cbd7d3d3cd..b9812bf4631e 100644 --- a/sdk/test-utils/perfstress/README.md +++ b/sdk/test-utils/perfstress/README.md @@ -15,7 +15,7 @@ Link to the wiki - [Writing-Performance-Tests](https://github.com/Azure/azure-sd - PerfStress tests are executed as many times as possible until the `duration` parameter is specified. This process may repeat as many `iterations` are given. Before each iteration, tests might be called for a period of time up to `warmup`, to adjust to possible runtime optimizations. In each iteration, as many as `parallel` instances of the same test are called without waiting for each other, letting the event loop decide which one is prioritized (it's not true parallelism, but it's an approximation that aligns with the design in other languages, we might improve it over time). - Each test can have a `globalSetup` method, which is called once before the process begins, a `globalCleanup` method, which is called once after the process finishes. - Each test can have a `setup` method, which is called as many times as test instances are created (up to `parallel`), and help specify local state for each test instance. A `cleanup` method is also optional, called the same amount of times, but after finishing running the tests. -- `test-proxy` url option - this option can be leveraged to avoid hitting throttling scenarios while testing the services. This option lets the requests go through the proxy server based on the url provided, we run runAsync method once in record mode to save the requests and responses in memory and then a ton of times in playback. Workflow with the test-proxy below. +- `test-proxies` url option - this option can be leveraged to avoid hitting throttling scenarios while testing the services. This option lets the requests go through proxy server(s) based on the url(s) provided, we run runAsync method once in record mode to save the requests and responses in memory and then a ton of times in playback. Workflow with the test-proxies below. ## Workflow with test proxy diff --git a/sdk/test-utils/perfstress/src/options.ts b/sdk/test-utils/perfstress/src/options.ts index 89745dbbcce5..9b3f4df4a25d 100644 --- a/sdk/test-utils/perfstress/src/options.ts +++ b/sdk/test-utils/perfstress/src/options.ts @@ -66,7 +66,7 @@ export interface DefaultPerfStressOptions { iterations: number; "no-cleanup": boolean; "milliseconds-to-log": number; - "test-proxy": string; + "test-proxies": string; insecure: boolean; } @@ -101,13 +101,13 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary this.recordAndStartPlayback(test))); @@ -301,7 +301,7 @@ export class PerfStressProgram { await this.runTest(i, Number(options.duration.value), "test"); } - if (this.tests[0].parsedOptions["test-proxy"].value) { + if (this.tests[0].parsedOptions["test-proxies"].value) { await Promise.all(this.tests.map((test) => this.stopPlayback(test))); } @@ -350,7 +350,7 @@ export class PerfStressProgram { * - delete the live resources that we have created before */ private async recordAndStartPlayback(test: PerfStressTest) { - // If test-proxy, + // If test proxy, // => then start record // => run the runAsync // => stop record diff --git a/sdk/test-utils/perfstress/src/tests.ts b/sdk/test-utils/perfstress/src/tests.ts index 4504e87a72dd..8aa9da5de7ba 100644 --- a/sdk/test-utils/perfstress/src/tests.ts +++ b/sdk/test-utils/perfstress/src/tests.ts @@ -34,10 +34,24 @@ export interface PerfStressTestConstructor { * (initializations are as many as the "parallel" command line parameter specifies). */ export abstract class PerfStressTest { + private readonly testProxy!: string; public testProxyHttpClient!: TestProxyHttpClient; public testProxyHttpClientV1!: TestProxyHttpClientV1; public abstract options: PerfStressOptionDictionary; + private static globalParallelIndex: number; + protected readonly parallelIndex: number; + + public constructor() { + this.parallelIndex = PerfStressTest.globalParallelIndex; + PerfStressTest.globalParallelIndex++; + + const testProxies = this.parsedOptions["test-proxies"].value; + if (testProxies) { + this.testProxy = testProxies.split(",")[this.parallelIndex]; + } + } + public get parsedOptions(): PerfStressOptionDictionary { // This cast is needed because TS thinks // PerfStressOptionDictionary @@ -67,9 +81,9 @@ export abstract class PerfStressTest { * Note: httpClient must be part of the options bag, it is required for the perf framework to update the underlying client properly */ public configureClientOptionsCoreV1(options: T & { httpClient?: HttpClient }): T { - if (this.parsedOptions["test-proxy"].value) { + if (this.testProxy) { this.testProxyHttpClientV1 = new TestProxyHttpClientV1( - this.parsedOptions["test-proxy"].value, + this.testProxy, this.parsedOptions["insecure"].value! ); options.httpClient = this.testProxyHttpClientV1; @@ -86,16 +100,15 @@ export abstract class PerfStressTest { * Note: Client must expose the pipeline property which is required for the perf framework to add its policies correctly */ public configureClient(client: T & { pipeline: Pipeline }): T { - const url = this.parsedOptions["test-proxy"].value; - if (url) { + if (this.testProxy) { this.testProxyHttpClient = new TestProxyHttpClient( - url, + this.testProxy, this.parsedOptions["insecure"].value! ); client.pipeline.addPolicy( testProxyHttpPolicy( this.testProxyHttpClient, - url.startsWith("https"), + this.testProxy.startsWith("https"), this.parsedOptions["insecure"].value! ) ); From 872d33c3f1d2c467735c075d6269eb73e1eced80 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:42:22 -0700 Subject: [PATCH 2/7] Split URLs on semicolon instead of comma --- sdk/test-utils/perfstress/src/options.ts | 2 +- sdk/test-utils/perfstress/src/tests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/test-utils/perfstress/src/options.ts b/sdk/test-utils/perfstress/src/options.ts index 9b3f4df4a25d..36bf2cfc82f9 100644 --- a/sdk/test-utils/perfstress/src/options.ts +++ b/sdk/test-utils/perfstress/src/options.ts @@ -102,7 +102,7 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary { const testProxies = this.parsedOptions["test-proxies"].value; if (testProxies) { - this.testProxy = testProxies.split(",")[this.parallelIndex]; + this.testProxy = testProxies.split(";")[this.parallelIndex]; } } From 410365b341d26e3c05d013b2c74cb2ad954719aa Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:49:59 -0700 Subject: [PATCH 3/7] Initialize globalParallelIndex to 0 --- sdk/test-utils/perfstress/src/tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test-utils/perfstress/src/tests.ts b/sdk/test-utils/perfstress/src/tests.ts index 4f5c624ee254..b1928d272e32 100644 --- a/sdk/test-utils/perfstress/src/tests.ts +++ b/sdk/test-utils/perfstress/src/tests.ts @@ -39,7 +39,7 @@ export abstract class PerfStressTest { public testProxyHttpClientV1!: TestProxyHttpClientV1; public abstract options: PerfStressOptionDictionary; - private static globalParallelIndex: number; + private static globalParallelIndex: number = 0; protected readonly parallelIndex: number; public constructor() { From 25573b5045d97d5695456852c6b2ccf1f85fa30a Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:56:09 -0700 Subject: [PATCH 4/7] Fix parallelIndex modulus math --- sdk/test-utils/perfstress/src/tests.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/test-utils/perfstress/src/tests.ts b/sdk/test-utils/perfstress/src/tests.ts index b1928d272e32..e102f8edfe15 100644 --- a/sdk/test-utils/perfstress/src/tests.ts +++ b/sdk/test-utils/perfstress/src/tests.ts @@ -48,7 +48,8 @@ export abstract class PerfStressTest { const testProxies = this.parsedOptions["test-proxies"].value; if (testProxies) { - this.testProxy = testProxies.split(";")[this.parallelIndex]; + const testProxyArray = testProxies.split(";"); + this.testProxy = testProxyArray[this.parallelIndex % testProxyArray.length]; } } From b3c0215b476762c91aac5a7408df567d0e54615c Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:56:58 -0700 Subject: [PATCH 5/7] Revert change to comment --- sdk/test-utils/perfstress/src/program.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/test-utils/perfstress/src/program.ts b/sdk/test-utils/perfstress/src/program.ts index 2a6f9dec21c6..fc31eec6d993 100644 --- a/sdk/test-utils/perfstress/src/program.ts +++ b/sdk/test-utils/perfstress/src/program.ts @@ -350,7 +350,7 @@ export class PerfStressProgram { * - delete the live resources that we have created before */ private async recordAndStartPlayback(test: PerfStressTest) { - // If test proxy, + // If test-proxy, // => then start record // => run the runAsync // => stop record From f2f9af21f4a83d330214572d7a2ae592ed8702ea Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:58:09 -0700 Subject: [PATCH 6/7] Rename local --- sdk/test-utils/perfstress/src/tests.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/test-utils/perfstress/src/tests.ts b/sdk/test-utils/perfstress/src/tests.ts index e102f8edfe15..fe5c97ccd11a 100644 --- a/sdk/test-utils/perfstress/src/tests.ts +++ b/sdk/test-utils/perfstress/src/tests.ts @@ -48,8 +48,8 @@ export abstract class PerfStressTest { const testProxies = this.parsedOptions["test-proxies"].value; if (testProxies) { - const testProxyArray = testProxies.split(";"); - this.testProxy = testProxyArray[this.parallelIndex % testProxyArray.length]; + const testProxiesArray = testProxies.split(";"); + this.testProxy = testProxiesArray[this.parallelIndex % testProxiesArray.length]; } } From 4c4cf5b868987d0ca7af7b1449498a63eebd9b00 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Mon, 4 Oct 2021 18:59:57 -0700 Subject: [PATCH 7/7] Add PR to changelog --- sdk/test-utils/perfstress/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/test-utils/perfstress/CHANGELOG.md b/sdk/test-utils/perfstress/CHANGELOG.md index f8e5e8a18df2..06b7266c85e9 100644 --- a/sdk/test-utils/perfstress/CHANGELOG.md +++ b/sdk/test-utils/perfstress/CHANGELOG.md @@ -5,6 +5,7 @@ ### 2021-10-05 - Support multiple test proxies + [#18031](https://github.com/Azure/azure-sdk-for-js/pull/18031) ### 2021-10-01