[Perf Tests] Updates to perf test framework

sdk/storage/storage-blob/package.json

@@ -52,6 +52,7 @@
     "lint:fix": "eslint package.json api-extractor.json src test --ext .ts --fix",
     "lint": "eslint package.json api-extractor.json src test --ext .ts -f html -o storage-blob-lintReport.html || exit 0",
     "pack": "npm pack 2>&1",
+    "perfstress-test:node": "npm run build && cross-env TS_NODE_COMPILER_OPTIONS=\"{\\\"module\\\": \\\"commonjs\\\"}\" ts-node test/perfstress/track-2/index.spec.ts",
     "prebuild": "npm run clean",
     "test:browser": "npm run clean && npm run build:test && npm run unit-test:browser",
     "test:node": "npm run clean && npm run build:test && npm run unit-test:node",
@@ -132,6 +133,7 @@
     "@azure/eslint-plugin-azure-sdk": "^3.0.0",
     "@azure/identity": "^1.1.0",
     "@azure/test-utils-recorder": "^1.0.0",
+    "@azure/test-utils-perfstress": "^1.0.0",
     "@microsoft/api-extractor": "7.7.11",
     "@rollup/plugin-multi-entry": "^3.0.0",
     "@rollup/plugin-replace": "^2.2.0",

sdk/storage/storage-blob/test/perfstress/track-2/download.spec.ts

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import {
+  PerfStressTest,
+  PerfStressOptionDictionary,
+  DefaultPerfStressOptions
+} from "@azure/test-utils-perfstress";
+
+import { BlobServiceClient, StorageSharedKeyCredential } from "../../../src";
+
+// Expects the .env file at the same level as the "test" folder
+import * as dotenv from "dotenv";
+dotenv.config();
+
+interface StorageBlobDownloadTestOptions extends DefaultPerfStressOptions {
+  size: number;
+}
+
+const account = process.env.ACCOUNT_NAME || "";
+const accountKey = process.env.ACCOUNT_KEY || "";
+
+const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey);
+
+const blobServiceClient = new BlobServiceClient(
+  // When using AnonymousCredential, following url should include a valid SAS or support public access
+  `https://${account}.blob.core.windows.net`,
+  sharedKeyCredential
+);
+const containerName = `newcontainer${new Date().getTime()}`;
+const blobName = `newblob${new Date().getTime()}`;
+const containerClient = blobServiceClient.getContainerClient(containerName);
+const blockBlobClient = blobServiceClient
+  .getContainerClient(containerName)
+  .getBlockBlobClient(blobName);
+
+export class StorageBlobDownloadTest extends PerfStressTest<StorageBlobDownloadTestOptions> {
+  public options: PerfStressOptionDictionary<StorageBlobDownloadTestOptions> = {
+    size: {
+      required: true,
+      description: "Required option",
+      shortName: "sz",
+      longName: "size",
+      defaultValue: 10
+    }
+  };
+
+  public async globalSetup() {
+    const createContainerResponse = await containerClient.create();
+    console.log(
+      `Create container ${containerName} successfully`,
+      createContainerResponse.requestId
+    );
+
+    // Create a blob
+    const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+    const uploadBlobResponse = await blockBlobClient.upload(
+      Buffer.alloc(this.options.size.value!),
+      this.options.size.value as number
+    );
+    console.log(`Upload block blob ${blobName} successfully`, uploadBlobResponse.requestId);
+  }
+
+  async runAsync(): Promise<void> {
+    await blockBlobClient.download(0);
+  }
+}

sdk/storage/storage-blob/test/perfstress/track-2/index.spec.ts

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { PerfStressProgram, selectPerfStressTest } from "@azure/test-utils-perfstress";
+import { StorageBlobDownloadTest } from "./download.spec";
+
+console.log("=== Starting the perfStress test ===");
+
+const perfStressProgram = new PerfStressProgram(selectPerfStressTest([StorageBlobDownloadTest]));
+
+perfStressProgram.run();

sdk/test-utils/perfstress/package.json

@@ -2,7 +2,7 @@
   "name": "@azure/test-utils-perfstress",
   "version": "1.0.0",
   "description": "Performance and stress test framework for the Azure SDK for JavaScript and TypeScript",
-  "main": "dist/index.js",
+  "main": "dist-esm/src/index.js",
   "module": "dist-esm/src/index.js",
   "browser": {},
   "types": "./typings/src/index.d.ts",
@@ -24,7 +24,7 @@
     "lint": "eslint package.json src test --ext .ts -f html -o perfstress-lintReport.html || exit 0",
     "pack": "npm pack 2>&1",
     "prebuild": "npm run clean",
-    "perfstress-test:node": "rushx build && node dist-esm/test/index.spec.js",
+    "perfstress-test:node": "npm run build && node dist-esm/test/index.spec.js",
     "unit-test:browser": "echo skipped",
     "unit-test:node": "echo skipped",
     "unit-test": "npm run unit-test:node && npm run unit-test:browser",

sdk/test-utils/perfstress/src/options.ts

@@ -3,15 +3,10 @@
 
 import { default as minimist, ParsedArgs as MinimistParsedArgs } from "minimist";
 
-/**
- * Possible values for each PerfStress option
- */
-export type PerfStressOptionValue = string | number | boolean | undefined;
-
 /**
  * The structure of a PerfStress option. They represent command line parameters.
  */
-export interface PerfStressOption {
+export interface OptionDetails<TType> {
   /**
    * Whether the option is required or not.
    */
@@ -30,49 +25,59 @@ export interface PerfStressOption {
   longName?: string;
   /**
    * The default value that is going to be assigned to the option.
+   * Expected: string | number | boolean | undefined
    */
-  defaultValue?: PerfStressOptionValue;
+  defaultValue?: TType;
   /**
    * The value specified by the user from the command line after either the shortName or the longName.
    * If no value was specified, the defaultValue will be used.
    * If the shortName or longName was specified, but no value was provided, "true" will be set.
    */
-  value?: PerfStressOptionValue;
+  value?: TType;
   /**
    * The description of each option. Descriptions of the assigned options will be shown at the beginning of the test call.
    * Descriptions of all the available options will be shown if the user sends either --help or -h.
    */
   description: string;
 }
 
+// TODO: Add docs
+type PerfStressOptionDictionaryOptional<TOptions extends DefaultPerfStressOptions> = {
+  [longName in keyof TOptions]?: OptionDetails<TOptions[longName]>;
+};
+
 /**
  * A group of options is called PerfStressOptionDictionary,
  * and is shaped as a plain object to make it easier to access them.
+ * // TODO: Update docs
  *
  * TNames defines the names of the options. This is necessary to allow TypeScript to suggest the appropriate names
  * for the options.
  */
-export type PerfStressOptionDictionary<TNames extends string> = {
-  [longName in TNames]: PerfStressOption;
-};
+export type PerfStressOptionDictionary<
+  TOptions extends DefaultPerfStressOptions
+> = PerfStressOptionDictionaryOptional<TOptions> &
+  Pick<
+    Required<PerfStressOptionDictionaryOptional<TOptions>>,
+    keyof Omit<TOptions, keyof DefaultPerfStressOptions>
+  >;
 
-/**
- * These are the default options longNames.
- */
-export type DefaultPerfStressOptionNames =
-  | "help"
-  | "parallel"
-  | "duration"
-  | "warmup"
-  | "iterations"
-  | "no-cleanup"
-  | "milliseconds-to-log"
-  | "sync";
+// TODO: Add docs
+export interface DefaultPerfStressOptions {
+  help: string;
+  parallel: number;
+  duration: number;
+  warmup: number;
+  iterations: number;
+  "no-cleanup": boolean;
+  "milliseconds-to-log": number;
+  sync: boolean;
+}
 
 /**
  * These are the default options in full.
  */
-export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStressOptionNames> = {
+export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStressOptions> = {
   help: {
     description: "Shows all of the available options",
     shortName: "h"
@@ -114,14 +119,17 @@ export const defaultPerfStressOptions: PerfStressOptionDictionary<DefaultPerfStr
 /**
  * Parses the given options by extracting their values through `minimist`, or setting the default value defined in each option.
  * It also overwrites any present longName with the property name of each option.
+ *
+ * // TODO: parse to the expected type
+ * // TODO: Change T to something else
  * @param options A dictionary of options to parse using minimist.
  * @returns A new options dictionary.
  */
-export function parsePerfStressOption(
-  options: PerfStressOptionDictionary<string>
-): PerfStressOptionDictionary<string> {
+export function parsePerfStressOption<
+  T extends PerfStressOptionDictionary<DefaultPerfStressOptions> | {}
+>(options: T): Required<T> {
   const minimistResult: MinimistParsedArgs = minimist(process.argv);
-  const result: PerfStressOptionDictionary<string> = {};
+  const result = {};
 
   for (const longName of Object.keys(options)) {
     const option = (options as any)[longName];
@@ -139,5 +147,5 @@ export function parsePerfStressOption(
     };
   }
 
-  return result;
+  return result as any;
 }

sdk/test-utils/perfstress/src/program.ts

@@ -7,7 +7,7 @@ import {
   PerfStressOptionDictionary,
   parsePerfStressOption,
   defaultPerfStressOptions,
-  DefaultPerfStressOptionNames
+  DefaultPerfStressOptions
 } from "./options";
 import { PerfStressParallel } from "./parallel";
 
@@ -35,9 +35,10 @@ export type TestType = "";
  */
 export class PerfStressProgram {
   private testName: string;
-  private options: PerfStressOptionDictionary<DefaultPerfStressOptionNames>;
+  // TODO: Re-evaluate "Required" for options (maybe come up with parsedOptions)
+  private options: Required<PerfStressOptionDictionary<DefaultPerfStressOptions>>;
   private parallelNumber: number;
-  private tests: PerfStressTest<string>[];
+  private tests: PerfStressTest<DefaultPerfStressOptions>[];
 
   /**
    * Receives a test class to instantiate and execute.
@@ -47,13 +48,13 @@ export class PerfStressProgram {
    *
    * @param testClass The testClass to be instantiated.
    */
-  constructor(testClass: PerfStressTestConstructor<string>) {
+  constructor(testClass: PerfStressTestConstructor<DefaultPerfStressOptions>) {
     this.testName = testClass.name;
     this.options = parsePerfStressOption(defaultPerfStressOptions);
     this.parallelNumber = Number(this.options.parallel.value);
 
     console.log(`=== Creating ${this.parallelNumber} instance(s) of ${this.testName} ===`);
-    this.tests = new Array<PerfStressTest<string>>(this.parallelNumber);
+    this.tests = new Array<PerfStressTest<DefaultPerfStressOptions>>(this.parallelNumber);
 
     for (let i = 0; i < this.parallelNumber; i++) {
       const test = new testClass();
@@ -108,7 +109,7 @@ export class PerfStressProgram {
    * @param abortController Allows us to send through a signal determining when to abort any execution.
    */
   private runLoopSync(
-    test: PerfStressTest<string>,
+    test: PerfStressTest<DefaultPerfStressOptions>,
     parallel: PerfStressParallel,
     durationMilliseconds: number,
     abortController: AbortController
@@ -149,7 +150,7 @@ export class PerfStressProgram {
    * @param abortController Allows us to send through a signal determining when to abort any execution.
    */
   private async runLoopAsync(
-    test: PerfStressTest<string>,
+    test: PerfStressTest<DefaultPerfStressOptions>,
     parallel: PerfStressParallel,
     durationMilliseconds: number,
     abortController: AbortController

sdk/test-utils/perfstress/src/tests.ts

@@ -3,13 +3,17 @@
 
 import { AbortSignalLike } from "@azure/abort-controller";
 import { default as minimist, ParsedArgs as MinimistParsedArgs } from "minimist";
-import { PerfStressOptionDictionary, parsePerfStressOption } from "./options";
+import {
+  PerfStressOptionDictionary,
+  parsePerfStressOption,
+  DefaultPerfStressOptions
+} from "./options";
 
 /**
  * Defines the behavior of the PerfStressTest constructor, to use the class as a value.
  */
-export interface PerfStressTestConstructor<TOptionsNames extends string> {
-  new (): PerfStressTest<TOptionsNames>;
+export interface PerfStressTestConstructor<TOptions extends DefaultPerfStressOptions> {
+  new (): PerfStressTest<TOptions>;
 }
 
 /**
@@ -21,11 +25,13 @@ export interface PerfStressTestConstructor<TOptionsNames extends string> {
  * and at a local level, which happens once for each initialization of the test class
  * (initializations are as many as the "parallel" command line parameter specifies).
  */
-export abstract class PerfStressTest<TOptionsNames extends string> {
-  public abstract options: PerfStressOptionDictionary<TOptionsNames>;
+export abstract class PerfStressTest<TOptions extends DefaultPerfStressOptions> {
+  public abstract options: PerfStressOptionDictionary<TOptions>;
 
   public parseOptions() {
-    this.options = parsePerfStressOption(this.options) as PerfStressOptionDictionary<TOptionsNames>;
+    // TODO: remove this casting
+    // TODO: remove the ! workarounds in the tests
+    this.options = parsePerfStressOption(this.options) as PerfStressOptionDictionary<TOptions>;
   }
 
   // Before and after running a bunch of the same test.
@@ -45,8 +51,8 @@ export abstract class PerfStressTest<TOptionsNames extends string> {
  * @param tests An array of classes that extend PerfStressTest
  */
 export function selectPerfStressTest(
-  tests: PerfStressTestConstructor<string>[]
-): PerfStressTestConstructor<string> {
+  tests: PerfStressTestConstructor<DefaultPerfStressOptions>[]
+): PerfStressTestConstructor<DefaultPerfStressOptions> {
   const testsNames: string[] = tests.map((test) => test.name);
   const minimistResult: MinimistParsedArgs = minimist(process.argv);
   const testName = minimistResult._[minimistResult._.length - 1];

sdk/test-utils/perfstress/test/delay.spec.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { PerfStressTest } from "../src";
+import { DefaultPerfStressOptions, PerfStressTest } from "../src";
 import { delay } from "@azure/core-http";
 
 /**
@@ -32,7 +32,7 @@ import { delay } from "@azure/core-http";
  * Completed 8 operations in a weighted-average of 4.00s (2.00 ops/s 0.501 s/op)
  * ```
  */
-export class Delay500ms extends PerfStressTest<string> {
+export class Delay500ms extends PerfStressTest<DefaultPerfStressOptions> {
   /**
    * This test doesn't receive command line parameters.
    */

sdk/test-utils/perfstress/test/exception.spec.ts

@@ -1,14 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-import { PerfStressTest } from "../src";
+import { DefaultPerfStressOptions, PerfStressTest } from "../src";
 
 /**
  * Exception is designed to test the response speed of the PerfStress test framework
  * If the option "sync" is passed, errors will be thrown on every test call, where the test being called is simple function.
  * Otherwise, errors thrown on every test call, where the test being called is a function returning a promise (an asynchronous function).
  */
-export class Exception extends PerfStressTest<string> {
+export class Exception extends PerfStressTest<DefaultPerfStressOptions> {
   public options = {};
 
   run(): void {