diff --git a/docs/api/index.md b/docs/api/index.md index 818085f43d62..21fe1f949910 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -906,10 +906,23 @@ Vitest provides a way to run all tests in random order via CLI flag [`--sequence ```ts import { describe, test } from 'vitest' +// or describe('suite', { shuffle: true }, ...) describe.shuffle('suite', () => { test('random test 1', async () => { /* ... */ }) test('random test 2', async () => { /* ... */ }) test('random test 3', async () => { /* ... */ }) + + // `shuffle` is inherited + describe('still random', () => { + test('random 4.1', async () => { /* ... */ }) + test('random 4.2', async () => { /* ... */ }) + }) + + // disable shuffle inside + describe('not random', { shuffle: false }, () => { + test('in order 5.1', async () => { /* ... */ }) + test('in order 5.2', async () => { /* ... */ }) + }) }) // order depends on sequence.seed option in config (Date.now() by default) ``` diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index f0f4eaabc1b3..ce7b2301d667 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -32,6 +32,7 @@ export async function collectTests( const testLocations = typeof spec === 'string' ? undefined : spec.testLocations const file = createFileTask(filepath, config.root, config.name, runner.pool) + file.shuffle = config.sequence.shuffle runner.onCollectStart?.(file) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 4ef1498d94c2..0972a45cc321 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -412,7 +412,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise group.type === 'suite', diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 1a568eea48f8..a59423a0b887 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -207,8 +207,7 @@ export function getRunner(): VitestRunner { function createDefaultSuite(runner: VitestRunner) { const config = runner.config.sequence - const api = config.shuffle ? suite.shuffle : suite - return api('', { concurrent: config.concurrent }, () => {}) + return suite('', { concurrent: config.concurrent }, () => {}) } export function clearCollectorContext( @@ -292,7 +291,6 @@ function createSuiteCollector( name: string, factory: SuiteFactory = () => {}, mode: RunMode, - shuffle?: boolean, each?: boolean, suiteOptions?: TestOptions, ) { @@ -331,9 +329,7 @@ function createSuiteCollector( ) { task.concurrent = true } - if (shuffle) { - task.shuffle = true - } + task.shuffle = suiteOptions?.shuffle const context = createTestContext(task, runner) // create test context @@ -425,7 +421,7 @@ function createSuiteCollector( mode, each, file: undefined!, - shuffle, + shuffle: suiteOptions?.shuffle, tasks: [], meta: Object.create(null), concurrent: suiteOptions?.concurrent, @@ -523,8 +519,10 @@ function createSuite() { const isSequentialSpecified = options.sequential || this.sequential || options.concurrent === false // inherit options from current suite - if (currentSuite?.options) { - options = { ...currentSuite.options, ...options } + options = { + ...currentSuite?.options, + ...options, + shuffle: this.shuffle ?? options.shuffle ?? currentSuite?.options?.shuffle ?? runner?.config.sequence.shuffle, } // inherit concurrent / sequential from suite @@ -537,7 +535,6 @@ function createSuite() { formatName(name), factory, mode, - this.shuffle, this.each, options, ) diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 19645f304668..3fbfbff43128 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -286,16 +286,16 @@ interface EachFunctionReturn { ( name: string | Function, fn: (...args: T) => Awaitable, - options: TestOptions + options: TestCollectorOptions ): void ( name: string | Function, fn: (...args: T) => Awaitable, - options?: number | TestOptions + options?: number | TestCollectorOptions ): void ( name: string | Function, - options: TestOptions, + options: TestCollectorOptions, fn: (...args: T) => Awaitable ): void } @@ -316,7 +316,7 @@ interface TestForFunctionReturn { ): void ( name: string | Function, - options: TestOptions, + options: TestCollectorOptions, fn: (args: Arg, context: Context) => Awaitable ): void } @@ -347,16 +347,16 @@ interface TestCollectorCallable { ( name: string | Function, fn: TestFunction, - options: TestOptions + options: TestCollectorOptions ): void ( name: string | Function, fn?: TestFunction, - options?: number | TestOptions + options?: number | TestCollectorOptions ): void ( name: string | Function, - options?: TestOptions, + options?: TestCollectorOptions, fn?: TestFunction ): void } @@ -370,6 +370,8 @@ type ChainableTestAPI = ChainableFunction< } > +type TestCollectorOptions = Omit + export interface TestOptions { /** * Test timeout. @@ -399,6 +401,10 @@ export interface TestOptions { * Tests inherit `sequential` from `describe()` and nested `describe()` will inherit from parent's `sequential`. */ sequential?: boolean + /** + * Whether the tasks of the suite run in a random order. + */ + shuffle?: boolean /** * Whether the test should be skipped. */ diff --git a/test/config/fixtures/shuffle/basic.test.ts b/test/config/fixtures/shuffle/basic.test.ts new file mode 100644 index 000000000000..673479ea80cf --- /dev/null +++ b/test/config/fixtures/shuffle/basic.test.ts @@ -0,0 +1,27 @@ +import { afterAll, describe, expect, test } from 'vitest' + +const numbers: number[] = [] + +test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(10 + v) +}) + +describe("inherit shuffle", () => { + test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(20 + v) + }) +}) + +describe('unshuffle', { shuffle: false }, () => { + test.for([1, 2, 3, 4, 5])('test %s', (v) => { + numbers.push(30 + v) + }) +}) + +afterAll(() => { + expect(numbers).toEqual([ + 11, 14, 13, 15, 12, + 31, 32, 33, 34, 35, + 21, 24, 23, 25, 22 + ]) +}) diff --git a/test/config/fixtures/shuffle/vitest.config.ts b/test/config/fixtures/shuffle/vitest.config.ts new file mode 100644 index 000000000000..4f7af402e973 --- /dev/null +++ b/test/config/fixtures/shuffle/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + sequence: { + seed: 101, + shuffle: true, + } + } +}) diff --git a/test/config/test/shuffle-options.test.ts b/test/config/test/shuffle-options.test.ts index 63e5a12c7b10..f9659f04819e 100644 --- a/test/config/test/shuffle-options.test.ts +++ b/test/config/test/shuffle-options.test.ts @@ -47,3 +47,18 @@ test.each([ const { ctx } = await run({ shuffle, sequencer: CustomSequencer }) expect(ctx?.config.sequence.sequencer.name).toBe('CustomSequencer') }) + +test('shuffle', async () => { + const { stderr, ctx } = await runVitest({ + root: './fixtures/shuffle', + }) + expect(stderr).toBe('') + expect(ctx?.state.getFiles().map(f => [f.name, f.result?.state])).toMatchInlineSnapshot(` + [ + [ + "basic.test.ts", + "pass", + ], + ] + `) +}) diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts index b4f32f44cece..90e46b251c9b 100644 --- a/test/core/test/random.test.ts +++ b/test/core/test/random.test.ts @@ -1,18 +1,31 @@ -import { afterAll, describe, expect, test } from 'vitest' +import { describe, expect, test } from 'vitest' // tests use seed of 101, so they have deterministic random order const numbers: number[] = [] describe.shuffle('random tests', () => { - describe('inside', () => { - // shuffle is not inherited from parent - + describe('suite unshuffle', { shuffle: false }, () => { test('inside 1', () => { numbers.push(1) }) + test('inside 1.5', () => { + numbers.push(1.5) + }) test('inside 2', () => { numbers.push(2) }) + + describe('suite shuffle', { shuffle: true }, () => { + test('inside 2.1', () => { + numbers.push(2.1) + }) + test('inside 2.2', () => { + numbers.push(2.2) + }) + test('inside 2.3', () => { + numbers.push(2.3) + }) + }) }) test('test 1', () => { @@ -24,8 +37,20 @@ describe.shuffle('random tests', () => { test('test 3', () => { numbers.push(5) }) +}) - afterAll(() => { - expect(numbers).toStrictEqual([4, 5, 3, 1, 2]) - }) +test('assert', () => { + expect(numbers).toMatchInlineSnapshot(` + [ + 4, + 5, + 3, + 1, + 1.5, + 2, + 2.2, + 2.3, + 2.1, + ] + `) })