Schedule previously failing tests to run first

Fixes #1546.

Co-authored-by: Mark Wubben <mark@novemberborn.net>

lib/api.js

@@ -18,6 +18,7 @@ const fork = require('./fork');
 const serializeError = require('./serialize-error');
 const {getApplicableLineNumbers} = require('./line-numbers');
 const sharedWorkers = require('./plugin-support/shared-workers');
+const scheduler = require('./scheduler');
 
 function resolveModules(modules) {
 	return arrify(modules).map(name => {
@@ -142,6 +143,8 @@ class Api extends Emittery {
 				runStatus = new RunStatus(selectedFiles.length, null);
 			}
 
+			selectedFiles = scheduler.failingTestsFirst(selectedFiles, this._getLocalCacheDir(), this.options.cacheEnabled);
+
 			const debugWithoutSpecificFile = Boolean(this.options.debug) && !this.options.debug.active && selectedFiles.length !== 1;
 
 			await this.emit('run', {
@@ -243,6 +246,7 @@ class Api extends Emittery {
 
 			// Allow shared workers to clean up before the run ends.
 			await Promise.all(deregisteredSharedWorkers);
+			scheduler.storeFailedTestFiles(runStatus, this.options.cacheEnabled === false ? null : this._createCacheDir());
 		} catch (error) {
 			if (error && error.name === 'AggregateError') {
 				for (const error_ of error) {
@@ -257,14 +261,18 @@ class Api extends Emittery {
 		return runStatus;
 	}
 
+	_getLocalCacheDir() {
+		return path.join(this.options.projectDir, 'node_modules', '.cache', 'ava');
+	}
+
 	_createCacheDir() {
 		if (this._cacheDir) {
 			return this._cacheDir;
 		}
 
 		const cacheDir = this.options.cacheEnabled === false ?
 			fs.mkdtempSync(`${tempDir}${path.sep}`) :
-			path.join(this.options.projectDir, 'node_modules', '.cache', 'ava');
+			this._getLocalCacheDir();
 
 		// Ensure cacheDir exists
 		fs.mkdirSync(cacheDir, {recursive: true});

lib/run-status.js

@@ -194,6 +194,10 @@ class RunStatus extends Emittery {
 			this.pendingTests.get(event.testFile).delete(event.title);
 		}
 	}
+
+	getFailedTestFiles() {
+		return [...this.stats.byFile].filter(statByFile => statByFile[1].failedTests).map(statByFile => statByFile[0]);
+	}
 }
 
 module.exports = RunStatus;

lib/scheduler.js

@@ -0,0 +1,45 @@
+const fs = require('fs');
+const path = require('path');
+const writeFileAtomic = require('write-file-atomic');
+const isCi = require('./is-ci');
+
+const FILE_NAME_FAILING_TEST = 'failing-test.json';
+
+module.exports.storeFailedTestFiles = (runStatus, cacheDir) => {
+	if (isCi || !cacheDir) {
+		return;
+	}
+
+	writeFileAtomic(path.join(cacheDir, FILE_NAME_FAILING_TEST), JSON.stringify(runStatus.getFailedTestFiles()));
+};
+
+// Order test-files, so that files with failing tests come first
+module.exports.failingTestsFirst = (selectedFiles, cacheDir, cacheEnabled) => {
+	if (isCi || cacheEnabled === false) {
+		return selectedFiles;
+	}
+
+	const filePath = path.join(cacheDir, FILE_NAME_FAILING_TEST);
+	let failedTestFiles;
+	try {
+		failedTestFiles = JSON.parse(fs.readFileSync(filePath));
+	} catch {
+		return selectedFiles;
+	}
+
+	return [...selectedFiles].sort((f, s) => {
+		if (failedTestFiles.some(tf => tf === f) && failedTestFiles.some(tf => tf === s)) {
+			return 0;
+		}
+
+		if (failedTestFiles.some(tf => tf === f)) {
+			return -1;
+		}
+
+		if (failedTestFiles.some(tf => tf === s)) {
+			return 1;
+		}
+
+		return 0;
+	});
+};

test-tap/helper/report.js

@@ -108,7 +108,7 @@ const run = (type, reporter, {match = [], filter} = {}) => {
 		failWithoutAssertions: false,
 		serial: type === 'failFast' || type === 'failFast2',
 		require: [],
-		cacheEnabled: true,
+		cacheEnabled: false,
 		experiments: {},
 		match,
 		providers,

test/helpers/exec.js

@@ -119,6 +119,7 @@ exports.fixture = async (args, options = {}) => {
 				const statObject = {title, file: normalizePath(cwd, testFile)};
 				errors.set(statObject, statusEvent.err);
 				stats.failed.push(statObject);
+				logs.set(statObject, statusEvent.logs);
 				break;
 			}
 

test/scheduler/fixtures/1pass.js

@@ -0,0 +1,6 @@
+const test = require('ava');
+
+test('pass', t => {
+	t.log(Date.now());
+	t.pass();
+});

test/scheduler/fixtures/2fail.js

@@ -0,0 +1,6 @@
+const test = require('ava');
+
+test('fail', t => {
+	t.log(Date.now());
+	t.fail();
+});

test/scheduler/fixtures/disabled-cache.cjs

@@ -0,0 +1,6 @@
+module.exports = {
+	files: [
+		"*.js"
+	],
+	cache: false
+};

test/scheduler/fixtures/package.json

@@ -0,0 +1,7 @@
+{
+  "ava": {
+    "files": [
+      "*.js"
+    ]
+  }
+}

test/scheduler/test.js

@@ -0,0 +1,61 @@
+const test = require('@ava/test');
+const exec = require('../helpers/exec');
+
+const options = {
+	// The scheduler only works when not in CI, so trick it into believing it is
+	// not in CI even when it's being tested by AVA's CI.
+	env: {AVA_FORCE_CI: 'not-ci'}
+};
+
+function getTimestamps(stats) {
+	return {passed: BigInt(stats.getLogs(stats.passed[0])), failed: BigInt(stats.getLogs(stats.failed[0]))};
+}
+
+test.serial('failing tests come first', async t => {
+	try {
+		await exec.fixture(['1pass.js', '2fail.js'], options);
+	} catch {}
+
+	try {
+		await exec.fixture(['-t', '--concurrency=1', '1pass.js', '2fail.js'], options);
+	} catch (error) {
+		const timestamps = getTimestamps(error.stats);
+		t.true(timestamps.failed < timestamps.passed);
+	}
+});
+
+test.serial('scheduler disabled when cache empty', async t => {
+	await exec.fixture(['reset-cache'], options); // `ava reset-cache` resets the cache but does not run tests.
+	try {
+		await exec.fixture(['-t', '--concurrency=1', '1pass.js', '2fail.js'], options);
+	} catch (error) {
+		const timestamps = getTimestamps(error.stats);
+		t.true(timestamps.passed < timestamps.failed);
+	}
+});
+
+test.serial('scheduler disabled when cache disabled', async t => {
+	try {
+		await exec.fixture(['1pass.js', '2fail.js'], options);
+	} catch {}
+
+	try {
+		await exec.fixture(['-t', '--concurrency=1', '--config', 'disabled-cache.cjs', '1pass.js', '2fail.js'], options);
+	} catch (error) {
+		const timestamps = getTimestamps(error.stats);
+		t.true(timestamps.passed < timestamps.failed);
+	}
+});
+
+test.serial('scheduler disabled in CI', async t => {
+	try {
+		await exec.fixture(['1pass.js', '2fail.js'], {env: {AVA_FORCE_CI: 'ci'}});
+	} catch {}
+
+	try {
+		await exec.fixture(['-t', '--concurrency=1', '--config', 'disabled-cache.cjs', '1pass.js', '2fail.js'], options);
+	} catch (error) {
+		const timestamps = getTimestamps(error.stats);
+		t.true(timestamps.passed < timestamps.failed);
+	}
+});