Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/test meta remarks #4732

Open
wants to merge 6 commits into
base: 3.x
Choose a base branch
from
+420 −297
Open

Feat/test meta remarks #4732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9aa62fe
3.7.0-beta.1
Jan 8, 2025
b88077e
Merge branch '3.x' of github.com:codeceptjs/CodeceptJS into 3.x
Jan 8, 2025
c561dc9
added notes for test object, use notes for heal reports
Jan 8, 2025
6edc423
Merge branch '3.x' of github.com:codeceptjs/CodeceptJS into 3.x
Jan 10, 2025
b62dd35
added notes & metadata for tests, improved output
Jan 10, 2025
9075ba8
Merge branch '3.x' into feat/test-meta-remarks
Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 88 additions & 95 deletions docs/ai.md
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion lib/command/run-workers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ module.exports = async function (workerCount, selectedRuns, options) {
output.print(`CodeceptJS v${require('../codecept').version()} ${output.standWithUkraine()}`)
output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`)
output.print()
store.hasWorkers = true

const workers = new Workers(numberOfWorkers, config)
workers.overrideConfig(overrideConfigs)
Expand Down Expand Up @@ -100,7 +101,6 @@ module.exports = async function (workerCount, selectedRuns, options) {
if (options.verbose || options.debug) store.debugMode = true

if (options.verbose) {
global.debugMode = true
const { getMachineInfo } = require('./info')
await getMachineInfo()
}
Expand Down
221 changes: 112 additions & 109 deletions lib/command/workers/runTests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,120 +1,120 @@
const tty = require('tty');
const tty = require('tty')

if (!tty.getWindowSize) {
// this is really old method, long removed from Node, but Mocha
// reporters fall back on it if they cannot use `process.stdout.getWindowSize`
// we need to polyfill it.
tty.getWindowSize = () => [40, 80];
tty.getWindowSize = () => [40, 80]
}

const { parentPort, workerData } = require('worker_threads');
const event = require('../../event');
const container = require('../../container');
const { getConfig } = require('../utils');
const { tryOrDefault, deepMerge } = require('../../utils');
const { parentPort, workerData } = require('worker_threads')
const event = require('../../event')
const container = require('../../container')
const { getConfig } = require('../utils')
const { tryOrDefault, deepMerge } = require('../../utils')

let stdout = '';
let stdout = ''

const stderr = '';
const stderr = ''

// Requiring of Codecept need to be after tty.getWindowSize is available.
const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept');
const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept')

const { options, tests, testRoot, workerIndex } = workerData;
const { options, tests, testRoot, workerIndex } = workerData

// hide worker output
if (!options.debug && !options.verbose)
process.stdout.write = string => {
stdout += string;
return true;
};
stdout += string
return true
}

const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {});
const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {})

// important deep merge so dynamic things e.g. functions on config are not overridden
const config = deepMerge(getConfig(options.config || testRoot), overrideConfigs);
const config = deepMerge(getConfig(options.config || testRoot), overrideConfigs)

// Load test and run
const codecept = new Codecept(config, options);
codecept.init(testRoot);
codecept.loadTests();
const mocha = container.mocha();
filterTests();
const codecept = new Codecept(config, options)
codecept.init(testRoot)
codecept.loadTests()
const mocha = container.mocha()
filterTests()

// run tests
(async function () {
;(async function () {
if (mocha.suite.total()) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this code format adds this unnecessarily

await runTests();
await runTests()
}
})();
})()

async function runTests() {
try {
await codecept.bootstrap();
await codecept.bootstrap()
} catch (err) {
throw new Error(`Error while running bootstrap file :${err}`);
throw new Error(`Error while running bootstrap file :${err}`)
}
listenToParentThread();
initializeListeners();
disablePause();
listenToParentThread()
initializeListeners()
disablePause()
try {
await codecept.run();
await codecept.run()
} finally {
await codecept.teardown();
await codecept.teardown()
}
}

function filterTests() {
const files = codecept.testFiles;
mocha.files = files;
mocha.loadFiles();
const files = codecept.testFiles
mocha.files = files
mocha.loadFiles()

for (const suite of mocha.suite.suites) {
suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0);
suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0)
}
}

function initializeListeners() {
function simplifyError(error) {
if (error) {
const { stack, uncaught, message, actual, expected } = error;
const { stack, uncaught, message, actual, expected } = error

return {
stack,
uncaught,
message,
actual,
expected,
};
}
}

return null;
return null
}
function simplifyTest(test, err = null) {
test = { ...test };
test = { ...test }

if (test.start && !test.duration) {
const end = new Date();
test.duration = end - test.start;
const end = new Date()
test.duration = end - test.start
}

if (test.err) {
err = simplifyError(test.err);
test.status = 'failed';
err = simplifyError(test.err)
test.status = 'failed'
} else if (err) {
err = simplifyError(err);
test.status = 'failed';
err = simplifyError(err)
test.status = 'failed'
}
const parent = {};
const parent = {}
if (test.parent) {
parent.title = test.parent.title;
parent.title = test.parent.title
}

if (test.opts) {
Object.keys(test.opts).forEach(k => {
if (typeof test.opts[k] === 'object') delete test.opts[k];
if (typeof test.opts[k] === 'function') delete test.opts[k];
});
if (typeof test.opts[k] === 'object') delete test.opts[k]
if (typeof test.opts[k] === 'function') delete test.opts[k]
})
}

return {
Expand All @@ -125,30 +125,33 @@ function initializeListeners() {
retries: test._retries,
title: test.title,
status: test.status,
notes: test.notes || [],
meta: test.meta || {},
artifacts: test.artifacts || [],
duration: test.duration || 0,
err,
parent,
steps: test.steps && test.steps.length > 0 ? simplifyStepsInTestObject(test.steps, err) : [],
};
}
}

function simplifyStepsInTestObject(steps, err) {
steps = [...steps];
const _steps = [];
steps = [...steps]
const _steps = []

for (step of steps) {
const _args = [];
const _args = []

if (step.args) {
for (const arg of step.args) {
// check if arg is a JOI object
if (arg && arg.$_root) {
_args.push(JSON.stringify(arg).slice(0, 300));
_args.push(JSON.stringify(arg).slice(0, 300))
// check if arg is a function
} else if (arg && typeof arg === 'function') {
_args.push(arg.name);
_args.push(arg.name)
} else {
_args.push(arg);
_args.push(arg)
}
}
}
Expand All @@ -164,38 +167,38 @@ function initializeListeners() {
finishedAt: step.finishedAt,
duration: step.duration,
err,
});
})
}

return _steps;
return _steps
}

function simplifyStep(step, err = null) {
step = { ...step };
step = { ...step }

if (step.startTime && !step.duration) {
const end = new Date();
step.duration = end - step.startTime;
const end = new Date()
step.duration = end - step.startTime
}

if (step.err) {
err = simplifyError(step.err);
step.status = 'failed';
err = simplifyError(step.err)
step.status = 'failed'
} else if (err) {
err = simplifyError(err);
step.status = 'failed';
err = simplifyError(err)
step.status = 'failed'
}

const parent = {};
const parent = {}
if (step.metaStep) {
parent.title = step.metaStep.actor;
parent.title = step.metaStep.actor
}

if (step.opts) {
Object.keys(step.opts).forEach(k => {
if (typeof step.opts[k] === 'object') delete step.opts[k];
if (typeof step.opts[k] === 'function') delete step.opts[k];
});
if (typeof step.opts[k] === 'object') delete step.opts[k]
if (typeof step.opts[k] === 'function') delete step.opts[k]
})
}

return {
Expand All @@ -207,43 +210,43 @@ function initializeListeners() {
err,
parent,
test: simplifyTest(step.test),
};
}
}

collectStats();
collectStats()
// suite
event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: simplifyTest(suite) }));
event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: simplifyTest(suite) }));
event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: simplifyTest(suite) }))
event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: simplifyTest(suite) }))

// calculate duration
event.dispatcher.on(event.test.started, test => (test.start = new Date()));
event.dispatcher.on(event.test.started, test => (test.start = new Date()))

// tests
event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: simplifyTest(test) }));
event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: simplifyTest(test) }));
event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: simplifyTest(test) }))
event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: simplifyTest(test) }))
// we should force-send correct errors to prevent race condition
event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: simplifyTest(test, err) }));
event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: simplifyTest(test, err) }));
event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: simplifyTest(test, err) }));
event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: simplifyTest(test) }));
event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: simplifyTest(test) }));
event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: simplifyTest(test, err) }))
event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: simplifyTest(test, err) }))
event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: simplifyTest(test, err) }))
event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: simplifyTest(test) }))
event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: simplifyTest(test) }))

// steps
event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: simplifyStep(step) }));
event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: simplifyStep(step) }));
event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: simplifyStep(step) }));
event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: simplifyStep(step) }));
event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: simplifyStep(step) }))
event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: simplifyStep(step) }))
event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: simplifyStep(step) }))
event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: simplifyStep(step) }))

event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: simplifyTest(test, err) }));
event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: simplifyTest(test, err) }));
event.dispatcher.on(event.all.failures, data => sendToParentThread({ event: event.all.failures, workerIndex, data }));
event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: simplifyTest(test, err) }))
event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: simplifyTest(test, err) }))
event.dispatcher.on(event.all.failures, data => sendToParentThread({ event: event.all.failures, workerIndex, data }))

// all
event.dispatcher.once(event.all.result, () => parentPort.close());
event.dispatcher.once(event.all.result, () => parentPort.close())
}

function disablePause() {
global.pause = () => {};
global.pause = () => {}
}

function collectStats() {
Expand All @@ -253,36 +256,36 @@ function collectStats() {
skipped: 0,
tests: 0,
pending: 0,
};
}
event.dispatcher.on(event.test.skipped, () => {
stats.skipped++;
});
stats.skipped++
})
event.dispatcher.on(event.test.passed, () => {
stats.passes++;
});
stats.passes++
})
event.dispatcher.on(event.test.failed, test => {
if (test.ctx._runnable.title.includes('hook: AfterSuite')) {
stats.failedHooks += 1;
stats.failedHooks += 1
}
stats.failures++;
});
stats.failures++
})
event.dispatcher.on(event.test.skipped, () => {
stats.pending++;
});
stats.pending++
})
event.dispatcher.on(event.test.finished, () => {
stats.tests++;
});
stats.tests++
})
event.dispatcher.once(event.all.after, () => {
sendToParentThread({ event: event.all.after, data: stats });
});
sendToParentThread({ event: event.all.after, data: stats })
})
}

function sendToParentThread(data) {
parentPort.postMessage(data);
parentPort.postMessage(data)
}

function listenToParentThread() {
parentPort.on('message', eventData => {
container.append({ support: eventData.data });
});
container.append({ support: eventData.data })
})
}
Loading
Loading