From 9aa62feab39904b9a8c6c0729c9e86750fef6f83 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Wed, 8 Jan 2025 03:30:49 +0200 Subject: [PATCH 1/4] 3.7.0-beta.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8fff055b3..b7816d11d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeceptjs", - "version": "3.6.10", + "version": "3.7.0-beta.1", "description": "Supercharged End 2 End Testing Framework for NodeJS", "keywords": [ "acceptance", From c561dc9c888845acdcac90dd64bb5bece6321a4e Mon Sep 17 00:00:00 2001 From: DavertMik Date: Wed, 8 Jan 2025 05:50:45 +0200 Subject: [PATCH 2/4] added notes for test object, use notes for heal reports --- lib/command/workers/runTests.js | 221 ++++++++++++++++---------------- lib/heal.js | 9 ++ lib/mocha/test.js | 2 + lib/mocha/types.d.ts | 3 + typings/index.d.ts | 6 +- 5 files changed, 131 insertions(+), 110 deletions(-) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 9331007d2..f32725d35 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -1,83 +1,83 @@ -const tty = require('tty'); +const tty = require('tty') if (!tty.getWindowSize) { // this is really old method, long removed from Node, but Mocha // reporters fall back on it if they cannot use `process.stdout.getWindowSize` // we need to polyfill it. - tty.getWindowSize = () => [40, 80]; + tty.getWindowSize = () => [40, 80] } -const { parentPort, workerData } = require('worker_threads'); -const event = require('../../event'); -const container = require('../../container'); -const { getConfig } = require('../utils'); -const { tryOrDefault, deepMerge } = require('../../utils'); +const { parentPort, workerData } = require('worker_threads') +const event = require('../../event') +const container = require('../../container') +const { getConfig } = require('../utils') +const { tryOrDefault, deepMerge } = require('../../utils') -let stdout = ''; +let stdout = '' -const stderr = ''; +const stderr = '' // Requiring of Codecept need to be after tty.getWindowSize is available. -const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept'); +const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept') -const { options, tests, testRoot, workerIndex } = workerData; +const { options, tests, testRoot, workerIndex } = workerData // hide worker output if (!options.debug && !options.verbose) process.stdout.write = string => { - stdout += string; - return true; - }; + stdout += string + return true + } -const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {}); +const overrideConfigs = tryOrDefault(() => JSON.parse(options.override), {}) // important deep merge so dynamic things e.g. functions on config are not overridden -const config = deepMerge(getConfig(options.config || testRoot), overrideConfigs); +const config = deepMerge(getConfig(options.config || testRoot), overrideConfigs) // Load test and run -const codecept = new Codecept(config, options); -codecept.init(testRoot); -codecept.loadTests(); -const mocha = container.mocha(); -filterTests(); +const codecept = new Codecept(config, options) +codecept.init(testRoot) +codecept.loadTests() +const mocha = container.mocha() +filterTests() // run tests -(async function () { +;(async function () { if (mocha.suite.total()) { - await runTests(); + await runTests() } -})(); +})() async function runTests() { try { - await codecept.bootstrap(); + await codecept.bootstrap() } catch (err) { - throw new Error(`Error while running bootstrap file :${err}`); + throw new Error(`Error while running bootstrap file :${err}`) } - listenToParentThread(); - initializeListeners(); - disablePause(); + listenToParentThread() + initializeListeners() + disablePause() try { - await codecept.run(); + await codecept.run() } finally { - await codecept.teardown(); + await codecept.teardown() } } function filterTests() { - const files = codecept.testFiles; - mocha.files = files; - mocha.loadFiles(); + const files = codecept.testFiles + mocha.files = files + mocha.loadFiles() for (const suite of mocha.suite.suites) { - suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0); + suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0) } } function initializeListeners() { function simplifyError(error) { if (error) { - const { stack, uncaught, message, actual, expected } = error; + const { stack, uncaught, message, actual, expected } = error return { stack, @@ -85,36 +85,36 @@ function initializeListeners() { message, actual, expected, - }; + } } - return null; + return null } function simplifyTest(test, err = null) { - test = { ...test }; + test = { ...test } if (test.start && !test.duration) { - const end = new Date(); - test.duration = end - test.start; + const end = new Date() + test.duration = end - test.start } if (test.err) { - err = simplifyError(test.err); - test.status = 'failed'; + err = simplifyError(test.err) + test.status = 'failed' } else if (err) { - err = simplifyError(err); - test.status = 'failed'; + err = simplifyError(err) + test.status = 'failed' } - const parent = {}; + const parent = {} if (test.parent) { - parent.title = test.parent.title; + parent.title = test.parent.title } if (test.opts) { Object.keys(test.opts).forEach(k => { - if (typeof test.opts[k] === 'object') delete test.opts[k]; - if (typeof test.opts[k] === 'function') delete test.opts[k]; - }); + if (typeof test.opts[k] === 'object') delete test.opts[k] + if (typeof test.opts[k] === 'function') delete test.opts[k] + }) } return { @@ -125,30 +125,33 @@ function initializeListeners() { retries: test._retries, title: test.title, status: test.status, + notes: test.notes || [], + meta: test.meta || {}, + artifacts: test.artifacts || [], duration: test.duration || 0, err, parent, steps: test.steps && test.steps.length > 0 ? simplifyStepsInTestObject(test.steps, err) : [], - }; + } } function simplifyStepsInTestObject(steps, err) { - steps = [...steps]; - const _steps = []; + steps = [...steps] + const _steps = [] for (step of steps) { - const _args = []; + const _args = [] if (step.args) { for (const arg of step.args) { // check if arg is a JOI object if (arg && arg.$_root) { - _args.push(JSON.stringify(arg).slice(0, 300)); + _args.push(JSON.stringify(arg).slice(0, 300)) // check if arg is a function } else if (arg && typeof arg === 'function') { - _args.push(arg.name); + _args.push(arg.name) } else { - _args.push(arg); + _args.push(arg) } } } @@ -164,38 +167,38 @@ function initializeListeners() { finishedAt: step.finishedAt, duration: step.duration, err, - }); + }) } - return _steps; + return _steps } function simplifyStep(step, err = null) { - step = { ...step }; + step = { ...step } if (step.startTime && !step.duration) { - const end = new Date(); - step.duration = end - step.startTime; + const end = new Date() + step.duration = end - step.startTime } if (step.err) { - err = simplifyError(step.err); - step.status = 'failed'; + err = simplifyError(step.err) + step.status = 'failed' } else if (err) { - err = simplifyError(err); - step.status = 'failed'; + err = simplifyError(err) + step.status = 'failed' } - const parent = {}; + const parent = {} if (step.metaStep) { - parent.title = step.metaStep.actor; + parent.title = step.metaStep.actor } if (step.opts) { Object.keys(step.opts).forEach(k => { - if (typeof step.opts[k] === 'object') delete step.opts[k]; - if (typeof step.opts[k] === 'function') delete step.opts[k]; - }); + if (typeof step.opts[k] === 'object') delete step.opts[k] + if (typeof step.opts[k] === 'function') delete step.opts[k] + }) } return { @@ -207,43 +210,43 @@ function initializeListeners() { err, parent, test: simplifyTest(step.test), - }; + } } - collectStats(); + collectStats() // suite - event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: simplifyTest(suite) })); - event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: simplifyTest(suite) })); + event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: simplifyTest(suite) })) + event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: simplifyTest(suite) })) // calculate duration - event.dispatcher.on(event.test.started, test => (test.start = new Date())); + event.dispatcher.on(event.test.started, test => (test.start = new Date())) // tests - event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: simplifyTest(test) })); - event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: simplifyTest(test) })); + event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: simplifyTest(test) })) + event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: simplifyTest(test) })) // we should force-send correct errors to prevent race condition - event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: simplifyTest(test, err) })); - event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: simplifyTest(test, err) })); - event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: simplifyTest(test, err) })); - event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: simplifyTest(test) })); - event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: simplifyTest(test) })); + event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: simplifyTest(test, err) })) + event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: simplifyTest(test, err) })) + event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: simplifyTest(test, err) })) + event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: simplifyTest(test) })) + event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: simplifyTest(test) })) // steps - event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: simplifyStep(step) })); - event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: simplifyStep(step) })); - event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: simplifyStep(step) })); - event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: simplifyStep(step) })); + event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: simplifyStep(step) })) + event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: simplifyStep(step) })) + event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: simplifyStep(step) })) + event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: simplifyStep(step) })) - event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: simplifyTest(test, err) })); - event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: simplifyTest(test, err) })); - event.dispatcher.on(event.all.failures, data => sendToParentThread({ event: event.all.failures, workerIndex, data })); + event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: simplifyTest(test, err) })) + event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: simplifyTest(test, err) })) + event.dispatcher.on(event.all.failures, data => sendToParentThread({ event: event.all.failures, workerIndex, data })) // all - event.dispatcher.once(event.all.result, () => parentPort.close()); + event.dispatcher.once(event.all.result, () => parentPort.close()) } function disablePause() { - global.pause = () => {}; + global.pause = () => {} } function collectStats() { @@ -253,36 +256,36 @@ function collectStats() { skipped: 0, tests: 0, pending: 0, - }; + } event.dispatcher.on(event.test.skipped, () => { - stats.skipped++; - }); + stats.skipped++ + }) event.dispatcher.on(event.test.passed, () => { - stats.passes++; - }); + stats.passes++ + }) event.dispatcher.on(event.test.failed, test => { if (test.ctx._runnable.title.includes('hook: AfterSuite')) { - stats.failedHooks += 1; + stats.failedHooks += 1 } - stats.failures++; - }); + stats.failures++ + }) event.dispatcher.on(event.test.skipped, () => { - stats.pending++; - }); + stats.pending++ + }) event.dispatcher.on(event.test.finished, () => { - stats.tests++; - }); + stats.tests++ + }) event.dispatcher.once(event.all.after, () => { - sendToParentThread({ event: event.all.after, data: stats }); - }); + sendToParentThread({ event: event.all.after, data: stats }) + }) } function sendToParentThread(data) { - parentPort.postMessage(data); + parentPort.postMessage(data) } function listenToParentThread() { parentPort.on('message', eventData => { - container.append({ support: eventData.data }); - }); + container.append({ support: eventData.data }) + }) } diff --git a/lib/heal.js b/lib/heal.js index 0c5280dca..f274567a8 100644 --- a/lib/heal.js +++ b/lib/heal.js @@ -129,6 +129,15 @@ class Heal { snippet: codeSnippet, }) + if (failureContext?.test) { + let note = `This test was healed by '${suggestion.name}'` + note += `\n\nReplace the failed code:\n\n` + note += colors.red(`- ${failedStep.toCode()}\n`) + note += colors.green(`+ ${codeSnippet}\n`) + failureContext.notes.push(note) + failureContext.test.meta.healed = true + } + recorder.add('healed', () => output.print(colors.bold.green(` Code healed successfully by ${suggestion.name}`), colors.gray('(no errors thrown)'))) this.numHealed++ // recorder.session.restore(); diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 836f1f6cf..fdeb3ca27 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -40,6 +40,8 @@ function enhanceMochaTest(test) { suite.addTest(testWrapper(this)) test.tags = [...(test.tags || []), ...(suite.tags || [])] test.fullTitle = () => `${suite.title}: ${test.title}` + test.meta = {} + test.notes = [] } test.applyOptions = function (opts) { diff --git a/lib/mocha/types.d.ts b/lib/mocha/types.d.ts index 36e3e72ce..9cafa7d1e 100644 --- a/lib/mocha/types.d.ts +++ b/lib/mocha/types.d.ts @@ -3,9 +3,12 @@ import { Test as MochaTest, Suite as MochaSuite } from 'mocha' declare global { namespace CodeceptJS { interface Test extends MochaTest { + uid: string title: string tags: string[] steps: string[] + meta: Record + notes: string[] config: Record artifacts: string[] inject: Record diff --git a/typings/index.d.ts b/typings/index.d.ts index fb6a68aac..6b2aa917e 100644 --- a/typings/index.d.ts +++ b/typings/index.d.ts @@ -502,7 +502,7 @@ declare namespace CodeceptJS { (title: string, opts: { [key: string]: any }, callback: HookCallback): ScenarioConfig } interface IHook { - (callback: HookCallback): void + (callback: HookCallback): HookConfig } interface Globals { @@ -516,6 +516,10 @@ declare namespace CodeceptJS { useForSnippets?: boolean preferForRegexpMatch?: boolean } + + interface HookConfig { + retry(retries?: number): HookConfig + } } // Globals From b62dd352995d400c3815878517bd886ad78fcf08 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 10 Jan 2025 05:42:22 +0200 Subject: [PATCH 3/4] added notes & metadata for tests, improved output --- docs/ai.md | 183 +++++++++++++++++------------------- lib/command/run-workers.js | 2 +- lib/heal.js | 5 +- lib/helper/Playwright.js | 10 ++ lib/mocha/cli.js | 27 +++++- lib/mocha/featureConfig.js | 13 +++ lib/mocha/scenarioConfig.js | 11 +++ lib/mocha/test.js | 15 ++- lib/mocha/types.d.ts | 6 +- lib/output.js | 147 +++++++++++++++-------------- lib/plugin/heal.js | 30 ++++++ lib/workers.js | 14 +-- test/unit/mocha/ui_test.js | 20 ++++ 13 files changed, 293 insertions(+), 190 deletions(-) diff --git a/docs/ai.md b/docs/ai.md index 9d1776b77..2a0b5ffb8 100644 --- a/docs/ai.md +++ b/docs/ai.md @@ -11,7 +11,6 @@ Think of it as your testing co-pilot built into the testing framework > 🪄 **AI features for testing are experimental**. AI works only for web based testing with Playwright, WebDriver, etc. Those features will be improved based on user's experience. - ## How AI Improves Automated Testing LLMs like ChatGPT can technically write automated tests for you. However, ChatGPT misses the context of your application so it will guess elements on page, instead of writing the code that works. @@ -22,10 +21,10 @@ So, instead of asking "write me a test" it can ask "write a test for **this** pa CodeceptJS AI can do the following: -* 🏋️‍♀️ **assist writing tests** in `pause()` or interactive shell mode -* 📃 **generate page objects** in `pause()` or interactive shell mode -* 🚑 **self-heal failing tests** (can be used on CI) -* 💬 send arbitrary prompts to AI provider from any tested page attaching its HTML contents +- 🏋️‍♀️ **assist writing tests** in `pause()` or interactive shell mode +- 📃 **generate page objects** in `pause()` or interactive shell mode +- 🚑 **self-heal failing tests** (can be used on CI) +- 💬 send arbitrary prompts to AI provider from any tested page attaching its HTML contents ![](/img/fill_form.gif) @@ -39,18 +38,16 @@ Even though, the HTML is still quite big and may exceed the token limit. So we r > ❗AI features require sending HTML contents to AI provider. Choosing one may depend on the descurity policy of your company. Ask your security department which AI providers you can use. - - ## Set up AI Provider To enable AI features in CodeceptJS you should pick an AI provider and add `ai` section to `codecept.conf` file. This section should contain `request` function which will take a prompt from CodeceptJS, send it to AI provider and return a result. ```js ai: { - request: async (messages) => { + request: async messages => { // implement OpenAI or any other provider like this const ai = require('my-ai-provider') - return ai.send(messages); + return ai.send(messages) } } ``` @@ -58,7 +55,7 @@ ai: { In `request` function `messages` is an array of prompt messages in format ```js -[{ role: 'user', content: 'prompt text'}] +;[{ role: 'user', content: 'prompt text' }] ``` Which is natively supported by OpenAI, Anthropic, and others. You can adjust messages to expected format before sending a request. The expected response from AI provider is a text in markdown format with code samples, which can be interpreted by CodeceptJS. @@ -71,33 +68,33 @@ npx codeceptjs run --ai Below we list sample configuration for popular AI providers -#### OpenAI GPT +### OpenAI GPT Prerequisite: -* Install `openai` package -* obtain `OPENAI_API_KEY` from OpenAI -* set `OPENAI_API_KEY` as environment variable +- Install `openai` package +- obtain `OPENAI_API_KEY` from OpenAI +- set `OPENAI_API_KEY` as environment variable Sample OpenAI configuration: ```js ai: { - request: async (messages) => { - const OpenAI = require('openai'); + request: async messages => { + const OpenAI = require('openai') const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] }) const completion = await openai.chat.completions.create({ model: 'gpt-3.5-turbo-0125', messages, - }); + }) - return completion?.choices[0]?.message?.content; + return completion?.choices[0]?.message?.content } } ``` -#### Mixtral +### Mixtral Mixtral is opensource and can be used via Cloudflare, Google Cloud, Azure or installed locally. @@ -105,74 +102,77 @@ The simplest way to try Mixtral on your case is using [Groq Cloud](https://groq. Prerequisite: -* Install `groq-sdk` package -* obtain `GROQ_API_KEY` from OpenAI -* set `GROQ_API_KEY` as environment variable +- Install `groq-sdk` package +- obtain `GROQ_API_KEY` from Groq Cloud +- set `GROQ_API_KEY` as environment variable Sample Groq configuration with Mixtral model: ```js ai: { - request: async (messages) => { + request: async messages => { + const Groq = require('groq-sdk') + + const client = new Groq({ + apiKey: process.env['GROQ_API_KEY'], // This is the default and can be omitted + }) + const chatCompletion = await groq.chat.completions.create({ - messages, - model: "mixtral-8x7b-32768", - }); - return chatCompletion.choices[0]?.message?.content || ""; + messages, + model: 'mixtral-8x7b-32768', + }) + return chatCompletion.choices[0]?.message?.content || '' } } ``` > Groq also provides access to other opensource models like llama or gemma -#### Anthropic Claude +### Anthropic Claude Prerequisite: -* Install `@anthropic-ai/sdk` package -* obtain `CLAUDE_API_KEY` from Anthropic -* set `CLAUDE_API_KEY` as environment variable +- Install `@anthropic-ai/sdk` package +- obtain `CLAUDE_API_KEY` from Anthropic +- set `CLAUDE_API_KEY` as environment variable ```js ai: { - request: async(messages) => { - const Anthropic = require('@anthropic-ai/sdk'); + request: async messages => { + const Anthropic = require('@anthropic-ai/sdk') const anthropic = new Anthropic({ apiKey: process.env.CLAUDE_API_KEY, - }); + }) const resp = await anthropic.messages.create({ model: 'claude-2.1', max_tokens: 1024, - messages - }); - return resp.content.map((c) => c.text).join('\n\n'); + messages, + }) + return resp.content.map(c => c.text).join('\n\n') } } ``` -#### Azure OpenAI +### Azure OpenAI When your setup using Azure API key Prerequisite: -* Install `@azure/openai` package -* obtain `Azure API key`, `resource name` and `deployment ID` +- Install `@azure/openai` package +- obtain `Azure API key`, `resource name` and `deployment ID` ```js ai: { - request: async(messages) => { - const { OpenAIClient, AzureKeyCredential } = require("@azure/openai"); + request: async messages => { + const { OpenAIClient, AzureKeyCredential } = require('@azure/openai') - const client = new OpenAIClient( - "https://.openai.azure.com/", - new AzureKeyCredential("") - ); - const { choices } = await client.getCompletions("", messages); + const client = new OpenAIClient('https://.openai.azure.com/', new AzureKeyCredential('')) + const { choices } = await client.getCompletions('', messages) - return choices[0]?.message?.content; + return choices[0]?.message?.content } } ``` @@ -181,29 +181,29 @@ When your setup using `bearer token` Prerequisite: -* Install `@azure/openai`, `@azure/identity` packages -* obtain `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` +- Install `@azure/openai`, `@azure/identity` packages +- obtain `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET` ```js ai: { - request: async (messages) => { + request: async messages => { try { - const { OpenAIClient} = require("@azure/openai"); - const { DefaultAzureCredential } = require("@azure/identity"); + const { OpenAIClient } = require('@azure/openai') + const { DefaultAzureCredential } = require('@azure/identity') - const endpoint = process.env.API_ENDPOINT; - const deploymentId = process.env.DEPLOYMENT_ID; + const endpoint = process.env.API_ENDPOINT + const deploymentId = process.env.DEPLOYMENT_ID - const client = new OpenAIClient(endpoint, new DefaultAzureCredential()); + const client = new OpenAIClient(endpoint, new DefaultAzureCredential()) const result = await client.getCompletions(deploymentId, { prompt: messages, - model: 'gpt-3.5-turbo' // your preferred model - }); + model: 'gpt-3.5-turbo', // your preferred model + }) - return result.choices[0]?.text; + return result.choices[0]?.text } catch (error) { - console.error("Error calling API:", error); - throw error; + console.error('Error calling API:', error) + throw error } } } @@ -273,12 +273,12 @@ npx codeceptjs gt Name a test and write the code. We will use `Scenario.only` instead of Scenario to execute only this exact test. ```js -Feature('ai'); +Feature('ai') Scenario.only('test ai features', ({ I }) => { I.amOnPage('https://getbootstrap.com/docs/5.1/examples/checkout/') - pause(); -}); + pause() +}) ``` Now run the test in debug mode with AI enabled: @@ -289,7 +289,6 @@ npx codeceptjs run --debug --ai When pause mode started you can ask GPT to fill in the fields on this page. Use natural language to describe your request, and provide enough details that AI could operate with it. It is important to include at least a space char in your input, otherwise, CodeceptJS will consider the input to be JavaScript code. - ``` I.fill checkout form with valid values without submitting it ``` @@ -310,16 +309,14 @@ Please keep in mind that GPT can't react to page changes and operates with stati In large test suites, the cost of maintaining tests goes exponentially. That's why any effort that can improve the stability of tests pays itself. That's why CodeceptJS has concept of [heal recipes](./heal), functions that can be executed on a test failure. Those functions can try to revive the test and continue execution. When combined with AI, heal recipe can ask AI provider how to fix the test. It will provide error message, step being executed and HTML context of a page. Based on this information AI can suggest the code to be executed to fix the failing test. - AI healing can solve exactly one problem: if a locator of an element has changed, and an action can't be performed, **it matches a new locator, tries a command again, and continues executing a test**. For instance, if the "Sign in" button was renamed to "Login" or changed its class, it will detect a new locator of the button and will retry execution. > You can define your own [heal recipes](./heal) that won't use AI to revive failing tests. -Heal actions **work only on actions like `click`, `fillField`, etc, and won't work on assertions, waiters, grabbers, etc. Assertions can't be guessed by AI, the same way as grabbers, as this may lead to unpredictable results. +Heal actions \*\*work only on actions like `click`, `fillField`, etc, and won't work on assertions, waiters, grabbers, etc. Assertions can't be guessed by AI, the same way as grabbers, as this may lead to unpredictable results. If Heal plugin successfully fixes the step, it will print a suggested change at the end of execution. Take it as actionable advice and use it to update the codebase. Heal plugin is supposed to be used on CI, and works automatically without human assistance. - To start, make sure [AI provider is connected](#set-up-ai-provider), and [heal recipes were created](/heal#how-to-start-healing) by running this command: ``` @@ -357,7 +354,6 @@ npx codeceptjs run --ai When execution finishes, you will receive information on token usage and code suggestions proposed by AI. By evaluating this information you will be able to check how effective AI can be for your case. - ## Arbitrary Prompts What if you want to take AI on the journey of test automation and ask it questions while browsing pages? @@ -377,23 +373,23 @@ helpers: { AI helper will be automatically attached to Playwright, WebDriver, or another web helper you use. It includes the following methods: -* `askGptOnPage` - sends GPT prompt attaching the HTML of the page. Large pages will be split into chunks, according to `chunkSize` config. You will receive responses for all chunks. -* `askGptOnPageFragment` - sends GPT prompt attaching the HTML of the specific element. This method is recommended over `askGptOnPage` as you can reduce the amount of data to be processed. -* `askGptGeneralPrompt` - sends GPT prompt without HTML. -* `askForPageObject` - creates PageObject for you, explained in next section. +- `askGptOnPage` - sends GPT prompt attaching the HTML of the page. Large pages will be split into chunks, according to `chunkSize` config. You will receive responses for all chunks. +- `askGptOnPageFragment` - sends GPT prompt attaching the HTML of the specific element. This method is recommended over `askGptOnPage` as you can reduce the amount of data to be processed. +- `askGptGeneralPrompt` - sends GPT prompt without HTML. +- `askForPageObject` - creates PageObject for you, explained in next section. `askGpt` methods won't remove non-interactive elements, so it is recommended to manually control the size of the sent HTML. Here are some good use cases for this helper: -* get page summaries -* inside pause mode navigate through your application and ask to document pages -* etc... +- get page summaries +- inside pause mode navigate through your application and ask to document pages +- etc... ```js // use it inside test or inside interactive pause // pretend you are technical writer asking for documentation -const pageDoc = await I.askGptOnPageFragment('Act as technical writer, describe what is this page for', '#container'); +const pageDoc = await I.askGptOnPageFragment('Act as technical writer, describe what is this page for', '#container') ``` As of now, those use cases do not apply to test automation but maybe you can apply them to your testing setup. @@ -508,7 +504,7 @@ ai: { prompts: { writeStep: (html, input) => [{ role: 'user', content: 'As a test engineer...' }] healStep: (html, { step, error, prevSteps }) => [{ role: 'user', content: 'As a test engineer...' }] - generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{ role: 'user', content: 'As a test engineer...' }] + generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{ role: 'user', content: 'As a test engineer...' }] } } ``` @@ -531,34 +527,33 @@ ai: { } ``` -* `maxLength`: the size of HTML to cut to not reach the token limit. 50K is the current default but you may try to increase it or even set it to null. -* `simplify`: should we process HTML before sending to GPT. This will remove all non-interactive elements from HTML. -* `minify`: should HTML be additionally minified. This removed empty attributes, shortens notations, etc. -* `interactiveElements`: explicit list of all elements that are considered interactive. -* `textElements`: elements that contain text which can be used for test automation. -* `allowedAttrs`: explicit list of attributes that may be used to construct locators. If you use special `data-` attributes to enable locators, add them to the list. -* `allowedRoles`: list of roles that make standard elements interactive. +- `maxLength`: the size of HTML to cut to not reach the token limit. 50K is the current default but you may try to increase it or even set it to null. +- `simplify`: should we process HTML before sending to GPT. This will remove all non-interactive elements from HTML. +- `minify`: should HTML be additionally minified. This removed empty attributes, shortens notations, etc. +- `interactiveElements`: explicit list of all elements that are considered interactive. +- `textElements`: elements that contain text which can be used for test automation. +- `allowedAttrs`: explicit list of attributes that may be used to construct locators. If you use special `data-` attributes to enable locators, add them to the list. +- `allowedRoles`: list of roles that make standard elements interactive. It is recommended to try HTML processing on one of your web pages before launching AI features of CodeceptJS. - To do that open the common page of your application and using DevTools copy the outerHTML of `` element. Don't use `Page Source` for that, as it may not include dynamically added HTML elements. Save this HTML into a file and create a NodeJS script: ```js -const { removeNonInteractiveElements } = require('codeceptjs/lib/html'); -const fs = require('fs'); +const { removeNonInteractiveElements } = require('codeceptjs/lib/html') +const fs = require('fs') const htmlOpts = { interactiveElements: ['a', 'input', 'button', 'select', 'textarea', 'label', 'option'], allowedAttrs: ['id', 'for', 'class', 'name', 'type', 'value', 'aria-labelledby', 'aria-label', 'label', 'placeholder', 'title', 'alt', 'src', 'role'], textElements: ['label', 'h1', 'h2'], allowedRoles: ['button', 'checkbox', 'search', 'textbox', 'tab'], -}; +} -html = fs.readFileSync('saved.html', 'utf8'); -const result = removeNonInteractiveElements(html, htmlOpts); +html = fs.readFileSync('saved.html', 'utf8') +const result = removeNonInteractiveElements(html, htmlOpts) -console.log(result); +console.log(result) ``` Tune the options until you are satisfied with the results and use this as `html` config for `ai` section inside `codecept.conf` file. @@ -571,9 +566,7 @@ For instance, if you use `data-qa` attributes to specify locators and you want t // inside codecept.conf.js ai: { html: { - allowedAttrs: [ - 'data-qa', 'id', 'for', 'class', 'name', 'type', 'value', 'aria-labelledby', 'aria-label', 'label', 'placeholder', 'title', 'alt', 'src', 'role' - ] + allowedAttrs: ['data-qa', 'id', 'for', 'class', 'name', 'type', 'value', 'aria-labelledby', 'aria-label', 'label', 'placeholder', 'title', 'alt', 'src', 'role'] } } } diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 03b0eff8e..31a2c598e 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -30,6 +30,7 @@ module.exports = async function (workerCount, selectedRuns, options) { output.print(`CodeceptJS v${require('../codecept').version()} ${output.standWithUkraine()}`) output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`) output.print() + store.hasWorkers = true const workers = new Workers(numberOfWorkers, config) workers.overrideConfig(overrideConfigs) @@ -100,7 +101,6 @@ module.exports = async function (workerCount, selectedRuns, options) { if (options.verbose || options.debug) store.debugMode = true if (options.verbose) { - global.debugMode = true const { getMachineInfo } = require('./info') await getMachineInfo() } diff --git a/lib/heal.js b/lib/heal.js index f274567a8..f1d24b6df 100644 --- a/lib/heal.js +++ b/lib/heal.js @@ -130,12 +130,13 @@ class Heal { }) if (failureContext?.test) { + const test = failureContext.test let note = `This test was healed by '${suggestion.name}'` note += `\n\nReplace the failed code:\n\n` note += colors.red(`- ${failedStep.toCode()}\n`) note += colors.green(`+ ${codeSnippet}\n`) - failureContext.notes.push(note) - failureContext.test.meta.healed = true + test.addNote('heal', note) + test.meta.healed = true } recorder.add('healed', () => output.print(colors.bold.green(` Code healed successfully by ${suggestion.name}`), colors.gray('(no errors thrown)'))) diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index f074c710f..bc3c9559c 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -482,6 +482,7 @@ class Playwright extends Helper { async _before(test) { this.currentRunningTest = test + recorder.retry({ retries: process.env.FAILED_STEP_RETRIES || 3, when: err => { @@ -552,6 +553,15 @@ class Playwright extends Helper { await this._setPage(mainPage) + try { + // set metadata for reporting + test.meta.browser = this.browser.browserType().name() + test.meta.browserVersion = this.browser.version() + test.meta.windowSize = `${this.page.viewportSize().width}x${this.page.viewportSize().height}` + } catch (e) { + this.debug('Failed to set metadata for reporting') + } + if (this.options.trace) await this.browserContext.tracing.start({ screenshots: true, snapshots: true }) return this.browser diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 5e1328c75..64fd72d49 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -1,6 +1,7 @@ const { reporters: { Base }, } = require('mocha') +const figures = require('figures') const ms = require('ms') const event = require('../event') const AssertionFailedError = require('../assert/error') @@ -163,33 +164,49 @@ class Cli extends Base { const err = test.err let log = '' + let originalMessage = err.message if (err instanceof AssertionFailedError) { err.message = err.inspect() } + // multi-line error messages + err.message = '\n ' + (err.message || '').replace(/^/gm, ' ').trim() + const steps = test.steps || (test.ctx && test.ctx.test.steps) if (steps && steps.length) { let scenarioTrace = '' steps.reverse().forEach(step => { - const line = `- ${step.toCode()} ${step.line()}` - // if (step.status === 'failed') line = '' + line; + const hasFailed = step.status === 'failed' + let line = `${hasFailed ? output.styles.bold(figures.cross) : figures.tick} ${step.toCode()} ${step.line()}` + if (hasFailed) line = output.styles.bold(line) scenarioTrace += `\n${line}` }) - log += `${output.styles.bold('Scenario Steps')}:${scenarioTrace}\n` + log += `${output.styles.basic(figures.circle)} ${output.styles.section('Scenario Steps')}:${scenarioTrace}\n` } // display artifacts in debug mode if (test?.artifacts && Object.keys(test.artifacts).length) { - log += `\n${output.styles.bold('Artifacts:')}` + log += `\n${output.styles.basic(figures.circle)} ${output.styles.section('Artifacts:')}` for (const artifact of Object.keys(test.artifacts)) { log += `\n- ${artifact}: ${test.artifacts[artifact]}` } } + // display metadata + if (test.meta && Object.keys(test.meta).length) { + log += `\n\n${output.styles.basic(figures.circle)} ${output.styles.section('Metadata:')}` + for (const [key, value] of Object.entries(test.meta)) { + log += `\n- ${key}: ${value}` + } + } + try { - let stack = err.stack ? err.stack.split('\n') : [] + let stack = err.stack + stack = stack.replace(originalMessage, '') + stack = stack ? stack.split('\n') : [] + if (stack[0] && stack[0].includes(err.message)) { stack.shift() } diff --git a/lib/mocha/featureConfig.js b/lib/mocha/featureConfig.js index a80ab1dfe..dec3da2da 100644 --- a/lib/mocha/featureConfig.js +++ b/lib/mocha/featureConfig.js @@ -7,6 +7,19 @@ class FeatureConfig { this.suite = suite } + /** + * Set metadata for this suite + * @param {string} key + * @param {string} value + * @returns {this} + */ + meta(key, value) { + this.suite.tests.forEach(test => { + test.meta[key] = value + }) + return this + } + /** * Retry this test for number of times * diff --git a/lib/mocha/scenarioConfig.js b/lib/mocha/scenarioConfig.js index 33ca01f28..6813e786e 100644 --- a/lib/mocha/scenarioConfig.js +++ b/lib/mocha/scenarioConfig.js @@ -42,6 +42,17 @@ class ScenarioConfig { return this } + /** + * Set metadata for this test + * @param {string} key + * @param {string} value + * @returns {this} + */ + meta(key, value) { + this.test.meta[key] = value + return this + } + /** * Set timeout for this test * @param {number} timeout diff --git a/lib/mocha/test.js b/lib/mocha/test.js index fdeb3ca27..7d5377d3f 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -1,4 +1,5 @@ const Test = require('mocha/lib/test') +const Suite = require('mocha/lib/suite') const { test: testWrapper } = require('./asyncWrapper') const { enhanceMochaSuite } = require('./suite') @@ -31,6 +32,11 @@ function enhanceMochaTest(test) { test.inject = {} test.opts = {} + test.notes = [] + test.addNote = (type, note) => { + test.notes.push({ type, text: note }) + } + // Add new methods /** * @param {Mocha.Suite} suite - The Mocha suite to add this test to @@ -41,12 +47,12 @@ function enhanceMochaTest(test) { test.tags = [...(test.tags || []), ...(suite.tags || [])] test.fullTitle = () => `${suite.title}: ${test.title}` test.meta = {} - test.notes = [] } test.applyOptions = function (opts) { if (!opts) opts = {} test.opts = opts + test.meta = opts.meta || {} test.totalTimeout = opts.timeout if (opts.retries) this.retries(opts.retries) } @@ -54,7 +60,14 @@ function enhanceMochaTest(test) { return test } +function repackTestForWorkersTransport(test) { + test = Object.assign(new Test(test.title || '', () => {}), test) + test.parent = Object.assign(new Suite(test.parent.title), test.parent) + return test +} + module.exports = { createTest, enhanceMochaTest, + repackTestForWorkersTransport, } diff --git a/lib/mocha/types.d.ts b/lib/mocha/types.d.ts index 9cafa7d1e..2bdb55a56 100644 --- a/lib/mocha/types.d.ts +++ b/lib/mocha/types.d.ts @@ -8,7 +8,10 @@ declare global { tags: string[] steps: string[] meta: Record - notes: string[] + notes: Array<{ + type: string + text: string + }> config: Record artifacts: string[] inject: Record @@ -17,6 +20,7 @@ declare global { totalTimeout?: number addToSuite(suite: Mocha.Suite): void applyOptions(opts: Record): void + addNote(type: string, note: string): void codeceptjs: boolean } diff --git a/lib/output.js b/lib/output.js index 436d493b5..fc209d043 100644 --- a/lib/output.js +++ b/lib/output.js @@ -1,6 +1,6 @@ -const colors = require('chalk'); -const figures = require('figures'); -const { maskSensitiveData } = require('invisi-data'); +const colors = require('chalk') +const figures = require('figures') +const { maskSensitiveData } = require('invisi-data') const styles = { error: colors.bgRed.white.bold, @@ -10,11 +10,12 @@ const styles = { debug: colors.cyan, log: colors.grey, bold: colors.bold, -}; + section: colors.white.dim.bold, +} -let outputLevel = 0; -let outputProcess = ''; -let newline = true; +let outputLevel = 0 +let outputProcess = '' +let newline = true /** * @alias output @@ -28,7 +29,7 @@ module.exports = { stepShift: 0, standWithUkraine() { - return `#${colors.bold.yellow('StandWith')}${colors.bold.cyan('Ukraine')}`; + return `#${colors.bold.yellow('StandWith')}${colors.bold.cyan('Ukraine')}` }, /** @@ -37,8 +38,8 @@ module.exports = { * @returns {number} */ level(level) { - if (level !== undefined) outputLevel = level; - return outputLevel; + if (level !== undefined) outputLevel = level + return outputLevel }, /** @@ -48,9 +49,9 @@ module.exports = { * @returns {string} */ process(process) { - if (process === null) return (outputProcess = ''); - if (process) outputProcess = String(process).length === 1 ? `[0${process}]` : `[${process}]`; - return outputProcess; + if (process === null) return (outputProcess = '') + if (process) outputProcess = String(process).length === 1 ? `[0${process}]` : `[${process}]` + return outputProcess }, /** @@ -58,9 +59,9 @@ module.exports = { * @param {string} msg */ debug(msg) { - const _msg = isMaskedData() ? maskSensitiveData(msg) : msg; + const _msg = isMaskedData() ? maskSensitiveData(msg) : msg if (outputLevel >= 2) { - print(' '.repeat(this.stepShift), styles.debug(`${figures.pointerSmall} ${_msg}`)); + print(' '.repeat(this.stepShift), styles.debug(`${figures.pointerSmall} ${_msg}`)) } }, @@ -69,9 +70,9 @@ module.exports = { * @param {string} msg */ log(msg) { - const _msg = isMaskedData() ? maskSensitiveData(msg) : msg; + const _msg = isMaskedData() ? maskSensitiveData(msg) : msg if (outputLevel >= 3) { - print(' '.repeat(this.stepShift), styles.log(truncate(` ${_msg}`, this.spaceShift))); + print(' '.repeat(this.stepShift), styles.log(truncate(` ${_msg}`, this.spaceShift))) } }, @@ -80,7 +81,7 @@ module.exports = { * @param {string} msg */ error(msg) { - print(styles.error(msg)); + print(styles.error(msg)) }, /** @@ -88,7 +89,7 @@ module.exports = { * @param {string} msg */ success(msg) { - print(styles.success(msg)); + print(styles.success(msg)) }, /** @@ -97,7 +98,7 @@ module.exports = { * @param {string} msg */ plugin(pluginName, msg = '') { - this.debug(`<${pluginName}> ${msg}`); + this.debug(`<${pluginName}> ${msg}`) }, /** @@ -105,26 +106,26 @@ module.exports = { * @param {CodeceptJS.Step} step */ step(step) { - if (outputLevel === 0) return; - if (!step) return; + if (outputLevel === 0) return + if (!step) return // Avoid to print non-gherkin steps, when gherkin is running for --steps mode if (outputLevel === 1) { if (typeof step === 'object' && step.hasBDDAncestor()) { - return; + return } } - let stepLine = step.toCliStyled(); + let stepLine = step.toCliStyled() if (step.metaStep && outputLevel >= 1) { // this.stepShift += 2; - stepLine = colors.dim(truncate(stepLine, this.spaceShift)); + stepLine = colors.dim(truncate(stepLine, this.spaceShift)) } if (step.comment) { - stepLine += colors.grey(step.comment.split('\n').join('\n' + ' '.repeat(4))); + stepLine += colors.grey(step.comment.split('\n').join('\n' + ' '.repeat(4))) } - const _stepLine = isMaskedData() ? maskSensitiveData(stepLine) : stepLine; - print(' '.repeat(this.stepShift), truncate(_stepLine, this.spaceShift)); + const _stepLine = isMaskedData() ? maskSensitiveData(stepLine) : stepLine + print(' '.repeat(this.stepShift), truncate(_stepLine, this.spaceShift)) }, /** @namespace */ @@ -133,9 +134,9 @@ module.exports = { * @param {Mocha.Suite} suite */ started: suite => { - if (!suite.title) return; - print(`${colors.bold(suite.title)} --`); - if (suite.comment) print(suite.comment); + if (!suite.title) return + print(`${colors.bold(suite.title)} --`) + if (suite.comment) print(suite.comment) }, }, @@ -145,25 +146,25 @@ module.exports = { * @param {Mocha.Test} test */ started(test) { - print(` ${colors.magenta.bold(test.title)}`); + print(` ${colors.magenta.bold(test.title)}`) }, /** * @param {Mocha.Test} test */ passed(test) { - print(` ${colors.green.bold(figures.tick)} ${test.title} ${colors.grey(`in ${test.duration}ms`)}`); + print(` ${colors.green.bold(figures.tick)} ${test.title} ${colors.grey(`in ${test.duration}ms`)}`) }, /** * @param {Mocha.Test} test */ failed(test) { - print(` ${colors.red.bold(figures.cross)} ${test.title} ${colors.grey(`in ${test.duration}ms`)}`); + print(` ${colors.red.bold(figures.cross)} ${test.title} ${colors.grey(`in ${test.duration}ms`)}`) }, /** * @param {Mocha.Test} test */ skipped(test) { - print(` ${colors.yellow.bold('S')} ${test.title}`); + print(` ${colors.yellow.bold('S')} ${test.title}`) }, }, @@ -174,38 +175,38 @@ module.exports = { */ started(test) { - if (outputLevel < 1) return; - print(` ${colors.dim.bold('Scenario()')}`); + if (outputLevel < 1) return + print(` ${colors.dim.bold('Scenario()')}`) }, /** * @param {Mocha.Test} test */ passed(test) { - print(` ${colors.green.bold(`${figures.tick} OK`)} ${colors.grey(`in ${test.duration}ms`)}`); - print(); + print(` ${colors.green.bold(`${figures.tick} OK`)} ${colors.grey(`in ${test.duration}ms`)}`) + print() }, /** * @param {Mocha.Test} test */ failed(test) { - print(` ${colors.red.bold(`${figures.cross} FAILED`)} ${colors.grey(`in ${test.duration}ms`)}`); - print(); + print(` ${colors.red.bold(`${figures.cross} FAILED`)} ${colors.grey(`in ${test.duration}ms`)}`) + print() }, }, hook: { started(hook) { - if (outputLevel < 1) return; - print(` ${colors.dim.bold(hook.toCode())}`); + if (outputLevel < 1) return + print(` ${colors.dim.bold(hook.toCode())}`) }, passed(hook) { - if (outputLevel < 1) return; - print(); + if (outputLevel < 1) return + print() }, failed(hook) { - if (outputLevel < 1) return; - print(` ${colors.red.bold(hook.toCode())}`); + if (outputLevel < 1) return + print(` ${colors.red.bold(hook.toCode())}`) }, }, @@ -217,9 +218,9 @@ module.exports = { */ say(message, color = 'cyan') { if (colors[color] === undefined) { - color = 'cyan'; + color = 'cyan' } - if (outputLevel >= 1) print(` ${colors[color].bold(message)}`); + if (outputLevel >= 1) print(` ${colors[color].bold(message)}`) }, /** @@ -229,54 +230,54 @@ module.exports = { * @param {number|string} duration */ result(passed, failed, skipped, duration, failedHooks = 0) { - let style = colors.bgGreen; - let msg = ` ${passed || 0} passed`; - let status = style.bold(' OK '); + let style = colors.bgGreen + let msg = ` ${passed || 0} passed` + let status = style.bold(' OK ') if (failed) { - style = style.bgRed; - status = style.bold(' FAIL '); - msg += `, ${failed} failed`; + style = style.bgRed + status = style.bold(' FAIL ') + msg += `, ${failed} failed` } if (failedHooks > 0) { - style = style.bgRed; - status = style.bold(' FAIL '); - msg += `, ${failedHooks} failedHooks`; + style = style.bgRed + status = style.bold(' FAIL ') + msg += `, ${failedHooks} failedHooks` } - status += style.grey(' |'); + status += style.grey(' |') if (skipped) { - if (!failed) style = style.bgYellow; - msg += `, ${skipped} skipped`; + if (!failed) style = style.bgYellow + msg += `, ${skipped} skipped` } - msg += ' '; - print(status + style(msg) + colors.grey(` // ${duration}`)); + msg += ' ' + print(status + style(msg) + colors.grey(` // ${duration}`)) }, -}; +} function print(...msg) { if (outputProcess) { - msg.unshift(outputProcess); + msg.unshift(outputProcess) } if (!newline) { - console.log(); - newline = true; + console.log() + newline = true } - console.log.apply(this, msg); + console.log.apply(this, msg) } function truncate(msg, gap = 0) { if (msg.indexOf('\n') > 0 || outputLevel >= 3) { - return msg; // don't cut multi line steps or on verbose log level + return msg // don't cut multi line steps or on verbose log level } - const width = (process.stdout.columns || 200) - gap - 4; + const width = (process.stdout.columns || 200) - gap - 4 if (msg.length > width) { - msg = msg.substr(0, width - 1) + figures.ellipsis; + msg = msg.substr(0, width - 1) + figures.ellipsis } - return msg; + return msg } function isMaskedData() { - return global.maskSensitiveData === true || false; + return global.maskSensitiveData === true || false } diff --git a/lib/plugin/heal.js b/lib/plugin/heal.js index 022297e44..35644f875 100644 --- a/lib/plugin/heal.js +++ b/lib/plugin/heal.js @@ -5,6 +5,7 @@ const event = require('../event') const output = require('../output') const heal = require('../heal') const store = require('../store') +const container = require('../container') const defaultConfig = { healLimit: 2, @@ -115,4 +116,33 @@ module.exports = function (config = {}) { i++ } }) + + event.dispatcher.on(event.workers.result, ({ tests }) => { + const { print } = output + + const healedTests = Object.values(tests) + .flat() + .filter(test => test.notes.some(note => note.type === 'heal')) + if (!healedTests.length) return + + setTimeout(() => { + print('') + print('===================') + print(colors.bold.green('Self-Healing Report:')) + + print('') + print('Suggested changes:') + print('') + + healedTests.forEach(test => { + print(`${colors.bold.magenta(test.title)}`) + test.notes + .filter(note => note.type === 'heal') + .forEach(note => { + print(note.text) + print('') + }) + }) + }, 0) + }) } diff --git a/lib/workers.js b/lib/workers.js index 3618f9c35..c66aa2c40 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -1,11 +1,6 @@ const path = require('path') const mkdirp = require('mkdirp') const { Worker } = require('worker_threads') -const { - Suite, - Test, - reporters: { Base }, -} = require('mocha') const { EventEmitter } = require('events') const ms = require('ms') const Codecept = require('./codecept') @@ -17,6 +12,7 @@ const { replaceValueDeep, deepClone } = require('./utils') const mainConfig = require('./config') const output = require('./output') const event = require('./event') +const { repackTestForWorkersTransport: repackTest } = require('./mocha/test') const recorder = require('./recorder') const runHook = require('./hooks') const WorkerStorage = require('./workerStorage') @@ -78,12 +74,6 @@ const simplifyObject = object => { }, {}) } -const repackTest = test => { - test = Object.assign(new Test(test.title || '', () => {}), test) - test.parent = Object.assign(new Suite(test.parent.title), test.parent) - return test -} - const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => { selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) { @@ -459,7 +449,7 @@ class Workers extends EventEmitter { } _finishRun() { - event.dispatcher.emit(event.workers.after) + event.dispatcher.emit(event.workers.after, { tests: this.workers.map(worker => worker.tests) }) if (this.isFailed()) { process.exitCode = 1 } else { diff --git a/test/unit/mocha/ui_test.js b/test/unit/mocha/ui_test.js index 6ad5f3afa..1de8064f8 100644 --- a/test/unit/mocha/ui_test.js +++ b/test/unit/mocha/ui_test.js @@ -4,6 +4,7 @@ import('chai').then(chai => { }) const Mocha = require('mocha/lib/mocha') const Suite = require('mocha/lib/suite') +const { createTest } = require('../../../lib/mocha/test') global.codeceptjs = require('../../../lib') const makeUI = require('../../../lib/mocha/ui') @@ -132,6 +133,19 @@ describe('ui', () => { suiteConfig = context.Feature('not skipped suite', { key: 'value' }) expect(suiteConfig.suite.opts).to.deep.eq({ key: 'value' }, 'Features should have passed options') }) + + it('should be able to set metadata', () => { + suiteConfig = context.Feature('suite') + const test1 = createTest('test', () => {}) + const test2 = createTest('test2', () => {}) + test1.addToSuite(suiteConfig.suite) + test2.addToSuite(suiteConfig.suite) + + suiteConfig.meta('key', 'value') + + expect(test1.meta.key).eq('value') + expect(test2.meta.key).eq('value') + }) }) describe('Scenario', () => { @@ -169,6 +183,12 @@ describe('ui', () => { expect(scenarioConfig.test.inject.Data).eq('data') }) + it('should be able to set metadata', () => { + scenarioConfig = context.Scenario('scenario') + scenarioConfig.meta('key', 'value') + expect(scenarioConfig.test.meta.key).eq('value') + }) + describe('todo', () => { it('should inject skipInfo to opts', () => { scenarioConfig = context.Scenario.todo('scenario', () => { From 3135bd2612a4909dcd6e82446ac3c10ebab344a4 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 12 Jan 2025 20:22:13 +0200 Subject: [PATCH 4/4] fixed styles in failing tests --- test/runner/pageobject_test.js | 10 ++++++---- test/runner/step_timeout_test.js | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/runner/pageobject_test.js b/test/runner/pageobject_test.js index d0f08fc7b..49fb4eb86 100644 --- a/test/runner/pageobject_test.js +++ b/test/runner/pageobject_test.js @@ -1,7 +1,8 @@ const path = require('path') const exec = require('child_process').exec const { expect } = require('expect') - +const figures = require('figures') +const debug = require('debug')('codeceptjs:test') const runner = path.join(__dirname, '/../../bin/codecept.js') const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/pageObjects') const codecept_run = `${runner} run` @@ -16,13 +17,14 @@ describe('CodeceptJS PageObject', () => { it('should fail if page objects was failed', done => { exec(`${config_run_config('codecept.fail_po.js')} --debug`, (err, stdout) => { const lines = stdout.split('\n') + debug(stdout) expect(lines).toEqual( expect.arrayContaining([ expect.stringContaining('File notexistfile.js not found in'), expect.stringContaining('-- FAILURES'), - expect.stringContaining('- I.seeFile("notexistfile.js")'), - expect.stringContaining('- I.seeFile("codecept.class.js")'), - expect.stringContaining('- I.amInPath(".")'), + expect.stringContaining(figures.cross + ' I.seeFile("notexistfile.js")'), + expect.stringContaining(figures.tick + ' I.seeFile("codecept.class.js")'), + expect.stringContaining(figures.tick + ' I.amInPath(".")'), ]), ) expect(stdout).toContain('FAIL | 0 passed, 1 failed') diff --git a/test/runner/step_timeout_test.js b/test/runner/step_timeout_test.js index 6cc27d147..e055c7ffc 100644 --- a/test/runner/step_timeout_test.js +++ b/test/runner/step_timeout_test.js @@ -1,7 +1,7 @@ const { expect } = require('expect') const exec = require('child_process').exec const { codecept_dir, codecept_run } = require('./consts') - +const figures = require('figures') const debug_this_test = false const config_run_config = (config, grep, verbose = false) => `${codecept_run} ${verbose || debug_this_test ? '--verbose' : ''} --config ${codecept_dir}/configs/step_timeout/${config} ${grep ? `--grep "${grep}"` : ''}` @@ -13,7 +13,7 @@ describe('CodeceptJS Steps', function () { exec(config_run_config('codecept-1000.conf.js', 'Default command timeout'), (err, stdout) => { expect(stdout).toContain('Action exceededByTimeout: 1500 was interrupted on step timeout 1000ms') expect(stdout).toContain('0 passed, 1 failed') - expect(stdout).toContain('- I.exceededByTimeout(1500)') + expect(stdout).toContain(figures.cross + ' I.exceededByTimeout(1500)') expect(err).toBeTruthy() done() })