diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 0ca1a1f54..34c7d20cd 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -98,6 +98,7 @@ definitions: default: 200 content_type: type: string + default: application/json payload: $ref: '#/definitions/Payload' required: [ status ] @@ -115,6 +116,9 @@ definitions: message: type: string description: Error message for non 2XX responses. + error: + type: object + description: Error object. required: [ status, content_type, payload ] additionalProperties: false diff --git a/tests/indices/index_lifecycle.yaml b/tests/indices/index_lifecycle.yaml deleted file mode 100644 index f974b928a..000000000 --- a/tests/indices/index_lifecycle.yaml +++ /dev/null @@ -1,84 +0,0 @@ -$schema: ../../json_schemas/test_story.schema.yaml - -skip: false -description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion. -epilogues: - - path: /books,movies,games - method: DELETE - ignore_errors: false -chapters: - - synopsis: Create an index named `books` with mappings and settings. - path: /{index} - method: PUT - parameters: - index: books - request_body: - payload: - mappings: - properties: - name: - type: keyword - age: - type: integer - settings: - number_of_shards: 5 - number_of_replicas: 2 - response: - status: 200 - - - synopsis: Create an index named `games` with default settings, - path: /{index} - method: PUT - parameters: - index: games - response: - status: 200 - - - synopsis: Check if the index `books` exists. It should. - path: /{index} - method: HEAD - parameters: - index: books - response: - status: 200 - - - synopsis: Check if the index `movies` exists. It should not. - path: /{index} - method: HEAD - parameters: - index: movies - response: - status: 404 - - - synopsis: Retrieve the mappings and settings of the `books` and `games` indices. - path: /{index} - method: GET - parameters: - index: books,games - flat_settings: true - response: - status: 200 - - - synopsis: Close the `books` index. - path: /{index}/_close - method: POST - parameters: - index: books - response: - status: 200 - - - synopsis: Open the `books` index. - path: /{index}/_open - method: POST - parameters: - index: books - response: - status: 200 - - - synopsis: Delete the `books` and `games` indices. - path: /{index} - method: DELETE - parameters: - index: books,games - response: - status: 200 diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 3eedb6d52..26dbf490d 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -3,7 +3,7 @@ import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.ty import { type ParsedOperation } from './types/spec.types' import { overall_result } from './helpers' import type ChapterReader from './ChapterReader' -import { ResponseError } from './ChapterReader' +import { ServerError } from './ChapterReader' import SharedResources from './SharedResources' import type SpecParser from './SpecParser' import type SchemaValidator from './SchemaValidator' @@ -24,7 +24,7 @@ export default class ChapterEvaluator { async evaluate (skipped: boolean): Promise { try { - if (skipped) return { result: Result.SKIPPED, title: this.chapter.synopsis } + if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } } const operation = this.spec_parser.locate_operation(this.chapter) const response = await this.chapter_reader.read(this.chapter, true) const params = this.#evaluate_parameters(operation) @@ -33,13 +33,13 @@ export default class ChapterEvaluator { const payload = this.#evaluate_payload(operation, response) return { title: this.chapter.synopsis, + overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) }, request: { parameters: params, requestBody: request_body }, - response: { status, payload }, - result: overall_result(Object.values(params).concat([request_body, status, payload])) + response: { status, payload } } } catch (error) { - if (!(error instanceof ResponseError)) throw error - return { result: Result.ERROR, title: this.chapter.synopsis, message: error.message } + if (!(error instanceof ServerError)) throw error + return { title: this.chapter.synopsis, overall: { result: Result.ERROR, message: error.message, error: error.original_error } } } } @@ -56,7 +56,7 @@ export default class ChapterEvaluator { if (!this.chapter.request_body) return { result: Result.PASSED } const content_type = this.chapter.request_body.content_type ?? 'application/json' const schema = operation.requestBody?.content[content_type]?.schema - if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found.` } + if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` } return this.schema_validator.validate(schema, this.chapter.request_body?.payload ?? {}) } @@ -64,15 +64,16 @@ export default class ChapterEvaluator { const expected_status = this.chapter.response?.status ?? 200 if (response.status === expected_status) return { result: Result.PASSED } this.skip_payload = true - return { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}: ${response.message}.` } + return { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.` } } #evaluate_payload (operation: ParsedOperation, response: ActualResponse): Evaluation { if (this.skip_payload) return { result: Result.SKIPPED } - const content = operation.responses[response.status]?.content[response.content_type] + const content_type = response.content_type ?? 'application/json' + const content = operation.responses[response.status]?.content[content_type] const schema = content?.schema if (schema == null && content != null) return { result: Result.PASSED } - if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found.` } + if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` } return this.schema_validator.validate(schema, response.payload) } } diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 0a6fbc990..26841f4bb 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -1,7 +1,13 @@ import axios from 'axios' import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types' -export class ResponseError extends Error {} +export class ServerError extends Error { + original_error: Error + constructor (message: string, error: Error) { + super(message) + this.original_error = error + } +} // A lightweight client for testing the API export default class ChapterReader { @@ -25,11 +31,12 @@ export default class ChapterReader { response.payload = r.data }).catch(e => { if (e.response == null) throw e - if (!ignore_errors) throw new ResponseError(e.response.data.error.reason as string) - response.status = e.response.status + if (!ignore_errors) throw new ServerError(e.response.data.error.reason as string, e as Error) response.content_type = e.response.headers['content-type'].split(';')[0] response.payload = e.response.data?.error response.message = e.response.data?.error?.reason + response.status = e.response.status + response.error = e }) return response as ActualResponse } diff --git a/tools/src/tester/ResultsDisplayer.ts b/tools/src/tester/ResultsDisplayer.ts index 3950618f3..ab745bf17 100644 --- a/tools/src/tester/ResultsDisplayer.ts +++ b/tools/src/tester/ResultsDisplayer.ts @@ -16,17 +16,25 @@ function cyan (text: string): string { return `\x1b[36m${text}\x1b[0m` } function gray (text: string): string { return `\x1b[90m${text}\x1b[0m` } function magenta (text: string): string { return `\x1b[35m${text}\x1b[0m` } +interface ResultDisplayerOptions { + gap?: number + verbose?: boolean + ignored_results: Set +} + export default class ResultsDisplayer { - gap: number evaluation: StoryEvaluation skip_components: boolean + gap: number ignored_results: Set + verbose: boolean - constructor (evaluation: StoryEvaluation, ignored_results: Set, gap: number = 4) { + constructor (evaluation: StoryEvaluation, opts: ResultDisplayerOptions) { this.evaluation = evaluation - this.ignored_results = ignored_results this.skip_components = [Result.PASSED, Result.SKIPPED].includes(evaluation.result) - this.gap = gap + this.ignored_results = opts.ignored_results + this.gap = opts.gap ?? 4 + this.verbose = opts.verbose ?? false } display (): void { @@ -46,15 +54,15 @@ export default class ResultsDisplayer { #display_chapters (evaluations: ChapterEvaluation[], title: string): void { if (this.skip_components || evaluations.length === 0) return - const result = overall_result(evaluations) + const result = overall_result(evaluations.map(e => e.overall)) this.#display_evaluation({ result }, title, this.gap) if (result === Result.PASSED) return for (const evaluation of evaluations) this.#display_chapter(evaluation) } #display_chapter (chapter: ChapterEvaluation): void { - this.#display_evaluation(chapter, i(chapter.title), this.gap * 2) - if (chapter.result === Result.PASSED || chapter.result === Result.SKIPPED) return + this.#display_evaluation(chapter.overall, i(chapter.title), this.gap * 2) + if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return this.#display_parameters(chapter.request?.parameters ?? {}) this.#display_request_body(chapter.request?.requestBody) @@ -93,6 +101,11 @@ export default class ResultsDisplayer { const result = padding(this.#result(evaluation.result), 0, prefix) const message = evaluation.message != null ? `${gray('(' + evaluation.message + ')')}` : '' console.log(`${result} ${title} ${message}`) + if (evaluation.error && this.verbose) { + console.log('-'.repeat(100)) + console.error(evaluation.error) + console.log('-'.repeat(100)) + } } #result (r: Result): string { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 4c1028eb9..fffffd96b 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -2,7 +2,7 @@ import { type Chapter, type Story, type SupplementalChapter } from './types/stor import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/eval.types' import ChapterEvaluator from './ChapterEvaluator' import type ChapterReader from './ChapterReader' -import { ResponseError } from './ChapterReader' +import { ServerError } from './ChapterReader' import SharedResources from './SharedResources' export interface StoryFile { @@ -56,9 +56,9 @@ export default class StoryEvaluator { for (const chapter of chapters) { const evaluator = new ChapterEvaluator(chapter) const evaluation = await evaluator.evaluate(has_errors) - has_errors = has_errors || evaluation.result === Result.ERROR - if (evaluation.result === Result.FAILED) this.result = Result.FAILED - if (evaluation.result === Result.ERROR) this.result = Result.ERROR + has_errors = has_errors || evaluation.overall.result === Result.ERROR + if (evaluation.overall.result === Result.FAILED) this.result = Result.FAILED + if (evaluation.overall.result === Result.ERROR) this.result = Result.ERROR evaluations.push(evaluation) } @@ -71,12 +71,12 @@ export default class StoryEvaluator { const title = `${chapter.method} ${chapter.path}` try { await this.chapter_reader.read(chapter, chapter.ignore_errors) - evaluations.push({ title, result: Result.PASSED }) + evaluations.push({ title, overall: { result: Result.PASSED } }) } catch (error) { - if (!(error instanceof ResponseError)) throw error + if (!(error instanceof ServerError)) throw error this.result = Result.ERROR this.has_errors = true - evaluations.push({ title, result: Result.ERROR, message: (error).message }) + evaluations.push({ title, overall: { result: Result.ERROR, message: (error).message, error: error.original_error } }) } } return evaluations diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts index b91091f42..3a4073477 100644 --- a/tools/src/tester/TestsRunner.ts +++ b/tools/src/tester/TestsRunner.ts @@ -34,7 +34,7 @@ export default class TestsRunner { async run (): Promise { const evaluations = await this.evaluate() for (const evaluation of evaluations) { - const displayer = new ResultsDisplayer(evaluation, this.opts.ignored_results) + const displayer = new ResultsDisplayer(evaluation, this.opts) displayer.display() } } diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index e47e11276..95f9e7af7 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -13,8 +13,7 @@ export interface StoryEvaluation { export interface ChapterEvaluation { title: string - result: Result - message?: string + overall: Evaluation request?: { parameters?: Record requestBody?: Evaluation @@ -28,6 +27,7 @@ export interface ChapterEvaluation { export interface Evaluation { result: Result message?: string + error?: Error } export enum Result { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 4cc483584..881d19ebf 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -104,4 +104,8 @@ export interface ActualResponse { * Error message for non 2XX responses. */ message?: string; + /** + * Error object. + */ + error?: {}; }