# implemented verbose feature

Signed-off-by: Theo Truong <theotr@amazon.com>

json_schemas/test_story.schema.yaml

@@ -98,6 +98,7 @@ definitions:
         default: 200
       content_type:
         type: string
+        default: application/json
       payload:
         $ref: '#/definitions/Payload'
     required: [ status ]
@@ -115,6 +116,9 @@ definitions:
       message:
         type: string
         description: Error message for non 2XX responses.
+      error:
+        type: object
+        description: Error object.
     required: [ status, content_type, payload ]
     additionalProperties: false
 

tools/src/tester/ChapterEvaluator.ts

@@ -3,7 +3,7 @@ import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.ty
 import { type ParsedOperation } from './types/spec.types'
 import { overall_result } from './helpers'
 import type ChapterReader from './ChapterReader'
-import { ResponseError } from './ChapterReader'
+import { ServerError } from './ChapterReader'
 import SharedResources from './SharedResources'
 import type SpecParser from './SpecParser'
 import type SchemaValidator from './SchemaValidator'
@@ -24,7 +24,7 @@ export default class ChapterEvaluator {
 
   async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
     try {
-      if (skipped) return { result: Result.SKIPPED, title: this.chapter.synopsis }
+      if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
       const operation = this.spec_parser.locate_operation(this.chapter)
       const response = await this.chapter_reader.read(this.chapter, true)
       const params = this.#evaluate_parameters(operation)
@@ -33,13 +33,13 @@ export default class ChapterEvaluator {
       const payload = this.#evaluate_payload(operation, response)
       return {
         title: this.chapter.synopsis,
+        overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
         request: { parameters: params, requestBody: request_body },
-        response: { status, payload },
-        result: overall_result(Object.values(params).concat([request_body, status, payload]))
+        response: { status, payload }
       }
     } catch (error) {
-      if (!(error instanceof ResponseError)) throw error
-      return { result: Result.ERROR, title: this.chapter.synopsis, message: error.message }
+      if (!(error instanceof ServerError)) throw error
+      return { title: this.chapter.synopsis, overall: { result: Result.ERROR, message: error.message, error: error.original_error } }
     }
   }
 
@@ -56,23 +56,24 @@ export default class ChapterEvaluator {
     if (!this.chapter.request_body) return { result: Result.PASSED }
     const content_type = this.chapter.request_body.content_type ?? 'application/json'
     const schema = operation.requestBody?.content[content_type]?.schema
-    if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found.` }
+    if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` }
     return this.schema_validator.validate(schema, this.chapter.request_body?.payload ?? {})
   }
 
   #evaluate_status (response: ActualResponse): Evaluation {
     const expected_status = this.chapter.response?.status ?? 200
     if (response.status === expected_status) return { result: Result.PASSED }
     this.skip_payload = true
-    return { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}: ${response.message}.` }
+    return { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.` }
   }
 
   #evaluate_payload (operation: ParsedOperation, response: ActualResponse): Evaluation {
     if (this.skip_payload) return { result: Result.SKIPPED }
-    const content = operation.responses[response.status]?.content[response.content_type]
+    const content_type = response.content_type ?? 'application/json'
+    const content = operation.responses[response.status]?.content[content_type]
     const schema = content?.schema
     if (schema == null && content != null) return { result: Result.PASSED }
-    if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found.` }
+    if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` }
     return this.schema_validator.validate(schema, response.payload)
   }
 }

tools/src/tester/ChapterReader.ts

@@ -1,7 +1,13 @@
 import axios from 'axios'
 import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types'
 
-export class ResponseError extends Error {}
+export class ServerError extends Error {
+  original_error: Error
+  constructor (message: string, error: Error) {
+    super(message)
+    this.original_error = error
+  }
+}
 
 // A lightweight client for testing the API
 export default class ChapterReader {
@@ -25,11 +31,12 @@ export default class ChapterReader {
       response.payload = r.data
     }).catch(e => {
       if (e.response == null) throw e
-      if (!ignore_errors) throw new ResponseError(e.response.data.error.reason as string)
-      response.status = e.response.status
+      if (!ignore_errors) throw new ServerError(e.response.data.error.reason as string, e as Error)
       response.content_type = e.response.headers['content-type'].split(';')[0]
       response.payload = e.response.data?.error
       response.message = e.response.data?.error?.reason
+      response.status = e.response.status
+      response.error = e
     })
     return response as ActualResponse
   }

tools/src/tester/ResultsDisplayer.ts

@@ -16,17 +16,25 @@ function cyan (text: string): string { return `\x1b[36m${text}\x1b[0m` }
 function gray (text: string): string { return `\x1b[90m${text}\x1b[0m` }
 function magenta (text: string): string { return `\x1b[35m${text}\x1b[0m` }
 
+interface ResultDisplayerOptions {
+  gap?: number
+  verbose?: boolean
+  ignored_results: Set<Result>
+}
+
 export default class ResultsDisplayer {
-  gap: number
   evaluation: StoryEvaluation
   skip_components: boolean
+  gap: number
   ignored_results: Set<Result>
+  verbose: boolean
 
-  constructor (evaluation: StoryEvaluation, ignored_results: Set<Result>, gap: number = 4) {
+  constructor (evaluation: StoryEvaluation, opts: ResultDisplayerOptions) {
     this.evaluation = evaluation
-    this.ignored_results = ignored_results
     this.skip_components = [Result.PASSED, Result.SKIPPED].includes(evaluation.result)
-    this.gap = gap
+    this.ignored_results = opts.ignored_results
+    this.gap = opts.gap ?? 4
+    this.verbose = opts.verbose ?? false
   }
 
   display (): void {
@@ -46,15 +54,15 @@ export default class ResultsDisplayer {
 
   #display_chapters (evaluations: ChapterEvaluation[], title: string): void {
     if (this.skip_components || evaluations.length === 0) return
-    const result = overall_result(evaluations)
+    const result = overall_result(evaluations.map(e => e.overall))
     this.#display_evaluation({ result }, title, this.gap)
     if (result === Result.PASSED) return
     for (const evaluation of evaluations) this.#display_chapter(evaluation)
   }
 
   #display_chapter (chapter: ChapterEvaluation): void {
-    this.#display_evaluation(chapter, i(chapter.title), this.gap * 2)
-    if (chapter.result === Result.PASSED || chapter.result === Result.SKIPPED) return
+    this.#display_evaluation(chapter.overall, i(chapter.title), this.gap * 2)
+    if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return
 
     this.#display_parameters(chapter.request?.parameters ?? {})
     this.#display_request_body(chapter.request?.requestBody)
@@ -93,6 +101,11 @@ export default class ResultsDisplayer {
     const result = padding(this.#result(evaluation.result), 0, prefix)
     const message = evaluation.message != null ? `${gray('(' + evaluation.message + ')')}` : ''
     console.log(`${result} ${title} ${message}`)
+    if (evaluation.error && this.verbose) {
+      console.log('-'.repeat(100))
+      console.error(evaluation.error)
+      console.log('-'.repeat(100))
+    }
   }
 
   #result (r: Result): string {

tools/src/tester/StoryEvaluator.ts

@@ -2,7 +2,7 @@ import { type Chapter, type Story, type SupplementalChapter } from './types/stor
 import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/eval.types'
 import ChapterEvaluator from './ChapterEvaluator'
 import type ChapterReader from './ChapterReader'
-import { ResponseError } from './ChapterReader'
+import { ServerError } from './ChapterReader'
 import SharedResources from './SharedResources'
 
 export interface StoryFile {
@@ -56,9 +56,9 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const evaluator = new ChapterEvaluator(chapter)
       const evaluation = await evaluator.evaluate(has_errors)
-      has_errors = has_errors || evaluation.result === Result.ERROR
-      if (evaluation.result === Result.FAILED) this.result = Result.FAILED
-      if (evaluation.result === Result.ERROR) this.result = Result.ERROR
+      has_errors = has_errors || evaluation.overall.result === Result.ERROR
+      if (evaluation.overall.result === Result.FAILED) this.result = Result.FAILED
+      if (evaluation.overall.result === Result.ERROR) this.result = Result.ERROR
       evaluations.push(evaluation)
     }
 
@@ -71,12 +71,12 @@ export default class StoryEvaluator {
       const title = `${chapter.method} ${chapter.path}`
       try {
         await this.chapter_reader.read(chapter, chapter.ignore_errors)
-        evaluations.push({ title, result: Result.PASSED })
+        evaluations.push({ title, overall: { result: Result.PASSED } })
       } catch (error) {
-        if (!(error instanceof ResponseError)) throw error
+        if (!(error instanceof ServerError)) throw error
         this.result = Result.ERROR
         this.has_errors = true
-        evaluations.push({ title, result: Result.ERROR, message: (error).message })
+        evaluations.push({ title, overall: { result: Result.ERROR, message: (error).message, error: error.original_error } })
       }
     }
     return evaluations

tools/src/tester/TestsRunner.ts

@@ -34,7 +34,7 @@ export default class TestsRunner {
   async run (): Promise<void> {
     const evaluations = await this.evaluate()
     for (const evaluation of evaluations) {
-      const displayer = new ResultsDisplayer(evaluation, this.opts.ignored_results)
+      const displayer = new ResultsDisplayer(evaluation, this.opts)
       displayer.display()
     }
   }

tools/src/tester/types/eval.types.ts

@@ -13,8 +13,7 @@ export interface StoryEvaluation {
 
 export interface ChapterEvaluation {
   title: string
-  result: Result
-  message?: string
+  overall: Evaluation
   request?: {
     parameters?: Record<string, Evaluation>
     requestBody?: Evaluation
@@ -28,6 +27,7 @@ export interface ChapterEvaluation {
 export interface Evaluation {
   result: Result
   message?: string
+  error?: Error
 }
 
 export enum Result {

tools/src/tester/types/story.types.ts

@@ -104,4 +104,8 @@ export interface ActualResponse {
    * Error message for non 2XX responses.
    */
   message?: string;
+  /**
+   * Error object.
+   */
+  error?: {};
 }