Allow response payload to reference variables.

CHANGELOG.md

@@ -41,7 +41,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396))
-- Add `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408))
+- Added `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408))
 - Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409))
 - Added support for skipping tests using semver range ([#410](https://github.com/opensearch-project/opensearch-api-specification/pull/410))
 - Added `cluster_manager_timeout` to `HEAD /{index}` ([#421](https://github.com/opensearch-project/opensearch-api-specification/pull/421))
@@ -65,7 +65,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462))
 - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462))
 - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465))
-
+- Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471))
+
 ### Changed
 
 - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))

TESTING_GUIDE.md

@@ -167,6 +167,8 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml)
 ```
 As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group.
 
+You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example.
+
 ### Managing Versions
 
 It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows.

tests/nodes/plugins/index_state_management.yaml

@@ -0,0 +1,30 @@
+$schema: ../../../json_schemas/test_story.schema.yaml
+
+description: Get index_state_management node info settings.
+prologues:
+  - path: /_cat/nodes
+    id: node
+    method: GET
+    parameters:
+      full_id: true
+      size: 1
+      format: json
+      h: id
+    output:
+      id: payload[0].id
+chapters:
+  - synopsis: Get node info.
+    path: /_nodes/{node_id_or_metric}
+    method: GET
+    parameters:
+      node_id_or_metric: ${node.id}
+    response:
+      status: 200
+      payload:
+        nodes:
+          ${node.id}:
+            settings:
+              plugins:
+                index_state_management:
+                  job_interval: '1'
+

tools/src/tester/ChapterEvaluator.ts

@@ -66,9 +66,12 @@ export default class ChapterEvaluator {
     const params = this.#evaluate_parameters(chapter, operation)
     const request = this.#evaluate_request(chapter, operation)
     const status = this.#evaluate_status(chapter, response)
-    const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED }
     const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
     const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } }
+    const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
+    const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED }
+
+    if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
 
     const evaluations = _.compact(_.concat(
       Object.values(params),

tools/src/tester/ChapterOutput.ts

@@ -30,19 +30,17 @@ export class ChapterOutput {
     if (!output) return { evaluation: { result: Result.SKIPPED } }
     const chapter_output = new ChapterOutput({})
     for (const [name, path] of Object.entries(output)) {
-      const [source, ...rest] = path.split('.')
-      const keys = rest.join('.')
       let value: any
-      if (source === 'payload') {
+      if (path == 'payload' || path.startsWith('payload.') || path.match(/^payload\[\d*\]/)) {
         if (response.payload === undefined) {
           return { evaluation: { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path } }
         }
-        value = keys.length === 0 ? response.payload : _.get(response.payload, keys)
+        value = _.get(response, path)
         if (value === undefined) {
           return { evaluation: { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` } }
         }
       } else {
-        return { evaluation: { result: Result.ERROR, message: 'Unknown output source: ' + source } }
+        return { evaluation: { result: Result.ERROR, message: `Unknown output source: ${path.split('.')[0]}.` } }
       }
       chapter_output.set(name, value)
     }

tools/src/tester/StoryOutputs.ts

@@ -68,12 +68,14 @@ export class StoryOutputs {
             resolved_array.push(this.resolve_value(value))
           }
           return resolved_array
-        } else {
+        } else if (payload !== null) {
           const resolved_obj: Record<string, any> = {}
           for (const [key, value] of Object.entries(payload as Record<string, any>)) {
-            resolved_obj[key] = this.resolve_value(value)
+            resolved_obj[this.resolve_value(key)] = this.resolve_value(value)
           }
           return resolved_obj
+        } else {
+          return payload
         }
       default:
         return payload

tools/src/tester/SupplementalChapterEvaluator.ts

@@ -14,19 +14,24 @@ import { StoryOutputs } from "./StoryOutputs";
 import { overall_result } from "./helpers";
 import { ChapterEvaluation, Result } from "./types/eval.types";
 import { SupplementalChapter } from "./types/story.types";
+import { Logger } from "../Logger";
+import { to_json } from "../helpers";
 
 export default class SupplementalChapterEvaluator {
   private readonly _chapter_reader: ChapterReader;
+  private readonly logger: Logger;
 
-  constructor(chapter_reader: ChapterReader) {
+  constructor(chapter_reader: ChapterReader, logger: Logger) {
     this._chapter_reader = chapter_reader;
+    this.logger = logger
   }
 
   async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<{ evaluation: ChapterEvaluation, evaluation_error: boolean }> {
     const title = `${chapter.method} ${chapter.path}`
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const status = chapter.status ?? [200, 201]
     const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
+    if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
     let response_evaluation: ChapterEvaluation
     const passed_evaluation = { title, overall: { result: Result.PASSED } }
     if (status.includes(response.status)) {

tools/src/tester/test.ts

@@ -57,7 +57,7 @@ const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Lo
 const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts }))
 const chapter_reader = new ChapterReader(http_client, logger)
 const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger)
-const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
+const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger)
 const story_validator = new StoryValidator()
 const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
 const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose)

tools/tests/tester/ChapterOutput.test.ts

@@ -0,0 +1,158 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { ChapterOutput } from 'tester/ChapterOutput'
+import { EvaluationWithOutput, Result } from 'tester/types/eval.types'
+import { ActualResponse } from 'tester/types/story.types'
+
+function create_response(payload: any): ActualResponse {
+  return {
+    status: 200,
+    content_type: 'application/json',
+    payload
+  }
+}
+
+function passed_output(output: Record<string, any>): EvaluationWithOutput {
+  return {
+    evaluation: { result: Result.PASSED },
+    output: new ChapterOutput(output)
+  }
+}
+
+describe('with an object response', () => {
+  const response: ActualResponse = create_response({
+    a: {
+      b: {
+        c: 1
+      },
+      arr: [
+        { d: 2 },
+        { e: 3 }
+      ]
+    }
+  })
+
+  test('returns nested values', () => {
+    const output = {
+      c: 'payload.a.b.c',
+      d: 'payload.a.arr[0].d',
+      e: 'payload.a.arr[1].e'
+    }
+
+    expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({
+      c: 1,
+      d: 2,
+      e: 3
+    }))
+  })
+
+  test('extracts complete payload', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual(
+      passed_output({ x: response.payload })
+    )
+  })
+
+  test('errors on undefined value', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({
+      evaluation: {
+        result: Result.ERROR,
+        message: 'Expected to find non undefined value at `payload.a.b.x[0]`.'
+      }
+    })
+  })
+
+  test('errors on invalid source', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({
+      evaluation: {
+        result: Result.ERROR,
+        message: 'Unknown output source: foobar.'
+      }
+    })
+  })
+})
+
+describe('with an array response', () => {
+  const response: ActualResponse = create_response([
+    {
+      a: {
+        b: {
+          c: 1
+        },
+        arr: [
+          { d: 2 },
+          { e: 3 }
+        ]
+      }
+    },{
+      a: {
+        b: {
+          c: 2
+        },
+        arr: [
+          { d: 3 },
+          { e: 4 }
+        ]
+      }
+    },
+  ])
+
+  test('returns nested values', () => {
+    const output = {
+      c1: 'payload[0].a.b.c',
+      d1: 'payload[0].a.arr[0].d',
+      e1: 'payload[0].a.arr[1].e',
+      c2: 'payload[1].a.b.c',
+      d2: 'payload[1].a.arr[0].d',
+      e2: 'payload[1].a.arr[1].e'
+    }
+
+    expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({
+      c1: 1,
+      d1: 2,
+      e1: 3,
+      c2: 2,
+      d2: 3,
+      e2: 4
+    }))
+  })
+
+  test('extracts complete payload', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual(
+      passed_output({ x: response.payload })
+    )
+  })
+
+  test('errors on undefined value', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'payload[0].a.b.x[0]' })).toEqual({
+      evaluation: {
+        result: Result.ERROR,
+        message: 'Expected to find non undefined value at `payload[0].a.b.x[0]`.'
+      }
+    })
+  })
+
+  test('errors on invalid source', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({
+      evaluation: {
+        result: Result.ERROR,
+        message: 'Unknown output source: foobar.'
+      }
+    })
+  })
+
+  test('errors on invalid index', () => {
+    expect(ChapterOutput.extract_output_values(response, { x: 'payload[2]' })).toEqual({
+      evaluation: {
+        result: Result.ERROR,
+        message: 'Expected to find non undefined value at `payload[2]`.'
+      }
+    })
+  })
+})

tools/tests/tester/StoryOutputs.test.ts

@@ -32,7 +32,8 @@ test('resolve_value', () => {
       e: 'str',
       f: true
     },
-    g: 123
+    g: 123,
+    '${chapter_id.x}': 345
   }
   expect(story_outputs.resolve_value(value)).toEqual(
     {
@@ -43,7 +44,8 @@ test('resolve_value', () => {
         e: 'str',
         f: true
       },
-      g: 123
+      g: 123,
+      1: 345
     }
   )
 })

tools/tests/tester/helpers.ts

@@ -48,7 +48,7 @@ export function construct_tester_components (spec_path: string): {
   const chapter_reader = new ChapterReader(opensearch_http_client, logger)
   const schema_validator = new SchemaValidator(specification, logger)
   const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger)
-  const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
+  const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger)
   const story_validator = new StoryValidator()
   const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
   const result_logger = new NoOpResultLogger()