diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a28aadc4..1bebc33e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395)) - Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395)) - Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396)) -- Add `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408)) +- Added `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408)) - Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409)) - Added support for skipping tests using semver range ([#410](https://github.com/opensearch-project/opensearch-api-specification/pull/410)) - Added `cluster_manager_timeout` to `HEAD /{index}` ([#421](https://github.com/opensearch-project/opensearch-api-specification/pull/421)) @@ -65,7 +65,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - +- Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) + ### Changed - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189)) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index ad79f85d6..a6d4456cf 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -167,6 +167,8 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) ``` As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group. +You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example. + ### Managing Versions It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. diff --git a/tests/nodes/plugins/index_state_management.yaml b/tests/nodes/plugins/index_state_management.yaml new file mode 100644 index 000000000..4f95866d6 --- /dev/null +++ b/tests/nodes/plugins/index_state_management.yaml @@ -0,0 +1,30 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Get index_state_management node info settings. +prologues: + - path: /_cat/nodes + id: node + method: GET + parameters: + full_id: true + size: 1 + format: json + h: id + output: + id: payload[0].id +chapters: + - synopsis: Get node info. + path: /_nodes/{node_id_or_metric} + method: GET + parameters: + node_id_or_metric: ${node.id} + response: + status: 200 + payload: + nodes: + ${node.id}: + settings: + plugins: + index_state_management: + job_interval: '1' + diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index e06fca445..0be147f4b 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -66,9 +66,12 @@ export default class ChapterEvaluator { const params = this.#evaluate_parameters(chapter, operation) const request = this.#evaluate_request(chapter, operation) const status = this.#evaluate_status(chapter, response) - const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED } const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } } + const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload + const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED } + + if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) const evaluations = _.compact(_.concat( Object.values(params), diff --git a/tools/src/tester/ChapterOutput.ts b/tools/src/tester/ChapterOutput.ts index 5a8e07b28..d8fd6d115 100644 --- a/tools/src/tester/ChapterOutput.ts +++ b/tools/src/tester/ChapterOutput.ts @@ -30,19 +30,17 @@ export class ChapterOutput { if (!output) return { evaluation: { result: Result.SKIPPED } } const chapter_output = new ChapterOutput({}) for (const [name, path] of Object.entries(output)) { - const [source, ...rest] = path.split('.') - const keys = rest.join('.') let value: any - if (source === 'payload') { + if (path == 'payload' || path.startsWith('payload.') || path.match(/^payload\[\d*\]/)) { if (response.payload === undefined) { return { evaluation: { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path } } } - value = keys.length === 0 ? response.payload : _.get(response.payload, keys) + value = _.get(response, path) if (value === undefined) { return { evaluation: { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` } } } } else { - return { evaluation: { result: Result.ERROR, message: 'Unknown output source: ' + source } } + return { evaluation: { result: Result.ERROR, message: `Unknown output source: ${path.split('.')[0]}.` } } } chapter_output.set(name, value) } diff --git a/tools/src/tester/StoryOutputs.ts b/tools/src/tester/StoryOutputs.ts index c55ac21d6..41b8d9e71 100644 --- a/tools/src/tester/StoryOutputs.ts +++ b/tools/src/tester/StoryOutputs.ts @@ -68,12 +68,14 @@ export class StoryOutputs { resolved_array.push(this.resolve_value(value)) } return resolved_array - } else { + } else if (payload !== null) { const resolved_obj: Record = {} for (const [key, value] of Object.entries(payload as Record)) { - resolved_obj[key] = this.resolve_value(value) + resolved_obj[this.resolve_value(key)] = this.resolve_value(value) } return resolved_obj + } else { + return payload } default: return payload diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index 81b680bef..e507e9915 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -14,12 +14,16 @@ import { StoryOutputs } from "./StoryOutputs"; import { overall_result } from "./helpers"; import { ChapterEvaluation, Result } from "./types/eval.types"; import { SupplementalChapter } from "./types/story.types"; +import { Logger } from "../Logger"; +import { to_json } from "../helpers"; export default class SupplementalChapterEvaluator { private readonly _chapter_reader: ChapterReader; + private readonly logger: Logger; - constructor(chapter_reader: ChapterReader) { + constructor(chapter_reader: ChapterReader, logger: Logger) { this._chapter_reader = chapter_reader; + this.logger = logger } async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<{ evaluation: ChapterEvaluation, evaluation_error: boolean }> { @@ -27,6 +31,7 @@ export default class SupplementalChapterEvaluator { const response = await this._chapter_reader.read(chapter, story_outputs) const status = chapter.status ?? [200, 201] const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output) + if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) let response_evaluation: ChapterEvaluation const passed_evaluation = { title, overall: { result: Result.PASSED } } if (status.includes(response.status)) { diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 98a875e09..601d3ff5d 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -57,7 +57,7 @@ const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Lo const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) -const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) +const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) diff --git a/tools/tests/tester/ChapterOutput.test.ts b/tools/tests/tester/ChapterOutput.test.ts new file mode 100644 index 000000000..f5ace6ed7 --- /dev/null +++ b/tools/tests/tester/ChapterOutput.test.ts @@ -0,0 +1,158 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { ChapterOutput } from 'tester/ChapterOutput' +import { EvaluationWithOutput, Result } from 'tester/types/eval.types' +import { ActualResponse } from 'tester/types/story.types' + +function create_response(payload: any): ActualResponse { + return { + status: 200, + content_type: 'application/json', + payload + } +} + +function passed_output(output: Record): EvaluationWithOutput { + return { + evaluation: { result: Result.PASSED }, + output: new ChapterOutput(output) + } +} + +describe('with an object response', () => { + const response: ActualResponse = create_response({ + a: { + b: { + c: 1 + }, + arr: [ + { d: 2 }, + { e: 3 } + ] + } + }) + + test('returns nested values', () => { + const output = { + c: 'payload.a.b.c', + d: 'payload.a.arr[0].d', + e: 'payload.a.arr[1].e' + } + + expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({ + c: 1, + d: 2, + e: 3 + })) + }) + + test('extracts complete payload', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( + passed_output({ x: response.payload }) + ) + }) + + test('errors on undefined value', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload.a.b.x[0]`.' + } + }) + }) + + test('errors on invalid source', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Unknown output source: foobar.' + } + }) + }) +}) + +describe('with an array response', () => { + const response: ActualResponse = create_response([ + { + a: { + b: { + c: 1 + }, + arr: [ + { d: 2 }, + { e: 3 } + ] + } + },{ + a: { + b: { + c: 2 + }, + arr: [ + { d: 3 }, + { e: 4 } + ] + } + }, + ]) + + test('returns nested values', () => { + const output = { + c1: 'payload[0].a.b.c', + d1: 'payload[0].a.arr[0].d', + e1: 'payload[0].a.arr[1].e', + c2: 'payload[1].a.b.c', + d2: 'payload[1].a.arr[0].d', + e2: 'payload[1].a.arr[1].e' + } + + expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({ + c1: 1, + d1: 2, + e1: 3, + c2: 2, + d2: 3, + e2: 4 + })) + }) + + test('extracts complete payload', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( + passed_output({ x: response.payload }) + ) + }) + + test('errors on undefined value', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload[0].a.b.x[0]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload[0].a.b.x[0]`.' + } + }) + }) + + test('errors on invalid source', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Unknown output source: foobar.' + } + }) + }) + + test('errors on invalid index', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload[2]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload[2]`.' + } + }) + }) +}) diff --git a/tools/tests/tester/StoryOutputs.test.ts b/tools/tests/tester/StoryOutputs.test.ts index 13d4a082a..3017b7a66 100644 --- a/tools/tests/tester/StoryOutputs.test.ts +++ b/tools/tests/tester/StoryOutputs.test.ts @@ -32,7 +32,8 @@ test('resolve_value', () => { e: 'str', f: true }, - g: 123 + g: 123, + '${chapter_id.x}': 345 } expect(story_outputs.resolve_value(value)).toEqual( { @@ -43,7 +44,8 @@ test('resolve_value', () => { e: 'str', f: true }, - g: 123 + g: 123, + 1: 345 } ) }) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 5ee331a14..dd6758d2a 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -48,7 +48,7 @@ export function construct_tester_components (spec_path: string): { const chapter_reader = new ChapterReader(opensearch_http_client, logger) const schema_validator = new SchemaValidator(specification, logger) const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger) - const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) + const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new NoOpResultLogger() diff --git a/tools/tests/tester/test.test.ts b/tools/tests/tester/test.test.ts index ee76c19b9..e6677185d 100644 --- a/tools/tests/tester/test.test.ts +++ b/tools/tests/tester/test.test.ts @@ -10,9 +10,8 @@ import { spawnSync } from 'child_process' import * as ansi from 'tester/Ansi' import * as path from 'path' -import { type Chapter, type ChapterRequest, type Output, type Request, type ActualResponse, Story } from 'tester/types/story.types' -import { type EvaluationWithOutput, Result, ChapterEvaluation, StoryEvaluation } from 'tester/types/eval.types' -import { ChapterOutput } from 'tester/ChapterOutput' +import { type Chapter, type ChapterRequest, type Output, type Request, Story } from 'tester/types/story.types' +import { ChapterEvaluation, Result, StoryEvaluation } from 'tester/types/eval.types' import StoryEvaluator from 'tester/StoryEvaluator' const spec = (args: string[]): any => { @@ -45,54 +44,6 @@ test('invalid story', () => { ) }) -function create_response(payload: any): ActualResponse { - return { - status: 200, - content_type: 'application/json', - payload - } -} - -function passed_output(output: Record): EvaluationWithOutput { - return { - evaluation: { result: Result.PASSED }, - output: new ChapterOutput(output) - } -} - -test('extract_output_values', () => { - const response: ActualResponse = create_response({ - a: { - b: { - c: 1 - }, - arr: [ - { d: 2 }, - { e: 3 } - ] - } - }) - const output1 = { - c: 'payload.a.b.c', - d: 'payload.a.arr[0].d', - e: 'payload.a.arr[1].e' - } - expect(ChapterOutput.extract_output_values(response, output1)).toEqual(passed_output({ - c: 1, - d: 2, - e: 3 - })) - expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( - passed_output({ x: response.payload }) - ) - expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({ - evaluation: { - result: Result.ERROR, - message: 'Expected to find non undefined value at `payload.a.b.x[0]`.' - } - }) -}) - function dummy_chapter_request(id?: string, output?: Output): ChapterRequest { return { id,