Specify semver range for entire stories.

Signed-off-by: dblock <dblock@amazon.com>
opensearch-project · Jul 11, 2024 · 31288b8 · 31288b8
1 parent b510523
commit 31288b8
Show file tree

Hide file tree

Showing 9 changed files with 33 additions and 5 deletions.
diff --git a/SPECIFICATION_TESTING.md b/SPECIFICATION_TESTING.md
@@ -138,7 +138,7 @@ As you can see, the `output` field in the first chapter saves the `model_group_i
 
 ### Managing Versions
 
-It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your tests as follows.
+It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows.
 
 ```yaml
 - synopsis: Search with `phase_took` added in OpenSearch 2.12.

diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml
@@ -18,6 +18,8 @@ properties:
     type: array
     items:
       $ref: '#/definitions/Chapter'
+  version:
+    $ref: '#/definitions/Version'
 required: [description, chapters]
 additionalProperties: false
 

diff --git a/tests/cat/pit_segments.yaml b/tests/cat/pit_segments.yaml
@@ -1,6 +1,7 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
 description: Test cat/pit_segments endpoints.
+version: '>= 2.4.0'
 epilogues:
   - path: /games
     method: DELETE

diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
@@ -26,6 +26,16 @@ export default class StoryEvaluator {
   }
 
   async evaluate({ story, display_path, full_path }: StoryFile, version: string, dry_run: boolean = false): Promise<StoryEvaluation> {
+    if (story.version !== undefined && !semver.satisfies(version, story.version)) {
+      return {
+        result: Result.SKIPPED,
+        display_path,
+        full_path,
+        description: story.description,
+        message: `Skipped because version ${version} does not satisfy ${story.version}.`
+      }
+    }
+
     const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path)
     if (variables_error !== undefined) {
       return variables_error
@@ -53,7 +63,7 @@ export default class StoryEvaluator {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } })
       } else if (chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because ${version} does not satisfy ${chapter.version}.`, error: undefined } })
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
         has_errors = has_errors || evaluation.overall.result === Result.ERROR

diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts
@@ -65,11 +65,12 @@ export type ReadChapter = Chapter & {
 };
 
 export interface Story {
-  $schema: string;
+  $schema?: string;
   description: string;
   prologues?: SupplementalChapter[];
   epilogues?: SupplementalChapter[];
   chapters: Chapter[];
+  version?: Version;
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema

diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml
@@ -136,7 +136,7 @@ chapters:
   - title: This GET /_cat/health should be skipped (> 2.999.0).
     overall:
       result: SKIPPED
-      message: Skipped because 2.15.0 does not satisfy >= 2.999.0.
+      message: Skipped because version 2.15.0 does not satisfy >= 2.999.0.
 epilogues:
   - title: DELETE /books
     overall:

diff --git a/tools/tests/tester/fixtures/evals/skipped/semver.yaml b/tools/tests/tester/fixtures/evals/skipped/semver.yaml
@@ -0,0 +1,6 @@
+display_path: skipped/semver.yaml
+full_path: tools/tests/tester/fixtures/stories/skipped/semver.yaml
+
+result: SKIPPED
+description: This story should be skipped because of version.
+message: Skipped because version 2.15.0 does not satisfy >= 2.999.0.
diff --git a/tools/tests/tester/fixtures/stories/skipped/semver.yaml b/tools/tests/tester/fixtures/stories/skipped/semver.yaml
@@ -0,0 +1,7 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should be skipped because of version.
+version: '>= 2.999.0'
+prologues: []
+epilogues: []
+chapters: []
diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts
@@ -29,11 +29,12 @@ test('stories folder', async () => {
   }
 
   const passed = load_expected_evaluation('passed', true)
+  const skipped = load_expected_evaluation('skipped/semver', true)
   const not_found = load_expected_evaluation('failed/not_found', true)
   const invalid_data = load_expected_evaluation('failed/invalid_data', true)
   const chapter_error = load_expected_evaluation('error/chapter_error', true)
   const prologue_error = load_expected_evaluation('error/prologue_error', true)
 
-  const expected_evaluations = [passed, chapter_error, prologue_error, invalid_data, not_found]
+  const expected_evaluations = [passed, chapter_error, prologue_error, invalid_data, not_found, skipped]
   expect(actual_evaluations).toEqual(expected_evaluations)
 })