Fixed POST and DELETE /_dangling/{index_uuid} returning 202 (#686)

* Added dangling index tests.

Signed-off-by: dblock <dblock@amazon.com>

* Start the cluster once.

Signed-off-by: dblock <dblock@amazon.com>

* Removed replicas and node2 from voting configuration.

Signed-off-by: dblock <dblock@amazon.com>

* Fixed POST and DELETE /_dangling/{index_uuid} returning 202.

Signed-off-by: dblock <dblock@amazon.com>

* Fix: link.

Signed-off-by: dblock <dblock@amazon.com>

* Include hidden files in preparing for vale.

Signed-off-by: dblock <dblock@amazon.com>

* Security is disabled in this cluster.

Signed-off-by: dblock <dblock@amazon.com>

---------

Signed-off-by: dblock <dblock@amazon.com>

.github/workflows/test-spec.yml

@@ -37,6 +37,9 @@ jobs:
             tests: routing
           - version: 2.18.0
             tests: snapshot
+          - version: 2.18.0
+            tests: dangling
+            url: http://localhost:9200
           - version: 2.18.0
             tests: plugins/replication
             url: http://localhost:9200

CHANGELOG.md

@@ -37,6 +37,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed `refresh` options to allow `boolean` and `string` ([#673](https://github.com/opensearch-project/opensearch-api-specification/pull/673))
 - Fixed `value` type in `ExplanationDetail` and added `_type` to `explain@200` ([#685](https://github.com/opensearch-project/opensearch-api-specification/pull/685))
 - Fixed index settings types ([#684](https://github.com/opensearch-project/opensearch-api-specification/pull/684))
+- Fixed `POST` and `DELETE /_dangling/{index_uuid}` returning `202` ([#686](https://github.com/opensearch-project/opensearch-api-specification/pull/686))
 
 ### Changed
 - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))

spec/namespaces/dangling_indices.yaml

@@ -31,8 +31,8 @@ paths:
         - $ref: '#/components/parameters/dangling_indices.import_dangling_index::query.master_timeout'
         - $ref: '#/components/parameters/dangling_indices.import_dangling_index::query.timeout'
       responses:
-        '200':
-          $ref: '#/components/responses/dangling_indices.import_dangling_index@200'
+        '202':
+          $ref: '#/components/responses/dangling_indices.import_dangling_index@202'
     delete:
       operationId: dangling_indices.delete_dangling_index.0
       x-operation-group: dangling_indices.delete_dangling_index
@@ -47,17 +47,17 @@ paths:
         - $ref: '#/components/parameters/dangling_indices.delete_dangling_index::query.master_timeout'
         - $ref: '#/components/parameters/dangling_indices.delete_dangling_index::query.timeout'
       responses:
-        '200':
-          $ref: '#/components/responses/dangling_indices.delete_dangling_index@200'
+        '202':
+          $ref: '#/components/responses/dangling_indices.delete_dangling_index@202'
 components:
   requestBodies: {}
   responses:
-    dangling_indices.delete_dangling_index@200:
+    dangling_indices.delete_dangling_index@202:
       content:
         application/json:
           schema:
             $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase'
-    dangling_indices.import_dangling_index@200:
+    dangling_indices.import_dangling_index@202:
       content:
         application/json:
           schema:

tests/dangling/.data/README.md

@@ -0,0 +1,89 @@
+This container contains data from a detached cluster with a `movies` index. It was created as follows.
+
+Data will be stored in [opensearch-data1](./opensearch-data1).
+
+From this `.data` directory.
+
+Create a single-node cluster (without security) and open a shell in it.
+
+```
+docker run --name opensearch-single-node-cluster\
+  -it --entrypoint /bin/bash \
+  -p 9200:9200 -p 9600:9600 \
+  -e "discovery.type=single-node" \
+  --ulimit memlock=-1:-1 \
+  --ulimit nofile=65536:65536 \
+  -e DISABLE_INSTALL_DEMO_CONFIG=true \
+  -e DISABLE_SECURITY_PLUGIN=true \
+  -v $(pwd)/opensearch-data1:/usr/share/opensearch/data \
+  opensearchproject/opensearch:latest
+```
+
+Start the service.
+
+```
+./opensearch-docker-entrypoint.sh
+```
+
+From another shell, add data.
+
+```
+curl -X POST http://localhost:9200/movies/_doc --json '{"director":"Bennett Miller","title":"The Cruise","year":1998}'
+
+{"_index":"movies","_id":"XBDMTpMBjBlaZgUqAkGm","_version":1,"result":"created","_shards":{"total":2,"successful":1,"failed":0},"_seq_no":0,"_primary_term":1}
+```
+
+Remove replicas.
+
+```
+curl -X PUT  http://localhost:9200/movies/_settings --json '{"index.number_of_replicas":0}'
+
+{"acknowledged":true}
+```
+
+Stop the service with Ctrl+C.
+
+Detach cluster.
+
+```
+./bin/opensearch-node detach-cluster
+
+Confirm [y/N] y
+Node was successfully detached from the cluster
+```
+
+The cluster in [docker-compose](docker-compose.yml) uses this data. Start it.
+
+```
+docker compose up
+```
+
+After the service starts the `movies` index is dangling.
+
+To make it usable in tests we need to turn it to a single node cluster.
+
+Remove replicas.
+
+```
+curl -X PUT  http://localhost:9200/_settings --json '{"index.number_of_replicas":0}'
+
+{"acknowledged":true}
+```
+
+Remove node2 from voting.
+
+```
+curl -X POST http://localhost:9200/_cluster/voting_config_exclusions?node_names=opensearch-node2
+
+{"acknowledged":true}
+```
+
+Stop the cluster.
+
+Remove lock files, those will cause an "Underlying file changed by an external force" error when copied to the docker container.
+
+```
+find . -name *.lock | xargs rm 
+```
+
+Now you can run the [single-node docker-compose](../docker-compose.yml) in the folder above.

tests/dangling/.data/docker-compose.yml

@@ -0,0 +1,42 @@
+version: '3'
+services:
+  opensearch-node1:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    environment:
+      - DISABLE_INSTALL_DEMO_CONFIG=true
+      - DISABLE_SECURITY_PLUGIN=true
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2
+      - cluster.name=opensearch-cluster
+      - discovery.seed_hosts=opensearch-node1,opensearch-node2
+      - node.name=opensearch-node1
+    volumes:
+      - ./opensearch-data1:/usr/share/opensearch/data
+    ports:
+      - 9200:9200
+      - 9600:9600
+    networks:
+      - opensearch-net
+  opensearch-node2:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    environment:
+      - DISABLE_INSTALL_DEMO_CONFIG=true
+      - DISABLE_SECURITY_PLUGIN=true
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2
+      - cluster.name=opensearch-cluster
+      - discovery.seed_hosts=opensearch-node1,opensearch-node2
+      - node.name=opensearch-node2
+    volumes:
+      - opensearch-data2:/usr/share/opensearch/data
+    networks:
+      - opensearch-net
+
+volumes:
+  opensearch-data1: {}
+  opensearch-data2: {}
+
+networks:
+  opensearch-net: {}

tests/dangling/.data/opensearch-data1/batch_metrics_enabled.conf

@@ -0,0 +1 @@
+false

tests/dangling/.data/opensearch-data1/logging_enabled.conf

@@ -0,0 +1 @@
+false

tests/dangling/.data/opensearch-data1/performance_analyzer_enabled.conf

@@ -0,0 +1 @@
+false

tests/dangling/.data/opensearch-data1/rca_enabled.conf

@@ -0,0 +1 @@
+false

tests/dangling/.data/opensearch-data1/thread_contention_monitoring_enabled.conf

@@ -0,0 +1 @@
+false

tests/dangling/Dockerfile

@@ -0,0 +1,7 @@
+ARG OPENSEARCH_DOCKER_HUB_PROJECT
+ARG OPENSEARCH_VERSION
+ARG OPENSEARCH_DOCKER_REF
+
+FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF}
+
+COPY --chown=1000:1000 .data/opensearch-data1 /usr/share/opensearch/data

tests/dangling/core/dangling.yaml

@@ -0,0 +1,33 @@
+$schema: ../../../json_schemas/test_story.schema.yaml
+
+description: Test dangling indexes.
+distributions:
+  excluded:
+    - amazon-managed
+    - amazon-serverless
+chapters:
+  - synopsis: Get dangling indexes.
+    path: /_dangling
+    method: GET
+    response:
+      status: 200
+      payload:
+        dangling_indices:
+          - index_name: .opensearch-observability
+          - index_name: movies
+  - synopsis: Import a dangling index.
+    path: /_dangling/{index_uuid}
+    method: POST  
+    parameters:
+      index_uuid: p6tliBznQO-FzTdslShrwA # movies
+      accept_data_loss: true
+    response:
+      status: 202
+  - synopsis: Delete a dangling index.
+    path: /_dangling/{index_uuid}
+    method: DELETE
+    parameters:
+      index_uuid: _bJfm1BsRdG-N91LRxI1Vg # .opensearch-observability
+      accept_data_loss: true
+    response:
+      status: 202

tests/dangling/docker-compose.yml

@@ -0,0 +1,18 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    build:
+      context: .
+      args:
+        - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}
+        - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF}
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - DISABLE_INSTALL_DEMO_CONFIG=true
+      - DISABLE_SECURITY_PLUGIN=true
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node

tools/src/prepare-for-vale/KeepDescriptions.ts

@@ -25,7 +25,7 @@ export default class KeepDescriptions {
 
   process(): void {
     this.root_folder
-    const files = fg.globSync([`${this.root_folder}/**/*.{yaml,yml}`])
+    const files = fg.globSync([`${this.root_folder}/**/*.{yaml,yml}`], { dot: true })
     files.forEach((path) => {
       this.logger.log(path)
       this.process_file(path)

tools/tests/prepare-for-vale/KeepDescriptions.test.ts

@@ -11,11 +11,12 @@ import KeepDescriptions from 'prepare-for-vale/KeepDescriptions'
 import fs from 'fs'
 import fg from 'fast-glob'
 import tmp from 'tmp'
+import path from 'path'
 
 describe('KeepDescriptions', () => {
   var temp: tmp.DirResult
   var fixture_path: string = './tools/tests/prepare-for-vale/fixtures'
-  var fixtures = fg.globSync(`${fixture_path}/**/*.{yaml,yml}`)
+  var fixtures = fg.globSync(`${fixture_path}/**/*.{yaml,yml}`, { dot: true })
 
   describe('defaults', () => {
     beforeAll(() => {
@@ -32,7 +33,8 @@ describe('KeepDescriptions', () => {
       fixtures.forEach((filename) => {
         test(filename, () => {
           const processed_yaml = filename.replace(fixture_path, temp.name)
-          const filename_txt = processed_yaml.replace(".yaml", ".txt")
+          const basename = path.basename(processed_yaml, path.extname(processed_yaml))
+          const filename_txt = path.join(path.dirname(processed_yaml), basename + '.txt')
           expect(fs.readFileSync(processed_yaml, 'utf8')).toEqual(fs.readFileSync(filename_txt, 'utf8'))
           fs.rmSync(processed_yaml)
           fs.rmSync(filename_txt)

tools/tests/prepare-for-vale/fixtures/.docker-compose.txt

@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+

tools/tests/prepare-for-vale/fixtures/.docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node