Skip to content

Commit 73e2228

Browse files
q-andyq-andy
committed
Add bwc tests for stats
Signed-off-by: Andy Qin <qinandy@amazon.com>
1 parent c56a110 commit 73e2228

File tree

4 files changed

+208
-0
lines changed

4 files changed

+208
-0
lines changed

‎qa/restart-upgrade/build.gradle

+19
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ def versionsBelow2_13 = versionsBelow2_12 + "2.12"
6161
def versionsBelow2_14 = versionsBelow2_13 + "2.13"
6262
def versionsBelow2_15 = versionsBelow2_14 + "2.14"
6363
def versionsBelow2_16 = versionsBelow2_15 + "2.15"
64+
def versionsBelow2_17 = versionsBelow2_16 + "2.16"
65+
def versionsBelow2_18 = versionsBelow2_17 + "2.17"
66+
def versionsBelow2_19 = versionsBelow2_18 + "2.18"
67+
def versionsBelow2_20 = versionsBelow2_19 + "2.19"
68+
def versionsBelow3_0 = versionsBelow2_20 + "2.20"
6469

6570
// Task to run BWC tests against the old cluster
6671
task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
@@ -114,6 +119,13 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
114119
}
115120
}
116121

122+
// Excluding stats tests because we introduce this feature in 3.0
123+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
124+
filter {
125+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.restart.RestNeuralStatsActionIT.*"
126+
}
127+
}
128+
117129
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
118130
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
119131
systemProperty 'tests.security.manager', 'false'
@@ -179,6 +191,13 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) {
179191
}
180192
}
181193

194+
// Excluding stats tests because we introduce this feature in 3.0
195+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
196+
filter {
197+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.restart.RestNeuralStatsActionIT.*"
198+
}
199+
}
200+
182201
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
183202
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
184203
systemProperty 'tests.security.manager', 'false'

‎qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/restart/RestNeuralStatsActionIT.java

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.neuralsearch.bwc.restart;
7+
8+
import org.opensearch.neuralsearch.stats.events.EventStatName;
9+
import org.opensearch.neuralsearch.stats.info.InfoStatName;
10+
11+
import java.nio.file.Files;
12+
import java.nio.file.Path;
13+
import java.util.ArrayList;
14+
import java.util.Map;
15+
16+
import static org.opensearch.neuralsearch.util.TestUtils.NODES_BWC_CLUSTER;
17+
import static org.opensearch.neuralsearch.util.TestUtils.TEXT_EMBEDDING_PROCESSOR;
18+
import static org.opensearch.neuralsearch.util.TestUtils.getModelId;
19+
20+
public class RestNeuralStatsActionIT extends AbstractRestartUpgradeRestTestCase {
21+
private static final String PIPELINE_NAME = "nlp-pipeline";
22+
private static final String TEST_FIELD = "passage_text";
23+
private static final String TEXT = "Hello world";
24+
private static final String TEXT_1 = "Hello world a";
25+
26+
// Test restart-upgrade with neural stats
27+
// Enabled/disabled settings should persist between restarts
28+
// Event stats should be reset on restart
29+
// Info stats based on persistent constructs should be persisted between restarts
30+
public void testNeuralStats_E2EFlow() throws Exception {
31+
waitForClusterHealthGreen(NODES_BWC_CLUSTER);
32+
updateClusterSettings("plugins.neural_search.stats_enabled", true);
33+
34+
// Currently using text embedding processor executions stat since that's the only one implemented
35+
// Once other stats are implemented, it may be smarter to use those instead of text embedding processor
36+
// to avoid having to upload a model and run inference.
37+
if (isRunningAgainstOldCluster()) {
38+
String modelId = uploadTextEmbeddingModel();
39+
loadModel(modelId);
40+
createPipelineProcessor(modelId, PIPELINE_NAME);
41+
createIndexWithConfiguration(
42+
getIndexNameForTest(),
43+
Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())),
44+
PIPELINE_NAME
45+
);
46+
addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT, null, null);
47+
48+
// Get stats request
49+
String responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
50+
Map<String, Object> infoStats = parseInfoStatsResponse(responseBody);
51+
Map<String, Object> aggregatedNodeStats = parseAggregatedNodeStatsResponse(responseBody);
52+
53+
assertEquals(1, getNestedValue(aggregatedNodeStats, EventStatName.TEXT_EMBEDDING_PROCESSOR_EXECUTIONS));
54+
assertEquals(1, getNestedValue(infoStats, InfoStatName.TEXT_EMBEDDING_PROCESSORS));
55+
} else {
56+
String modelId = null;
57+
try {
58+
modelId = getModelId(getIngestionPipeline(PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR);
59+
loadModel(modelId);
60+
addDocument(getIndexNameForTest(), "1", TEST_FIELD, TEXT_1, null, null);
61+
addDocument(getIndexNameForTest(), "2", TEST_FIELD, TEXT_1, null, null);
62+
addDocument(getIndexNameForTest(), "3", TEST_FIELD, TEXT_1, null, null);
63+
64+
// Get stats request
65+
String responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
66+
Map<String, Object> infoStats = parseInfoStatsResponse(responseBody);
67+
Map<String, Object> aggregatedNodeStats = parseAggregatedNodeStatsResponse(responseBody);
68+
69+
assertEquals(3, getNestedValue(aggregatedNodeStats, EventStatName.TEXT_EMBEDDING_PROCESSOR_EXECUTIONS));
70+
assertEquals(1, getNestedValue(infoStats, InfoStatName.TEXT_EMBEDDING_PROCESSORS));
71+
} finally {
72+
wipeOfTestResources(getIndexNameForTest(), PIPELINE_NAME, modelId, null);
73+
}
74+
}
75+
}
76+
}

‎qa/rolling-upgrade/build.gradle

+33
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ def versionsBelow2_13 = versionsBelow2_12 + "2.12"
6161
def versionsBelow2_14 = versionsBelow2_13 + "2.13"
6262
def versionsBelow2_15 = versionsBelow2_14 + "2.14"
6363
def versionsBelow2_16 = versionsBelow2_15 + "2.15"
64+
def versionsBelow2_17 = versionsBelow2_16 + "2.16"
65+
def versionsBelow2_18 = versionsBelow2_17 + "2.17"
66+
def versionsBelow2_19 = versionsBelow2_18 + "2.18"
67+
def versionsBelow2_20 = versionsBelow2_19 + "2.19"
68+
def versionsBelow3_0 = versionsBelow2_20 + "2.20"
6469

6570
// Task to run BWC tests against the old cluster
6671
task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
@@ -75,6 +80,13 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
7580
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
7681
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
7782
systemProperty 'tests.security.manager', 'false'
83+
84+
// Excluding stats tests because we introduce this feature in 3.0
85+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
86+
filter {
87+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.rolling.RestNeuralStatsActionIT.*"
88+
}
89+
}
7890
}
7991

8092
// Part of rolling upgrade. Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
@@ -100,6 +112,13 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
100112
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
101113
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
102114
systemProperty 'tests.security.manager', 'false'
115+
116+
// Excluding stats tests because we introduce this feature in 3.0
117+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
118+
filter {
119+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.rolling.RestNeuralStatsActionIT.*"
120+
}
121+
}
103122
}
104123

105124
// Part of rolling upgrade. Upgrades the second node to new OpenSearch version with upgraded plugin version after the
@@ -124,6 +143,13 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) {
124143
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
125144
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
126145
systemProperty 'tests.security.manager', 'false'
146+
147+
// Excluding stats tests because we introduce this feature in 3.0
148+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
149+
filter {
150+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.rolling.RestNeuralStatsActionIT.*"
151+
}
152+
}
127153
}
128154

129155
// Part of rolling upgrade. Upgrades the third node to new OpenSearch version with upgraded plugin version after the
@@ -148,4 +174,11 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) {
148174
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
149175
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
150176
systemProperty 'tests.security.manager', 'false'
177+
178+
// Excluding stats tests because we introduce this feature in 3.0
179+
if (versionsBelow3_0.any { ext.neural_search_bwc_version.startsWith(it) }){
180+
filter {
181+
excludeTestsMatching "org.opensearch.neuralsearch.bwc.rolling.RestNeuralStatsActionIT.*"
182+
}
183+
}
151184
}

‎qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/rolling/RestNeuralStatsActionIT.java

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package org.opensearch.neuralsearch.bwc.rolling;
6+
7+
import org.opensearch.neuralsearch.stats.events.EventStatName;
8+
import org.opensearch.neuralsearch.stats.info.InfoStatName;
9+
10+
import java.nio.file.Files;
11+
import java.nio.file.Path;
12+
import java.util.ArrayList;
13+
import java.util.Map;
14+
15+
import static org.opensearch.neuralsearch.util.TestUtils.NODES_BWC_CLUSTER;
16+
17+
public class RestNeuralStatsActionIT extends AbstractRollingUpgradeTestCase {
18+
private static final String PIPELINE_NAME = "nlp-pipeline";
19+
private static final String TEST_FIELD = "passage_text";
20+
private static final String TEXT = "Hello world";
21+
private static final String TEXT_MIXED = "Hello world mixed";
22+
private static final String TEXT_UPGRADED = "Hello world upgraded";
23+
private static final int NUM_DOCS_PER_ROUND = 1;
24+
private static String modelId = "";
25+
26+
// Test rolling-upgrade neural stats action
27+
// Create Text Embedding Processor, Ingestion Pipeline and add document
28+
// Validate stats are correct during upgrade
29+
// When new stats are added, we will also want to validate handling fetching stats from previous versions
30+
// that don't have those stats.
31+
public void testStats_E2EFlow() throws Exception {
32+
waitForClusterHealthGreen(NODES_BWC_CLUSTER, 90);
33+
updateClusterSettings("plugins.neural_search.stats_enabled", true);
34+
35+
switch (getClusterType()) {
36+
case OLD:
37+
modelId = uploadTextEmbeddingModel();
38+
loadModel(modelId);
39+
createPipelineProcessor(modelId, PIPELINE_NAME);
40+
createIndexWithConfiguration(
41+
getIndexNameForTest(),
42+
Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())),
43+
PIPELINE_NAME
44+
);
45+
addDocument(getIndexNameForTest(), "0", TEST_FIELD, TEXT, null, null);
46+
47+
// Get stats request
48+
String responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
49+
Map<String, Object> infoStats = parseInfoStatsResponse(responseBody);
50+
Map<String, Object> aggregatedNodeStats = parseAggregatedNodeStatsResponse(responseBody);
51+
52+
assertEquals(1, getNestedValue(aggregatedNodeStats, EventStatName.TEXT_EMBEDDING_PROCESSOR_EXECUTIONS));
53+
assertEquals(1, getNestedValue(infoStats, InfoStatName.TEXT_EMBEDDING_PROCESSORS));
54+
break;
55+
case MIXED:
56+
// Get stats request
57+
responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
58+
infoStats = parseInfoStatsResponse(responseBody);
59+
60+
assertEquals(1, getNestedValue(infoStats, InfoStatName.TEXT_EMBEDDING_PROCESSORS));
61+
break;
62+
case UPGRADED:
63+
try {
64+
// Get stats request
65+
responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
66+
infoStats = parseInfoStatsResponse(responseBody);
67+
aggregatedNodeStats = parseAggregatedNodeStatsResponse(responseBody);
68+
69+
// After all nodes have be restarted, all event stats should be reset as well
70+
assertEquals(0, getNestedValue(aggregatedNodeStats, EventStatName.TEXT_EMBEDDING_PROCESSOR_EXECUTIONS));
71+
assertEquals(1, getNestedValue(infoStats, InfoStatName.TEXT_EMBEDDING_PROCESSORS));
72+
} finally {
73+
wipeOfTestResources(getIndexNameForTest(), PIPELINE_NAME, modelId, null);
74+
}
75+
break;
76+
default:
77+
throw new IllegalStateException("Unexpected value: " + getClusterType());
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)