From 6c7a3f494a447ceab07a4ed78e8525b3ff7d2573 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Mon, 26 Feb 2024 14:22:16 +0800 Subject: [PATCH 01/27] feature: implement default model id for neural sparse Signed-off-by: zhichao-aws --- .../query/ModelInferenceQueryBuilder.java | 27 +++++++++++++++++++ .../query/NeuralQueryBuilder.java | 2 +- .../query/NeuralSparseQueryBuilder.java | 2 +- .../visitor/NeuralSearchQueryVisitor.java | 20 +++++++------- 4 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java diff --git a/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java new file mode 100644 index 000000000..a1001c455 --- /dev/null +++ b/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.query; + +/** + * Query builders which calls ml-commons API to do model inference. + * The model inference result is used for search on target field. + */ + +public interface ModelInferenceQueryBuilder { + /** + * Get the model id used by ml-commons model inference. Return null if the model id is absent. + */ + public String modelId(); + + /** + * Set a new model id for the query builder. + */ + public ModelInferenceQueryBuilder modelId(String modelId); + + /** + * Get the field name for search. + */ + public String fieldName(); +} diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java index cda01767e..d74378617 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java @@ -58,7 +58,7 @@ @Accessors(chain = true, fluent = true) @NoArgsConstructor @AllArgsConstructor -public class NeuralQueryBuilder extends AbstractQueryBuilder { +public class NeuralQueryBuilder extends AbstractQueryBuilder implements ModelInferenceQueryBuilder { public static final String NAME = "neural"; diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 226594a87..48c722011 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -55,7 +55,7 @@ @Accessors(chain = true, fluent = true) @NoArgsConstructor @AllArgsConstructor -public class NeuralSparseQueryBuilder extends AbstractQueryBuilder { +public class NeuralSparseQueryBuilder extends AbstractQueryBuilder implements ModelInferenceQueryBuilder { public static final String NAME = "neural_sparse"; @VisibleForTesting static final ParseField QUERY_TEXT_FIELD = new ParseField("query_text"); diff --git a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java index 9dab0a695..6fd4d0708 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java +++ b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java @@ -9,12 +9,12 @@ import org.apache.lucene.search.BooleanClause; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilderVisitor; -import org.opensearch.neuralsearch.query.NeuralQueryBuilder; +import org.opensearch.neuralsearch.query.ModelInferenceQueryBuilder; import lombok.AllArgsConstructor; /** - * Neural Search Query Visitor. It visits each and every component of query buikder tree. + * Neural Search Query Visitor. It visits each and every component of query builder tree. */ @AllArgsConstructor public class NeuralSearchQueryVisitor implements QueryBuilderVisitor { @@ -28,16 +28,16 @@ public class NeuralSearchQueryVisitor implements QueryBuilderVisitor { */ @Override public void accept(QueryBuilder queryBuilder) { - if (queryBuilder instanceof NeuralQueryBuilder) { - NeuralQueryBuilder neuralQueryBuilder = (NeuralQueryBuilder) queryBuilder; - if (neuralQueryBuilder.modelId() == null) { + if (queryBuilder instanceof ModelInferenceQueryBuilder) { + ModelInferenceQueryBuilder modelInferenceQueryBuilder = (ModelInferenceQueryBuilder) queryBuilder; + if (modelInferenceQueryBuilder.modelId() == null) { if (neuralFieldMap != null - && neuralQueryBuilder.fieldName() != null - && neuralFieldMap.get(neuralQueryBuilder.fieldName()) != null) { - String fieldDefaultModelId = (String) neuralFieldMap.get(neuralQueryBuilder.fieldName()); - neuralQueryBuilder.modelId(fieldDefaultModelId); + && modelInferenceQueryBuilder.fieldName() != null + && neuralFieldMap.get(modelInferenceQueryBuilder.fieldName()) != null) { + String fieldDefaultModelId = (String) neuralFieldMap.get(modelInferenceQueryBuilder.fieldName()); + modelInferenceQueryBuilder.modelId(fieldDefaultModelId); } else if (modelId != null) { - neuralQueryBuilder.modelId(modelId); + modelInferenceQueryBuilder.modelId(modelId); } else { throw new IllegalArgumentException( "model id must be provided in neural query or a default model id must be set in search request processor" From 1e7037bd291a8bdafcb5a33aad804aedf407dc40 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Mon, 26 Feb 2024 16:10:17 +0800 Subject: [PATCH 02/27] feature: implement default model id for neural sparse Signed-off-by: zhichao-aws --- .../query/NeuralSparseQueryBuilder.java | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 48c722011..6e87f4c38 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -18,6 +18,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.opensearch.Version; import org.opensearch.common.SetOnce; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; @@ -32,6 +33,7 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor; +import org.opensearch.neuralsearch.util.NeuralSearchClusterUtil; import org.opensearch.neuralsearch.util.TokenWeightUtil; import com.google.common.annotations.VisibleForTesting; @@ -72,6 +74,7 @@ public static void initialize(MLCommonsClientAccessor mlClient) { private String queryText; private String modelId; private Supplier> queryTokensSupplier; + private static final Version MINIMAL_SUPPORTED_VERSION_DEFAULT_MODEL_ID = Version.V_2_13_0; /** * Constructor from stream input @@ -83,7 +86,11 @@ public NeuralSparseQueryBuilder(StreamInput in) throws IOException { super(in); this.fieldName = in.readString(); this.queryText = in.readString(); - this.modelId = in.readString(); + if (isClusterOnOrAfterMinReqVersionForDefaultModelIdSupport()) { + this.modelId = in.readOptionalString(); + } else { + this.modelId = in.readString(); + } if (in.readBoolean()) { Map queryTokens = in.readMap(StreamInput::readString, StreamInput::readFloat); this.queryTokensSupplier = () -> queryTokens; @@ -94,7 +101,11 @@ public NeuralSparseQueryBuilder(StreamInput in) throws IOException { protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeString(queryText); - out.writeString(modelId); + if (isClusterOnOrAfterMinReqVersionForDefaultModelIdSupport()) { + out.writeOptionalString(this.modelId); + } else { + out.writeString(this.modelId); + } if (!Objects.isNull(queryTokensSupplier) && !Objects.isNull(queryTokensSupplier.get())) { out.writeBoolean(true); out.writeMap(queryTokensSupplier.get(), StreamOutput::writeString, StreamOutput::writeFloat); @@ -108,7 +119,9 @@ protected void doXContent(XContentBuilder xContentBuilder, Params params) throws xContentBuilder.startObject(NAME); xContentBuilder.startObject(fieldName); xContentBuilder.field(QUERY_TEXT_FIELD.getPreferredName(), queryText); - xContentBuilder.field(MODEL_ID_FIELD.getPreferredName(), modelId); + if (modelId != null) { + xContentBuilder.field(MODEL_ID_FIELD.getPreferredName(), modelId); + } printBoostAndQueryName(xContentBuilder); xContentBuilder.endObject(); xContentBuilder.endObject(); @@ -152,11 +165,12 @@ public static NeuralSparseQueryBuilder fromXContent(XContentParser parser) throw sparseEncodingQueryBuilder.queryText(), String.format(Locale.ROOT, "%s field must be provided for [%s] query", QUERY_TEXT_FIELD.getPreferredName(), NAME) ); - requireValue( - sparseEncodingQueryBuilder.modelId(), - String.format(Locale.ROOT, "%s field must be provided for [%s] query", MODEL_ID_FIELD.getPreferredName(), NAME) - ); - + if (!isClusterOnOrAfterMinReqVersionForDefaultModelIdSupport()) { + requireValue( + sparseEncodingQueryBuilder.modelId(), + String.format(Locale.ROOT, "%s field must be provided for [%s] query", MODEL_ID_FIELD.getPreferredName(), NAME) + ); + } return sparseEncodingQueryBuilder; } @@ -291,4 +305,8 @@ protected int doHashCode() { public String getWriteableName() { return NAME; } + + private static boolean isClusterOnOrAfterMinReqVersionForDefaultModelIdSupport() { + return NeuralSearchClusterUtil.instance().getClusterMinVersion().onOrAfter(MINIMAL_SUPPORTED_VERSION_DEFAULT_MODEL_ID); + } } From 7caf01e6bb1c743ab102a27440c5ca1383bb6011 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Tue, 27 Feb 2024 10:31:27 +0800 Subject: [PATCH 03/27] add ut Signed-off-by: zhichao-aws --- .../query/NeuralSparseQueryBuilderTests.java | 50 +++++++++++++++++-- .../NeuralSearchQueryVisitorTests.java | 26 ++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java index 2220ce326..89bcd57d7 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilderTests.java @@ -22,7 +22,10 @@ import java.util.function.BiConsumer; import java.util.function.Supplier; +import org.junit.Before; +import org.opensearch.Version; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.SetOnce; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.XContentFactory; @@ -38,6 +41,8 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.neuralsearch.ml.MLCommonsClientAccessor; +import org.opensearch.neuralsearch.util.NeuralSearchClusterTestUtils; +import org.opensearch.neuralsearch.util.NeuralSearchClusterUtil; import org.opensearch.test.OpenSearchTestCase; import lombok.SneakyThrows; @@ -51,6 +56,11 @@ public class NeuralSparseQueryBuilderTests extends OpenSearchTestCase { private static final String QUERY_NAME = "queryName"; private static final Supplier> QUERY_TOKENS_SUPPLIER = () -> Map.of("hello", 1.f, "world", 2.f); + @Before + public void setupClusterServiceToCurrentVersion() { + setUpClusterService(Version.CURRENT); + } + @SneakyThrows public void testFromXContent_whenBuiltWithQueryText_thenBuildSuccessfully() { /* @@ -162,7 +172,7 @@ public void testFromXContent_whenBuildWithMissingQuery_thenFail() { } @SneakyThrows - public void testFromXContent_whenBuildWithMissingModelId_thenFail() { + public void testFromXContent_whenBuildWithMissingModelIdInCurrentVersion_thenSuccess() { /* { "VECTOR_FIELD": { @@ -177,6 +187,30 @@ public void testFromXContent_whenBuildWithMissingModelId_thenFail() { .endObject() .endObject(); + XContentParser contentParser = createParser(xContentBuilder); + contentParser.nextToken(); + NeuralSparseQueryBuilder sparseEncodingQueryBuilder = NeuralSparseQueryBuilder.fromXContent(contentParser); + + assertNull(sparseEncodingQueryBuilder.modelId()); + } + + @SneakyThrows + public void testFromXContent_whenBuildWithMissingModelIdInOldVersion_thenFail() { + /* + { + "VECTOR_FIELD": { + "query_text": "string" + } + } + */ + setUpClusterService(Version.V_2_12_0); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject(FIELD_NAME) + .field(QUERY_TEXT_FIELD.getPreferredName(), QUERY_TEXT) + .endObject() + .endObject(); + XContentParser contentParser = createParser(xContentBuilder); contentParser.nextToken(); expectThrows(IllegalArgumentException.class, () -> NeuralSparseQueryBuilder.fromXContent(contentParser)); @@ -241,6 +275,11 @@ public void testToXContent() { assertEquals(QUERY_TEXT, secondInnerMap.get(QUERY_TEXT_FIELD.getPreferredName())); } + public void testStreams_whenMinVersionIsBeforeDefaultModelId_thenSuccess() { + setUpClusterService(Version.V_2_12_0); + testStreams(); + } + @SneakyThrows public void testStreams() { NeuralSparseQueryBuilder original = new NeuralSparseQueryBuilder(); @@ -436,10 +475,15 @@ public void testRewrite_whenQueryTokensSupplierSet_thenReturnSelf() { .modelId(MODEL_ID) .queryTokensSupplier(QUERY_TOKENS_SUPPLIER); QueryBuilder queryBuilder = sparseEncodingQueryBuilder.doRewrite(null); - assertTrue(queryBuilder == sparseEncodingQueryBuilder); + assertSame(queryBuilder, sparseEncodingQueryBuilder); sparseEncodingQueryBuilder.queryTokensSupplier(() -> null); queryBuilder = sparseEncodingQueryBuilder.doRewrite(null); - assertTrue(queryBuilder == sparseEncodingQueryBuilder); + assertSame(queryBuilder, sparseEncodingQueryBuilder); + } + + private void setUpClusterService(Version version) { + ClusterService clusterService = NeuralSearchClusterTestUtils.mockClusterService(version); + NeuralSearchClusterUtil.instance().initialize(clusterService); } } diff --git a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java index e513ab035..ba890959c 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.BooleanClause; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; +import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; import org.opensearch.test.OpenSearchTestCase; public class NeuralSearchQueryVisitorTests extends OpenSearchTestCase { @@ -38,11 +39,36 @@ public void testAccept_whenNeuralQueryBuilderWithoutFieldModelId_thenSetFieldMod assertEquals("bdcvjkcdjvkddcjxdjsc", neuralQueryBuilder.modelId()); } + public void testAccept_whenNeuralSparseQueryBuilderWithoutModelId_thenSetModelId() { + String modelId = "bdcvjkcdjvkddcjxdjsc"; + NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); + neuralSparseQueryBuilder.fieldName("passage_text"); + + NeuralSearchQueryVisitor neuralSearchQueryVisitor = new NeuralSearchQueryVisitor(modelId, null); + neuralSearchQueryVisitor.accept(neuralSparseQueryBuilder); + + assertEquals(modelId, neuralSparseQueryBuilder.modelId()); + } + + public void testAccept_whenNeuralSparseQueryBuilderWithoutFieldModelId_thenSetFieldModelId() { + Map neuralInfoMap = new HashMap<>(); + neuralInfoMap.put("passage_text", "bdcvjkcdjvkddcjxdjsc"); + NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); + neuralSparseQueryBuilder.fieldName("passage_text"); + + NeuralSearchQueryVisitor neuralSearchQueryVisitor = new NeuralSearchQueryVisitor(null, neuralInfoMap); + neuralSearchQueryVisitor.accept(neuralSparseQueryBuilder); + + assertEquals("bdcvjkcdjvkddcjxdjsc", neuralSparseQueryBuilder.modelId()); + } + public void testAccept_whenNullValuesInVisitor_thenFail() { NeuralQueryBuilder neuralQueryBuilder = new NeuralQueryBuilder(); + NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); NeuralSearchQueryVisitor neuralSearchQueryVisitor = new NeuralSearchQueryVisitor(null, null); expectThrows(IllegalArgumentException.class, () -> neuralSearchQueryVisitor.accept(neuralQueryBuilder)); + expectThrows(IllegalArgumentException.class, () -> neuralSearchQueryVisitor.accept(neuralSparseQueryBuilder)); } public void testGetChildVisitor() { From 5f7f7c54fcabad26755df4b6c267f6ad619e51eb Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Tue, 27 Feb 2024 13:06:39 +0800 Subject: [PATCH 04/27] add ut it Signed-off-by: zhichao-aws --- .../NeuralQueryEnricherProcessorIT.java | 29 +++++++++++++++++++ .../NeuralSearchQueryVisitorTests.java | 13 +++++++++ 2 files changed, 42 insertions(+) diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index cd8300928..74d0912d6 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -9,6 +9,7 @@ import static org.opensearch.neuralsearch.TestUtils.createRandomVector; import java.util.Collections; +import java.util.List; import java.util.Map; import org.apache.hc.core5.http.io.entity.EntityUtils; @@ -26,13 +27,16 @@ import com.google.common.primitives.Floats; import lombok.SneakyThrows; +import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; public class NeuralQueryEnricherProcessorIT extends BaseNeuralSearchIT { private static final String index = "my-nlp-index"; + private static final String sparseIndex = "my-nlp-index-sparse"; private static final String search_pipeline = "search-pipeline"; private static final String ingest_pipeline = "nlp-pipeline"; private static final String TEST_KNN_VECTOR_FIELD_NAME_1 = "test-knn-vector-1"; + private static final String TEST_RANK_FEATURES_FIELD_NAME_1 = "test-rank-features-1"; private final float[] testVector = createRandomVector(TEST_DIMENSION); @Before @@ -61,6 +65,25 @@ public void testNeuralQueryEnricherProcessor_whenNoModelIdPassed_thenSuccess() { } } + @SneakyThrows + public void testNeuralQueryEnricherProcessor_whenNoModelIdPassedInNeuralSparseQuery_thenSuccess() { + String modelId = null; + try { + initializeIndexIfNotExist(sparseIndex); + modelId = prepareSparseEncodingModel(); + createSearchRequestProcessor(modelId, search_pipeline); + createPipelineProcessor(modelId, ingest_pipeline, ProcessorType.TEXT_EMBEDDING); + updateIndexSettings(sparseIndex, Settings.builder().put("index.search.default_pipeline", search_pipeline)); + NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); + neuralSparseQueryBuilder.fieldName(TEST_RANK_FEATURES_FIELD_NAME_1); + neuralSparseQueryBuilder.queryText("hello"); + Map response = search(sparseIndex, neuralSparseQueryBuilder, 2); + assertFalse(response.isEmpty()); + } finally { + wipeOfTestResources(sparseIndex, ingest_pipeline, modelId, search_pipeline); + } + } + @SneakyThrows public void testNeuralQueryEnricherProcessor_whenGetEmptyQueryBody_thenSuccess() { String modelId = null; @@ -120,5 +143,11 @@ private void initializeIndexIfNotExist(String indexName) { ); assertEquals(1, getDocCount(indexName)); } + + if (sparseIndex.equals(indexName) && !indexExists(indexName)) { + prepareSparseEncodingIndex(indexName, List.of(TEST_RANK_FEATURES_FIELD_NAME_1)); + addSparseEncodingDoc(indexName, "1", List.of(TEST_RANK_FEATURES_FIELD_NAME_1), List.of(Map.of("hi", 1.0f, "hello", 1.1f))); + assertEquals(1, getDocCount(indexName)); + } } } diff --git a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java index ba890959c..26dbb289b 100644 --- a/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java +++ b/src/test/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitorTests.java @@ -39,6 +39,19 @@ public void testAccept_whenNeuralQueryBuilderWithoutFieldModelId_thenSetFieldMod assertEquals("bdcvjkcdjvkddcjxdjsc", neuralQueryBuilder.modelId()); } + public void testAccept_whenNeuralSparseQueryBuilderWithModelId_theDoNothing() { + String modelId1 = "bdcvjkcdjvkddcjxdjsc"; + String modelId2 = "45dfsnfoiqwrjcjxdjsc"; + NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); + neuralSparseQueryBuilder.fieldName("passage_text"); + neuralSparseQueryBuilder.modelId(modelId1); + + NeuralSearchQueryVisitor neuralSearchQueryVisitor = new NeuralSearchQueryVisitor(modelId2, null); + neuralSearchQueryVisitor.accept(neuralSparseQueryBuilder); + + assertEquals(modelId1, neuralSparseQueryBuilder.modelId()); + } + public void testAccept_whenNeuralSparseQueryBuilderWithoutModelId_thenSetModelId() { String modelId = "bdcvjkcdjvkddcjxdjsc"; NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); From f6657fc04a2f3ffb41088aa5d0d6141364f9f466 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Tue, 27 Feb 2024 13:34:49 +0800 Subject: [PATCH 05/27] add changelog Signed-off-by: zhichao-aws --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 528231b07..9c7b56855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/neural-search/compare/2.x...HEAD) ### Features +- Enabled support for applying default modelId in neural sparse query ([#614](https://github.com/opensearch-project/neural-search/pull/614) ### Enhancements ### Bug Fixes - Fix async actions are left in neural_sparse query ([#438](https://github.com/opensearch-project/neural-search/pull/438)) From 0815301a120826da3c1940ad0b36c04148b793b3 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 29 Feb 2024 11:59:54 +0800 Subject: [PATCH 06/27] nit Signed-off-by: zhichao-aws --- .../neuralsearch/query/ModelInferenceQueryBuilder.java | 9 ++++----- .../processor/NeuralQueryEnricherProcessorIT.java | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java index a1001c455..23db567d4 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/ModelInferenceQueryBuilder.java @@ -8,7 +8,6 @@ * Query builders which calls ml-commons API to do model inference. * The model inference result is used for search on target field. */ - public interface ModelInferenceQueryBuilder { /** * Get the model id used by ml-commons model inference. Return null if the model id is absent. @@ -16,12 +15,12 @@ public interface ModelInferenceQueryBuilder { public String modelId(); /** - * Set a new model id for the query builder. - */ + * Set a new model id for the query builder. + */ public ModelInferenceQueryBuilder modelId(String modelId); /** - * Get the field name for search. - */ + * Get the field name for search. + */ public String fieldName(); } diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index 74d0912d6..0024eb8c5 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -23,11 +23,11 @@ import org.opensearch.neuralsearch.BaseNeuralSearchIT; import org.opensearch.neuralsearch.query.HybridQueryBuilder; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; +import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; import com.google.common.primitives.Floats; import lombok.SneakyThrows; -import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; public class NeuralQueryEnricherProcessorIT extends BaseNeuralSearchIT { From a6b255c5dddfaf1b4b64a06ca7ae122ab3059565 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 10:54:38 +0800 Subject: [PATCH 07/27] fix ingest pipeline in it Signed-off-by: zhichao-aws --- .../neuralsearch/processor/NeuralQueryEnricherProcessorIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java index 0024eb8c5..f5ff75ac6 100644 --- a/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java +++ b/src/test/java/org/opensearch/neuralsearch/processor/NeuralQueryEnricherProcessorIT.java @@ -72,7 +72,7 @@ public void testNeuralQueryEnricherProcessor_whenNoModelIdPassedInNeuralSparseQu initializeIndexIfNotExist(sparseIndex); modelId = prepareSparseEncodingModel(); createSearchRequestProcessor(modelId, search_pipeline); - createPipelineProcessor(modelId, ingest_pipeline, ProcessorType.TEXT_EMBEDDING); + createPipelineProcessor(modelId, ingest_pipeline, ProcessorType.SPARSE_ENCODING); updateIndexSettings(sparseIndex, Settings.builder().put("index.search.default_pipeline", search_pipeline)); NeuralSparseQueryBuilder neuralSparseQueryBuilder = new NeuralSparseQueryBuilder(); neuralSparseQueryBuilder.fieldName(TEST_RANK_FEATURES_FIELD_NAME_1); From a143e2b7ad973ca72ac575751c4c6220e4b89410 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 14:46:35 +0800 Subject: [PATCH 08/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- .../bwc/NeuralQueryEnricherProcessorIT.java | 94 +++++++++++++++++++ .../SearchRequestPipelineConfiguration.json | 11 +++ .../neuralsearch/BaseNeuralSearchIT.java | 10 ++ 3 files changed, 115 insertions(+) create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java create mode 100644 qa/restart-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java new file mode 100644 index 000000000..b90382838 --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.neuralsearch.bwc; + +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.SPARSE_ENCODING_PROCESSOR; + +import org.opensearch.Version; +import org.opensearch.client.ResponseException; +import org.opensearch.neuralsearch.TestUtils; +import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTestCase { + private static final String SPARSE_INGEST_PIPELINE_NAME = "nlp-ingest-pipeline-sparse"; + private static final String SPARSE_SEARCH_PIPELINE_NAME = "nlp-ingest-pipeline-sparse"; + private static final String TEST_SPARSE_ENCODING_FIELD = "passage_embedding"; + private static final String TEST_TEXT_FIELD = "passage_text"; + private static final String TEXT_1 = "Hello world a b"; + private static final String TEXT_2 = "Hello planet"; + private static final List TEST_TOKENS_1 = List.of("hello", "world", "a", "b", "c"); + private static final List TEST_TOKENS_2 = List.of("hello", "planet", "a", "b", "c"); + private final Map testRankFeaturesDoc1 = TestUtils.createRandomTokenWeightMap(TEST_TOKENS_1); + private final Map testRankFeaturesDoc2 = TestUtils.createRandomTokenWeightMap(TEST_TOKENS_2); + + // Test restart-upgrade test sparse embedding processor + // Create Sparse Encoding Processor, Ingestion Pipeline and add document + // Validate process , pipeline and document count in restart-upgrade scenario + public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + Version bwcVersion = parseVersionFromString(getBWCVersion().get()); + logger.info("bwc version: " + bwcVersion.toString()); + if (isRunningAgainstOldCluster()) { + String modelId = uploadSparseEncodingModel(); + loadModel(modelId); + createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/SparseIndexMappings.json").toURI())), + SPARSE_INGEST_PIPELINE_NAME + ); + + addSparseEncodingDoc( + getIndexNameForTest(), + "0", + List.of(TEST_SPARSE_ENCODING_FIELD), + List.of(testRankFeaturesDoc1), + List.of(TEST_TEXT_FIELD), + List.of(TEXT_1) + ); + + // skip the test before we introduce neural_query_enricher + if (bwcVersion.onOrAfter(Version.V_2_11_0)) { + createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { + // after we support default model id in neural_sparse query + // do nothing here. need to add test codes after finishing backport + ; + } else { + // before we support default model id in neural_sparse query + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_SPARSE_ENCODING_FIELD + ).queryText(TEXT_1); + + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); + } + } + } else { + String modelId = null; + try { + modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); + loadModel(modelId); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_SPARSE_ENCODING_FIELD + ).queryText(TEXT_1); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_SPARSE_ENCODING_FIELD + ).queryText(TEXT_1).modelId(modelId); + assertEquals( + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") + ); + } finally { + wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); + } + } + } +} diff --git a/qa/restart-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json b/qa/restart-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json new file mode 100644 index 000000000..5cd9988cb --- /dev/null +++ b/qa/restart-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json @@ -0,0 +1,11 @@ +{ + "request_processors": [ + { + "neural_query_enricher": { + "tag": "tag1", + "description": "This processor is going to restrict to publicly visible documents", + "default_model_id": "%s" + } + } + ] +} diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index ffbbed2bc..50f97053a 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -4,6 +4,7 @@ */ package org.opensearch.neuralsearch; +import org.opensearch.Version; import org.opensearch.ml.common.model.MLModelState; import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray; @@ -1064,6 +1065,15 @@ protected void wipeOfTestResources( } } + // parse the version from string like: "2.12.0","2.13.0-SNAPSHOT" + protected Version parseVersionFromString(String version) { + /* snapshot labels are not supported in OpenSearch Version.fromStringSlow */ + if (version.endsWith("-SNAPSHOT")) { + version = version.substring(0, version.length() - "-SNAPSHOT".length()); + } + return Version.fromString(version); + } + /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From 19420d5b61e50f636be9e00d3cbc68a20dabd234 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 16:28:33 +0800 Subject: [PATCH 09/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 8 +++-- .../bwc/NeuralQueryEnricherProcessorIT.java | 29 ++++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index a4ca03b65..13cab7bdc 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -65,12 +65,14 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherIT.*" } } @@ -98,12 +100,14 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherIT.*" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index b90382838..c358a0af8 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -9,6 +9,7 @@ import org.opensearch.Version; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.neuralsearch.TestUtils; import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; @@ -36,6 +37,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); logger.info("bwc version: " + bwcVersion.toString()); + if (isRunningAgainstOldCluster()) { String modelId = uploadSparseEncodingModel(); loadModel(modelId); @@ -55,22 +57,21 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws List.of(TEXT_1) ); - // skip the test before we introduce neural_query_enricher - if (bwcVersion.onOrAfter(Version.V_2_11_0)) { - createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); - if (bwcVersion.onOrAfter(Version.V_2_13_0)) { - // after we support default model id in neural_sparse query - // do nothing here. need to add test codes after finishing backport - ; - } else { - // before we support default model id in neural_sparse query - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_SPARSE_ENCODING_FIELD - ).queryText(TEXT_1); + createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME)); + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { + // after we support default model id in neural_sparse query + // do nothing here. need to add test codes after finishing backport + ; + } else { + // before we support default model id in neural_sparse query + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_SPARSE_ENCODING_FIELD + ).queryText(TEXT_1); - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); - } + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } + } else { String modelId = null; try { From 4fe59c6dbbe77687364ec6a007c76724013545b4 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 16:43:58 +0800 Subject: [PATCH 10/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 4 ++-- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 13cab7bdc..96c9a9762 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -72,7 +72,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" - excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } @@ -107,7 +107,7 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" - excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index c358a0af8..5a73f4055 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -39,8 +39,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws logger.info("bwc version: " + bwcVersion.toString()); if (isRunningAgainstOldCluster()) { - String modelId = uploadSparseEncodingModel(); - loadModel(modelId); + String modelId = prepareSparseEncodingModel(); createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), @@ -51,8 +50,8 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws addSparseEncodingDoc( getIndexNameForTest(), "0", - List.of(TEST_SPARSE_ENCODING_FIELD), - List.of(testRankFeaturesDoc1), + List.of(), + List.of(), List.of(TEST_TEXT_FIELD), List.of(TEXT_1) ); From ae882f44d7d7e90bad9c86f4cb9e2937e4e5464d Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 17:20:53 +0800 Subject: [PATCH 11/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 4 +-- .../bwc/NeuralQueryEnricherProcessorIT.java | 28 +++++++------------ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 96c9a9762..10bd29602 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -65,7 +65,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherIT tests from neural search version 2.9 and 2.10 + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { @@ -100,7 +100,7 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherIT tests from neural search version 2.9 and 2.10 + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 5a73f4055..6c679d43a 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Map; public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTestCase { private static final String SPARSE_INGEST_PIPELINE_NAME = "nlp-ingest-pipeline-sparse"; @@ -25,21 +24,16 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTe private static final String TEST_TEXT_FIELD = "passage_text"; private static final String TEXT_1 = "Hello world a b"; private static final String TEXT_2 = "Hello planet"; - private static final List TEST_TOKENS_1 = List.of("hello", "world", "a", "b", "c"); - private static final List TEST_TOKENS_2 = List.of("hello", "planet", "a", "b", "c"); - private final Map testRankFeaturesDoc1 = TestUtils.createRandomTokenWeightMap(TEST_TOKENS_1); - private final Map testRankFeaturesDoc2 = TestUtils.createRandomTokenWeightMap(TEST_TOKENS_2); - // Test restart-upgrade test sparse embedding processor - // Create Sparse Encoding Processor, Ingestion Pipeline and add document - // Validate process , pipeline and document count in restart-upgrade scenario + // Test restart-upgrade neural_query_enricher in restart-upgrade scenario public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); logger.info("bwc version: " + bwcVersion.toString()); if (isRunningAgainstOldCluster()) { - String modelId = prepareSparseEncodingModel(); + String modelId = uploadSparseEncodingModel(); + loadModel(modelId); createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), @@ -47,22 +41,20 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws SPARSE_INGEST_PIPELINE_NAME ); - addSparseEncodingDoc( - getIndexNameForTest(), - "0", - List.of(), - List.of(), - List.of(TEST_TEXT_FIELD), - List.of(TEXT_1) - ); + addSparseEncodingDoc(getIndexNameForTest(), "0", List.of(), List.of(), List.of(TEST_TEXT_FIELD), List.of(TEXT_1)); createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) + ); if (bwcVersion.onOrAfter(Version.V_2_13_0)) { // after we support default model id in neural_sparse query // do nothing here. need to add test codes after finishing backport ; } else { + // deploy model + loadModel(modelId); // before we support default model id in neural_sparse query NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( TEST_SPARSE_ENCODING_FIELD From 79c169d84741ad56945c7fdc6a51154819fa3bcc Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 17:57:49 +0800 Subject: [PATCH 12/27] fix undeploy with retry Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 3 --- .../org/opensearch/neuralsearch/BaseNeuralSearchIT.java | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 6c679d43a..ab1a9e60c 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -53,8 +53,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws // do nothing here. need to add test codes after finishing backport ; } else { - // deploy model - loadModel(modelId); // before we support default model id in neural_sparse query NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( TEST_SPARSE_ENCODING_FIELD @@ -62,7 +60,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } - } else { String modelId = null; try { diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 50f97053a..6e76fd22b 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -1058,7 +1058,13 @@ protected void wipeOfTestResources( deleteSearchPipeline(searchPipeline); } if (modelId != null) { - deleteModel(modelId); + try { + deleteModel(modelId); + } catch (AssertionError e) { + // sometimes we have flaky test that the model state doesn't change after call undeploy api + // for this case we can call undeploy api one more time + deleteModel(modelId); + } } if (indexName != null) { deleteIndex(indexName); From 6e5bcc345223398cb144d3d1d9a39692f000650e Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 6 Mar 2024 18:17:02 +0800 Subject: [PATCH 13/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- .../bwc/NeuralQueryEnricherProcessorIT.java | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index ab1a9e60c..88d8d76e0 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -6,11 +6,13 @@ import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; import static org.opensearch.neuralsearch.TestUtils.SPARSE_ENCODING_PROCESSOR; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; import org.opensearch.Version; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.neuralsearch.TestUtils; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; import java.nio.file.Files; @@ -18,9 +20,12 @@ import java.util.List; public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTestCase { - private static final String SPARSE_INGEST_PIPELINE_NAME = "nlp-ingest-pipeline-sparse"; - private static final String SPARSE_SEARCH_PIPELINE_NAME = "nlp-ingest-pipeline-sparse"; - private static final String TEST_SPARSE_ENCODING_FIELD = "passage_embedding"; + // add prefix to avoid conflicts with other IT class, since we don't wipe resources after first round + private static final String SPARSE_INGEST_PIPELINE_NAME = "nqep-nlp-ingest-pipeline-sparse"; + private static final String DENSE_INGEST_PIPELINE_NAME = "nqep-nlp-ingest-pipeline-dense"; + private static final String SPARSE_SEARCH_PIPELINE_NAME = "nqep-nlp-search-pipeline-sparse"; + private static final String DENSE_SEARCH_PIPELINE_NAME = "nqep-nlp-search-pipeline-dense"; + private static final String TEST_ENCODING_FIELD = "passage_embedding"; private static final String TEST_TEXT_FIELD = "passage_text"; private static final String TEXT_1 = "Hello world a b"; private static final String TEXT_2 = "Hello planet"; @@ -55,7 +60,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } else { // before we support default model id in neural_sparse query NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_SPARSE_ENCODING_FIELD + TEST_ENCODING_FIELD ).queryText(TEXT_1); expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); @@ -66,10 +71,10 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(modelId); NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_SPARSE_ENCODING_FIELD + TEST_ENCODING_FIELD ).queryText(TEXT_1); NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_SPARSE_ENCODING_FIELD + TEST_ENCODING_FIELD ).queryText(TEXT_1).modelId(modelId); assertEquals( search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), @@ -80,4 +85,53 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } } } + + public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + Version bwcVersion = parseVersionFromString(getBWCVersion().get()); + logger.info("bwc version: " + bwcVersion.toString()); + + if (isRunningAgainstOldCluster()) { + String modelId = uploadTextEmbeddingModel(); + loadModel(modelId); + createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())), + DENSE_INGEST_PIPELINE_NAME + ); + + addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); + + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + + NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); + NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1) + .modelId(modelId); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } else { + String modelId = null; + try { + modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); + loadModel(modelId); + + NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1); + NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1) + .modelId(modelId); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } finally { + wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); + } + } + } } From 8df05d250cf22d75752cffb53b639184b8e95704 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 11:15:12 +0800 Subject: [PATCH 14/27] add it for bwc restart-upgrade Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 10 ++-- .../bwc/NeuralQueryEnricherProcessorIT.java | 47 ++++++++++++------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 10bd29602..58bb6dfe9 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -65,14 +65,12 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 - // because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" - excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } @@ -100,18 +98,16 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 - // because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" - excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' -} +} \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 88d8d76e0..0e74deb60 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -34,7 +34,11 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTe public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - logger.info("bwc version: " + bwcVersion.toString()); + + // skip this test before we have neural_sparse search in 2.11 + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + return; + } if (isRunningAgainstOldCluster()) { String modelId = uploadSparseEncodingModel(); @@ -53,16 +57,17 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); + + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_ENCODING_FIELD + ).queryText(TEXT_1); + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { // after we support default model id in neural_sparse query // do nothing here. need to add test codes after finishing backport ; } else { // before we support default model id in neural_sparse query - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } } else { @@ -89,7 +94,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - logger.info("bwc version: " + bwcVersion.toString()); if (isRunningAgainstOldCluster()) { String modelId = uploadTextEmbeddingModel(); @@ -103,23 +107,34 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); - NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) - .queryText(TEXT_1) - .modelId(modelId); - assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") - ); + .queryText(TEXT_1) + .modelId(modelId); + + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + // before we have neural_query_enricher + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); + } else { + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } } else { String modelId = null; try { modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); + // create the neural_query_enricher processor if we don't create them before + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + } + NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) @@ -130,7 +145,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } finally { - wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); + wipeOfTestResources(getIndexNameForTest(), DENSE_INGEST_PIPELINE_NAME, modelId, DENSE_SEARCH_PIPELINE_NAME); } } } From 046c3b236a56d0920ba0f754bda108f8a5d85630 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 13:27:08 +0800 Subject: [PATCH 15/27] optimize it code structure Signed-off-by: zhichao-aws --- .../bwc/NeuralQueryEnricherProcessorIT.java | 44 +++++++++---------- .../bwc/NeuralQueryEnricherProcessorIT.java | 2 + 2 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 0e74deb60..4aafcad54 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -28,7 +28,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTe private static final String TEST_ENCODING_FIELD = "passage_embedding"; private static final String TEST_TEXT_FIELD = "passage_text"; private static final String TEXT_1 = "Hello world a b"; - private static final String TEXT_2 = "Hello planet"; // Test restart-upgrade neural_query_enricher in restart-upgrade scenario public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { @@ -40,9 +39,19 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws return; } + String modelId = null; + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_ENCODING_FIELD + ).queryText(TEXT_1); + // will set the model_id after we obtain the id + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_ENCODING_FIELD + ).queryText(TEXT_1); + if (isRunningAgainstOldCluster()) { - String modelId = uploadSparseEncodingModel(); + modelId = uploadSparseEncodingModel(); loadModel(modelId); + sparseEncodingQueryBuilderWithModelId.modelId(modelId); createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), @@ -58,9 +67,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { // after we support default model id in neural_sparse query @@ -71,16 +78,10 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } } else { - String modelId = null; try { modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(modelId); - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1).modelId(modelId); + sparseEncodingQueryBuilderWithModelId.modelId(modelId); assertEquals( search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") @@ -95,9 +96,14 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); + String modelId = null; + NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); + NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); + if (isRunningAgainstOldCluster()) { - String modelId = uploadTextEmbeddingModel(); + modelId = uploadTextEmbeddingModel(); loadModel(modelId); + neuralQueryBuilderWithModelId.modelId(modelId); createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), @@ -107,11 +113,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); - NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) - .queryText(TEXT_1) - .modelId(modelId); - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { // before we have neural_query_enricher expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); @@ -124,10 +125,10 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep ); } } else { - String modelId = null; try { modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); + neuralQueryBuilderWithModelId.modelId(modelId); // create the neural_query_enricher processor if we don't create them before if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { @@ -135,11 +136,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); } - NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) - .queryText(TEXT_1); - NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD) - .queryText(TEXT_1) - .modelId(modelId); assertEquals( search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java new file mode 100644 index 000000000..cc70ddb7d --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -0,0 +1,2 @@ +package org.opensearch.neuralsearch.bwc;public class NeuralQueryEnricherProcessorIT { +} From d11d3c467e169a903b57e4b6bbc2d48cc2861d6d Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 13:27:32 +0800 Subject: [PATCH 16/27] add it for bwc rolling-upgrade Signed-off-by: zhichao-aws --- .../bwc/NeuralQueryEnricherProcessorIT.java | 184 +++++++++++++++++- .../SearchRequestPipelineConfiguration.json | 11 ++ 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 qa/rolling-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index cc70ddb7d..e1af684a3 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -1,2 +1,184 @@ -package org.opensearch.neuralsearch.bwc;public class NeuralQueryEnricherProcessorIT { +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.neuralsearch.bwc; + +import org.opensearch.Version; +import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; +import org.opensearch.neuralsearch.TestUtils; +import org.opensearch.neuralsearch.query.NeuralQueryBuilder; +import org.opensearch.neuralsearch.query.NeuralSparseQueryBuilder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.neuralsearch.TestUtils.SPARSE_ENCODING_PROCESSOR; +import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; + +public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCase { + // add prefix to avoid conflicts with other IT class, since we don't wipe resources after first round + private static final String SPARSE_INGEST_PIPELINE_NAME = "nqep-nlp-ingest-pipeline-sparse"; + private static final String DENSE_INGEST_PIPELINE_NAME = "nqep-nlp-ingest-pipeline-dense"; + private static final String SPARSE_SEARCH_PIPELINE_NAME = "nqep-nlp-search-pipeline-sparse"; + private static final String DENSE_SEARCH_PIPELINE_NAME = "nqep-nlp-search-pipeline-dense"; + private static final String TEST_ENCODING_FIELD = "passage_embedding"; + private static final String TEST_TEXT_FIELD = "passage_text"; + private static final String TEXT_1 = "Hello world a b"; + + public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + Version bwcVersion = parseVersionFromString(getBWCVersion().get()); + // skip this test before we have neural_sparse search in 2.11 + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + return; + } + + String modelId = null; + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_ENCODING_FIELD + ).queryText(TEXT_1); + // will set the model_id after we obtain the id + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( + TEST_ENCODING_FIELD + ).queryText(TEXT_1); + + switch (getClusterType()) { + case OLD: + modelId = uploadSparseEncodingModel(); + loadModel(modelId); + sparseEncodingQueryBuilderWithModelId.modelId(modelId); + createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/SparseIndexMappings.json").toURI())), + SPARSE_INGEST_PIPELINE_NAME + ); + + addSparseEncodingDoc(getIndexNameForTest(), "0", List.of(), List.of(), List.of(TEST_TEXT_FIELD), List.of(TEXT_1)); + createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) + ); + + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { + // after we support default model id in neural_sparse query + // do nothing here. need to add test codes after finishing backport + ; + } else { + // before we support default model id in neural_sparse query + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); + } + break; + case MIXED: + modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); + loadModel(modelId); + sparseEncodingQueryBuilderWithModelId.modelId(modelId); + if (bwcVersion.onOrAfter(Version.V_2_13_0)) { + // after we support default model id in neural_sparse query + // do nothing here. need to add test codes after finishing backport + ; + } else { + // before we support default model id in neural_sparse query + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); + } + break; + case UPGRADED: + try { + modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); + loadModel(modelId); + sparseEncodingQueryBuilderWithModelId.modelId(modelId); + assertEquals( + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") + ); + } finally { + wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); + } + break; + default: + throw new IllegalStateException("Unexpected value: " + getClusterType()); + } + } + + public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + Version bwcVersion = parseVersionFromString(getBWCVersion().get()); + + String modelId = null; + NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); + NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); + + switch (getClusterType()) { + case OLD: + modelId = uploadTextEmbeddingModel(); + loadModel(modelId); + neuralQueryBuilderWithModelId.modelId(modelId); + createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); + createIndexWithConfiguration( + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())), + DENSE_INGEST_PIPELINE_NAME + ); + + addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); + + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + // before we have neural_query_enricher + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); + } else { + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } + break; + case MIXED: + modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); + loadModel(modelId); + neuralQueryBuilderWithModelId.modelId(modelId); + + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + // before we have neural_query_enricher + expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); + } else { + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } + break; + case UPGRADED: + try { + modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); + loadModel(modelId); + neuralQueryBuilderWithModelId.modelId(modelId); + + // create the neural_query_enricher processor if we don't create them before + if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + } + + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); + } finally { + wipeOfTestResources(getIndexNameForTest(), DENSE_INGEST_PIPELINE_NAME, modelId, DENSE_SEARCH_PIPELINE_NAME); + } + break; + default: + throw new IllegalStateException("Unexpected value: " + getClusterType()); + } + } } diff --git a/qa/rolling-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json b/qa/rolling-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json new file mode 100644 index 000000000..5cd9988cb --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/processor/SearchRequestPipelineConfiguration.json @@ -0,0 +1,11 @@ +{ + "request_processors": [ + { + "neural_query_enricher": { + "tag": "tag1", + "description": "This processor is going to restrict to publicly visible documents", + "default_model_id": "%s" + } + } + ] +} From f818bd533bfbe7005583e4bb2d4698563e04b3d1 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 13:28:25 +0800 Subject: [PATCH 17/27] tidy Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 2 +- .../bwc/NeuralQueryEnricherProcessorIT.java | 26 +++++---- .../bwc/NeuralQueryEnricherProcessorIT.java | 58 ++++++++++--------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 58bb6dfe9..a4ca03b65 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -110,4 +110,4 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' -} \ No newline at end of file +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 4aafcad54..e7df3ff92 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -40,13 +40,11 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } String modelId = null; - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1); // will set the model_id after we obtain the id - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1); if (isRunningAgainstOldCluster()) { modelId = uploadSparseEncodingModel(); @@ -67,8 +65,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - - if (bwcVersion.onOrAfter(Version.V_2_13_0)) { // after we support default model id in neural_sparse query // do nothing here. need to add test codes after finishing backport @@ -118,10 +114,13 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } } else { @@ -133,7 +132,10 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep // create the neural_query_enricher processor if we don't create them before if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); } assertEquals( diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index e1af684a3..77d56564f 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -2,7 +2,6 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.neuralsearch.bwc; import org.opensearch.Version; @@ -39,13 +38,11 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } String modelId = null; - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1); // will set the model_id after we obtain the id - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName( - TEST_ENCODING_FIELD - ).queryText(TEXT_1); + NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) + .queryText(TEXT_1); switch (getClusterType()) { case OLD: @@ -54,16 +51,16 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws sparseEncodingQueryBuilderWithModelId.modelId(modelId); createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( - getIndexNameForTest(), - Files.readString(Path.of(classLoader.getResource("processor/SparseIndexMappings.json").toURI())), - SPARSE_INGEST_PIPELINE_NAME + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/SparseIndexMappings.json").toURI())), + SPARSE_INGEST_PIPELINE_NAME ); addSparseEncodingDoc(getIndexNameForTest(), "0", List.of(), List.of(), List.of(TEST_TEXT_FIELD), List.of(TEXT_1)); createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); if (bwcVersion.onOrAfter(Version.V_2_13_0)) { @@ -94,8 +91,8 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws loadModel(modelId); sparseEncodingQueryBuilderWithModelId.modelId(modelId); assertEquals( - search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") ); } finally { wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); @@ -121,9 +118,9 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep neuralQueryBuilderWithModelId.modelId(modelId); createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( - getIndexNameForTest(), - Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())), - DENSE_INGEST_PIPELINE_NAME + getIndexNameForTest(), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())), + DENSE_INGEST_PIPELINE_NAME ); addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); @@ -133,10 +130,13 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } break; @@ -150,10 +150,13 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } break; @@ -166,12 +169,15 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep // create the neural_query_enricher processor if we don't create them before if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); } assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } finally { wipeOfTestResources(getIndexNameForTest(), DENSE_INGEST_PIPELINE_NAME, modelId, DENSE_SEARCH_PIPELINE_NAME); From c57704aabf9a2e72f15301567549180130ede0d2 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 13:51:57 +0800 Subject: [PATCH 18/27] update index mapping in it Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 77d56564f..ac421d034 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -119,7 +119,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), - Files.readString(Path.of(classLoader.getResource("processor/IndexMappingMultipleShard.json").toURI())), + Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), DENSE_INGEST_PIPELINE_NAME ); From 66c4994b4142e9ca16224d11b14f85c27d68e1c1 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Thu, 7 Mar 2024 14:09:49 +0800 Subject: [PATCH 19/27] nit Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 6 +++--- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index e7df3ff92..87aed551f 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -35,7 +35,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Version bwcVersion = parseVersionFromString(getBWCVersion().get()); // skip this test before we have neural_sparse search in 2.11 - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { return; } @@ -109,7 +109,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { // before we have neural_query_enricher expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { @@ -130,7 +130,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep neuralQueryBuilderWithModelId.modelId(modelId); // create the neural_query_enricher processor if we don't create them before - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); updateIndexSettings( getIndexNameForTest(), diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index ac421d034..41e1f64f5 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -33,7 +33,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); // skip this test before we have neural_sparse search in 2.11 - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { return; } @@ -125,7 +125,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { // before we have neural_query_enricher expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { @@ -145,7 +145,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep loadModel(modelId); neuralQueryBuilderWithModelId.modelId(modelId); - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { // before we have neural_query_enricher expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); } else { @@ -167,7 +167,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep neuralQueryBuilderWithModelId.modelId(modelId); // create the neural_query_enricher processor if we don't create them before - if (!bwcVersion.onOrAfter(Version.V_2_11_0)) { + if (bwcVersion.before(Version.V_2_11_0)) { createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); updateIndexSettings( getIndexNameForTest(), From 8db736ad3b8ee293c65e183541f569f6a7c489f1 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Fri, 8 Mar 2024 09:05:57 +0800 Subject: [PATCH 20/27] move version check to build script Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 8 ++- .../bwc/NeuralQueryEnricherProcessorIT.java | 34 ++--------- qa/rolling-upgrade/build.gradle | 12 +++- .../bwc/NeuralQueryEnricherProcessorIT.java | 59 ++++++------------- 4 files changed, 39 insertions(+), 74 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index a4ca03b65..10bd29602 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -65,12 +65,14 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } @@ -98,12 +100,14 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 87aed551f..db6d43b8a 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -34,11 +34,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - // skip this test before we have neural_sparse search in 2.11 - if (bwcVersion.before(Version.V_2_11_0)) { - return; - } - String modelId = null; NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); @@ -109,35 +104,18 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - if (bwcVersion.before(Version.V_2_11_0)) { - // before we have neural_query_enricher - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); - } else { - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) - ); - assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") - ); - } + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings(getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME)); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); } else { try { modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); neuralQueryBuilderWithModelId.modelId(modelId); - // create the neural_query_enricher processor if we don't create them before - if (bwcVersion.before(Version.V_2_11_0)) { - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) - ); - } - assertEquals( search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 175924523..4ff3afe6b 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -65,12 +65,14 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version systemProperty 'tests.skip_delete_model_index', 'true' - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } @@ -99,12 +101,14 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } @@ -132,12 +136,14 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', ext.neural_search_bwc_version - //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT tests from neural search version 2.9 and 2.10 because these features were released in 2.11 version. + //Excluding MultiModalSearchIT, HybridSearchIT, NeuralSparseSearchIT, NeuralQueryEnricherProcessorIT tests from neural search version 2.9 and 2.10 + // because these features were released in 2.11 version. if (ext.neural_search_bwc_version.startsWith("2.9") || ext.neural_search_bwc_version.startsWith("2.10")){ filter { excludeTestsMatching "org.opensearch.neuralsearch.bwc.MultiModalSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.HybridSearchIT.*" excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralSparseSearchIT.*" + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.*" } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 41e1f64f5..7bbad4f34 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -32,10 +32,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCa public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - // skip this test before we have neural_sparse search in 2.11 - if (bwcVersion.before(Version.V_2_11_0)) { - return; - } String modelId = null; NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) @@ -125,40 +121,30 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - if (bwcVersion.before(Version.V_2_11_0)) { - // before we have neural_query_enricher - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); - } else { - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) - ); - assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") - ); - } + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); break; case MIXED: modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); neuralQueryBuilderWithModelId.modelId(modelId); - if (bwcVersion.before(Version.V_2_11_0)) { - // before we have neural_query_enricher - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1)); - } else { - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) - ); - assertEquals( - search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), - search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") - ); - } + createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + updateIndexSettings( + getIndexNameForTest(), + Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) + ); + assertEquals( + search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), + search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") + ); break; case UPGRADED: try { @@ -166,15 +152,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep loadModel(modelId); neuralQueryBuilderWithModelId.modelId(modelId); - // create the neural_query_enricher processor if we don't create them before - if (bwcVersion.before(Version.V_2_11_0)) { - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); - updateIndexSettings( - getIndexNameForTest(), - Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) - ); - } - assertEquals( search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") From fe9a870b882a9738d269a60414f8ba201eaaf74f Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Fri, 8 Mar 2024 09:20:25 +0800 Subject: [PATCH 21/27] resolve modelId Signed-off-by: zhichao-aws --- .../bwc/NeuralQueryEnricherProcessorIT.java | 10 ++-- .../bwc/NeuralQueryEnricherProcessorIT.java | 56 +++++++++---------- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index db6d43b8a..afb23dae0 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -34,7 +34,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - String modelId = null; NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id @@ -42,7 +41,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws .queryText(TEXT_1); if (isRunningAgainstOldCluster()) { - modelId = uploadSparseEncodingModel(); + String modelId = uploadSparseEncodingModel(); loadModel(modelId); sparseEncodingQueryBuilderWithModelId.modelId(modelId); createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); @@ -69,6 +68,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } } else { + String modelId = ""; try { modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(modelId); @@ -85,14 +85,11 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - - String modelId = null; NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); if (isRunningAgainstOldCluster()) { - modelId = uploadTextEmbeddingModel(); + String modelId = uploadTextEmbeddingModel(); loadModel(modelId); neuralQueryBuilderWithModelId.modelId(modelId); createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); @@ -111,6 +108,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } else { + String modelId = ""; try { modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 7bbad4f34..b683ef203 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -28,12 +28,13 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCa private static final String TEST_ENCODING_FIELD = "passage_embedding"; private static final String TEST_TEXT_FIELD = "passage_text"; private static final String TEXT_1 = "Hello world a b"; + private String sparseModelId = ""; + private String denseModelId = ""; public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - String modelId = null; NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id @@ -42,10 +43,10 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws switch (getClusterType()) { case OLD: - modelId = uploadSparseEncodingModel(); - loadModel(modelId); - sparseEncodingQueryBuilderWithModelId.modelId(modelId); - createPipelineForSparseEncodingProcessor(modelId, SPARSE_INGEST_PIPELINE_NAME); + sparseModelId = uploadSparseEncodingModel(); + loadModel(sparseModelId); + sparseEncodingQueryBuilderWithModelId.modelId(sparseModelId); + createPipelineForSparseEncodingProcessor(sparseModelId, SPARSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), Files.readString(Path.of(classLoader.getResource("processor/SparseIndexMappings.json").toURI())), @@ -53,7 +54,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws ); addSparseEncodingDoc(getIndexNameForTest(), "0", List.of(), List.of(), List.of(TEST_TEXT_FIELD), List.of(TEXT_1)); - createSearchRequestProcessor(modelId, SPARSE_SEARCH_PIPELINE_NAME); + createSearchRequestProcessor(sparseModelId, SPARSE_SEARCH_PIPELINE_NAME); updateIndexSettings( getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) @@ -69,9 +70,9 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } break; case MIXED: - modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); - loadModel(modelId); - sparseEncodingQueryBuilderWithModelId.modelId(modelId); + sparseModelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); + loadModel(sparseModelId); + sparseEncodingQueryBuilderWithModelId.modelId(sparseModelId); if (bwcVersion.onOrAfter(Version.V_2_13_0)) { // after we support default model id in neural_sparse query // do nothing here. need to add test codes after finishing backport @@ -83,15 +84,15 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws break; case UPGRADED: try { - modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); - loadModel(modelId); - sparseEncodingQueryBuilderWithModelId.modelId(modelId); + sparseModelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); + loadModel(sparseModelId); + sparseEncodingQueryBuilderWithModelId.modelId(sparseModelId); assertEquals( search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), sparseEncodingQueryBuilderWithModelId, 1).get("hits") ); } finally { - wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, modelId, SPARSE_SEARCH_PIPELINE_NAME); + wipeOfTestResources(getIndexNameForTest(), SPARSE_INGEST_PIPELINE_NAME, sparseModelId, SPARSE_SEARCH_PIPELINE_NAME); } break; default: @@ -101,18 +102,15 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - - String modelId = null; NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); NeuralQueryBuilder neuralQueryBuilderWithModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); switch (getClusterType()) { case OLD: - modelId = uploadTextEmbeddingModel(); - loadModel(modelId); - neuralQueryBuilderWithModelId.modelId(modelId); - createPipelineProcessor(modelId, DENSE_INGEST_PIPELINE_NAME); + denseModelId = uploadTextEmbeddingModel(); + loadModel(denseModelId); + neuralQueryBuilderWithModelId.modelId(denseModelId); + createPipelineProcessor(denseModelId, DENSE_INGEST_PIPELINE_NAME); createIndexWithConfiguration( getIndexNameForTest(), Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())), @@ -121,7 +119,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep addDocument(getIndexNameForTest(), "0", TEST_TEXT_FIELD, TEXT_1, null, null); - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + createSearchRequestProcessor(denseModelId, DENSE_SEARCH_PIPELINE_NAME); updateIndexSettings( getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) @@ -132,11 +130,11 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep ); break; case MIXED: - modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); - loadModel(modelId); - neuralQueryBuilderWithModelId.modelId(modelId); + denseModelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); + loadModel(denseModelId); + neuralQueryBuilderWithModelId.modelId(denseModelId); - createSearchRequestProcessor(modelId, DENSE_SEARCH_PIPELINE_NAME); + createSearchRequestProcessor(denseModelId, DENSE_SEARCH_PIPELINE_NAME); updateIndexSettings( getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", DENSE_SEARCH_PIPELINE_NAME) @@ -148,16 +146,16 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep break; case UPGRADED: try { - modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); - loadModel(modelId); - neuralQueryBuilderWithModelId.modelId(modelId); + denseModelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); + loadModel(denseModelId); + neuralQueryBuilderWithModelId.modelId(denseModelId); assertEquals( search(getIndexNameForTest(), neuralQueryBuilderWithoutModelId, 1).get("hits"), search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } finally { - wipeOfTestResources(getIndexNameForTest(), DENSE_INGEST_PIPELINE_NAME, modelId, DENSE_SEARCH_PIPELINE_NAME); + wipeOfTestResources(getIndexNameForTest(), DENSE_INGEST_PIPELINE_NAME, denseModelId, DENSE_SEARCH_PIPELINE_NAME); } break; default: From 9031db724f1b3125cf969a533c9b7b1ed8a638cd Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Fri, 8 Mar 2024 10:10:40 +0800 Subject: [PATCH 22/27] nit Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 1 - .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index afb23dae0..bb93a44f4 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -33,7 +33,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTe public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index b683ef203..e159e65cf 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -34,7 +34,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCa public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); Version bwcVersion = parseVersionFromString(getBWCVersion().get()); - NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id From 41d019f52d4272ec43e491f1eb2b8f5ca2c1dc41 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Fri, 8 Mar 2024 10:55:23 +0800 Subject: [PATCH 23/27] update init model id Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index bb93a44f4..db64ee6e7 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -67,7 +67,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); } } else { - String modelId = ""; + String modelId = null; try { modelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(modelId); @@ -107,7 +107,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Excep search(getIndexNameForTest(), neuralQueryBuilderWithModelId, 1).get("hits") ); } else { - String modelId = ""; + String modelId = null; try { modelId = TestUtils.getModelId(getIngestionPipeline(DENSE_INGEST_PIPELINE_NAME), TEXT_EMBEDDING_PROCESSOR); loadModel(modelId); From 5320961418a64e79728e8ee901ad623043e30f4d Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Fri, 8 Mar 2024 11:59:53 +0800 Subject: [PATCH 24/27] modify versions check logic in bwc test Signed-off-by: zhichao-aws --- qa/restart-upgrade/build.gradle | 14 +++++++++++++ .../bwc/NeuralQueryEnricherProcessorIT.java | 12 +---------- qa/rolling-upgrade/build.gradle | 21 +++++++++++++++++++ .../bwc/NeuralQueryEnricherProcessorIT.java | 21 ++----------------- .../neuralsearch/BaseNeuralSearchIT.java | 10 --------- 5 files changed, 38 insertions(+), 40 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 10bd29602..1a6d0a104 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -76,6 +76,13 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { } } + // Excluding the test because we introduce this feature in 2.13 + if (ext.neural_search_bwc_version.startsWith("2.11") || ext.neural_search_bwc_version.startsWith("2.12")){ + filter { + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -111,6 +118,13 @@ task testAgainstNewCluster(type: StandaloneRestIntegTestTask) { } } + // Excluding the test because we introduce this feature in 2.13 + if (ext.neural_search_bwc_version.startsWith("2.11") || ext.neural_search_bwc_version.startsWith("2.12")){ + filter { + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index db64ee6e7..6ef7b6d65 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -8,8 +8,6 @@ import static org.opensearch.neuralsearch.TestUtils.SPARSE_ENCODING_PROCESSOR; import static org.opensearch.neuralsearch.TestUtils.TEXT_EMBEDDING_PROCESSOR; -import org.opensearch.Version; -import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.neuralsearch.TestUtils; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -32,7 +30,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRestartUpgradeRestTe // Test restart-upgrade neural_query_enricher in restart-upgrade scenario public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - Version bwcVersion = parseVersionFromString(getBWCVersion().get()); NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id @@ -58,14 +55,7 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - if (bwcVersion.onOrAfter(Version.V_2_13_0)) { - // after we support default model id in neural_sparse query - // do nothing here. need to add test codes after finishing backport - ; - } else { - // before we support default model id in neural_sparse query - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); - } + // do nothing here. need to add test codes after finishing backport } else { String modelId = null; try { diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 4ff3afe6b..9d71b4205 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -76,6 +76,13 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { } } + // Excluding the test because we introduce this feature in 2.13 + if (ext.neural_search_bwc_version.startsWith("2.11") || ext.neural_search_bwc_version.startsWith("2.12")){ + filter { + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -112,6 +119,13 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { } } + // Excluding the test because we introduce this feature in 2.13 + if (ext.neural_search_bwc_version.startsWith("2.11") || ext.neural_search_bwc_version.startsWith("2.12")){ + filter { + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -147,6 +161,13 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { } } + // Excluding the test because we introduce this feature in 2.13 + if (ext.neural_search_bwc_version.startsWith("2.11") || ext.neural_search_bwc_version.startsWith("2.12")){ + filter { + excludeTestsMatching "org.opensearch.neuralsearch.bwc.NeuralQueryEnricherProcessorIT.testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index e159e65cf..332c3d294 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -4,8 +4,6 @@ */ package org.opensearch.neuralsearch.bwc; -import org.opensearch.Version; -import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.neuralsearch.TestUtils; import org.opensearch.neuralsearch.query.NeuralQueryBuilder; @@ -33,7 +31,6 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCa public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); - Version bwcVersion = parseVersionFromString(getBWCVersion().get()); NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) .queryText(TEXT_1); // will set the model_id after we obtain the id @@ -59,27 +56,13 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - if (bwcVersion.onOrAfter(Version.V_2_13_0)) { - // after we support default model id in neural_sparse query - // do nothing here. need to add test codes after finishing backport - ; - } else { - // before we support default model id in neural_sparse query - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); - } + // do nothing here. need to add test codes after finishing backport break; case MIXED: sparseModelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(sparseModelId); sparseEncodingQueryBuilderWithModelId.modelId(sparseModelId); - if (bwcVersion.onOrAfter(Version.V_2_13_0)) { - // after we support default model id in neural_sparse query - // do nothing here. need to add test codes after finishing backport - ; - } else { - // before we support default model id in neural_sparse query - expectThrows(ResponseException.class, () -> search(getIndexNameForTest(), sparseEncodingQueryBuilderWithoutModelId, 1)); - } + // do nothing here. need to add test codes after finishing backport break; case UPGRADED: try { diff --git a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java index 6e76fd22b..5cfba2518 100644 --- a/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java +++ b/src/testFixtures/java/org/opensearch/neuralsearch/BaseNeuralSearchIT.java @@ -4,7 +4,6 @@ */ package org.opensearch.neuralsearch; -import org.opensearch.Version; import org.opensearch.ml.common.model.MLModelState; import static org.opensearch.neuralsearch.common.VectorUtil.vectorAsListToArray; @@ -1071,15 +1070,6 @@ protected void wipeOfTestResources( } } - // parse the version from string like: "2.12.0","2.13.0-SNAPSHOT" - protected Version parseVersionFromString(String version) { - /* snapshot labels are not supported in OpenSearch Version.fromStringSlow */ - if (version.endsWith("-SNAPSHOT")) { - version = version.substring(0, version.length() - "-SNAPSHOT".length()); - } - return Version.fromString(version); - } - /** * Enumeration for types of pipeline processors, used to lookup resources like create * processor request as those are type specific From 1ade82f5566b96fe617ab1f6c5933ebad3258c6d Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Sat, 9 Mar 2024 18:07:17 +0800 Subject: [PATCH 25/27] add comments Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 332c3d294..ff54348d2 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -29,6 +29,8 @@ public class NeuralQueryEnricherProcessorIT extends AbstractRollingUpgradeTestCa private String sparseModelId = ""; private String denseModelId = ""; + // test of NeuralQueryEnricherProcessor supports neural_sparse query default model_id + // the feature is introduced from 2.13 public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); NeuralSparseQueryBuilder sparseEncodingQueryBuilderWithoutModelId = new NeuralSparseQueryBuilder().fieldName(TEST_ENCODING_FIELD) @@ -82,6 +84,8 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws } } + // test of NeuralQueryEnricherProcessor supports neural query default model_id + // the feature is introduced from 2.11 public void testNeuralQueryEnricherProcessor_NeuralSearch_E2EFlow() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); NeuralQueryBuilder neuralQueryBuilderWithoutModelId = new NeuralQueryBuilder().fieldName(TEST_ENCODING_FIELD).queryText(TEXT_1); From d41947c9cdc173929e9baee5e22e28377a645788 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 13 Mar 2024 10:47:27 +0800 Subject: [PATCH 26/27] nit Signed-off-by: zhichao-aws --- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 2 -- .../neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index 6ef7b6d65..d27b9d0f3 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -54,8 +54,6 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - - // do nothing here. need to add test codes after finishing backport } else { String modelId = null; try { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java index ff54348d2..7a98b7322 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/NeuralQueryEnricherProcessorIT.java @@ -57,14 +57,11 @@ public void testNeuralQueryEnricherProcessor_NeuralSparseSearch_E2EFlow() throws getIndexNameForTest(), Settings.builder().put("index.search.default_pipeline", SPARSE_SEARCH_PIPELINE_NAME) ); - - // do nothing here. need to add test codes after finishing backport break; case MIXED: sparseModelId = TestUtils.getModelId(getIngestionPipeline(SPARSE_INGEST_PIPELINE_NAME), SPARSE_ENCODING_PROCESSOR); loadModel(sparseModelId); sparseEncodingQueryBuilderWithModelId.modelId(sparseModelId); - // do nothing here. need to add test codes after finishing backport break; case UPGRADED: try { From 1f12c1a0e4a61b28ecbd6340d15b86f4c2237c62 Mon Sep 17 00:00:00 2001 From: zhichao-aws Date: Wed, 13 Mar 2024 11:09:52 +0800 Subject: [PATCH 27/27] updates for comments Signed-off-by: zhichao-aws --- CHANGELOG.md | 2 +- .../neuralsearch/query/NeuralQueryBuilder.java | 5 +++-- .../query/NeuralSparseQueryBuilder.java | 2 +- .../query/visitor/NeuralSearchQueryVisitor.java | 14 ++++++++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c7b56855..48ea9865c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/neural-search/compare/2.x...HEAD) ### Features -- Enabled support for applying default modelId in neural sparse query ([#614](https://github.com/opensearch-project/neural-search/pull/614) ### Enhancements ### Bug Fixes - Fix async actions are left in neural_sparse query ([#438](https://github.com/opensearch-project/neural-search/pull/438)) @@ -21,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/neural-search/compare/2.12...2.x) ### Features +- Enabled support for applying default modelId in neural sparse query ([#614](https://github.com/opensearch-project/neural-search/pull/614) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java index d74378617..d27061e36 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralQueryBuilder.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; import org.apache.commons.lang.StringUtils; @@ -133,11 +134,11 @@ protected void doXContent(XContentBuilder xContentBuilder, Params params) throws xContentBuilder.startObject(NAME); xContentBuilder.startObject(fieldName); xContentBuilder.field(QUERY_TEXT_FIELD.getPreferredName(), queryText); - if (modelId != null) { + if (Objects.nonNull(modelId)) { xContentBuilder.field(MODEL_ID_FIELD.getPreferredName(), modelId); } xContentBuilder.field(K_FIELD.getPreferredName(), k); - if (filter != null) { + if (Objects.nonNull(filter)) { xContentBuilder.field(FILTER_FIELD.getPreferredName(), filter); } printBoostAndQueryName(xContentBuilder); diff --git a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java index 6e87f4c38..58e912788 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java +++ b/src/main/java/org/opensearch/neuralsearch/query/NeuralSparseQueryBuilder.java @@ -119,7 +119,7 @@ protected void doXContent(XContentBuilder xContentBuilder, Params params) throws xContentBuilder.startObject(NAME); xContentBuilder.startObject(fieldName); xContentBuilder.field(QUERY_TEXT_FIELD.getPreferredName(), queryText); - if (modelId != null) { + if (Objects.nonNull(modelId)) { xContentBuilder.field(MODEL_ID_FIELD.getPreferredName(), modelId); } printBoostAndQueryName(xContentBuilder); diff --git a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java index 6fd4d0708..b869db10a 100644 --- a/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java +++ b/src/main/java/org/opensearch/neuralsearch/query/visitor/NeuralSearchQueryVisitor.java @@ -5,6 +5,7 @@ package org.opensearch.neuralsearch.query.visitor; import java.util.Map; +import java.util.Objects; import org.apache.lucene.search.BooleanClause; import org.opensearch.index.query.QueryBuilder; @@ -22,6 +23,13 @@ public class NeuralSearchQueryVisitor implements QueryBuilderVisitor { private final String modelId; private final Map neuralFieldMap; + private Boolean isFieldDefaultModelIdApplied(String fieldName) { + if (Objects.nonNull(neuralFieldMap) && Objects.nonNull(fieldName) && Objects.nonNull(neuralFieldMap.get(fieldName))) { + return true; + } + return false; + } + /** * Accept method accepts every query builder from the search request, * and processes it if the required conditions in accept method are satisfied. @@ -31,12 +39,10 @@ public void accept(QueryBuilder queryBuilder) { if (queryBuilder instanceof ModelInferenceQueryBuilder) { ModelInferenceQueryBuilder modelInferenceQueryBuilder = (ModelInferenceQueryBuilder) queryBuilder; if (modelInferenceQueryBuilder.modelId() == null) { - if (neuralFieldMap != null - && modelInferenceQueryBuilder.fieldName() != null - && neuralFieldMap.get(modelInferenceQueryBuilder.fieldName()) != null) { + if (isFieldDefaultModelIdApplied(modelInferenceQueryBuilder.fieldName())) { String fieldDefaultModelId = (String) neuralFieldMap.get(modelInferenceQueryBuilder.fieldName()); modelInferenceQueryBuilder.modelId(fieldDefaultModelId); - } else if (modelId != null) { + } else if (Objects.nonNull(modelId)) { modelInferenceQueryBuilder.modelId(modelId); } else { throw new IllegalArgumentException(