Add tests for ranking evaluation with aliases (elastic#29452)

The ranking evaluation requests so far were not tested against aliases but they should run regardless of the targeted index is a real index or an alias. This change adds cases for this to the integration and rest tests.
tomcallahan · Apr 19, 2018 · d0f6657 · d0f6657
1 parent 24763d8
commit d0f6657
Show file tree

Hide file tree

Showing 2 changed files with 73 additions and 67 deletions.
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -40,10 +41,13 @@
 import java.util.Set;
 
 import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class RankEvalRequestIT extends ESIntegTestCase {
 
+    private static final String TEST_INDEX = "test";
+    private static final String INDEX_ALIAS = "alias0";
     private static final int RELEVANT_RATING_1 = 1;
 
     @Override
@@ -58,20 +62,23 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
 
     @Before
     public void setup() {
-        createIndex("test");
+        createIndex(TEST_INDEX);
         ensureGreen();
 
-        client().prepareIndex("test", "testtype").setId("1")
+        client().prepareIndex(TEST_INDEX, "testtype").setId("1")
                 .setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
-        client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
-        client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
-        client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
-        client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
-        client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
+        client().prepareIndex(TEST_INDEX, "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
+        client().prepareIndex(TEST_INDEX, "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
+        client().prepareIndex(TEST_INDEX, "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
+        client().prepareIndex(TEST_INDEX, "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
+        client().prepareIndex(TEST_INDEX, "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
 
         // add another index for testing closed indices etc...
         client().prepareIndex("test2", "testtype").setId("7").setSource("text", "amsterdam", "population", 851573).get();
         refresh();
+
+        // set up an alias that can also be used in tests
+        assertAcked(client().admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(TEST_INDEX).alias(INDEX_ALIAS)));
     }
 
     /**
@@ -101,7 +108,8 @@ public void testPrecisionAtRequest() {
                 RankEvalAction.INSTANCE, new RankEvalRequest());
         builder.setRankEvalSpec(task);
 
-        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test"))
+        String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS;
+        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse))
                 .actionGet();
         // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
         // second is 1/6, divided by 2 to get the average
@@ -143,7 +151,7 @@ public void testPrecisionAtRequest() {
         metric = new PrecisionAtK(1, false, 3);
         task = new RankEvalSpec(specifications, metric);
 
-        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
         // if we look only at top 3 documente, the expected P@3 for the first query is
@@ -163,19 +171,19 @@ public void testDCGRequest() {
 
         List<RatedRequest> specifications = new ArrayList<>();
         List<RatedDocument> ratedDocs = Arrays.asList(
-                new RatedDocument("test", "1", 3),
-                new RatedDocument("test", "2", 2),
-                new RatedDocument("test", "3", 3),
-                new RatedDocument("test", "4", 0),
-                new RatedDocument("test", "5", 1),
-                new RatedDocument("test", "6", 2));
+                new RatedDocument(TEST_INDEX, "1", 3),
+                new RatedDocument(TEST_INDEX, "2", 2),
+                new RatedDocument(TEST_INDEX, "3", 3),
+                new RatedDocument(TEST_INDEX, "4", 0),
+                new RatedDocument(TEST_INDEX, "5", 1),
+                new RatedDocument(TEST_INDEX, "6", 2));
         specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery));
 
         DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10);
         RankEvalSpec task = new RankEvalSpec(specifications, metric);
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
-                new RankEvalRequest(task, new String[] { "test" }));
+                new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
         assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), 10E-14);
@@ -184,7 +192,7 @@ public void testDCGRequest() {
         metric = new DiscountedCumulativeGain(false, null, 3);
         task = new RankEvalSpec(specifications, metric);
 
-        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
         assertEquals(12.39278926071437, response.getEvaluationResult(), 10E-14);
@@ -203,7 +211,7 @@ public void testMRRRequest() {
         RankEvalSpec task = new RankEvalSpec(specifications, metric);
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
-                new RankEvalRequest(task, new String[] { "test" }));
+                new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
         // the expected reciprocal rank for the amsterdam_query is 1/5
@@ -216,7 +224,7 @@ public void testMRRRequest() {
         metric = new MeanReciprocalRank(1, 3);
         task = new RankEvalSpec(specifications, metric);
 
-        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
+        builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
 
         response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
         // limiting to top 3 results, the amsterdam_query has no relevant document in it
@@ -247,7 +255,7 @@ public void testBadQuery() {
         RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
 
         RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
-                new RankEvalRequest(task, new String[] { "test" }));
+                new RankEvalRequest(task, new String[] { TEST_INDEX }));
         builder.setRankEvalSpec(task);
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
@@ -267,7 +275,7 @@ public void testIndicesOptions() {
         specifications.add(new RatedRequest("amsterdam_query", relevantDocs, amsterdamQuery));
         RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
 
-        RankEvalRequest request = new RankEvalRequest(task, new String[] { "test", "test2" });
+        RankEvalRequest request = new RankEvalRequest(task, new String[] { TEST_INDEX, "test2" });
         request.setRankEvalSpec(task);
 
         RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, request).actionGet();

diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml
@@ -1,10 +1,4 @@
----
-"Response format":
-
-  - skip:
-      version: " - 6.2.99"
-      reason: response format was updated in 6.3
-
+setup: 
   - do:
      indices.create:
          index: foo
@@ -43,8 +37,21 @@
   - do:
       indices.refresh: {}
 
+  - do:
+      indices.put_alias:
+        index: foo
+        name: alias
+
+---
+"Response format":
+
+  - skip:
+      version: " - 6.2.99"
+      reason: response format was updated in 6.3
+
   - do:
       rank_eval:
+        index: foo,
         body: {  
           "requests" : [
             {
@@ -84,52 +91,43 @@
   - match: { details.berlin_query.hits.0.hit._id: "doc1" }
   - match: { details.berlin_query.hits.0.rating: 1}
   - match: { details.berlin_query.hits.1.hit._id: "doc4" }
-  - is_false: details.berlin_query.hits.1.rating 
+  - is_false: details.berlin_query.hits.1.rating
 
 ---
-"Mean Reciprocal Rank":
-
-  - skip:
-      version: " - 6.2.99"
-      reason: response format was updated in 6.3
+"Alias resolution":
 
   - do:
-     indices.create:
-         index: foo
-         body:
-           settings:
-             index:
-               number_of_shards: 1
-  - do:
-      index:
-        index:   foo
-        type:    bar
-        id:      doc1
-        body:    { "text": "berlin" }
+      rank_eval:
+        index: alias
+        body: {  
+          "requests" : [
+            {
+                "id": "amsterdam_query",
+                "request": { "query": { "match" : {"text" : "amsterdam" }}},
+                "ratings": [
+                    {"_index": "foo", "_id": "doc1", "rating": 0},
+                    {"_index": "foo", "_id": "doc2", "rating": 1},
+                    {"_index": "foo", "_id": "doc3", "rating": 1}]
+            },
+            {
+                "id" : "berlin_query",
+                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
+                "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
+            }
+          ],
+          "metric" : { "precision": { "ignore_unlabeled" : true }}
+        }
 
-  - do:
-      index:
-        index:  foo
-        type:   bar
-        id:     doc2
-        body:   { "text": "amsterdam" }
+  - match: { quality_level: 1}
+  - match: { details.amsterdam_query.quality_level: 1.0}
+  - match: { details.berlin_query.quality_level: 1.0}
 
-  - do:
-      index:
-        index:  foo
-        type:   bar
-        id:     doc3
-        body:   { "text": "amsterdam" }
-
-  - do:
-      index:
-        index:  foo
-        type:   bar
-        id:     doc4
-        body:   { "text": "something about amsterdam and berlin" }
+---
+"Mean Reciprocal Rank":
 
-  - do:
-      indices.refresh: {}
+  - skip:
+      version: " - 6.2.99"
+      reason: response format was updated in 6.3
 
   - do:
       rank_eval: