feat(es5): support for elasticsearch 5.x while also maintaining backw…

…ards compatibility with 2.x
pelias · Sep 28, 2018 · 5feb5c5 · 5feb5c5
1 parent f7445aa
commit 5feb5c5
Show file tree

Hide file tree

Showing 32 changed files with 1,798 additions and 949 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -11,13 +11,12 @@ matrix:
 env:
   matrix:
     - ES_VERSION=2.4.6
+    - ES_VERSION=5.6.12
 jdk:
   - oraclejdk8
 install:
   - mkdir /tmp/elasticsearch
-  - wget -O - https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${ES_VERSION}/elasticsearch-${ES_VERSION}.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
-  - /tmp/elasticsearch/bin/plugin install analysis-icu
-  - /tmp/elasticsearch/bin/elasticsearch --daemonize --path.data /tmp
+  - ./scripts/setup_ci.sh
   - npm i
 script:
   - npm run travis

diff --git a/README.md b/README.md
@@ -124,7 +124,7 @@ $ npm run integration
 Download the image and start an elasticsearch docker container:
 
 ```bash
-$ docker run --rm --name elastic-test -p 9200:9200 pelias/elasticsearch
+$ docker run --rm --name elastic-test -p 9200:9200 pelias/elasticsearch:5.6.12
 ```
 
 ### Continuous Integration

diff --git a/integration/address_matching.js b/integration/address_matching.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
     // index some docs

diff --git a/integration/admin_matching.js b/integration/admin_matching.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
     // index a document with all admin values

diff --git a/integration/analyzer_peliasAdmin.js b/integration/analyzer_peliasAdmin.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.analyze = function(test, common){
   test( 'analyze', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -34,7 +34,7 @@ module.exports.tests.analyze = function(test, common){
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -72,7 +72,7 @@ module.exports.tests.functional = function(test, common){
 module.exports.tests.synonyms = function(test, common){
   test( 'synonyms', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -99,7 +99,7 @@ module.exports.tests.synonyms = function(test, common){
 module.exports.tests.tokenizer = function(test, common){
   test( 'tokenizer', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -125,7 +125,7 @@ module.exports.tests.tokenizer = function(test, common){
 module.exports.tests.unicode = function(test, common){
   test( 'normalization', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 

diff --git a/integration/analyzer_peliasHousenumber.js b/integration/analyzer_peliasHousenumber.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.analyze = function(test, common){
   test( 'analyze', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasHousenumber' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -24,7 +24,7 @@ module.exports.tests.analyze = function(test, common){
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasHousenumber' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 

diff --git a/integration/analyzer_peliasIndexOneEdgeGram.js b/integration/analyzer_peliasIndexOneEdgeGram.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.analyze = function(test, common){
   test( 'analyze', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -65,7 +65,7 @@ module.exports.tests.analyze = function(test, common){
 module.exports.tests.address_suffix_expansions = function(test, common){
   test( 'address suffix expansions', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -93,7 +93,7 @@ module.exports.tests.address_suffix_expansions = function(test, common){
 module.exports.tests.stop_words = function(test, common){
   test( 'stop words', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -112,7 +112,7 @@ module.exports.tests.stop_words = function(test, common){
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -135,7 +135,7 @@ module.exports.tests.functional = function(test, common){
 module.exports.tests.address = function(test, common){
   test( 'address', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -159,7 +159,7 @@ module.exports.tests.address = function(test, common){
 module.exports.tests.unicode = function(test, common){
   test( 'normalization', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasIndexOneEdgeGram' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 

diff --git a/integration/analyzer_peliasPhrase.js b/integration/analyzer_peliasPhrase.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.analyze = function(test, common){
   test( 'analyze', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -54,7 +54,7 @@ module.exports.tests.analyze = function(test, common){
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -97,7 +97,7 @@ module.exports.tests.functional = function(test, common){
 module.exports.tests.tokenizer = function(test, common){
   test( 'tokenizer', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -124,7 +124,7 @@ module.exports.tests.tokenizer = function(test, common){
 module.exports.tests.slop = function(test, common){
   test( 'slop', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -142,7 +142,7 @@ module.exports.tests.slop = function(test, common){
 module.exports.tests.slop_query = function(test, common){
   test( 'slop query', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -179,31 +179,27 @@ module.exports.tests.slop_query = function(test, common){
     function buildQuery( i ){
       return {
         "query": {
-          "filtered": {
-            "query": {
-              "bool": {
-                "must": [
-                  {
-                    "match": {
-                      "name.default": {
-                        "query": i
-                      }
-                    }
+          "bool": {
+            "must": [
+              {
+                "match": {
+                  "name.default": {
+                    "query": i
                   }
-                ],
-                "should": [
-                  {
-                    "match": {
-                      "phrase.default": {
-                        "query": i,
-                        "type": "phrase",
-                        "slop": 2
-                      }
-                    }
+                }
+              }
+            ],
+            "should": [
+              {
+                "match": {
+                  "phrase.default": {
+                    "query": i,
+                    "type": "phrase",
+                    "slop": 2
                   }
-                ]
+                }
               }
-            }
+            ]
           }
         }
       };
@@ -213,17 +209,18 @@ module.exports.tests.slop_query = function(test, common){
       suite.client.search({
         index: suite.props.index,
         type: 'mytype',
+        searchType: 'dfs_query_then_fetch',
         body: buildQuery('Lake Cayuga')
       }, function( err, res ){
         t.equal( res.hits.total, 3 );
         var hits = res.hits.hits;
 
         t.equal( hits[0]._source.name.default, 'Lake Cayuga' );
-        t.equal( hits[1]._source.name.default, 'Cayuga Lake' );
-        t.equal( hits[2]._source.name.default, '7991 Lake Cayuga Dr' );
+        t.equal( hits[1]._source.name.default, '7991 Lake Cayuga Dr' );
+        t.equal( hits[2]._source.name.default, 'Cayuga Lake' );
 
-        t.true( hits[0]._score > hits[1]._score );
-        t.true( hits[1]._score > hits[2]._score );
+        t.true( hits[0]._score >= hits[1]._score );
+        t.true( hits[1]._score >= hits[2]._score );
         done();
       });
     });
@@ -236,7 +233,7 @@ module.exports.tests.slop_query = function(test, common){
 module.exports.tests.slop = function(test, common){
   test( 'slop', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
     // index a document
@@ -257,6 +254,7 @@ module.exports.tests.slop = function(test, common){
       suite.client.search({
         index: suite.props.index,
         type: 'test',
+        searchType: 'dfs_query_then_fetch',
         body: { query: { match: {
           'name.default': {
             'analyzer': 'peliasPhrase',
@@ -280,7 +278,7 @@ module.exports.tests.slop = function(test, common){
 module.exports.tests.unicode = function(test, common){
   test( 'normalization', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasPhrase' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 

diff --git a/integration/analyzer_peliasQueryFullToken.js b/integration/analyzer_peliasQueryFullToken.js
@@ -10,7 +10,7 @@ module.exports.tests = {};
 module.exports.tests.analyze = function(test, common){
   test( 'analyze', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -54,7 +54,7 @@ module.exports.tests.analyze = function(test, common){
 module.exports.tests.address_suffix_expansions = function(test, common){
   test( 'address suffix expansions', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -74,7 +74,7 @@ module.exports.tests.address_suffix_expansions = function(test, common){
 module.exports.tests.stop_words = function(test, common){
   test( 'stop words', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -89,7 +89,7 @@ module.exports.tests.stop_words = function(test, common){
 module.exports.tests.functional = function(test, common){
   test( 'functional', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -112,7 +112,7 @@ module.exports.tests.functional = function(test, common){
 module.exports.tests.tokenizer = function(test, common){
   test( 'tokenizer', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -135,7 +135,7 @@ module.exports.tests.tokenizer = function(test, common){
 module.exports.tests.slop = function(test, common){
   test( 'slop', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
     // index a document
@@ -178,7 +178,7 @@ module.exports.tests.slop = function(test, common){
 module.exports.tests.address = function(test, common){
   test( 'address', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
@@ -202,7 +202,7 @@ module.exports.tests.address = function(test, common){
 module.exports.tests.unicode = function(test, common){
   test( 'normalization', function(t){
 
-    var suite = new elastictest.Suite( null, { schema: schema } );
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
     var assertAnalysis = analyze.bind( null, suite, t, 'peliasQueryFullToken' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up