From 804a76ba426cc6b8b44c5d9f3694ae4a0281e90d Mon Sep 17 00:00:00 2001 From: Jack Mazanec Date: Thu, 14 Jan 2021 16:04:01 -0800 Subject: [PATCH] Backport nmslib 2.0.11 upgrade to 1.12 (#305) * Upgrade nmslib to v2.0.8 in JNI library (#291) * Upgrade nmslib to 2.0.11 (#302) --- .gitignore | 2 +- README.md | 2 +- jni/CMakeLists.txt | 6 ++--- jni/external/nmslib | 2 +- ...relasticsearch_knn_index_v2011_KNNIndex.h} | 26 +++++++++---------- ...lasticsearch_knn_index_v2011_KNNIndex.cpp} | 14 +++++----- .../knn/index/KNNIndexCache.java | 2 +- .../knn/index/KNNIndexShard.java | 2 +- .../knn/index/KNNWeight.java | 4 +-- .../KNN80Codec/KNN80DocValuesConsumer.java | 2 +- .../knn/index/util/NmsLibVersion.java | 8 +++--- .../knn/index/{v206 => v2011}/KNNIndex.java | 14 +++++----- .../plugin-metadata/plugin-security.policy | 2 +- .../knn/index/KNNIndexCacheTests.java | 2 +- .../knn/index/KNNJNITests.java | 20 +++++++------- .../knn/index/codec/KNNCodecTestCase.java | 4 +-- 16 files changed, 56 insertions(+), 56 deletions(-) rename jni/include/{com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.h => com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.h} (84%) rename jni/src/{com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.cpp => com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.cpp} (92%) rename src/main/java/com/amazon/opendistroforelasticsearch/knn/index/{v206 => v2011}/KNNIndex.java (93%) diff --git a/.gitignore b/.gitignore index 6f6860f2..05ad1026 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ build *.log out/ buildSrc/es-knn-offline-repo-* -buildSrc/libKNNIndexV2_0_6* +buildSrc/libKNNIndex* oss/* *.iml jni/CMakeCache.txt diff --git a/README.md b/README.md index d9887dec..1b3f749f 100644 --- a/README.md +++ b/README.md @@ -458,7 +458,7 @@ Second, it should first be confirmed that all of the graphs of interest are able ## Scoring During k-NN search, for each graph, NMSLIB will return up to `k` results. These results contain both the [document ID and the NMSLIB score](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNQueryResult.java#L21). -The score NMSLIB assigns to a result is related to the space type that is selected. For example, for cosine similarity, NMSLIB will return [`1 - normScalarProduct`](https://github.com/nmslib/nmslib/blob/master/similarity_search/src/method/hnsw_distfunc_opt.cc#L372). For euclidean distance, in almost all cases, it will return the euclidean distance between [the result and the query vector](https://github.com/nmslib/nmslib/blob/master/similarity_search/src/method/hnsw_distfunc_opt.cc#L131). However, when the dimension of the vector is divisible by 16 (i.e. `dimension % 16 == 0`), the score returned will actually be the [square of the euclidean distance](https://github.com/nmslib/nmslib/blob/master/similarity_search/src/method/hnsw_distfunc_opt.cc#L50). +The score NMSLIB assigns to a result is related to the space type that is selected. For example, for cosine similarity, NMSLIB will return [`1 - normScalarProduct`](https://github.com/nmslib/nmslib/blob/master/similarity_search/src/method/hnsw_distfunc_opt.cc#L372). For l2, the score returned will be the square of the euclidean distance. From the k-NN and NMSLIB perspective, a lower score equates to a closer and better result. This is the opposite of how Elasticsearch scores results, where a greater score equates to a better result. In order to convert from the NMSLIB score to the Elasticsearch score, we perform [the following conversion](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNWeight.java#L113): ``` diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index f54ce767..ed43eb7b 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -15,7 +15,7 @@ cmake_minimum_required(VERSION 2.8) -project(KNNIndexV2_0_6) +project(KNNIndexV2_0_11) # Corner case. For CMake 2.8, there is no option to specify set(CMAKE_CXX_STANDARD 11). Instead, the flag manually needs # to be set. @@ -27,7 +27,7 @@ else() endif() # Target Library to be built -set(KNN_INDEX KNNIndexV2_0_6) +set(KNN_INDEX KNNIndexV2_0_11) set(KNN_PACKAGE_NAME opendistro-knnlib) # Check if similarity search exists @@ -55,7 +55,7 @@ else() endif() # Compile the library -add_library(${KNN_INDEX} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.cpp) +add_library(${KNN_INDEX} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.cpp) target_link_libraries(${KNN_INDEX} NonMetricSpaceLib) target_include_directories(${KNN_INDEX} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include) diff --git a/jni/external/nmslib b/jni/external/nmslib index 5482e077..a2d6624e 160000 --- a/jni/external/nmslib +++ b/jni/external/nmslib @@ -1 +1 @@ -Subproject commit 5482e077d1c8637499f86231bcd3979cb7fa6aef +Subproject commit a2d6624e1315402662025debfdd614b505d9c3ef diff --git a/jni/include/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.h b/jni/include/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.h similarity index 84% rename from jni/include/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.h rename to jni/include/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.h index 43c88216..41766c6b 100644 --- a/jni/include/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.h +++ b/jni/include/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.h @@ -1,50 +1,50 @@ /* DO NOT EDIT THIS FILE - it is machine generated */ #include -/* Header for class com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex */ +/* Header for class com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex */ -#ifndef _Included_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex -#define _Included_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex +#ifndef _Included_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex +#define _Included_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex #ifdef __cplusplus extern "C" { #endif /* - * Class: com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex + * Class: com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex * Method: saveIndex * Signature: ([I[[FLjava/lang/String;[Ljava/lang/String;Ljava/lang/String;)V */ -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_saveIndex +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_saveIndex (JNIEnv *, jclass, jintArray, jobjectArray, jstring, jobjectArray, jstring); /* - * Class: com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex + * Class: com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex * Method: queryIndex * Signature: (J[FI)[Lcom/amazon/opendistroforelasticsearch/knn/index/KNNQueryResult; */ -JNIEXPORT jobjectArray JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_queryIndex +JNIEXPORT jobjectArray JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint); /* - * Class: com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex + * Class: com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex * Method: init * Signature: (Ljava/lang/String;[Ljava/lang/String;Ljava/lang/String;)J */ -JNIEXPORT jlong JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_init +JNIEXPORT jlong JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_init (JNIEnv *, jclass, jstring, jobjectArray, jstring); /* - * Class: com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex + * Class: com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex * Method: gc * Signature: (J)V */ -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_gc +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_gc (JNIEnv *, jclass, jlong); /* - * Class: com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex + * Class: com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex * Method: initLibrary * Signature: ()V */ -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_initLibrary +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_initLibrary (JNIEnv *, jclass); #ifdef __cplusplus diff --git a/jni/src/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.cpp b/jni/src/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.cpp similarity index 92% rename from jni/src/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.cpp rename to jni/src/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.cpp index 1494d0ea..886e164c 100644 --- a/jni/src/com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.cpp +++ b/jni/src/com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -#include "com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex.h" +#include "com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex.h" #include "init.h" #include "index.h" @@ -85,7 +85,7 @@ void catch_cpp_exception_and_throw_java(JNIEnv* env) } } -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_saveIndex(JNIEnv* env, jclass cls, jintArray ids, jobjectArray vectors, jstring indexPath, jobjectArray algoParams, jstring spaceType) +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_saveIndex(JNIEnv* env, jclass cls, jintArray ids, jobjectArray vectors, jstring indexPath, jobjectArray algoParams, jstring spaceType) { Space* space = NULL; ObjectVector dataset; @@ -144,7 +144,7 @@ JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206 } } -JNIEXPORT jobjectArray JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_queryIndex(JNIEnv* env, jclass cls, jlong indexPointer, jfloatArray queryVector, jint k) +JNIEXPORT jobjectArray JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_queryIndex(JNIEnv* env, jclass cls, jlong indexPointer, jfloatArray queryVector, jint k) { try { IndexWrapper *indexWrapper = reinterpret_cast(indexPointer); @@ -175,7 +175,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_amazon_opendistroforelasticsearch_knn_in return NULL; } -JNIEXPORT jlong JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_init(JNIEnv* env, jclass cls, jstring indexPath, jobjectArray algoParams, jstring spaceType) +JNIEXPORT jlong JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_init(JNIEnv* env, jclass cls, jstring indexPath, jobjectArray algoParams, jstring spaceType) { IndexWrapper *indexWrapper = NULL; try { @@ -215,7 +215,7 @@ JNIEXPORT jlong JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v20 return NULL; } -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_gc(JNIEnv* env, jclass cls, jlong indexPointer) +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_gc(JNIEnv* env, jclass cls, jlong indexPointer) { try { IndexWrapper *indexWrapper = reinterpret_cast(indexPointer); @@ -228,7 +228,7 @@ JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206 } } -JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v206_KNNIndex_initLibrary(JNIEnv *, jclass) +JNIEXPORT void JNICALL Java_com_amazon_opendistroforelasticsearch_knn_index_v2011_KNNIndex_initLibrary(JNIEnv *, jclass) { initLibrary(); diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCache.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCache.java index 961af6bc..cbb3ceeb 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCache.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCache.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.knn.index; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import com.amazon.opendistroforelasticsearch.knn.plugin.stats.StatNames; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexShard.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexShard.java index 22e347be..74c84e1e 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexShard.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexShard.java @@ -16,7 +16,7 @@ package com.amazon.opendistroforelasticsearch.knn.index; import com.amazon.opendistroforelasticsearch.knn.index.codec.KNNCodecUtil; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.FilterLeafReader; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNWeight.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNWeight.java index 862ae877..ff5851bd 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNWeight.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/KNNWeight.java @@ -16,7 +16,7 @@ package com.amazon.opendistroforelasticsearch.knn.index; import com.amazon.opendistroforelasticsearch.knn.index.codec.KNNCodecUtil; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.FieldInfo; @@ -92,7 +92,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { /** * TODO Add logic to pick up the right nmslib version based on the version - * in the name of the file. As of now we have one version 2.0.6 + * in the name of the file. As of now we have one version 2.0.11 * So deferring this to future releases */ diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index ee527520..ec86f4a3 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -37,7 +37,7 @@ import com.amazon.opendistroforelasticsearch.knn.index.KNNVectorFieldMapper; import com.amazon.opendistroforelasticsearch.knn.index.util.KNNConstants; import com.amazon.opendistroforelasticsearch.knn.index.util.NmsLibVersion; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import java.io.Closeable; import java.io.IOException; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/util/NmsLibVersion.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/util/NmsLibVersion.java index 1ce2749c..3be01ff9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/util/NmsLibVersion.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/util/NmsLibVersion.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -20,14 +20,14 @@ public enum NmsLibVersion { /** * Latest available nmslib version */ - V206("206"){ + V2011("2011"){ @Override public String indexLibraryVersion() { - return "KNNIndexV2_0_6"; + return "KNNIndexV2_0_11"; } }; - public static final NmsLibVersion LATEST = V206; + public static final NmsLibVersion LATEST = V2011; public String buildVersion; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v206/KNNIndex.java b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v2011/KNNIndex.java similarity index 93% rename from src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v206/KNNIndex.java rename to src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v2011/KNNIndex.java index 3a791491..ede3d1f9 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v206/KNNIndex.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/knn/index/v2011/KNNIndex.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -package com.amazon.opendistroforelasticsearch.knn.index.v206; +package com.amazon.opendistroforelasticsearch.knn.index.v2011; import com.amazon.opendistroforelasticsearch.knn.index.KNNQueryResult; import com.amazon.opendistroforelasticsearch.knn.index.util.NmsLibVersion; @@ -29,16 +29,16 @@ /** * JNI layer to communicate with the nmslib - * This class refers to the nms library build with version tag 2.0.6 - * See tag2.0.6 + * This class refers to the nms library build with version tag 2.0.11 + * See tag2.0.11 */ public class KNNIndex implements AutoCloseable { - public static NmsLibVersion VERSION = NmsLibVersion.V206; + public static NmsLibVersion VERSION = NmsLibVersion.V2011; static { AccessController.doPrivileged(new PrivilegedAction() { public Void run() { - System.loadLibrary(NmsLibVersion.V206.indexLibraryVersion()); + System.loadLibrary(NmsLibVersion.LATEST.indexLibraryVersion()); return null; } }); @@ -153,6 +153,6 @@ private static long computeFileSize(String indexPath) { // Deletes memory pointed to by index pointer (needs write lock) private static native void gc(long indexPointer); - // Calls nmslib's initLibrary function: https://github.com/nmslib/nmslib/blob/v2.0.6/similarity_search/include/init.h#L27 + // Calls nmslib's initLibrary function: https://github.com/nmslib/nmslib/blob/v2.0.11/similarity_search/include/init.h#L27 private static native void initLibrary(); } diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 37e25294..af59cec1 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,3 +1,3 @@ grant { - permission java.lang.RuntimePermission "loadLibrary.KNNIndexV2_0_6"; + permission java.lang.RuntimePermission "loadLibrary.KNNIndexV2_0_11"; }; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCacheTests.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCacheTests.java index 7d0602be..8bc490ac 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCacheTests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNIndexCacheTests.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.knn.index; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import com.amazon.opendistroforelasticsearch.knn.KNNTestCase; import com.amazon.opendistroforelasticsearch.knn.plugin.KNNPlugin; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNJNITests.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNJNITests.java index 8519652f..c8fa0235 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNJNITests.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNJNITests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ package com.amazon.opendistroforelasticsearch.knn.index; import com.amazon.opendistroforelasticsearch.knn.KNNTestCase; -import com.amazon.opendistroforelasticsearch.knn.index.v206.KNNIndex; +import com.amazon.opendistroforelasticsearch.knn.index.v2011.KNNIndex; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -102,12 +102,12 @@ public Void run() { /* * scores are evaluated using Euclidean distance. Distance of the documents with * respect to query vector are as follows - * doc0 = 11.224972, doc1 = 3.7416575, doc2 = 19.131126 + * doc0 = 126, doc1 = 14, doc2 = 366 * Nearest neighbor is doc1 then doc0 then doc2 */ - assertEquals(scores.get(0), 11.224972, 0.1); - assertEquals(scores.get(1), 3.7416575, 0.1); - assertEquals(scores.get(2), 19.131126, 0.1); + assertEquals(126.0, scores.get(0), 0.001); + assertEquals(14.0, scores.get(1), 0.001); + assertEquals(366.0, scores.get(2), 0.001); dir.close(); } @@ -225,12 +225,12 @@ public Void run() { /* * scores are evaluated using Euclidean distance. Distance of the documents with * respect to query vector are as follows - * doc0 = 11.224972, doc1 = 3.7416575, doc2 = 19.131126 + * doc0 = 126, doc1 = 14, doc2 = 366 * Nearest neighbor is doc1 then doc0 then doc2 */ - assertEquals(scores.get(0), 11.224972, 0.1); - assertEquals(scores.get(1), 3.7416575, 0.1); - assertEquals(scores.get(2), 19.131126, 0.1); + assertEquals(126.0, scores.get(0), 0.001); + assertEquals(14.0, scores.get(1), 0.001); + assertEquals(366.0, scores.get(2), 0.001); dir.close(); } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNNCodecTestCase.java index 7a61cb70..8d6664fd 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/knn/index/codec/KNNCodecTestCase.java @@ -153,8 +153,8 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { IndexSearcher searcher = new IndexSearcher(reader); float score = searcher.search(new KNNQuery("test_vector", new float[] {1.0f, 0.0f, 0.0f}, 1, "dummy"), 10).scoreDocs[0].score; float score1 = searcher.search(new KNNQuery("my_vector", new float[] {1.0f, 2.0f}, 1, "dummy"), 10).scoreDocs[0].score; - assertEquals(score, 0.1667f, 0.01f); - assertEquals(score1, 0.0714f, 0.01f); + assertEquals(1.0f/(1 + 25), score, 0.01f); + assertEquals(1.0f/(1 + 169), score1, 0.01f); // query to determine the hits assertEquals(1, searcher.count(new KNNQuery("test_vector", new float[] {1.0f, 0.0f, 0.0f}, 1, "dummy")));