diff --git a/build.gradle b/build.gradle index d7e042ddfe..d06ff4d120 100644 --- a/build.gradle +++ b/build.gradle @@ -302,6 +302,7 @@ task release(type: Copy, group: 'build') { //****************************************************************************/ dependencies { api "org.opensearch:opensearch:${opensearch_version}" + api project(":remote-index-build-client") compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}" api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1' api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre' @@ -321,9 +322,6 @@ dependencies { api "net.java.dev.jna:jna-platform:5.13.0" // OpenSearch core is using slf4j 1.7.36. Therefore, we cannot change the version here. implementation 'org.slf4j:slf4j-api:1.7.36' - api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" - api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" - api "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" } diff --git a/remote-index-build-client/LICENSE.txt b/remote-index-build-client/LICENSE.txt new file mode 100644 index 0000000000..62364a94c1 --- /dev/null +++ b/remote-index-build-client/LICENSE.txt @@ -0,0 +1,404 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--------------- +OpenSearch k-NN plugin includes the following third-party software/licensing: + +** faiss; version 1.7.6 -- +MIT License + +Copyright (c) Facebook, Inc. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +** OpenBLAS; version 0.3.3, 0.3.21 +Copyright (c) 2011-2014, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Cerberus + +ISC License + +Copyright (c) 2012-2016 Nicola Iarocci. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +------ +PyYAML + + +Copyright (c) 2017-2021 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------ + +Copyright (c) 2005-2021, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------ + +** libwinpthread; version 12.2.0 -- https://www.mingw-w64.org/ +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/remote-index-build-client/NOTICE.txt b/remote-index-build-client/NOTICE.txt new file mode 100644 index 0000000000..6653fac09c --- /dev/null +++ b/remote-index-build-client/NOTICE.txt @@ -0,0 +1,2 @@ +OpenSearch k-NN +Copyright OpenSearch Contributors \ No newline at end of file diff --git a/remote-index-build-client/build.gradle b/remote-index-build-client/build.gradle new file mode 100644 index 0000000000..4b01312ae5 --- /dev/null +++ b/remote-index-build-client/build.gradle @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java' + id 'java-library' + id 'jacoco' + id "io.freefair.lombok" + id 'com.diffplug.spotless' version '6.25.0' + id 'opensearch.build' +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +compileJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} +compileTestJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} + +dependencies { + compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + api group: 'commons-lang', name: 'commons-lang', version: '2.6' + api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" + api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}" + api "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}" + testImplementation "org.opensearch.test:framework:${opensearch_version}" +} + +check.dependsOn spotlessCheck + +publishing { + publications { + pluginZip(MavenPublication) { publication -> + pom { + name = "opensearch-knn-remote-index-build-client" + description = 'Client for connecting to remote index build endpoint' + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/k-NN/remote-index-build-client" + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexClient.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClient.java similarity index 71% rename from src/main/java/org/opensearch/knn/index/remote/RemoteIndexClient.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClient.java index dfdea6bf7b..913061ddfa 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexClient.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClient.java @@ -3,7 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.client; + +import org.opensearch.remoteindexbuild.model.RemoteBuildRequest; +import org.opensearch.remoteindexbuild.model.RemoteBuildResponse; +import org.opensearch.remoteindexbuild.model.RemoteStatusResponse; import java.io.IOException; @@ -12,6 +16,9 @@ */ public interface RemoteIndexClient { + String BUILD_ENDPOINT = "/_build"; + String STATUS_ENDPOINT = "/_status"; + /** * Submit a build to the Remote Vector Build Service. * @return RemoteBuildResponse from the server diff --git a/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClientFactory.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClientFactory.java new file mode 100644 index 0000000000..22bc319f90 --- /dev/null +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexClientFactory.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.remoteindexbuild.client; + +public class RemoteIndexClientFactory { + + // Default to HTTP client + public static RemoteIndexClient getRemoteIndexClient(final String endpoint) { + return new RemoteIndexHTTPClient(endpoint); + } +} diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClient.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClient.java similarity index 82% rename from src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClient.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClient.java index 42f19aec85..f4eba2375e 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClient.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClient.java @@ -3,10 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.client; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang.StringUtils; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -16,7 +17,6 @@ import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.entity.StringEntity; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.settings.SecureString; @@ -24,10 +24,9 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.codec.nativeindex.remote.RemoteIndexBuildStrategy; -import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.remoteindexbuild.model.RemoteBuildRequest; +import org.opensearch.remoteindexbuild.model.RemoteBuildResponse; +import org.opensearch.remoteindexbuild.model.RemoteStatusResponse; import java.io.Closeable; import java.io.IOException; @@ -37,9 +36,6 @@ import java.security.PrivilegedExceptionAction; import static org.apache.hc.core5.http.HttpStatus.SC_OK; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_CLIENT_PASSWORD_SETTING; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_CLIENT_USERNAME_SETTING; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.BUILD_ENDPOINT; /** * Class to handle all interactions with the remote vector build service. @@ -71,11 +67,9 @@ protected static CloseableHttpClient getHttpClient() { /** * Creates the client, setting the endpoint per-instance so the same endpoint is used per-build operation - * (per call to {@link RemoteIndexBuildStrategy#buildAndWriteIndex(BuildIndexParams)}) */ - public RemoteIndexHTTPClient() { - String endpoint = KNNSettings.getRemoteBuildServiceEndpoint(); - if (endpoint == null || endpoint.isEmpty()) { + public RemoteIndexHTTPClient(final String endpoint) { + if (StringUtils.isEmpty(endpoint)) { throw new IllegalArgumentException("No endpoint set for RemoteIndexClient"); } this.endpoint = endpoint; @@ -146,15 +140,12 @@ private String toJson(ToXContentObject object) throws IOException { } } - /** - * Set the global auth header to use the refreshed secure settings. - * Called by {@link KNNPlugin#reload(Settings)} when the nodes reload API is called. - * @param settings Settings to use to get the credentials - */ - public static void reloadAuthHeader(Settings settings) { - SecureString username = KNN_REMOTE_BUILD_CLIENT_USERNAME_SETTING.get(settings); - SecureString password = KNN_REMOTE_BUILD_CLIENT_PASSWORD_SETTING.get(settings); - + // /** + // * Set the global auth header to use the refreshed secure settings. + // * Called by {@link KNNPlugin#reload(Settings)} when the nodes reload API is called. + // * @param settings Settings to use to get the credentials + // */ + public static void reloadAuthHeader(SecureString username, SecureString password) { if (password != null && !password.isEmpty()) { if (username == null || username.isEmpty()) { throw new IllegalArgumentException("Username must be set if password is set"); diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientRetryStrategy.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClientRetryStrategy.java similarity index 98% rename from src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientRetryStrategy.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClientRetryStrategy.java index 2e1d93a431..905aa8b881 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientRetryStrategy.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/client/RemoteIndexHTTPClientRetryStrategy.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.client; import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; import org.apache.hc.core5.http.ConnectionClosedException; diff --git a/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/constants/KNNRemoteConstants.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/constants/KNNRemoteConstants.java new file mode 100644 index 0000000000..0a572280df --- /dev/null +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/constants/KNNRemoteConstants.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.remoteindexbuild.constants; + +// Public class to define the constants used by Remote Index Build in 2 or more classes. +public class KNNRemoteConstants { + // Build request keys + public static final String ALGORITHM = "algorithm"; + public static final String ALGORITHM_PARAMETERS = "algorithm_parameters"; + public static final String METHOD_PARAMETER_SPACE_TYPE = "space_type"; + public static final String METHOD_PARAMETER_EF_SEARCH = "ef_search"; + public static final String METHOD_PARAMETER_EF_CONSTRUCTION = "ef_construction"; + public static final String METHOD_PARAMETER_M = "m"; +} diff --git a/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildRequest.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildRequest.java new file mode 100644 index 0000000000..9532f2ba78 --- /dev/null +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildRequest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.remoteindexbuild.model; + +import lombok.Builder; +import lombok.Value; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Request object for sending build requests to the remote build service, encapsulating all the required parameters + * in a generic XContent format. + */ +@Value +@Builder +public class RemoteBuildRequest implements ToXContentObject { + private static final String DOC_COUNT = "doc_count"; + private static final String TENANT_ID = "tenant_id"; + private static final String DOC_ID_PATH = "doc_id_path"; + private static final String DIMENSION = "dimension"; + private static final String VECTOR_DATA_TYPE_FIELD = "data_type"; + private static final String KNN_ENGINE = "engine"; + + private static final String VECTOR_PATH = "vector_path"; + private static final String CONTAINER_NAME = "container_name"; + private static final String REPOSITORY_TYPE = "repository_type"; + private static final String INDEX_PARAMETERS = "index_parameters"; + + String repositoryType; + String containerName; + String vectorPath; + String docIdPath; + String tenantId; + int dimension; + int docCount; + String vectorDataType; + String engine; + RemoteIndexParameters indexParameters; + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(REPOSITORY_TYPE, repositoryType); + builder.field(CONTAINER_NAME, containerName); + builder.field(VECTOR_PATH, vectorPath); + builder.field(DOC_ID_PATH, docIdPath); + builder.field(TENANT_ID, tenantId); + builder.field(DIMENSION, dimension); + builder.field(DOC_COUNT, docCount); + builder.field(VECTOR_DATA_TYPE_FIELD, vectorDataType); + builder.field(KNN_ENGINE, engine); + builder.field(INDEX_PARAMETERS, indexParameters); + builder.endObject(); + return builder; + } +} diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteBuildResponse.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponse.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/remote/RemoteBuildResponse.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponse.java index 88bcdce3ad..4c08a28dd4 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteBuildResponse.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponse.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import lombok.Builder; import lombok.Value; diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParameters.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParameters.java similarity index 52% rename from src/main/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParameters.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParameters.java index aca33f298e..e6cb3f0c22 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParameters.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParameters.java @@ -3,28 +3,27 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import lombok.experimental.SuperBuilder; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.knn.common.KNNConstants; +import org.opensearch.remoteindexbuild.constants.KNNRemoteConstants; import java.io.IOException; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.ALGORITHM_PARAMETERS; - @SuperBuilder public class RemoteFaissHNSWIndexParameters extends RemoteIndexParameters { + int m; int efConstruction; int efSearch; @Override void addAlgorithmParameters(XContentBuilder builder) throws IOException { - builder.startObject(ALGORITHM_PARAMETERS); - builder.field(KNNConstants.METHOD_PARAMETER_M, m); - builder.field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); - builder.field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearch); + builder.startObject(KNNRemoteConstants.ALGORITHM_PARAMETERS); + builder.field(KNNRemoteConstants.METHOD_PARAMETER_M, m); + builder.field(KNNRemoteConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); + builder.field(KNNRemoteConstants.METHOD_PARAMETER_EF_SEARCH, efSearch); builder.endObject(); } } diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexParameters.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteIndexParameters.java similarity index 70% rename from src/main/java/org/opensearch/knn/index/remote/RemoteIndexParameters.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteIndexParameters.java index 1b7831f3c1..c97b70b98b 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexParameters.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteIndexParameters.java @@ -3,27 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import lombok.experimental.SuperBuilder; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.remoteindexbuild.constants.KNNRemoteConstants; import java.io.IOException; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.ALGORITHM; - @SuperBuilder public abstract class RemoteIndexParameters implements ToXContentObject { + String spaceType; String algorithm; @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(METHOD_PARAMETER_SPACE_TYPE, spaceType); - builder.field(ALGORITHM, algorithm); + builder.field(KNNRemoteConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType); + builder.field(KNNRemoteConstants.ALGORITHM, algorithm); addAlgorithmParameters(builder); builder.endObject(); return builder; diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteStatusResponse.java b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteStatusResponse.java similarity index 83% rename from src/main/java/org/opensearch/knn/index/remote/RemoteStatusResponse.java rename to remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteStatusResponse.java index 2751eee348..9accb5578c 100644 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteStatusResponse.java +++ b/remote-index-build-client/src/main/java/org/opensearch/remoteindexbuild/model/RemoteStatusResponse.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/TestConstants.java b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/TestConstants.java new file mode 100644 index 0000000000..42c12c49cb --- /dev/null +++ b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/TestConstants.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.remoteindexbuild; + +public class TestConstants { + public static final String TEST_BUCKET = "test-bucket"; + public static final String TEST_CLUSTER = "test-cluster"; + public static final String MOCK_JOB_ID_RESPONSE = "{\"job_id\": \"job-1739930402\"}"; + public static final String MOCK_JOB_ID = "job-1739930402"; + public static final String MOCK_BLOB_NAME = "blob"; + public static final String MOCK_ENDPOINT = "https://mock-build-service.com"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + public static final String L2_SPACE_TYPE = "l2"; + public static final String HNSW_ALGORITHM = "hnsw"; +} diff --git a/src/test/java/org/opensearch/knn/index/remote/RemoteBuildResponseTests.java b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponseTests.java similarity index 75% rename from src/test/java/org/opensearch/knn/index/remote/RemoteBuildResponseTests.java rename to remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponseTests.java index 42306e7bb5..ca52e41ec7 100644 --- a/src/test/java/org/opensearch/knn/index/remote/RemoteBuildResponseTests.java +++ b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteBuildResponseTests.java @@ -3,31 +3,29 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.knn.KNNTestCase; +import org.opensearch.remoteindexbuild.TestConstants; +import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; -import static org.opensearch.knn.index.remote.RemoteIndexHTTPClientTests.MOCK_JOB_ID; -import static org.opensearch.knn.index.remote.RemoteIndexHTTPClientTests.MOCK_JOB_ID_RESPONSE; - -public class RemoteBuildResponseTests extends KNNTestCase { +public class RemoteBuildResponseTests extends OpenSearchTestCase { public void testRemoteBuildResponseParsing() throws IOException { try ( XContentParser parser = JsonXContent.jsonXContent.createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - MOCK_JOB_ID_RESPONSE + TestConstants.MOCK_JOB_ID_RESPONSE ) ) { RemoteBuildResponse response = RemoteBuildResponse.fromXContent(parser); assertNotNull(response); - assertEquals(MOCK_JOB_ID, response.getJobId()); + assertEquals(TestConstants.MOCK_JOB_ID, response.getJobId()); } } diff --git a/src/test/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParametersTests.java b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParametersTests.java similarity index 64% rename from src/test/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParametersTests.java rename to remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParametersTests.java index 45bcdabadb..cf58d8a064 100644 --- a/src/test/java/org/opensearch/knn/index/remote/RemoteFaissHNSWIndexParametersTests.java +++ b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteFaissHNSWIndexParametersTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; @@ -11,8 +11,9 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.common.KNNConstants; +import org.opensearch.remoteindexbuild.TestConstants; +import org.opensearch.remoteindexbuild.constants.KNNRemoteConstants; +import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.Map; @@ -21,17 +22,14 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.index.SpaceType.L2; @SuppressWarnings("unchecked") -public class RemoteFaissHNSWIndexParametersTests extends KNNTestCase { +public class RemoteFaissHNSWIndexParametersTests extends OpenSearchTestCase { public void testToXContent() throws IOException { RemoteFaissHNSWIndexParameters params = spy( RemoteFaissHNSWIndexParameters.builder() - .spaceType(L2.getValue()) - .algorithm(METHOD_HNSW) + .spaceType(TestConstants.L2_SPACE_TYPE) + .algorithm(TestConstants.HNSW_ALGORITHM) .m(16) .efConstruction(88) .efSearch(99) @@ -50,13 +48,13 @@ public void testToXContent() throws IOException { ) { Map map = parser.map(); - assertEquals(L2.getValue(), map.get(METHOD_PARAMETER_SPACE_TYPE)); - assertEquals(METHOD_HNSW, map.get(KNNRemoteConstants.ALGORITHM)); + assertEquals(TestConstants.L2_SPACE_TYPE, map.get(KNNRemoteConstants.METHOD_PARAMETER_SPACE_TYPE)); + assertEquals(TestConstants.HNSW_ALGORITHM, map.get(KNNRemoteConstants.ALGORITHM)); Map algorithmParams = (Map) map.get(KNNRemoteConstants.ALGORITHM_PARAMETERS); - assertEquals(16, algorithmParams.get(KNNConstants.METHOD_PARAMETER_M)); - assertEquals(88, algorithmParams.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)); - assertEquals(99, algorithmParams.get(KNNConstants.METHOD_PARAMETER_EF_SEARCH)); + assertEquals(16, algorithmParams.get(KNNRemoteConstants.METHOD_PARAMETER_M)); + assertEquals(88, algorithmParams.get(KNNRemoteConstants.METHOD_PARAMETER_EF_CONSTRUCTION)); + assertEquals(99, algorithmParams.get(KNNRemoteConstants.METHOD_PARAMETER_EF_SEARCH)); } verify(params).addAlgorithmParameters(any(XContentBuilder.class)); diff --git a/src/test/java/org/opensearch/knn/index/remote/RemoteIndexParametersTests.java b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteIndexParametersTests.java similarity index 70% rename from src/test/java/org/opensearch/knn/index/remote/RemoteIndexParametersTests.java rename to remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteIndexParametersTests.java index 49f1213cb8..0b21e80853 100644 --- a/src/test/java/org/opensearch/knn/index/remote/RemoteIndexParametersTests.java +++ b/remote-index-build-client/src/test/java/org/opensearch/remoteindexbuild/model/RemoteIndexParametersTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.remote; +package org.opensearch.remoteindexbuild.model; import lombok.experimental.SuperBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -12,7 +12,9 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.knn.KNNTestCase; +import org.opensearch.remoteindexbuild.TestConstants; +import org.opensearch.remoteindexbuild.constants.KNNRemoteConstants; +import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.Map; @@ -21,13 +23,8 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.index.SpaceType.L2; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.ALGORITHM; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.ALGORITHM_PARAMETERS; -public class RemoteIndexParametersTests extends KNNTestCase { +public class RemoteIndexParametersTests extends OpenSearchTestCase { @SuperBuilder private static class TestRemoteIndexParameters extends RemoteIndexParameters { @@ -40,7 +37,7 @@ protected TestRemoteIndexParameters(RemoteIndexParametersBuilder b) { @Override void addAlgorithmParameters(XContentBuilder builder) throws IOException { - builder.startObject(ALGORITHM_PARAMETERS); + builder.startObject(KNNRemoteConstants.ALGORITHM_PARAMETERS); builder.field(TEST_PARAM, TEST_VALUE); builder.endObject(); } @@ -48,7 +45,9 @@ void addAlgorithmParameters(XContentBuilder builder) throws IOException { @SuppressWarnings("unchecked") public void testToXContent() throws IOException { - TestRemoteIndexParameters params = spy(TestRemoteIndexParameters.builder().spaceType(L2.getValue()).algorithm(METHOD_HNSW).build()); + TestRemoteIndexParameters params = spy( + TestRemoteIndexParameters.builder().spaceType(TestConstants.L2_SPACE_TYPE).algorithm(TestConstants.HNSW_ALGORITHM).build() + ); XContentBuilder builder = XContentFactory.jsonBuilder(); params.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -62,10 +61,10 @@ public void testToXContent() throws IOException { ) { Map map = parser.map(); - assertEquals(L2.getValue(), map.get(METHOD_PARAMETER_SPACE_TYPE)); - assertEquals(METHOD_HNSW, map.get(ALGORITHM)); + assertEquals(TestConstants.L2_SPACE_TYPE, map.get(KNNRemoteConstants.METHOD_PARAMETER_SPACE_TYPE)); + assertEquals(TestConstants.HNSW_ALGORITHM, map.get(KNNRemoteConstants.ALGORITHM)); - Map algorithmParams = (Map) map.get(ALGORITHM_PARAMETERS); + Map algorithmParams = (Map) map.get(KNNRemoteConstants.ALGORITHM_PARAMETERS); assertEquals(TestRemoteIndexParameters.TEST_VALUE, algorithmParams.get(TestRemoteIndexParameters.TEST_PARAM)); } verify(params).addAlgorithmParameters(any(XContentBuilder.class)); diff --git a/settings.gradle b/settings.gradle index 9056e382e0..db0f78ffd3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,4 +8,5 @@ rootProject.name = 'opensearch-knn' include ":qa" include ":qa:rolling-upgrade" include ":qa:restart-upgrade" +include ':remote-index-build-client' diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 14e95887c7..0992c1f603 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -165,4 +165,13 @@ public class KNNConstants { public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_KEY = "knn-derived-source-enabled"; public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_TRUE_VALUE = "true"; public static final String DERIVED_VECTOR_FIELD_ATTRIBUTE_FALSE_VALUE = "false"; + + // Repository filepath constants + public static final String VECTOR_BLOB_FILE_EXTENSION = ".knnvec"; + public static final String DOC_ID_FILE_EXTENSION = ".knndid"; + public static final String VECTORS_PATH = "_vectors"; + + // Repository-S3 + public static final String S3 = "s3"; + public static final String BUCKET = "bucket"; } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessor.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessor.java index 33098042ca..ea7a08492d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessor.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessor.java @@ -33,10 +33,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import static org.opensearch.knn.common.KNNConstants.VECTOR_BLOB_FILE_EXTENSION; +import static org.opensearch.knn.common.KNNConstants.VECTORS_PATH; +import static org.opensearch.knn.common.KNNConstants.DOC_ID_FILE_EXTENSION; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.initializeVectorValues; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.DOC_ID_FILE_EXTENSION; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.VECTORS_PATH; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.VECTOR_BLOB_FILE_EXTENSION; @Log4j2 @AllArgsConstructor diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/RemoteIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/RemoteIndexBuildStrategy.java index 3e7fc101fd..290501b738 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/RemoteIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/remote/RemoteIndexBuildStrategy.java @@ -6,6 +6,8 @@ package org.opensearch.knn.index.codec.nativeindex.remote; import lombok.extern.log4j.Log4j2; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.StopWatch; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.ExperimentalApi; @@ -13,11 +15,13 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.nativeindex.NativeIndexBuildStrategy; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.remote.RemoteBuildRequest; -import org.opensearch.knn.index.remote.RemoteBuildResponse; -import org.opensearch.knn.index.remote.RemoteIndexClient; -import org.opensearch.knn.index.remote.RemoteIndexClientFactory; -import org.opensearch.knn.index.remote.RemoteStatusResponse; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.remoteindexbuild.client.RemoteIndexClientFactory; +import org.opensearch.remoteindexbuild.client.RemoteIndexClient; +import org.opensearch.remoteindexbuild.model.RemoteBuildRequest; +import org.opensearch.remoteindexbuild.model.RemoteBuildResponse; +import org.opensearch.remoteindexbuild.model.RemoteStatusResponse; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryMissingException; @@ -26,6 +30,10 @@ import java.io.IOException; import java.util.function.Supplier; +import static org.opensearch.knn.common.KNNConstants.BUCKET; +import static org.opensearch.knn.common.KNNConstants.DOC_ID_FILE_EXTENSION; +import static org.opensearch.knn.common.KNNConstants.S3; +import static org.opensearch.knn.common.KNNConstants.VECTOR_BLOB_FILE_EXTENSION; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX_REMOTE_VECTOR_BUILD_SETTING; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX_REMOTE_VECTOR_BUILD_THRESHOLD_SETTING; import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_VECTOR_REPO_SETTING; @@ -125,15 +133,15 @@ public void buildAndWriteIndex(BuildIndexParams indexInfo) throws IOException { time_in_millis = stopWatch.stop().totalTime().millis(); log.debug("Repository write took {} ms for vector field [{}]", time_in_millis, indexInfo.getFieldName()); - RemoteIndexClient client = RemoteIndexClientFactory.getRemoteIndexClient(); - RemoteBuildRequest request = new RemoteBuildRequest(indexSettings, indexInfo, getRepository().getMetadata(), blobName); + final RemoteIndexClient client = RemoteIndexClientFactory.getRemoteIndexClient(KNNSettings.getRemoteBuildServiceEndpoint()); + final RemoteBuildRequest request = buildRemoteBuildRequest(indexSettings, indexInfo, getRepository().getMetadata(), blobName); stopWatch = new StopWatch().start(); - RemoteBuildResponse remoteBuildResponse = client.submitVectorBuild(request); + final RemoteBuildResponse remoteBuildResponse = client.submitVectorBuild(request); time_in_millis = stopWatch.stop().totalTime().millis(); log.debug("Submit vector build took {} ms for vector field [{}]", time_in_millis, indexInfo.getFieldName()); stopWatch = new StopWatch().start(); - RemoteStatusResponse remoteStatusResponse = client.awaitVectorBuild(remoteBuildResponse); + final RemoteStatusResponse remoteStatusResponse = client.awaitVectorBuild(remoteBuildResponse); time_in_millis = stopWatch.stop().totalTime().millis(); log.debug("Await vector build took {} ms for vector field [{}]", time_in_millis, indexInfo.getFieldName()); @@ -152,7 +160,7 @@ public void buildAndWriteIndex(BuildIndexParams indexInfo) throws IOException { * Gets the KNN repository container from the repository service. * * @return {@link RepositoriesService} - * @throws RepositoryMissingException if repository is not registered or if {@link KNN_REMOTE_VECTOR_REPO_SETTING} is not set + * @throws RepositoryMissingException if repository is not registered or if {@link KNNSettings#KNN_REMOTE_VECTOR_REPO_SETTING} is not set */ private BlobStoreRepository getRepository() throws RepositoryMissingException { RepositoriesService repositoriesService = repositoriesServiceSupplier.get(); @@ -165,4 +173,38 @@ private BlobStoreRepository getRepository() throws RepositoryMissingException { assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; return (BlobStoreRepository) repository; } + + private RemoteBuildRequest buildRemoteBuildRequest( + final IndexSettings indexSettings, + final BuildIndexParams indexInfo, + final RepositoryMetadata repositoryMetadata, + final String blobName + ) throws IOException { + String repositoryType = repositoryMetadata.type(); + String containerName; + switch (repositoryType) { + case S3 -> containerName = repositoryMetadata.settings().get(BUCKET); + default -> throw new IllegalArgumentException( + "Repository type " + repositoryType + " is not supported by the remote build service" + ); + } + String vectorDataType = indexInfo.getVectorDataType().getValue(); + + KNNVectorValues vectorValues = indexInfo.getKnnVectorValuesSupplier().get(); + KNNCodecUtil.initializeVectorValues(vectorValues); + assert (vectorValues.dimension() > 0); + + return RemoteBuildRequest.builder() + .repositoryType(repositoryType) + .containerName(containerName) + .vectorPath(blobName + VECTOR_BLOB_FILE_EXTENSION) + .docIdPath(blobName + DOC_ID_FILE_EXTENSION) + .tenantId(indexSettings.getSettings().get(ClusterName.CLUSTER_NAME_SETTING.getKey())) + .dimension(vectorValues.dimension()) + .docCount(indexInfo.getTotalLiveDocs()) + .vectorDataType(vectorDataType) + .engine(indexInfo.getKnnEngine().getName()) + .indexParameters(indexInfo.getKnnEngine().createRemoteIndexingParameters(indexInfo.getParameters())) + .build(); + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index c239fb9f3f..eceef98ad9 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -12,7 +12,7 @@ import org.opensearch.knn.index.engine.faiss.Faiss; import org.opensearch.knn.index.engine.lucene.Lucene; import org.opensearch.knn.index.engine.nmslib.Nmslib; -import org.opensearch.knn.index.remote.RemoteIndexParameters; +import org.opensearch.remoteindexbuild.model.RemoteIndexParameters; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index 50fac4897f..b32739fbb4 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -8,7 +8,7 @@ import org.opensearch.Version; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.remote.RemoteIndexParameters; +import org.opensearch.remoteindexbuild.model.RemoteIndexParameters; import java.util.Collections; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java index eb43ec6620..2c687f90d7 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java @@ -14,7 +14,7 @@ import org.opensearch.knn.index.engine.MethodResolver; import org.opensearch.knn.index.engine.NativeLibrary; import org.opensearch.knn.index.engine.ResolvedMethodContext; -import org.opensearch.knn.index.remote.RemoteIndexParameters; +import org.opensearch.remoteindexbuild.model.RemoteIndexParameters; import java.util.Map; import java.util.function.Function; diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index d105ca2568..e940dde1a6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -19,8 +19,8 @@ import org.opensearch.knn.index.engine.Parameter; import org.opensearch.knn.index.engine.TrainingConfigValidationInput; import org.opensearch.knn.index.engine.TrainingConfigValidationOutput; -import org.opensearch.knn.index.remote.RemoteFaissHNSWIndexParameters; -import org.opensearch.knn.index.remote.RemoteIndexParameters; +import org.opensearch.remoteindexbuild.model.RemoteFaissHNSWIndexParameters; +import org.opensearch.remoteindexbuild.model.RemoteIndexParameters; import java.util.Arrays; import java.util.Collections; diff --git a/src/main/java/org/opensearch/knn/index/remote/KNNRemoteConstants.java b/src/main/java/org/opensearch/knn/index/remote/KNNRemoteConstants.java deleted file mode 100644 index 3a00b6e6ff..0000000000 --- a/src/main/java/org/opensearch/knn/index/remote/KNNRemoteConstants.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.remote; - -// Public class to define the constants used by Remote Index Build in 2 or more classes. -public class KNNRemoteConstants { - // Repository filepath constants - public static final String VECTOR_BLOB_FILE_EXTENSION = ".knnvec"; - public static final String DOC_ID_FILE_EXTENSION = ".knndid"; - public static final String VECTORS_PATH = "_vectors"; - - // Repository-S3 - public static final String S3 = "s3"; - public static final String BUCKET = "bucket"; - - // Build request keys - public static final String ALGORITHM = "algorithm"; - public static final String ALGORITHM_PARAMETERS = "algorithm_parameters"; - public static final String INDEX_PARAMETERS = "index_parameters"; - - // HTTP implementation - public static final String BUILD_ENDPOINT = "/_build"; - public static final String STATUS_ENDPOINT = "/_status"; -} diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteBuildRequest.java b/src/main/java/org/opensearch/knn/index/remote/RemoteBuildRequest.java deleted file mode 100644 index d28e3dfe56..0000000000 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteBuildRequest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.remote; - -import lombok.Getter; -import org.opensearch.cluster.ClusterName; -import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.IndexSettings; -import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.codec.util.KNNCodecUtil; -import org.opensearch.knn.index.vectorvalues.KNNVectorValues; - -import java.io.IOException; - -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.BUCKET; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.DOC_ID_FILE_EXTENSION; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.INDEX_PARAMETERS; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.S3; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.VECTOR_BLOB_FILE_EXTENSION; - -/** - * Request object for sending build requests to the remote build service, encapsulating all the required parameters - * in a generic XContent format. - */ -@Getter -public class RemoteBuildRequest implements ToXContentObject { - private final String DOC_COUNT = "doc_count"; - private final String TENANT_ID = "tenant_id"; - private final String DOC_ID_PATH = "doc_id_path"; - public final String VECTOR_PATH = "vector_path"; - public final String CONTAINER_NAME = "container_name"; - public final String REPOSITORY_TYPE = "repository_type"; - - protected String repositoryType; - protected String containerName; - protected String vectorPath; - protected String docIdPath; - protected String tenantId; - protected int dimension; - protected int docCount; - protected String vectorDataType; - protected String engine; - protected RemoteIndexParameters indexParameters; - - /** - * Constructor for RemoteBuildRequest. - * - * @param indexSettings IndexSettings object - * @param indexInfo BuildIndexParams object - * @param repositoryMetadata RepositoryMetadata object - * @param blobName Name of the blob - * @throws IOException if an I/O error occurs - */ - public RemoteBuildRequest( - IndexSettings indexSettings, - BuildIndexParams indexInfo, - RepositoryMetadata repositoryMetadata, - String blobName - ) throws IOException { - String repositoryType = repositoryMetadata.type(); - String containerName; - switch (repositoryType) { - case S3 -> containerName = repositoryMetadata.settings().get(BUCKET); - default -> throw new IllegalArgumentException( - "Repository type " + repositoryType + " is not supported by the remote build service" - ); - } - String vectorDataType = indexInfo.getVectorDataType().getValue(); - - KNNVectorValues vectorValues = indexInfo.getKnnVectorValuesSupplier().get(); - KNNCodecUtil.initializeVectorValues(vectorValues); - assert (vectorValues.dimension() > 0); - - this.repositoryType = repositoryType; - this.containerName = containerName; - this.vectorPath = blobName + VECTOR_BLOB_FILE_EXTENSION; - this.docIdPath = blobName + DOC_ID_FILE_EXTENSION; - this.tenantId = indexSettings.getSettings().get(ClusterName.CLUSTER_NAME_SETTING.getKey()); - this.dimension = vectorValues.dimension(); - this.docCount = indexInfo.getTotalLiveDocs(); - this.vectorDataType = vectorDataType; - this.engine = indexInfo.getKnnEngine().getName(); - this.indexParameters = indexInfo.getKnnEngine().createRemoteIndexingParameters(indexInfo.getParameters()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(REPOSITORY_TYPE, repositoryType); - builder.field(CONTAINER_NAME, containerName); - builder.field(VECTOR_PATH, vectorPath); - builder.field(DOC_ID_PATH, docIdPath); - builder.field(TENANT_ID, tenantId); - builder.field(DIMENSION, dimension); - builder.field(DOC_COUNT, docCount); - builder.field(VECTOR_DATA_TYPE_FIELD, vectorDataType); - builder.field(KNN_ENGINE, engine); - builder.field(INDEX_PARAMETERS, indexParameters); - builder.endObject(); - return builder; - } -} diff --git a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexClientFactory.java b/src/main/java/org/opensearch/knn/index/remote/RemoteIndexClientFactory.java deleted file mode 100644 index ddf4a23984..0000000000 --- a/src/main/java/org/opensearch/knn/index/remote/RemoteIndexClientFactory.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.remote; - -public class RemoteIndexClientFactory { - - // Default to HTTP client - public static RemoteIndexClient getRemoteIndexClient() { - return new RemoteIndexHTTPClient(); - } -} diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 6aab16b734..0dc43b0102 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -21,6 +21,7 @@ import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; @@ -41,7 +42,6 @@ import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; -import org.opensearch.knn.index.remote.RemoteIndexHTTPClient; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; @@ -95,6 +95,7 @@ import org.opensearch.plugins.ScriptPlugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.plugins.SystemIndexPlugin; +import org.opensearch.remoteindexbuild.client.RemoteIndexHTTPClient; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; @@ -393,7 +394,9 @@ public void onNodeStarted(DiscoveryNode localNode) { @Override public void reload(Settings settings) { if (KNNFeatureFlags.isKNNRemoteVectorBuildEnabled()) { - RemoteIndexHTTPClient.reloadAuthHeader(settings); + SecureString username = KNNSettings.KNN_REMOTE_BUILD_CLIENT_USERNAME_SETTING.get(settings); + SecureString password = KNNSettings.KNN_REMOTE_BUILD_CLIENT_PASSWORD_SETTING.get(settings); + RemoteIndexHTTPClient.reloadAuthHeader(username, password); } } } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessorTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessorTests.java index ddb78f0efa..e6ffa48ad0 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessorTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/remote/DefaultVectorRepositoryAccessorTests.java @@ -36,8 +36,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.initializeVectorValues; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.DOC_ID_FILE_EXTENSION; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.VECTOR_BLOB_FILE_EXTENSION; +import static org.opensearch.knn.common.KNNConstants.DOC_ID_FILE_EXTENSION; +import static org.opensearch.knn.common.KNNConstants.VECTOR_BLOB_FILE_EXTENSION; public class DefaultVectorRepositoryAccessorTests extends RemoteIndexBuildTests { diff --git a/src/test/java/org/opensearch/knn/index/remote/RemoteBuildRequestTests.java b/src/test/java/org/opensearch/knn/index/remote/RemoteBuildRequestTests.java deleted file mode 100644 index 5db4fc164f..0000000000 --- a/src/test/java/org/opensearch/knn/index/remote/RemoteBuildRequestTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.remote; - -import org.junit.Before; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexSettings; -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.test.OpenSearchSingleNodeTestCase; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; -import static org.opensearch.knn.index.VectorDataType.FLOAT; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.DOC_ID_FILE_EXTENSION; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.S3; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.VECTOR_BLOB_FILE_EXTENSION; -import static org.opensearch.knn.index.remote.RemoteIndexHTTPClientTests.MOCK_BLOB_NAME; -import static org.opensearch.knn.index.remote.RemoteIndexHTTPClientTests.TEST_BUCKET; -import static org.opensearch.knn.index.remote.RemoteIndexHTTPClientTests.TEST_CLUSTER; - -public class RemoteBuildRequestTests extends OpenSearchSingleNodeTestCase { - @Mock - protected static ClusterService clusterService; - - protected AutoCloseable openMocks; - - @Before - public void setup() { - openMocks = MockitoAnnotations.openMocks(this); - clusterService = mock(ClusterService.class); - Set> defaultClusterSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - KNNSettings.state().setClusterService(clusterService); - when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, defaultClusterSettings)); - } - - /** - * Test the construction of the build request by comparing it to an explicitly created JSON object. - */ - public void testBuildRequest() { - RepositoryMetadata metadata = RemoteIndexHTTPClientTests.createTestRepositoryMetadata(); - KNNSettings knnSettingsMock = mock(KNNSettings.class); - IndexSettings mockIndexSettings = RemoteIndexHTTPClientTests.createTestIndexSettings(); - - try (MockedStatic knnSettingsStaticMock = Mockito.mockStatic(KNNSettings.class)) { - knnSettingsStaticMock.when(KNNSettings::state).thenReturn(knnSettingsMock); - KNNSettings.state().setClusterService(clusterService); - - BuildIndexParams indexInfo = RemoteIndexHTTPClientTests.createTestBuildIndexParams(); - - RemoteBuildRequest request = new RemoteBuildRequest(mockIndexSettings, indexInfo, metadata, MOCK_BLOB_NAME); - - assertEquals(S3, request.getRepositoryType()); - assertEquals(TEST_BUCKET, request.getContainerName()); - assertEquals(FAISS_NAME, request.getEngine()); - assertEquals(FLOAT.getValue(), request.getVectorDataType()); - assertEquals(MOCK_BLOB_NAME + VECTOR_BLOB_FILE_EXTENSION, request.getVectorPath()); - assertEquals(MOCK_BLOB_NAME + DOC_ID_FILE_EXTENSION, request.getDocIdPath()); - assertEquals(TEST_CLUSTER, request.getTenantId()); - assertEquals(2, request.getDocCount()); - assertEquals(2, request.getDimension()); - - String expectedJson = "{" - + "\"repository_type\":\"s3\"," - + "\"container_name\":\"test-bucket\"," - + "\"vector_path\":\"blob.knnvec\"," - + "\"doc_id_path\":\"blob.knndid\"," - + "\"tenant_id\":\"test-cluster\"," - + "\"dimension\":2," - + "\"doc_count\":2," - + "\"data_type\":\"float\"," - + "\"engine\":\"faiss\"," - + "\"index_parameters\":{" - + "\"space_type\":\"l2\"," - + "\"algorithm\":\"hnsw\"," - + "\"algorithm_parameters\":{" - + "\"ef_construction\":94," - + "\"ef_search\":89," - + "\"m\":14" - + "}" - + "}" - + "}"; - XContentParser expectedParser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - expectedJson - ); - Map expectedMap = expectedParser.map(); - - String jsonRequest; - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - request.toXContent(builder, ToXContentObject.EMPTY_PARAMS); - jsonRequest = builder.toString(); - } - - XContentParser generatedParser = JsonXContent.jsonXContent.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - jsonRequest - ); - Map generatedMap = generatedParser.map(); - - assertEquals(expectedMap, generatedMap); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/test/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientTests.java b/src/test/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientTests.java deleted file mode 100644 index 7a5cb8d533..0000000000 --- a/src/test/java/org/opensearch/knn/index/remote/RemoteIndexHTTPClientTests.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.knn.index.remote; - -import org.apache.commons.codec.binary.Base64; -import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.core5.http.Header; -import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.http.ProtocolException; -import org.apache.hc.core5.http.io.HttpClientResponseHandler; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.opensearch.cluster.ClusterName; -import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.MockSecureSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.IndexSettings; -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.vectorvalues.KNNVectorValues; -import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; -import org.opensearch.knn.index.vectorvalues.TestVectorValues; -import org.opensearch.test.OpenSearchSingleNodeTestCase; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; -import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_CLIENT_PASSWORD_SETTING; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_CLIENT_USERNAME_SETTING; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_SERVICE_ENDPOINT; -import static org.opensearch.knn.index.KNNSettings.KNN_REMOTE_BUILD_SERVICE_ENDPOINT_SETTING; -import static org.opensearch.knn.index.SpaceType.L2; -import static org.opensearch.knn.index.VectorDataType.FLOAT; -import static org.opensearch.knn.index.engine.faiss.FaissHNSWMethod.createRemoteIndexingParameters; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.BUCKET; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.BUILD_ENDPOINT; -import static org.opensearch.knn.index.remote.KNNRemoteConstants.S3; - -@SuppressWarnings("resource") -public class RemoteIndexHTTPClientTests extends OpenSearchSingleNodeTestCase { - public static final String TEST_BUCKET = "test-bucket"; - public static final String TEST_CLUSTER = "test-cluster"; - public static final String MOCK_JOB_ID_RESPONSE = "{\"job_id\": \"job-1739930402\"}"; - public static final String MOCK_JOB_ID = "job-1739930402"; - public static final String MOCK_BLOB_NAME = "blob"; - public static final String MOCK_ENDPOINT = "https://mock-build-service.com"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - - @Mock - protected static ClusterService clusterService; - - protected AutoCloseable openMocks; - - @Before - public void setup() { - openMocks = MockitoAnnotations.openMocks(this); - clusterService = mock(ClusterService.class); - Set> defaultClusterSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - KNNSettings.state().setClusterService(clusterService); - when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, defaultClusterSettings)); - } - - public void testGetAndCloseHttpclient_success() throws IOException { - setupTestClusterSettings(); - RemoteIndexHTTPClient client = new RemoteIndexHTTPClient(); - assertNotNull(client); - client.close(); - } - - public void testCreateRemoteIndexingParameters_Success() { - BuildIndexParams params = createTestBuildIndexParams(); - RemoteIndexParameters result = createRemoteIndexingParameters(params.getParameters()); - - assertNotNull(result); - assertTrue(result instanceof RemoteFaissHNSWIndexParameters); - - RemoteFaissHNSWIndexParameters hnswParams = (RemoteFaissHNSWIndexParameters) result; - assertEquals(METHOD_HNSW, hnswParams.algorithm); - assertEquals("l2", hnswParams.spaceType); - assertEquals(94, hnswParams.efConstruction); - assertEquals(89, hnswParams.efSearch); - assertEquals(14, hnswParams.m); - } - - public void testSubmitVectorBuild() throws IOException, URISyntaxException { - RepositoryMetadata metadata = createTestRepositoryMetadata(); - KNNSettings knnSettingsMock = mock(KNNSettings.class); - IndexSettings mockIndexSettings = createTestIndexSettings(); - setupTestClusterSettings(); - - CloseableHttpClient mockHttpClient = mock(CloseableHttpClient.class); - try (MockedStatic clientStaticMock = Mockito.mockStatic(RemoteIndexHTTPClient.class)) { - clientStaticMock.when(RemoteIndexHTTPClient::getHttpClient).thenReturn(mockHttpClient); - - try (MockedStatic knnSettingsStaticMock = Mockito.mockStatic(KNNSettings.class)) { - knnSettingsStaticMock.when(KNNSettings::state).thenReturn(knnSettingsMock); - when(KNNSettings.getRemoteBuildServiceEndpoint()).thenReturn(MOCK_ENDPOINT); - KNNSettings.state().setClusterService(clusterService); - - BuildIndexParams buildIndexParams = createTestBuildIndexParams(); - - when(mockHttpClient.execute(any(HttpPost.class), any(HttpClientResponseHandler.class))).thenAnswer( - response -> MOCK_JOB_ID_RESPONSE - ); - - RemoteIndexHTTPClient client = new RemoteIndexHTTPClient(); - - RemoteBuildResponse remoteBuildResponse = client.submitVectorBuild( - new RemoteBuildRequest(mockIndexSettings, buildIndexParams, metadata, MOCK_BLOB_NAME) - ); - assertEquals(MOCK_JOB_ID, remoteBuildResponse.getJobId()); - - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(HttpPost.class); - verify(mockHttpClient).execute(requestCaptor.capture(), any(HttpClientResponseHandler.class)); - HttpPost capturedRequest = requestCaptor.getValue(); - assertEquals(MOCK_ENDPOINT + BUILD_ENDPOINT, capturedRequest.getUri().toString()); - assertFalse(capturedRequest.containsHeader(HttpHeaders.AUTHORIZATION)); - } - } - } - - public void testSecureSettingsReloadAndException() throws IOException { - final MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(KNN_REMOTE_BUILD_CLIENT_USERNAME_SETTING.getKey(), USERNAME); - secureSettings.setString(KNN_REMOTE_BUILD_CLIENT_PASSWORD_SETTING.getKey(), PASSWORD); - final Settings settings = Settings.builder() - .setSecureSettings(secureSettings) - .put(KNN_REMOTE_BUILD_SERVICE_ENDPOINT, MOCK_ENDPOINT) - .build(); - - CloseableHttpClient mockHttpClient = mock(CloseableHttpClient.class); - try (MockedStatic clientStaticMock = Mockito.mockStatic(RemoteIndexHTTPClient.class)) { - clientStaticMock.when(RemoteIndexHTTPClient::getHttpClient).thenReturn(mockHttpClient); - - try (MockedStatic knnSettingsStaticMock = Mockito.mockStatic(KNNSettings.class)) { - KNNSettings knnSettingsMock = mock(KNNSettings.class); - knnSettingsStaticMock.when(KNNSettings::state).thenReturn(knnSettingsMock); - when(KNNSettings.getRemoteBuildServiceEndpoint()).thenReturn(MOCK_ENDPOINT); - KNNSettings.state().setClusterService(clusterService); - - when(mockHttpClient.execute(any(HttpPost.class), any(HttpClientResponseHandler.class))).thenAnswer( - response -> MOCK_JOB_ID_RESPONSE - ); - - RepositoryMetadata metadata = createTestRepositoryMetadata(); - IndexSettings mockIndexSettings = createTestIndexSettings(); - setupTestClusterSettings(); - BuildIndexParams buildIndexParams = createTestBuildIndexParams(); - - clientStaticMock.when(() -> RemoteIndexHTTPClient.reloadAuthHeader(any(Settings.class))).thenCallRealMethod(); - - RemoteIndexHTTPClient client = new RemoteIndexHTTPClient(); - RemoteIndexHTTPClient.reloadAuthHeader(settings); - - ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(HttpPost.class); - client.submitVectorBuild(new RemoteBuildRequest(mockIndexSettings, buildIndexParams, metadata, MOCK_BLOB_NAME)); - - verify(mockHttpClient).execute(requestCaptor.capture(), any(HttpClientResponseHandler.class)); - HttpPost capturedRequest = requestCaptor.getValue(); - Header authHeader = capturedRequest.getHeader(HttpHeaders.AUTHORIZATION); - assertNotNull("Auth header should be set", authHeader); - assertEquals( - "Basic " + Base64.encodeBase64String((USERNAME + ":" + PASSWORD).getBytes(StandardCharsets.ISO_8859_1)), - authHeader.getValue() - ); - } catch (ProtocolException e) { - throw new RuntimeException(e); - } - } - clearAuthHeader(); - } - - // Utility methods to populate settings for build requests - - static BuildIndexParams createTestBuildIndexParams() { - List vectorValues = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }); - final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( - vectorValues - ); - final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(FLOAT, randomVectorValues); - - Map encoderParams = new HashMap<>(); - encoderParams.put(NAME, ENCODER_FLAT); - encoderParams.put(PARAMETERS, Map.of()); - - Map algorithmParams = new HashMap<>(); - algorithmParams.put(METHOD_PARAMETER_EF_SEARCH, 89); - algorithmParams.put(METHOD_PARAMETER_EF_CONSTRUCTION, 94); - algorithmParams.put(ENCODER_FLAT, encoderParams); - - Map parameters = new HashMap<>(); - parameters.put(NAME, METHOD_HNSW); - parameters.put(VECTOR_DATA_TYPE_FIELD, FLOAT.getValue()); - parameters.put(INDEX_DESCRIPTION_PARAMETER, "HNSW14,Flat"); - parameters.put(SPACE_TYPE, L2.getValue()); - parameters.put(PARAMETERS, algorithmParams); - - return BuildIndexParams.builder() - .knnEngine(KNNEngine.FAISS) - .vectorDataType(FLOAT) - .parameters(parameters) - .knnVectorValuesSupplier(() -> knnVectorValues) - .totalLiveDocs(vectorValues.size()) - .build(); - } - - static RepositoryMetadata createTestRepositoryMetadata() { - RepositoryMetadata metadata = mock(RepositoryMetadata.class); - Settings repoSettings = Settings.builder().put(BUCKET, TEST_BUCKET).build(); - when(metadata.type()).thenReturn(S3); - when(metadata.settings()).thenReturn(repoSettings); - return metadata; - } - - static IndexSettings createTestIndexSettings() { - IndexSettings mockIndexSettings = mock(IndexSettings.class); - Settings indexSettingsSettings = Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), TEST_CLUSTER).build(); - when(mockIndexSettings.getSettings()).thenReturn(indexSettingsSettings); - return mockIndexSettings; - } - - static void setupTestClusterSettings() { - Settings settings = Settings.builder().put(KNN_REMOTE_BUILD_SERVICE_ENDPOINT_SETTING.getKey(), MOCK_ENDPOINT).build(); - Set> settingsSet = new HashSet<>(); - settingsSet.add(KNN_REMOTE_BUILD_SERVICE_ENDPOINT_SETTING); - ClusterSettings clusterSettings = new ClusterSettings(settings, settingsSet); - doReturn(clusterSettings).when(clusterService).getClusterSettings(); - doReturn(settings).when(clusterService).getSettings(); - KNNSettings.state().setClusterService(clusterService); - } - - void clearAuthHeader() { - final MockSecureSettings secureSettings = new MockSecureSettings(); - final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - RemoteIndexHTTPClient.reloadAuthHeader(settings); - } -}