From f76a10b942911d1c4f93b38fcc5b666ab757fbeb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 07:34:22 -0700 Subject: [PATCH] Refactoring datasource changes to a new module. (#1504) (#1510) * Reafactor into new datasources module Signed-off-by: vamsi-amazon * Refactored all the datasource releated code to new datasources module Signed-off-by: vamsi-amazon * More unit tests Signed-off-by: vamsi-amazon --------- Signed-off-by: vamsi-amazon (cherry picked from commit 7584f798beaf017700d9237f8843ad2e5c54f327) Co-authored-by: Vamsi Manohar --- common/build.gradle | 1 - config/checkstyle/google_checks.xml | 3 +- .../datasource/model/DataSourceMetadata.java | 3 - .../sql/storage/DataSourceFactory.java | 1 - datasources/build.gradle | 82 +++ datasources/lombok.config | 3 + .../datasources}/auth/AuthenticationType.java | 2 +- .../DataSourceUserAuthorizationHelper.java | 8 +- ...DataSourceUserAuthorizationHelperImpl.java | 11 +- .../sql/datasources}/encryptor/Encryptor.java | 2 +- .../datasources}/encryptor/EncryptorImpl.java | 10 +- .../DataSourceNotFoundException.java | 2 +- .../datasources/exceptions/ErrorMessage.java | 78 ++ .../CreateDataSourceActionRequest.java | 2 +- .../CreateDataSourceActionResponse.java | 2 +- .../DeleteDataSourceActionRequest.java | 2 +- .../DeleteDataSourceActionResponse.java | 2 +- .../GetDataSourceActionRequest.java | 2 +- .../GetDataSourceActionResponse.java | 2 +- .../UpdateDataSourceActionRequest.java | 4 +- .../UpdateDataSourceActionResponse.java | 2 +- .../rest/RestDataSourceQueryAction.java | 53 +- .../service}/DataSourceLoaderCache.java | 2 +- .../service}/DataSourceLoaderCacheImpl.java | 2 +- .../service}/DataSourceMetadataStorage.java | 3 +- .../service}/DataSourceServiceImpl.java | 14 +- .../settings}/DataSourceSettings.java | 2 +- .../OpenSearchDataSourceMetadataStorage.java | 55 +- .../TransportCreateDataSourceAction.java | 34 +- .../TransportDeleteDataSourceAction.java | 28 +- .../TransportGetDataSourceAction.java | 22 +- .../TransportUpdateDataSourceAction.java | 32 +- .../sql/datasources}/utils/Scheduler.java | 2 +- .../utils/XContentParserUtils.java | 9 +- .../resources/datasources-index-mapping.yml | 0 .../resources/datasources-index-settings.yml | 0 .../auth/AuthenticationTypeTest.java | 2 +- ...SourceUserAuthorizationHelperImplTest.java | 46 +- .../encryptor/EncryptorImplTest.java | 87 +++ .../DataSourceLoaderCacheImplTest.java | 2 +- .../service}/DataSourceServiceImplTest.java | 6 +- ...enSearchDataSourceMetadataStorageTest.java | 670 ++++++++++++++++++ .../TransportCreateDataSourceActionTest.java | 86 +++ .../TransportDeleteDataSourceActionTest.java | 78 ++ .../TransportGetDataSourceActionTest.java | 137 ++++ .../TransportUpdateDataSourceActionTest.java | 87 +++ .../sql/datasources}/utils/SchedulerTest.java | 26 +- .../utils/XContentParserUtilsTest.java | 101 +++ .../org/opensearch/sql/ppl/StandaloneIT.java | 6 +- plugin/build.gradle | 4 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 34 +- .../transport/TransportPPLQueryAction.java | 2 +- ...enSearchDataSourceMetadataStorageTest.java | 259 ------- prometheus/build.gradle | 1 + .../storage/PrometheusStorageFactory.java | 2 +- settings.gradle | 1 + 56 files changed, 1625 insertions(+), 494 deletions(-) create mode 100644 datasources/build.gradle create mode 100644 datasources/lombok.config rename {core/src/main/java/org/opensearch/sql/datasource/model => datasources/src/main/java/org/opensearch/sql/datasources}/auth/AuthenticationType.java (94%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/auth}/DataSourceUserAuthorizationHelper.java (79%) rename {plugin/src/main/java/org/opensearch/sql/plugin/datasource => datasources/src/main/java/org/opensearch/sql/datasources/auth}/DataSourceUserAuthorizationHelperImpl.java (80%) rename {common/src/main/java/org/opensearch/sql/common => datasources/src/main/java/org/opensearch/sql/datasources}/encryptor/Encryptor.java (90%) rename {common/src/main/java/org/opensearch/sql/common => datasources/src/main/java/org/opensearch/sql/datasources}/encryptor/EncryptorImpl.java (87%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources}/exceptions/DataSourceNotFoundException.java (84%) create mode 100644 datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/CreateDataSourceActionRequest.java (96%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/CreateDataSourceActionResponse.java (92%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/DeleteDataSourceActionRequest.java (96%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/DeleteDataSourceActionResponse.java (92%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/GetDataSourceActionRequest.java (95%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/GetDataSourceActionResponse.java (92%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/UpdateDataSourceActionRequest.java (90%) rename {plugin/src/main/java/org/opensearch/sql/plugin/model => datasources/src/main/java/org/opensearch/sql/datasources/model/transport}/UpdateDataSourceActionResponse.java (92%) rename {plugin/src/main/java/org/opensearch/sql/plugin => datasources/src/main/java/org/opensearch/sql/datasources}/rest/RestDataSourceQueryAction.java (82%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCache.java (91%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCacheImpl.java (97%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceMetadataStorage.java (95%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceServiceImpl.java (93%) rename {plugin/src/main/java/org/opensearch/sql/plugin/datasource => datasources/src/main/java/org/opensearch/sql/datasources/settings}/DataSourceSettings.java (92%) rename {plugin/src/main/java/org/opensearch/sql/plugin/datasource => datasources/src/main/java/org/opensearch/sql/datasources/storage}/OpenSearchDataSourceMetadataStorage.java (89%) rename {plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource => datasources/src/main/java/org/opensearch/sql/datasources/transport}/TransportCreateDataSourceAction.java (62%) rename {plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource => datasources/src/main/java/org/opensearch/sql/datasources/transport}/TransportDeleteDataSourceAction.java (64%) rename {plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource => datasources/src/main/java/org/opensearch/sql/datasources/transport}/TransportGetDataSourceAction.java (83%) rename {plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource => datasources/src/main/java/org/opensearch/sql/datasources/transport}/TransportUpdateDataSourceAction.java (61%) rename {plugin/src/main/java/org/opensearch/sql/plugin => datasources/src/main/java/org/opensearch/sql/datasources}/utils/Scheduler.java (95%) rename {plugin/src/main/java/org/opensearch/sql/plugin => datasources/src/main/java/org/opensearch/sql/datasources}/utils/XContentParserUtils.java (94%) rename {plugin => datasources}/src/main/resources/datasources-index-mapping.yml (100%) rename {plugin => datasources}/src/main/resources/datasources-index-settings.yml (100%) rename {core/src/test/java/org/opensearch/sql/datasource/model => datasources/src/test/java/org/opensearch/sql/datasources}/auth/AuthenticationTypeTest.java (93%) rename {plugin/src/test/java/org/opensearch/sql/plugin/datasource => datasources/src/test/java/org/opensearch/sql/datasources/auth}/DataSourceUserAuthorizationHelperImplTest.java (61%) create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java rename {core/src/test/java/org/opensearch/sql/datasource => datasources/src/test/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCacheImplTest.java (98%) rename {core/src/test/java/org/opensearch/sql/datasource => datasources/src/test/java/org/opensearch/sql/datasources/service}/DataSourceServiceImplTest.java (98%) create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java rename {plugin/src/test/java/org/opensearch/sql/plugin => datasources/src/test/java/org/opensearch/sql/datasources}/utils/SchedulerTest.java (56%) create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java delete mode 100644 plugin/src/test/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorageTest.java diff --git a/common/build.gradle b/common/build.gradle index da6b591961..369a649cde 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -36,7 +36,6 @@ dependencies { api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' - api 'com.amazonaws:aws-encryption-sdk-java:2.4.0' testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1' diff --git a/config/checkstyle/google_checks.xml b/config/checkstyle/google_checks.xml index a0c7d90fd9..12c90f8495 100644 --- a/config/checkstyle/google_checks.xml +++ b/config/checkstyle/google_checks.xml @@ -39,8 +39,9 @@ - + + diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java index 27d06d8151..7945f8aec3 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java @@ -12,8 +12,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; -import com.google.gson.Gson; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -21,7 +19,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; import lombok.Setter; import org.opensearch.sql.datasource.DataSourceService; diff --git a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java index d0f24d0e5a..8512eddbe3 100644 --- a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java +++ b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java @@ -7,7 +7,6 @@ package org.opensearch.sql.storage; -import java.util.Map; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/datasources/build.gradle b/datasources/build.gradle new file mode 100644 index 0000000000..ef52db2305 --- /dev/null +++ b/datasources/build.gradle @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'jacoco' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation project(':core') + implementation project(':protocol') + implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" + implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" + implementation group: 'commons-io', name: 'commons-io', version: '2.8.0' + implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.0' + + testImplementation group: 'junit', name: 'junit', version: '4.13.2' + testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.12.13' + testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.2.0' + testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '5.2.0' + testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2' +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +jacocoTestReport { + reports { + html.enabled true + xml.enabled true + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +test.finalizedBy(project.tasks.jacocoTestReport) + +jacocoTestCoverageVerification { + violationRules { + rule { + element = 'CLASS' + excludes = [ + 'org.opensearch.sql.datasources.settings.DataSourceSettings', + 'org.opensearch.sql.datasources.exceptions.*', + 'org.opensearch.sql.datasources.model.*', + 'org.opensearch.sql.datasources.rest.*' + ] + limit { + counter = 'LINE' + minimum = 1.0 + } + limit { + counter = 'BRANCH' + minimum = 0.9 + } + } + } + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it) + })) + } +} +check.dependsOn jacocoTestCoverageVerification +jacocoTestCoverageVerification.dependsOn jacocoTestReport diff --git a/datasources/lombok.config b/datasources/lombok.config new file mode 100644 index 0000000000..aac13295bd --- /dev/null +++ b/datasources/lombok.config @@ -0,0 +1,3 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true \ No newline at end of file diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java similarity index 94% rename from core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java rename to datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java index 9cf3e01509..715e72c0c3 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasource.model.auth; +package org.opensearch.sql.datasources.auth; import java.util.Collections; import java.util.HashMap; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java similarity index 79% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java rename to datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java index dbbe82a527..adcfb0bdfd 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java @@ -1,6 +1,10 @@ -package org.opensearch.sql.datasource; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.datasources.auth; -import java.util.List; import org.opensearch.sql.datasource.model.DataSourceMetadata; /** diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java similarity index 80% rename from plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java index 41ad450f68..cd55991d00 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java @@ -3,16 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.datasource; +package org.opensearch.sql.datasources.auth; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import java.util.List; import lombok.AllArgsConstructor; import org.opensearch.client.Client; +import org.opensearch.commons.ConfigConstants; import org.opensearch.commons.authuser.User; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasource.model.DataSourceMetadata; @AllArgsConstructor @@ -21,13 +20,15 @@ public class DataSourceUserAuthorizationHelperImpl implements DataSourceUserAuth private Boolean isAuthorizationRequired() { String userString = client.threadPool() - .getThreadContext().getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); + .getThreadContext().getTransient( + ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); return userString != null; } private List getUserRoles() { String userString = client.threadPool() - .getThreadContext().getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); + .getThreadContext().getTransient( + ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); User user = User.parse(userString); return user.getRoles(); } diff --git a/common/src/main/java/org/opensearch/sql/common/encryptor/Encryptor.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java similarity index 90% rename from common/src/main/java/org/opensearch/sql/common/encryptor/Encryptor.java rename to datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java index a886b72328..55dc1ef18f 100644 --- a/common/src/main/java/org/opensearch/sql/common/encryptor/Encryptor.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.common.encryptor; +package org.opensearch.sql.datasources.encryptor; public interface Encryptor { diff --git a/common/src/main/java/org/opensearch/sql/common/encryptor/EncryptorImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java similarity index 87% rename from common/src/main/java/org/opensearch/sql/common/encryptor/EncryptorImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java index 05a0d358fd..4838cd41a5 100644 --- a/common/src/main/java/org/opensearch/sql/common/encryptor/EncryptorImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.common.encryptor; +package org.opensearch.sql.datasources.encryptor; import com.amazonaws.encryptionsdk.AwsCrypto; import com.amazonaws.encryptionsdk.CommitmentPolicy; @@ -29,8 +29,8 @@ public String encrypt(String plainText) { .build(); JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "", - "AES/GCM/NoPadding"); + = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", + "opensearch.config.master.key", "AES/GCM/NoPadding"); final CryptoResult encryptResult = crypto.encryptData(jceMasterKey, plainText.getBytes(StandardCharsets.UTF_8)); @@ -44,8 +44,8 @@ public String decrypt(String encryptedText) { .build(); JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "", - "AES/GCM/NoPadding"); + = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", + "opensearch.config.master.key", "AES/GCM/NoPadding"); final CryptoResult decryptedResult = crypto.decryptData(jceMasterKey, Base64.getDecoder().decode(encryptedText)); diff --git a/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java similarity index 84% rename from core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java rename to datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java index 0a068ccdfc..484b0b92b2 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasource.exceptions; +package org.opensearch.sql.datasources.exceptions; /** * DataSourceNotFoundException. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java new file mode 100644 index 0000000000..6dbd9bcfb5 --- /dev/null +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.datasources.exceptions; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import lombok.Getter; +import org.opensearch.rest.RestStatus; + +/** + * Error Message. + */ +public class ErrorMessage { + + protected Throwable exception; + + private final int status; + + @Getter + private final String type; + + @Getter + private final String reason; + + @Getter + private final String details; + + /** + * Error Message Constructor. + */ + public ErrorMessage(Throwable exception, int status) { + this.exception = exception; + this.status = status; + + this.type = fetchType(); + this.reason = fetchReason(); + this.details = fetchDetails(); + } + + private String fetchType() { + return exception.getClass().getSimpleName(); + } + + protected String fetchReason() { + return status == RestStatus.BAD_REQUEST.getStatus() + ? "Invalid Request" + : "There was internal problem at backend"; + } + + protected String fetchDetails() { + // Some exception prints internal information (full class name) which is security concern + return emptyStringIfNull(exception.getLocalizedMessage()); + } + + private String emptyStringIfNull(String str) { + return str != null ? str : ""; + } + + @Override + public String toString() { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("status", status); + jsonObject.add("error", getErrorAsJson()); + return new Gson().toJson(jsonObject); + } + + private JsonObject getErrorAsJson() { + JsonObject errorJson = new JsonObject(); + errorJson.addProperty("type", type); + errorJson.addProperty("reason", reason); + errorJson.addProperty("details", details); + return errorJson; + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java similarity index 96% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java index d6a15e3a0c..333564c10a 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java similarity index 92% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java index 1d8d9aa9b7..4531c3d9fe 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/CreateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java similarity index 96% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java index 7377f25dcb..6bcbd7a561 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java similarity index 92% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java index 3a2f136e31..c6847ed9ed 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/DeleteDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java similarity index 95% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java index 41b854888e..6cafe1972a 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java similarity index 92% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java index e419ad1e08..030493cb51 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/GetDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java similarity index 90% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java index 73140c6b29..fe66483edd 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java @@ -5,19 +5,17 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import java.io.IOException; import lombok.Getter; -import org.json.JSONObject; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; public class UpdateDataSourceActionRequest extends ActionRequest { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java similarity index 92% rename from plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java index f2ae4e6472..faa3b1139b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/model/UpdateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java similarity index 82% rename from plugin/src/main/java/org/opensearch/sql/plugin/rest/RestDataSourceQueryAction.java rename to datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java index 2468d004a9..c75170c355 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestDataSourceQueryAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.plugin.rest; +package org.opensearch.sql.datasources.rest; import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; @@ -14,7 +14,6 @@ import static org.opensearch.rest.RestStatus.BAD_REQUEST; import static org.opensearch.rest.RestStatus.NOT_FOUND; import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; -import static org.opensearch.sql.plugin.utils.Scheduler.schedule; import com.google.common.collect.ImmutableList; import java.io.IOException; @@ -29,24 +28,24 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.legacy.metrics.MetricName; -import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory; -import org.opensearch.sql.plugin.model.CreateDataSourceActionRequest; -import org.opensearch.sql.plugin.model.CreateDataSourceActionResponse; -import org.opensearch.sql.plugin.model.DeleteDataSourceActionRequest; -import org.opensearch.sql.plugin.model.DeleteDataSourceActionResponse; -import org.opensearch.sql.plugin.model.GetDataSourceActionRequest; -import org.opensearch.sql.plugin.model.GetDataSourceActionResponse; -import org.opensearch.sql.plugin.model.UpdateDataSourceActionRequest; -import org.opensearch.sql.plugin.model.UpdateDataSourceActionResponse; -import org.opensearch.sql.plugin.transport.datasource.TransportCreateDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportDeleteDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportGetDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportUpdateDataSourceAction; -import org.opensearch.sql.plugin.utils.XContentParserUtils; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasources.exceptions.ErrorMessage; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportUpdateDataSourceAction; +import org.opensearch.sql.datasources.utils.Scheduler; +import org.opensearch.sql.datasources.utils.XContentParserUtils; + public class RestDataSourceQueryAction extends BaseRestHandler { @@ -136,7 +135,7 @@ private RestChannelConsumer executePostRequest(RestRequest restRequest, DataSourceMetadata dataSourceMetadata = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); - return restChannel -> schedule(nodeClient, + return restChannel -> Scheduler.schedule(nodeClient, () -> nodeClient.execute(TransportCreateDataSourceAction.ACTION_TYPE, new CreateDataSourceActionRequest(dataSourceMetadata), new ActionListener<>() { @@ -158,7 +157,7 @@ public void onFailure(Exception e) { private RestChannelConsumer executeGetRequest(RestRequest restRequest, NodeClient nodeClient) { String dataSourceName = restRequest.param("dataSourceName"); - return restChannel -> schedule(nodeClient, + return restChannel -> Scheduler.schedule(nodeClient, () -> nodeClient.execute(TransportGetDataSourceAction.ACTION_TYPE, new GetDataSourceActionRequest(dataSourceName), new ActionListener<>() { @@ -180,7 +179,7 @@ private RestChannelConsumer executeUpdateRequest(RestRequest restRequest, NodeClient nodeClient) throws IOException { DataSourceMetadata dataSourceMetadata = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); - return restChannel -> schedule(nodeClient, + return restChannel -> Scheduler.schedule(nodeClient, () -> nodeClient.execute(TransportUpdateDataSourceAction.ACTION_TYPE, new UpdateDataSourceActionRequest(dataSourceMetadata), new ActionListener<>() { @@ -203,7 +202,7 @@ private RestChannelConsumer executeDeleteRequest(RestRequest restRequest, NodeClient nodeClient) { String dataSourceName = restRequest.param("dataSourceName"); - return restChannel -> schedule(nodeClient, + return restChannel -> Scheduler.schedule(nodeClient, () -> nodeClient.execute(TransportDeleteDataSourceAction.ACTION_TYPE, new DeleteDataSourceActionRequest(dataSourceName), new ActionListener<>() { @@ -228,14 +227,8 @@ private void handleException(Exception e, RestChannel restChannel) { } else { LOG.error("Error happened during request handling", e); if (isClientError(e)) { - Metrics.getInstance() - .getNumericalMetric(MetricName.DATASOURCE_FAILED_REQ_COUNT_CUS) - .increment(); reportError(restChannel, e, BAD_REQUEST); } else { - Metrics.getInstance() - .getNumericalMetric(MetricName.DATASOURCE_FAILED_REQ_COUNT_SYS) - .increment(); reportError(restChannel, e, SERVICE_UNAVAILABLE); } } @@ -244,7 +237,7 @@ private void handleException(Exception e, RestChannel restChannel) { private void reportError(final RestChannel channel, final Exception e, final RestStatus status) { channel.sendResponse( new BytesRestResponse( - status, ErrorMessageFactory.createErrorMessage(e, status.getStatus()).toString())); + status, new ErrorMessage(e, status.getStatus()).toString())); } private static boolean isClientError(Exception e) { diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java similarity index 91% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java index cce70fe584..3fe2954c12 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java similarity index 97% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java index 56b7bec08b..ba9520fc0c 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java similarity index 95% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java index 85ffd0a1b3..b54af3195e 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java @@ -5,11 +5,10 @@ * */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import java.util.List; import java.util.Optional; -import javax.xml.crypto.Data; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java similarity index 93% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java index bc3f2d0abf..86afa90c2b 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java @@ -3,30 +3,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import javax.xml.crypto.Data; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.storage.DataSourceFactory; /** diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java b/datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java similarity index 92% rename from plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java rename to datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java index a451ad30be..0dc18f409d 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/DataSourceSettings.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/settings/DataSourceSettings.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.datasource; +package org.opensearch.sql.datasources.settings; import java.io.InputStream; import org.opensearch.common.settings.SecureSetting; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java similarity index 89% rename from plugin/src/main/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorage.java rename to datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java index 7e2a2ad705..f76e1ba9dc 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java @@ -5,13 +5,12 @@ * */ -package org.opensearch.sql.plugin.datasource; +package org.opensearch.sql.datasources.storage; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -43,12 +42,12 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.sql.common.encryptor.Encryptor; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasource.model.auth.AuthenticationType; -import org.opensearch.sql.plugin.utils.XContentParserUtils; +import org.opensearch.sql.datasources.auth.AuthenticationType; +import org.opensearch.sql.datasources.encryptor.Encryptor; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; +import org.opensearch.sql.datasources.utils.XContentParserUtils; public class OpenSearchDataSourceMetadataStorage implements DataSourceMetadataStorage { @@ -92,6 +91,7 @@ public List getDataSourceMetadata() { public Optional getDataSourceMetadata(String datasourceName) { if (!this.clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) { createDataSourcesIndex(); + return Optional.empty(); } return searchInDataSourcesIndex(QueryBuilders.termQuery("name", datasourceName)) .stream() @@ -187,13 +187,12 @@ private void createDataSourcesIndex() { InputStream settingsFileStream = OpenSearchDataSourceMetadataStorage.class.getClassLoader() .getResourceAsStream(DATASOURCE_INDEX_SETTINGS_FILE_NAME); CreateIndexRequest createIndexRequest = new CreateIndexRequest(DATASOURCE_INDEX_NAME); - createIndexRequest - .mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), + createIndexRequest.mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), XContentType.YAML) .settings(IOUtils.toString(settingsFileStream, StandardCharsets.UTF_8), XContentType.YAML); ActionFuture createIndexResponseActionFuture; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() + try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() .stashContext()) { createIndexResponseActionFuture = client.admin().indices().create(createIndexRequest); } @@ -202,11 +201,11 @@ private void createDataSourcesIndex() { LOG.info("Index: {} creation Acknowledged", DATASOURCE_INDEX_NAME); } else { throw new RuntimeException( - String.format("Index: %s creation failed", DATASOURCE_INDEX_NAME)); + "Index creation is not acknowledged."); } } catch (Throwable e) { throw new RuntimeException( - "Internal server error while creating" + DATASOURCE_INDEX_NAME + " index" + "Internal server error while creating" + DATASOURCE_INDEX_NAME + " index:: " + e.getMessage()); } } @@ -219,7 +218,7 @@ private List searchInDataSourcesIndex(QueryBuilder query) { searchSourceBuilder.size(DATASOURCE_QUERY_RESULT_SIZE); searchRequest.source(searchSourceBuilder); ActionFuture searchResponseActionFuture; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() + try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() .stashContext()) { searchResponseActionFuture = client.search(searchRequest); } @@ -243,6 +242,7 @@ private List searchInDataSourcesIndex(QueryBuilder query) { } } + @SuppressWarnings("missingswitchdefault") private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata dataSourceMetadata, Boolean isEncryption) { Map propertiesMap = dataSourceMetadata.getProperties(); @@ -259,8 +259,6 @@ private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata d case AWSSIGV4AUTH: handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption); break; - default: - break; } } return dataSourceMetadata; @@ -268,14 +266,16 @@ private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata d private void handleBasicAuthPropertiesEncryptionDecryption(Map propertiesMap, Boolean isEncryption) { - Optional usernameKey = propertiesMap.keySet().stream() + ArrayList list = new ArrayList<>(); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.username")) - .findFirst(); - Optional passwordKey = propertiesMap.keySet().stream() + .findFirst() + .ifPresent(list::add); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.password")) - .findFirst(); - encryptOrDecrypt(propertiesMap, isEncryption, - Arrays.asList(usernameKey.get(), passwordKey.get())); + .findFirst() + .ifPresent(list::add); + encryptOrDecrypt(propertiesMap, isEncryption, list); } private void encryptOrDecrypt(Map propertiesMap, Boolean isEncryption, @@ -293,13 +293,16 @@ private void encryptOrDecrypt(Map propertiesMap, Boolean isEncry private void handleSigV4PropertiesEncryptionDecryption(Map propertiesMap, Boolean isEncryption) { - Optional accessKey = propertiesMap.keySet().stream() + ArrayList list = new ArrayList<>(); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.access_key")) - .findFirst(); - Optional secretKey = propertiesMap.keySet().stream() + .findFirst() + .ifPresent(list::add); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.secret_key")) - .findFirst(); - encryptOrDecrypt(propertiesMap, isEncryption, Arrays.asList(accessKey.get(), secretKey.get())); + .findFirst() + .ifPresent(list::add); + encryptOrDecrypt(propertiesMap, isEncryption, list); } } \ No newline at end of file diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportCreateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java similarity index 62% rename from plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportCreateDataSourceAction.java rename to datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java index 05d724ba2a..4d8c51fac7 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportCreateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java @@ -5,66 +5,56 @@ * */ -package org.opensearch.sql.plugin.transport.datasource; +package org.opensearch.sql.datasources.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.legacy.metrics.MetricName; -import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.plugin.model.CreateDataSourceActionRequest; -import org.opensearch.sql.plugin.model.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; public class TransportCreateDataSourceAction extends HandledTransportAction { - - private static final Logger LOG = LogManager.getLogger(); public static final String NAME = "cluster:admin/opensearch/ql/datasources/create"; public static final ActionType ACTION_TYPE = new ActionType<>(NAME, CreateDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportCreateDataSourceAction action for creating datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportCreateDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportCreateDataSourceAction.NAME, transportService, actionFilters, CreateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; - this.client = client; } @Override protected void doExecute(Task task, CreateDataSourceActionRequest request, ActionListener actionListener) { - - Metrics.getInstance().getNumericalMetric(MetricName.DATASOURCE_REQ_COUNT).increment(); - DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata(); - dataSourceService.createDataSource(dataSourceMetadata); - actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name " - + dataSourceMetadata.getName())); + try { + DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata(); + dataSourceService.createDataSource(dataSourceMetadata); + actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name " + + dataSourceMetadata.getName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportDeleteDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java similarity index 64% rename from plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportDeleteDataSourceAction.java rename to datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java index 9557daee4e..1d109ca7fc 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportDeleteDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java @@ -5,22 +5,17 @@ * */ -package org.opensearch.sql.plugin.transport.datasource; +package org.opensearch.sql.datasources.transport; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.legacy.metrics.MetricName; -import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.opensearch.security.SecurityAccess; -import org.opensearch.sql.plugin.model.DeleteDataSourceActionRequest; -import org.opensearch.sql.plugin.model.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -32,34 +27,33 @@ public class TransportDeleteDataSourceAction ACTION_TYPE = new ActionType<>(NAME, DeleteDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportDeleteDataSourceAction action for deleting datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportDeleteDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportDeleteDataSourceAction.NAME, transportService, actionFilters, DeleteDataSourceActionRequest::new); - this.client = client; this.dataSourceService = dataSourceService; } @Override protected void doExecute(Task task, DeleteDataSourceActionRequest request, ActionListener actionListener) { - Metrics.getInstance().getNumericalMetric(MetricName.DATASOURCE_REQ_COUNT).increment(); - dataSourceService.deleteDataSource(request.getDataSourceName()); - actionListener.onResponse(new DeleteDataSourceActionResponse("Deleted DataSource with name " - + request.getDataSourceName())); + try { + dataSourceService.deleteDataSource(request.getDataSourceName()); + actionListener.onResponse(new DeleteDataSourceActionResponse("Deleted DataSource with name " + + request.getDataSourceName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportGetDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java similarity index 83% rename from plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportGetDataSourceAction.java rename to datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java index 179535ff9f..33d08f7cd2 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportGetDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java @@ -5,23 +5,19 @@ * */ -package org.opensearch.sql.plugin.transport.datasource; - -import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +package org.opensearch.sql.datasources.transport; import java.util.Set; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.plugin.model.GetDataSourceActionRequest; -import org.opensearch.sql.plugin.model.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -34,24 +30,20 @@ public class TransportGetDataSourceAction ACTION_TYPE = new ActionType<>(NAME, GetDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportGetDataSourceAction action for getting datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportGetDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportGetDataSourceAction.NAME, transportService, actionFilters, GetDataSourceActionRequest::new); - this.client = client; this.dataSourceService = dataSourceService; } @@ -76,7 +68,8 @@ private String handleGetAllDataSourcesRequest() { String responseContent; Set dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(false); - responseContent = new JsonResponseFormatter>(PRETTY) { + responseContent = new JsonResponseFormatter>( + JsonResponseFormatter.Style.PRETTY) { @Override protected Object buildJsonObject(Set response) { return response; @@ -90,7 +83,8 @@ private String handleSingleDataSourceRequest(String datasourceName) { DataSourceMetadata dataSourceMetadata = dataSourceService .getDataSourceMetadata(datasourceName); - responseContent = new JsonResponseFormatter(PRETTY) { + responseContent = new JsonResponseFormatter( + JsonResponseFormatter.Style.PRETTY) { @Override protected Object buildJsonObject(DataSourceMetadata response) { return response; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportUpdateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java similarity index 61% rename from plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportUpdateDataSourceAction.java rename to datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java index dfc7311e77..4aece69e5b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/datasource/TransportUpdateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java @@ -5,65 +5,55 @@ * */ -package org.opensearch.sql.plugin.transport.datasource; +package org.opensearch.sql.datasources.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.legacy.metrics.MetricName; -import org.opensearch.sql.legacy.metrics.Metrics; -import org.opensearch.sql.opensearch.security.SecurityAccess; -import org.opensearch.sql.plugin.model.UpdateDataSourceActionRequest; -import org.opensearch.sql.plugin.model.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; public class TransportUpdateDataSourceAction extends HandledTransportAction { - private static final Logger LOG = LogManager.getLogger(); public static final String NAME = "cluster:admin/opensearch/ql/datasources/update"; public static final ActionType ACTION_TYPE = new ActionType<>(NAME, UpdateDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportUpdateDataSourceAction action for updating datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportUpdateDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportUpdateDataSourceAction.NAME, transportService, actionFilters, UpdateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; - this.client = client; } @Override protected void doExecute(Task task, UpdateDataSourceActionRequest request, ActionListener actionListener) { - - Metrics.getInstance().getNumericalMetric(MetricName.DATASOURCE_REQ_COUNT).increment(); - dataSourceService.updateDataSource(request.getDataSourceMetadata()); - actionListener.onResponse(new UpdateDataSourceActionResponse("Updated DataSource with name " - + request.getDataSourceMetadata().getName())); + try { + dataSourceService.updateDataSource(request.getDataSourceMetadata()); + actionListener.onResponse(new UpdateDataSourceActionResponse("Updated DataSource with name " + + request.getDataSourceMetadata().getName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/utils/Scheduler.java b/datasources/src/main/java/org/opensearch/sql/datasources/utils/Scheduler.java similarity index 95% rename from plugin/src/main/java/org/opensearch/sql/plugin/utils/Scheduler.java rename to datasources/src/main/java/org/opensearch/sql/datasources/utils/Scheduler.java index a4a87b1b12..0bc597ed4f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/utils/Scheduler.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/utils/Scheduler.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.utils; +package org.opensearch.sql.datasources.utils; import java.util.Map; import lombok.experimental.UtilityClass; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/utils/XContentParserUtils.java b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java similarity index 94% rename from plugin/src/main/java/org/opensearch/sql/plugin/utils/XContentParserUtils.java rename to datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java index cc1310ffc3..a8643a35f3 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/utils/XContentParserUtils.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.utils; +package org.opensearch.sql.datasources.utils; import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.experimental.UtilityClass; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; @@ -21,6 +22,10 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +/** + * Utitlity class to serialize and deserialize objects in XContent. + */ +@UtilityClass public class XContentParserUtils { public static final String NAME_FIELD = "name"; public static final String CONNECTOR_FIELD = "connector"; @@ -69,7 +74,7 @@ public static DataSourceMetadata toDataSourceMetadata(XContentParser parser) thr } } if (name == null || connector == null) { - throw new IllegalArgumentException("Missing required fields"); + throw new IllegalArgumentException("name and connector are required fields."); } return new DataSourceMetadata(name, connector, allowedRoles, properties); } diff --git a/plugin/src/main/resources/datasources-index-mapping.yml b/datasources/src/main/resources/datasources-index-mapping.yml similarity index 100% rename from plugin/src/main/resources/datasources-index-mapping.yml rename to datasources/src/main/resources/datasources-index-mapping.yml diff --git a/plugin/src/main/resources/datasources-index-settings.yml b/datasources/src/main/resources/datasources-index-settings.yml similarity index 100% rename from plugin/src/main/resources/datasources-index-settings.yml rename to datasources/src/main/resources/datasources-index-settings.yml diff --git a/core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java similarity index 93% rename from core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java index f9e4f3ce59..23bb4688e1 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource.model.auth; +package org.opensearch.sql.datasources.auth; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java similarity index 61% rename from plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImplTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java index fe57b06c6e..552bd0edf9 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/datasource/DataSourceUserAuthorizationHelperImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java @@ -3,25 +3,25 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.datasource; +package org.opensearch.sql.datasources.auth; -import static org.mockito.Mockito.when; import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT; import java.util.HashMap; import java.util.List; import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.Client; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class DataSourceUserAuthorizationHelperImplTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) @@ -34,8 +34,8 @@ public class DataSourceUserAuthorizationHelperImplTest { @Test public void testAuthorizeDataSourceWithAllowedRoles() { String userString = "myuser|bckrole1,bckrol2|prometheus_access|myTenant"; - when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); this.dataSourceUserAuthorizationHelper @@ -45,19 +45,41 @@ public void testAuthorizeDataSourceWithAllowedRoles() { @Test public void testAuthorizeDataSourceWithAdminRole() { String userString = "myuser|bckrole1,bckrol2|all_access|myTenant"; - when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); this.dataSourceUserAuthorizationHelper .authorizeDataSource(dataSourceMetadata); } + @Test + public void testAuthorizeDataSourceWithNullUserString() { + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .thenReturn(null); + DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); + this.dataSourceUserAuthorizationHelper + .authorizeDataSource(dataSourceMetadata); + } + + @Test + public void testAuthorizeDataSourceWithDefaultDataSource() { + String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .thenReturn(userString); + DataSourceMetadata dataSourceMetadata = + DataSourceMetadata.defaultOpenSearchDataSourceMetadata(); + this.dataSourceUserAuthorizationHelper + .authorizeDataSource(dataSourceMetadata); + } + @Test public void testAuthorizeDataSourceWithException() { String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; - when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); SecurityException securityException diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java new file mode 100644 index 0000000000..22f5b09255 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java @@ -0,0 +1,87 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.datasources.encryptor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + + +@ExtendWith(MockitoExtension.class) +public class EncryptorImplTest { + + @Test + public void testEncryptAndDecrypt() { + String masterKey = "1234567890123456"; + String input = "This is a test input"; + Encryptor encryptor = new EncryptorImpl(masterKey); + + String encrypted = encryptor.encrypt(input); + String decrypted = encryptor.decrypt(encrypted); + + assertEquals(input, decrypted); + } + + @Test + public void testMasterKeySize() { + String input = "This is a test input"; + String masterKey8 = "12345678"; + Encryptor encryptor8 = new EncryptorImpl(masterKey8); + assertThrows(AwsCryptoException.class, () -> { + encryptor8.encrypt(input); + }); + + String masterKey16 = "1234567812345678"; + Encryptor encryptor16 = new EncryptorImpl(masterKey16); + String encrypted = encryptor16.encrypt(input); + Assertions.assertEquals(input, encryptor16.decrypt(encrypted)); + + String masterKey24 = "123456781234567812345678"; + Encryptor encryptor24 = new EncryptorImpl(masterKey24); + encrypted = encryptor24.encrypt(input); + Assertions.assertEquals(input, encryptor24.decrypt(encrypted)); + + String masterKey17 = "12345678123456781"; + Encryptor encryptor17 = new EncryptorImpl(masterKey17); + assertThrows(AwsCryptoException.class, () -> { + encryptor17.encrypt(input); + }); + } + + @Test + public void testInvalidBase64String() { + String encrypted = "invalidBase64String"; + Encryptor encryptor = new EncryptorImpl("randomMasterKey"); + + assertThrows(BadCiphertextException.class, () -> { + encryptor.decrypt(encrypted); + }); + } + + @Test + public void testDecryptWithDifferentKey() { + + String masterKeyOne = "1234567890123456"; + String masterKeyTwo = "1234567890123455"; + String input = "This is a test input"; + Encryptor encryptor1 = new EncryptorImpl(masterKeyOne); + Encryptor encryptor2 = new EncryptorImpl(masterKeyTwo); + + String encrypted = encryptor1.encrypt(input); + + assertThrows(Exception.class, () -> { + encryptor2.decrypt(encrypted); + }); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java similarity index 98% rename from core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java index fae69e7feb..bf656857b0 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; diff --git a/core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java similarity index 98% rename from core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java index 98e17e9166..e1312ec582 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -34,10 +34,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.sql.storage.StorageEngine; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java new file mode 100644 index 0000000000..5a9efaba67 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java @@ -0,0 +1,670 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.datasources.storage; + +import static org.opensearch.sql.datasources.storage.OpenSearchDataSourceMetadataStorage.DATASOURCE_INDEX_NAME; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import lombok.SneakyThrows; +import org.apache.lucene.search.TotalHits; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionFuture; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.delete.DeleteResponse; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.update.UpdateResponse; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.index.engine.DocumentMissingException; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.index.shard.ShardId; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.encryptor.Encryptor; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; + +@ExtendWith(MockitoExtension.class) +public class OpenSearchDataSourceMetadataStorageTest { + + private static final String TEST_DATASOURCE_INDEX_NAME = "testDS"; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Client client; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private ClusterService clusterService; + @Mock + private Encryptor encryptor; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchResponse searchResponse; + @Mock + private ActionFuture searchResponseActionFuture; + @Mock + private ActionFuture createIndexResponseActionFuture; + @Mock + private ActionFuture indexResponseActionFuture; + @Mock + private IndexResponse indexResponse; + @Mock + private ActionFuture updateResponseActionFuture; + @Mock + private UpdateResponse updateResponse; + @Mock + private ActionFuture deleteResponseActionFuture; + @Mock + private DeleteResponse deleteResponse; + @Mock + private SearchHit searchHit; + @InjectMocks + private OpenSearchDataSourceMetadataStorage openSearchDataSourceMetadataStorage; + + + @SneakyThrows + @Test + public void testGetDataSourceMetadata() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn(getBasicDataSourceMetadataString()); + Mockito.when(encryptor.decrypt("password")).thenReturn("password"); + Mockito.when(encryptor.decrypt("username")).thenReturn("username"); + + Optional dataSourceMetadataOptional + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + + + Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); + DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + Assertions.assertEquals("password", + dataSourceMetadata.getProperties().get("prometheus.auth.password")); + Assertions.assertEquals("username", + dataSourceMetadata.getProperties().get("prometheus.auth.username")); + Assertions.assertEquals("basicauth", + dataSourceMetadata.getProperties().get("prometheus.auth.type")); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWith404SearchResponse() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.NOT_FOUND); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( + TEST_DATASOURCE_INDEX_NAME)); + Assertions.assertEquals( + "Fetching dataSource metadata information failed with status : NOT_FOUND", + runtimeException.getMessage()); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithParsingFailed() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn("..testDs"); + + Assertions.assertThrows(RuntimeException.class, + () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( + TEST_DATASOURCE_INDEX_NAME)); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithAWSSigV4() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn(getAWSSigv4DataSourceMetadataString()); + Mockito.when(encryptor.decrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.decrypt("access_key")).thenReturn("access_key"); + + Optional dataSourceMetadataOptional + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + + + Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); + DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + Assertions.assertEquals("secret_key", + dataSourceMetadata.getProperties().get("prometheus.auth.secret_key")); + Assertions.assertEquals("access_key", + dataSourceMetadata.getProperties().get("prometheus.auth.access_key")); + Assertions.assertEquals("awssigv4", + dataSourceMetadata.getProperties().get("prometheus.auth.type")); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithBasicAuth() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn(getDataSourceMetadataStringWithBasicAuthentication()); + Mockito.when(encryptor.decrypt("username")).thenReturn("username"); + Mockito.when(encryptor.decrypt("password")).thenReturn("password"); + + Optional dataSourceMetadataOptional + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + + + Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); + DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + Assertions.assertEquals("username", + dataSourceMetadata.getProperties().get("prometheus.auth.username")); + Assertions.assertEquals("password", + dataSourceMetadata.getProperties().get("prometheus.auth.password")); + Assertions.assertEquals("basicauth", + dataSourceMetadata.getProperties().get("prometheus.auth.type")); + } + + + @SneakyThrows + @Test + public void testGetDataSourceMetadataList() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn(getDataSourceMetadataStringWithNoAuthentication()); + + List dataSourceMetadataList + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); + + + Assertions.assertEquals(1, dataSourceMetadataList.size()); + DataSourceMetadata dataSourceMetadata = dataSourceMetadataList.get(0); + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + } + + + @SneakyThrows + @Test + public void testGetDataSourceMetadataListWithNoIndex() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + + List dataSourceMetadataList + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); + + Assertions.assertEquals(0, dataSourceMetadataList.size()); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithNoIndex() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + + Optional dataSourceMetadataOptional + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + + Assertions.assertFalse(dataSourceMetadataOptional.isPresent()); + } + + @Test + public void testCreateDataSourceMetadata() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + Mockito.when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata(dataSourceMetadata); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithOutCreatingIndex() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.TRUE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + Mockito.when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata(dataSourceMetadata); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + + @Test + public void testCreateDataSourceMetadataFailedWithNotFoundResponse() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + Mockito.when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithVersionConflict() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())) + .thenThrow(VersionConflictEngineException.class); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows(IllegalArgumentException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("A datasource already exists with name: testDS", + illegalArgumentException.getMessage()); + + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithException() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("error while indexing")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("java.lang.RuntimeException: error while indexing", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithIndexCreationFailed() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(false, false, DATASOURCE_INDEX_NAME)); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "Internal server error while creating.ql-datasources index:: " + + "Index creation is not acknowledged.", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + @Test + public void testUpdateDataSourceMetadata() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())).thenReturn(updateResponseActionFuture); + Mockito.when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + Mockito.when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.UPDATED); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata(dataSourceMetadata); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testUpdateDataSourceMetadataWithNotFoundResult() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())).thenReturn(updateResponseActionFuture); + Mockito.when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + Mockito.when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testUpdateDataSourceMetadataWithDocumentMissingException() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())).thenThrow(new DocumentMissingException( + ShardId.fromString("[2][2]"), "testDS")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + + + DataSourceNotFoundException dataSourceNotFoundException = + Assertions.assertThrows(DataSourceNotFoundException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Datasource with name: testDS doesn't exist", + dataSourceNotFoundException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testUpdateDataSourceMetadataWithRuntimeException() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("error message")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("java.lang.RuntimeException: error message", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testDeleteDataSourceMetadata() { + Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); + Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.DELETED); + + this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS"); + + Mockito.verifyNoInteractions(encryptor); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).delete(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + @Test + public void testDeleteDataSourceMetadataWhichisAlreadyDeleted() { + Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); + Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + + DataSourceNotFoundException dataSourceNotFoundException = + Assertions.assertThrows(DataSourceNotFoundException.class, + () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); + Assertions.assertEquals("Datasource with name: testDS doesn't exist", + dataSourceNotFoundException.getMessage()); + + + Mockito.verifyNoInteractions(encryptor); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).delete(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + @Test + public void testDeleteDataSourceMetadataWithUnexpectedResult() { + Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); + Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); + Assertions.assertEquals("Deleting dataSource metadata information failed with result : noop", + runtimeException.getMessage()); + + Mockito.verifyNoInteractions(encryptor); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).delete(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + private String getBasicDataSourceMetadataString() throws JsonProcessingException { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "username"); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + properties.put("prometheus.auth.password", "password"); + dataSourceMetadata.setProperties(properties); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataSourceMetadata); + } + + private String getAWSSigv4DataSourceMetadataString() throws JsonProcessingException { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.type", "awssigv4"); + properties.put("prometheus.auth.secret_key", "secret_key"); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + properties.put("prometheus.auth.access_key", "access_key"); + dataSourceMetadata.setProperties(properties); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataSourceMetadata); + } + + private String getDataSourceMetadataStringWithBasicAuthentication() + throws JsonProcessingException { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "username"); + properties.put("prometheus.auth.password", "password"); + dataSourceMetadata.setProperties(properties); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataSourceMetadata); + } + + private String getDataSourceMetadataStringWithNoAuthentication() throws JsonProcessingException { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + dataSourceMetadata.setProperties(properties); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataSourceMetadata); + } + + private DataSourceMetadata getDataSourceMetadata() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.type", "awssigv4"); + properties.put("prometheus.auth.secret_key", "secret_key"); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + properties.put("prometheus.auth.access_key", "access_key"); + dataSourceMetadata.setProperties(properties); + return dataSourceMetadata; + } + +} diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java new file mode 100644 index 0000000000..3dd5c21214 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java @@ -0,0 +1,86 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportCreateDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportCreateDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + @Captor + private ArgumentCaptor + createDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportCreateDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + CreateDataSourceActionRequest request = new CreateDataSourceActionRequest(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).createDataSource(dataSourceMetadata); + Mockito.verify(actionListener) + .onResponse(createDataSourceActionResponseArgumentCaptor.capture()); + CreateDataSourceActionResponse createDataSourceActionResponse + = createDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Created DataSource with name test_datasource", + createDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + doThrow(new RuntimeException("Error")).when(dataSourceService) + .createDataSource(dataSourceMetadata); + CreateDataSourceActionRequest request = new CreateDataSourceActionRequest(dataSourceMetadata); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).createDataSource(dataSourceMetadata); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java new file mode 100644 index 0000000000..9beeb1a9a9 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java @@ -0,0 +1,78 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportDeleteDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportDeleteDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor + deleteDataSourceActionResponseArgumentCaptor; + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + + @BeforeEach + public void setUp() { + action = new TransportDeleteDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DeleteDataSourceActionRequest request = new DeleteDataSourceActionRequest("test_datasource"); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).deleteDataSource("test_datasource"); + Mockito.verify(actionListener) + .onResponse(deleteDataSourceActionResponseArgumentCaptor.capture()); + DeleteDataSourceActionResponse deleteDataSourceActionResponse + = deleteDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Deleted DataSource with name test_datasource", + deleteDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + doThrow(new RuntimeException("Error")).when(dataSourceService).deleteDataSource("testDS"); + DeleteDataSourceActionRequest request = new DeleteDataSourceActionRequest("testDS"); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).deleteDataSource("testDS"); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java new file mode 100644 index 0000000000..d5506c0a45 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java @@ -0,0 +1,137 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportGetDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportGetDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor getDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportGetDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + GetDataSourceActionRequest request = new GetDataSourceActionRequest("test_datasource"); + when(dataSourceService.getDataSourceMetadata("test_datasource")) + .thenReturn(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata("test_datasource"); + Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); + GetDataSourceActionResponse getDataSourceActionResponse + = getDataSourceActionResponseArgumentCaptor.getValue(); + JsonResponseFormatter dataSourceMetadataJsonResponseFormatter = + new JsonResponseFormatter<>( + JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(DataSourceMetadata response) { + return response; + } + }; + Assertions.assertEquals(dataSourceMetadataJsonResponseFormatter.format(dataSourceMetadata), + getDataSourceActionResponse.getResult()); + DataSourceMetadata result = + new Gson().fromJson(getDataSourceActionResponse.getResult(), DataSourceMetadata.class); + Assertions.assertEquals("test_datasource", result.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, result.getConnector()); + } + + @Test + public void testDoExecuteForGetAllDataSources() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + + GetDataSourceActionRequest request = new GetDataSourceActionRequest(); + when(dataSourceService.getDataSourceMetadata(false)) + .thenReturn(Collections.singleton(dataSourceMetadata)); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata(false); + Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); + GetDataSourceActionResponse getDataSourceActionResponse + = getDataSourceActionResponseArgumentCaptor.getValue(); + JsonResponseFormatter> dataSourceMetadataJsonResponseFormatter = + new JsonResponseFormatter<>( + JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(Set response) { + return response; + } + }; + Type setType = new TypeToken>() { + }.getType(); + Assertions.assertEquals( + dataSourceMetadataJsonResponseFormatter.format(Collections.singleton(dataSourceMetadata)), + getDataSourceActionResponse.getResult()); + Set result = + new Gson().fromJson(getDataSourceActionResponse.getResult(), setType); + DataSourceMetadata resultDataSource = result.iterator().next(); + Assertions.assertEquals("test_datasource", resultDataSource.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, resultDataSource.getConnector()); + } + + @Test + public void testDoExecuteWithException() { + doThrow(new RuntimeException("Error")).when(dataSourceService).getDataSourceMetadata("testDS"); + GetDataSourceActionRequest request = new GetDataSourceActionRequest("testDS"); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata("testDS"); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java new file mode 100644 index 0000000000..fecab012d2 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java @@ -0,0 +1,87 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportUpdateDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportUpdateDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor + updateDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportUpdateDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + UpdateDataSourceActionRequest request = new UpdateDataSourceActionRequest(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).updateDataSource(dataSourceMetadata); + Mockito.verify(actionListener) + .onResponse(updateDataSourceActionResponseArgumentCaptor.capture()); + UpdateDataSourceActionResponse updateDataSourceActionResponse + = updateDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Updated DataSource with name test_datasource", + updateDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + doThrow(new RuntimeException("Error")).when(dataSourceService) + .updateDataSource(dataSourceMetadata); + UpdateDataSourceActionRequest request = new UpdateDataSourceActionRequest(dataSourceMetadata); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).updateDataSource(dataSourceMetadata); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/utils/SchedulerTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java similarity index 56% rename from plugin/src/test/java/org/opensearch/sql/plugin/utils/SchedulerTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java index c86f3341b1..d091e77044 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/utils/SchedulerTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java @@ -3,22 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.plugin.utils; - -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.when; +package org.opensearch.sql.datasources.utils; import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.Assert; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.node.NodeClient; import org.opensearch.threadpool.ThreadPool; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class SchedulerTest { @Mock @@ -29,19 +27,19 @@ public class SchedulerTest { @Test public void testSchedule() { - when(nodeClient.threadPool()).thenReturn(threadPool); + Mockito.when(nodeClient.threadPool()).thenReturn(threadPool); - doAnswer( + Mockito.doAnswer( invocation -> { Runnable task = invocation.getArgument(0); task.run(); return null; }) .when(threadPool) - .schedule(any(), any(), any()); + .schedule(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); AtomicBoolean isRun = new AtomicBoolean(false); Scheduler.schedule(nodeClient, () -> isRun.set(true)); - assertTrue(isRun.get()); + Assert.assertTrue(isRun.get()); } } \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java new file mode 100644 index 0000000000..605d641bda --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java @@ -0,0 +1,101 @@ +package org.opensearch.sql.datasources.utils; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.gson.Gson; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; + +@ExtendWith(MockitoExtension.class) +public class XContentParserUtilsTest { + + @SneakyThrows + @Test + public void testConvertToXContent() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + + XContentBuilder contentBuilder = XContentParserUtils.convertToXContent(dataSourceMetadata); + String contentString = BytesReference.bytes(contentBuilder).utf8ToString(); + Assertions.assertEquals("{\"name\":\"testDS\",\"connector\":\"PROMETHEUS\",\"allowedRoles\":[\"prometheus_access\"],\"properties\":{\"prometheus.uri\":\"https://localhost:9090\"}}", + contentString); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJson() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + DataSourceMetadata retrievedMetadata = XContentParserUtils.toDataSourceMetadata(json); + + Assertions.assertEquals(retrievedMetadata, dataSourceMetadata); + Assertions.assertEquals("prometheus_access", retrievedMetadata.getAllowedRoles().get(0)); + + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonWithoutName() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonWithoutConnector() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("name"); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonUsingUnknownObject() { + HashMap hashMap = new HashMap<>(); + hashMap.put("test", "test"); + Gson gson = new Gson(); + String json = gson.toJson(hashMap); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("Unknown field: test", exception.getMessage()); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 0c900ea234..cca7833d66 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -31,10 +31,10 @@ import org.opensearch.sql.analysis.ExpressionAnalyzer; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; diff --git a/plugin/build.gradle b/plugin/build.gradle index 1c5b4366f0..e318103859 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -122,14 +122,12 @@ dependencies { api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - api group: 'commons-io', name: 'commons-io', version: '2.8.0' - implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}" - implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}" api project(":ppl") api project(':legacy') api project(':opensearch') api project(':prometheus') + api project(':datasources') testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.12.13' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 2fc47082d7..fcb66e2e43 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -46,13 +46,23 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; -import org.opensearch.sql.common.encryptor.EncryptorImpl; -import org.opensearch.sql.datasource.DataSourceLoaderCache; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; -import org.opensearch.sql.datasource.model.DataSource; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; +import org.opensearch.sql.datasources.encryptor.EncryptorImpl; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.rest.RestDataSourceQueryAction; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.datasources.settings.DataSourceSettings; +import org.opensearch.sql.datasources.storage.OpenSearchDataSourceMetadataStorage; +import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction; +import org.opensearch.sql.datasources.transport.TransportUpdateDataSourceAction; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.executor.AsyncRestExecutor; import org.opensearch.sql.legacy.metrics.Metrics; @@ -65,24 +75,12 @@ import org.opensearch.sql.opensearch.storage.script.ExpressionScriptEngine; import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; -import org.opensearch.sql.plugin.datasource.DataSourceSettings; -import org.opensearch.sql.plugin.datasource.DataSourceUserAuthorizationHelperImpl; -import org.opensearch.sql.plugin.datasource.OpenSearchDataSourceMetadataStorage; -import org.opensearch.sql.plugin.model.CreateDataSourceActionResponse; -import org.opensearch.sql.plugin.model.DeleteDataSourceActionResponse; -import org.opensearch.sql.plugin.model.GetDataSourceActionResponse; -import org.opensearch.sql.plugin.model.UpdateDataSourceActionResponse; -import org.opensearch.sql.plugin.rest.RestDataSourceQueryAction; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; import org.opensearch.sql.plugin.rest.RestQuerySettingsAction; import org.opensearch.sql.plugin.transport.PPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; -import org.opensearch.sql.plugin.transport.datasource.TransportCreateDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportDeleteDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportGetDataSourceAction; -import org.opensearch.sql.plugin.transport.datasource.TransportUpdateDataSourceAction; import org.opensearch.sql.prometheus.storage.PrometheusStorageFactory; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.threadpool.ExecutorBuilder; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 6825b2ac92..a5c094e956 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -21,7 +21,7 @@ import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorageTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorageTest.java deleted file mode 100644 index 471cb7f9db..0000000000 --- a/plugin/src/test/java/org/opensearch/sql/plugin/datasource/OpenSearchDataSourceMetadataStorageTest.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.plugin.datasource; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.plugin.datasource.OpenSearchDataSourceMetadataStorage.DATASOURCE_INDEX_NAME; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import lombok.SneakyThrows; -import org.apache.lucene.search.TotalHits; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Answers; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.action.ActionFuture; -import org.opensearch.action.DocWriteResponse; -import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.delete.DeleteResponse; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.update.UpdateResponse; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.rest.RestStatus; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.sql.common.encryptor.Encryptor; -import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasource.model.DataSourceType; - -@RunWith(MockitoJUnitRunner.class) -public class OpenSearchDataSourceMetadataStorageTest { - - private static final String TEST_DATASOURCE_INDEX_NAME = "testDS"; - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Client client; - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private ClusterService clusterService; - @Mock - private Encryptor encryptor; - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private SearchResponse searchResponse; - @Mock - private ActionFuture searchResponseActionFuture; - @Mock - private ActionFuture createIndexResponseActionFuture; - @Mock - private ActionFuture indexResponseActionFuture; - @Mock - private IndexResponse indexResponse; - @Mock - private ActionFuture updateResponseActionFuture; - @Mock - private UpdateResponse updateResponse; - @Mock - private ActionFuture deleteResponseActionFuture; - @Mock - private DeleteResponse deleteResponse; - @Mock - private SearchHit searchHit; - @InjectMocks - private OpenSearchDataSourceMetadataStorage openSearchDataSourceMetadataStorage; - - - @SneakyThrows - @Test - public void testGetDataSourceMetadata() { - when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) - .thenReturn(true); - when(client.search(any())).thenReturn(searchResponseActionFuture); - when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); - when(searchResponse.status()).thenReturn(RestStatus.OK); - when(searchResponse.getHits()) - .thenReturn( - new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); - when(searchHit.getSourceAsString()) - .thenReturn(getBasicDataSourceMetadataString()); - when(encryptor.decrypt("password")).thenReturn("password"); - when(encryptor.decrypt("username")).thenReturn("username"); - - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - - - assertFalse(dataSourceMetadataOptional.isEmpty()); - DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); - assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); - assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - assertEquals("password", - dataSourceMetadata.getProperties().get("prometheus.auth.password")); - assertEquals("username", - dataSourceMetadata.getProperties().get("prometheus.auth.username")); - assertEquals("basicauth", - dataSourceMetadata.getProperties().get("prometheus.auth.type")); - } - - @SneakyThrows - @Test - public void testGetDataSourceMetadataWithAWSSigV4() { - when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) - .thenReturn(true); - when(client.search(any())).thenReturn(searchResponseActionFuture); - when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); - when(searchResponse.status()).thenReturn(RestStatus.OK); - when(searchResponse.getHits()) - .thenReturn( - new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); - when(searchHit.getSourceAsString()) - .thenReturn(getAWSSigv4DataSourceMetadataString()); - when(encryptor.decrypt("secret_key")).thenReturn("secret_key"); - when(encryptor.decrypt("access_key")).thenReturn("access_key"); - - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - - - assertFalse(dataSourceMetadataOptional.isEmpty()); - DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); - assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); - assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - assertEquals("secret_key", - dataSourceMetadata.getProperties().get("prometheus.auth.secret_key")); - assertEquals("access_key", - dataSourceMetadata.getProperties().get("prometheus.auth.access_key")); - assertEquals("awssigv4", - dataSourceMetadata.getProperties().get("prometheus.auth.type")); - } - - @Test - public void testCreateDataSourceMetadata() { - - when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) - .thenReturn(Boolean.FALSE); - when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); - when(encryptor.encrypt("access_key")).thenReturn("access_key"); - when(client.admin().indices().create(any())) - .thenReturn(createIndexResponseActionFuture); - when(createIndexResponseActionFuture.actionGet()) - .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); - when(client.index(any())).thenReturn(indexResponseActionFuture); - when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); - when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); - DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - - this.openSearchDataSourceMetadataStorage.createDataSourceMetadata(dataSourceMetadata); - - verify(encryptor, times(1)).encrypt("secret_key"); - verify(encryptor, times(1)).encrypt("access_key"); - verify(client.admin().indices(), times(1)).create(any()); - verify(client, times(1)).index(any()); - verify(client.threadPool().getThreadContext(), times(2)).stashContext(); - - - } - - @Test - public void testUpdateDataSourceMetadata() { - when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) - .thenReturn(Boolean.TRUE); - when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); - when(encryptor.encrypt("access_key")).thenReturn("access_key"); - when(client.update(any())).thenReturn(updateResponseActionFuture); - when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); - when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.UPDATED); - DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - - this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata(dataSourceMetadata); - - verify(encryptor, times(1)).encrypt("secret_key"); - verify(encryptor, times(1)).encrypt("access_key"); - verify(client.admin().indices(), times(0)).create(any()); - verify(client, times(1)).update(any()); - verify(client.threadPool().getThreadContext(), times(1)).stashContext(); - - } - - @Test - public void testDeleteDataSourceMetadata() { - when(client.delete(any())).thenReturn(deleteResponseActionFuture); - when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); - when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.DELETED); - DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - - this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS"); - - verifyNoInteractions(encryptor); - verify(client.admin().indices(), times(0)).create(any()); - verify(client, times(1)).delete(any()); - verify(client.threadPool().getThreadContext(), times(1)).stashContext(); - } - - private String getBasicDataSourceMetadataString() throws JsonProcessingException { - DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); - dataSourceMetadata.setName("testDS"); - dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); - dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); - Map properties = new HashMap<>(); - properties.put("prometheus.auth.type", "basicauth"); - properties.put("prometheus.auth.username", "username"); - properties.put("prometheus.auth.uri", "https://localhost:9090"); - properties.put("prometheus.auth.password", "password"); - dataSourceMetadata.setProperties(properties); - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.writeValueAsString(dataSourceMetadata); - } - - private String getAWSSigv4DataSourceMetadataString() throws JsonProcessingException { - DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); - dataSourceMetadata.setName("testDS"); - dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); - dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); - Map properties = new HashMap<>(); - properties.put("prometheus.auth.type", "awssigv4"); - properties.put("prometheus.auth.secret_key", "secret_key"); - properties.put("prometheus.auth.uri", "https://localhost:9090"); - properties.put("prometheus.auth.access_key", "access_key"); - dataSourceMetadata.setProperties(properties); - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.writeValueAsString(dataSourceMetadata); - } - - private DataSourceMetadata getDataSourceMetadata() { - DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); - dataSourceMetadata.setName("testDS"); - dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); - dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); - Map properties = new HashMap<>(); - properties.put("prometheus.auth.type", "awssigv4"); - properties.put("prometheus.auth.secret_key", "secret_key"); - properties.put("prometheus.auth.uri", "https://localhost:9090"); - properties.put("prometheus.auth.access_key", "access_key"); - dataSourceMetadata.setProperties(properties); - return dataSourceMetadata; - } - -} diff --git a/prometheus/build.gradle b/prometheus/build.gradle index ca70813e58..b0c05f1bc8 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -16,6 +16,7 @@ repositories { dependencies { api project(':core') + implementation project(':datasources') implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java index bb95ca93a2..4a0f52f4a5 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java @@ -21,7 +21,7 @@ import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; -import org.opensearch.sql.datasource.model.auth.AuthenticationType; +import org.opensearch.sql.datasources.auth.AuthenticationType; import org.opensearch.sql.prometheus.authinterceptors.AwsSigningInterceptor; import org.opensearch.sql.prometheus.authinterceptors.BasicAuthenticationInterceptor; import org.opensearch.sql.prometheus.client.PrometheusClient; diff --git a/settings.gradle b/settings.gradle index 7def8a746c..6f7214cb3a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,3 +19,4 @@ include 'legacy' include 'sql' include 'prometheus' include 'benchmarks' +include 'datasources' \ No newline at end of file