From d2c8b758b50ea3133a36aff829b6f0ccc8549244 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar Date: Tue, 18 Apr 2023 09:06:34 -0700 Subject: [PATCH] Integ Test Fix (#1541) (cherry picked from commit 4e9ca016f45ccebacd629aebcd12fdf4d3bbb2e8) --- integ-test/build.gradle | 1 - .../sql/datasource/DataSourceAPIsIT.java | 57 ++++++------------- .../sql/legacy/SQLIntegTestCase.java | 43 ++++++++++++++ .../sql/ppl/InformationSchemaCommandIT.java | 32 +++++++++-- .../ppl/PrometheusDataSourceCommandsIT.java | 33 +++++++++-- .../sql/ppl/ShowDataSourcesCommandIT.java | 31 ++++++++-- 6 files changed, 141 insertions(+), 56 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 511005397b..dd26c060e9 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -114,7 +114,6 @@ testClusters.all { testClusters.integTest { plugin ":opensearch-sql-plugin" - keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/datasource/", 'datasources.json') } task startPrometheus(type: SpawnProcessTask) { diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java index d559207cc1..0dbc03d5a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java @@ -12,12 +12,15 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; +import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; @@ -29,6 +32,21 @@ public class DataSourceAPIsIT extends PPLIntegTestCase { + @AfterClass + protected static void deleteDataSourcesCreated() throws IOException { + Request deleteRequest = getDeleteDataSourceRequest("create_prometheus"); + Response deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + + deleteRequest = getDeleteDataSourceRequest("update_prometheus"); + deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + + deleteRequest = getDeleteDataSourceRequest("get_all_prometheus"); + deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + } + @SneakyThrows @Test public void createDataSourceAPITest() { @@ -150,43 +168,4 @@ public void getAllDataSourceTest() { dataSourceMetadataList.stream().anyMatch(ds -> ds.getName().equals("get_all_prometheus"))); } - - private Request getCreateDataSourceRequest(DataSourceMetadata dataSourceMetadata) { - Request request = new Request("POST", "/_plugins/_query/_datasources"); - request.setJsonEntity(new Gson().toJson(dataSourceMetadata)); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - private Request getUpdateDataSourceRequest(DataSourceMetadata dataSourceMetadata) { - Request request = new Request("PUT", "/_plugins/_query/_datasources"); - request.setJsonEntity(new Gson().toJson(dataSourceMetadata)); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - private Request getFetchDataSourceRequest(String name) { - Request request = new Request("GET", "/_plugins/_query/_datasources" + "/" + name); - if (StringUtils.isEmpty(name)) { - request = new Request("GET", "/_plugins/_query/_datasources"); - } - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - - - private Request getDeleteDataSourceRequest(String name) { - Request request = new Request("DELETE", "/_plugins/_query/_datasources" + "/" + name); - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - return request; - } - } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index cb86ed6a11..35ae5d3675 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -7,6 +7,8 @@ package org.opensearch.sql.legacy; import com.google.common.base.Strings; +import com.google.gson.Gson; +import org.apache.commons.lang3.StringUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.AfterClass; @@ -30,6 +32,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Locale; +import org.opensearch.sql.datasource.model.DataSourceMetadata; import static com.google.common.base.Strings.isNullOrEmpty; import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient; @@ -441,6 +444,44 @@ protected JSONObject getSource(JSONObject hit) { return hit.getJSONObject("_source"); } + protected static Request getCreateDataSourceRequest(DataSourceMetadata dataSourceMetadata) { + Request request = new Request("POST", "/_plugins/_query/_datasources"); + request.setJsonEntity(new Gson().toJson(dataSourceMetadata)); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + + protected static Request getUpdateDataSourceRequest(DataSourceMetadata dataSourceMetadata) { + Request request = new Request("PUT", "/_plugins/_query/_datasources"); + request.setJsonEntity(new Gson().toJson(dataSourceMetadata)); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + + protected static Request getFetchDataSourceRequest(String name) { + Request request = new Request("GET", "/_plugins/_query/_datasources" + "/" + name); + if (StringUtils.isEmpty(name)) { + request = new Request("GET", "/_plugins/_query/_datasources"); + } + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + + + protected static Request getDeleteDataSourceRequest(String name) { + Request request = new Request("DELETE", "/_plugins/_query/_datasources" + "/" + name); + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + return request; + } + /** * Enum for associating test index with relevant mapping and data. */ @@ -637,5 +678,7 @@ public String getMapping() { public String getDataSet() { return this.dataSet; } + + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java index 6b845bd68e..448bebd377 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java @@ -7,28 +7,31 @@ package org.opensearch.sql.ppl; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import org.json.JSONObject; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; public class InformationSchemaCommandIT extends PPLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.DATASOURCES); - } - /** * Integ tests are dependent on self generated metrics in prometheus instance. * When running individual integ tests there @@ -41,6 +44,23 @@ protected static void metricGenerationWait() throws InterruptedException { Thread.sleep(10000); } + @Override + protected void init() throws InterruptedException, IOException { + DataSourceMetadata createDSM = + new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS, + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090")); + Request createRequest = getCreateDataSourceRequest(createDSM); + Response response = client().performRequest(createRequest); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + } + + @After + protected void deleteDataSourceMetadata() throws IOException { + Request deleteRequest = getDeleteDataSourceRequest("my_prometheus"); + Response deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + } + @Test public void testSearchTablesFromPrometheusCatalog() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java index 9ed66d34fe..4ac94a0d78 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java @@ -12,24 +12,28 @@ import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; public class PrometheusDataSourceCommandsIT extends PPLIntegTestCase { - - @Override - protected void init() throws Exception { - loadIndex(Index.DATASOURCES); - } - /** * Integ tests are dependent on self generated metrics in prometheus instance. * When running individual integ tests there @@ -42,6 +46,23 @@ protected static void metricGenerationWait() throws InterruptedException { Thread.sleep(10000); } + @Override + protected void init() throws InterruptedException, IOException { + DataSourceMetadata createDSM = + new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS, + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090")); + Request createRequest = getCreateDataSourceRequest(createDSM); + Response response = client().performRequest(createRequest); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + } + + @After + protected void deleteDataSourceMetadata() throws IOException { + Request deleteRequest = getDeleteDataSourceRequest("my_prometheus"); + Response deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + } + @Test @SneakyThrows public void testSourceMetricCommand() { diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java index 61d8932013..4190e4274b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java @@ -7,21 +7,27 @@ package org.opensearch.sql.ppl; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import org.json.JSONObject; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; public class ShowDataSourcesCommandIT extends PPLIntegTestCase { - @Override - protected void init() throws Exception { - loadIndex(Index.DATASOURCES); - } /** * Integ tests are dependent on self generated metrics in prometheus instance. @@ -35,6 +41,23 @@ protected static void metricGenerationWait() throws InterruptedException { Thread.sleep(10000); } + @Override + protected void init() throws InterruptedException, IOException { + DataSourceMetadata createDSM = + new DataSourceMetadata("my_prometheus", DataSourceType.PROMETHEUS, + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "http://localhost:9090")); + Request createRequest = getCreateDataSourceRequest(createDSM); + Response response = client().performRequest(createRequest); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + } + + @After + protected void deleteDataSourceMetadata() throws IOException { + Request deleteRequest = getDeleteDataSourceRequest("my_prometheus"); + Response deleteResponse = client().performRequest(deleteRequest); + Assert.assertEquals(204, deleteResponse.getStatusLine().getStatusCode()); + } + @Test public void testShowDataSourcesCommands() throws IOException { JSONObject result = executeQuery("show datasources");