From 7530534425c91938f902403cd4b25ea8c9cd87dd Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 25 Oct 2023 20:43:25 +0000 Subject: [PATCH 1/6] Clean up how async test cases are written Signed-off-by: Peter Nied --- .../security/ResourceFocusedTests.java | 106 +++-------------- .../security/rest/CompressionTests.java | 107 +++++++----------- .../test/framework/AsyncActions.java | 64 +++++++++++ 3 files changed, 121 insertions(+), 156 deletions(-) create mode 100644 src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java index a25423471f..d68d6a23d6 100644 --- a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -16,17 +16,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryUsage; import java.nio.charset.StandardCharsets; -import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.zip.GZIPOutputStream; @@ -35,12 +27,14 @@ import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.http.HttpStatus; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; +import org.opensearch.test.framework.AsyncActions; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.User; import org.opensearch.test.framework.cluster.ClusterManager; @@ -93,9 +87,8 @@ public void testUnauthenticatedFewBig() { final String requestPath = "/*/_search"; final int parrallelism = 5; final int totalNumberOfRequests = 100; - final boolean statsPrinter = false; - runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests, statsPrinter); + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); } @Test @@ -105,9 +98,8 @@ public void testUnauthenticatedManyMedium() { final String requestPath = "/*/_search"; final int parrallelism = 20; final int totalNumberOfRequests = 10_000; - final boolean statsPrinter = false; - runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests, statsPrinter); + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); } @Test @@ -117,61 +109,26 @@ public void testUnauthenticatedTonsSmall() { final String requestPath = "/*/_search"; final int parrallelism = 100; final int totalNumberOfRequests = 1_000_000; - final boolean statsPrinter = false; - runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests, statsPrinter); + runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); } - private Long runResourceTest( + private void runResourceTest( final RequestBodySize size, final String requestPath, final int parrallelism, - final int totalNumberOfRequests, - final boolean statsPrinter + final int totalNumberOfRequests ) { final byte[] compressedRequestBody = createCompressedRequestBody(size); try (final TestRestClient client = cluster.getRestClient(new BasicHeader("Content-Encoding", "gzip"))) { - - if (statsPrinter) { - printStats(); - } - final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); - post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); - - final ForkJoinPool forkJoinPool = new ForkJoinPool(parrallelism); - - final List> waitingOn = IntStream.rangeClosed(1, totalNumberOfRequests) - .boxed() - .map(i -> CompletableFuture.runAsync(() -> client.executeRequest(post), forkJoinPool)) - .collect(Collectors.toList()); - Supplier getCount = () -> waitingOn.stream().filter(cf -> cf.isDone() && !cf.isCompletedExceptionally()).count(); - - CompletableFuture statPrinter = statsPrinter ? CompletableFuture.runAsync(() -> { - while (true) { - printStats(); - System.err.println(" & Succesful completions: " + getCount.get()); - try { - Thread.sleep(500); - } catch (Exception e) { - break; - } - } - }, forkJoinPool) : CompletableFuture.completedFuture(null); - - final CompletableFuture allOfThem = CompletableFuture.allOf(waitingOn.toArray(new CompletableFuture[0])); - - try { - allOfThem.get(30, TimeUnit.SECONDS); - statPrinter.cancel(true); - } catch (final Exception e) { - // Ignored - } - - if (statsPrinter) { - printStats(); - System.err.println(" & Succesful completions: " + getCount.get()); - } - return getCount.get(); + final var requests = AsyncActions.generate(() -> { + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parrallelism, totalNumberOfRequests); + + AsyncActions.getAll(requests, 30, TimeUnit.SECONDS) + .forEach((response) -> { response.assertStatusCode(HttpStatus.SC_UNAUTHORIZED); }); } } @@ -231,37 +188,4 @@ private byte[] createCompressedRequestBody(final RequestBodySize size) { throw new RuntimeException(ioe); } } - - private void printStats() { - System.err.println("** Stats "); - printMemory(); - printMemoryPools(); - printGCPools(); - } - - private void printMemory() { - final Runtime runtime = Runtime.getRuntime(); - - final long totalMemory = runtime.totalMemory(); // Total allocated memory - final long freeMemory = runtime.freeMemory(); // Amount of free memory - final long usedMemory = totalMemory - freeMemory; // Amount of used memory - - System.err.println(" Memory Total: " + totalMemory + " Free:" + freeMemory + " Used:" + usedMemory); - } - - private void printMemoryPools() { - List memoryPools = ManagementFactory.getMemoryPoolMXBeans(); - for (MemoryPoolMXBean memoryPool : memoryPools) { - MemoryUsage usage = memoryPool.getUsage(); - System.err.println(" " + memoryPool.getName() + " USED: " + usage.getUsed() + " MAX: " + usage.getMax()); - } - } - - private void printGCPools() { - List garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans(); - for (GarbageCollectorMXBean garbageCollector : garbageCollectors) { - System.err.println(" " + garbageCollector.getName() + " COLLECTION TIME: " + garbageCollector.getCollectionTime()); - } - } - } diff --git a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java index cf07f93ad8..9283ac1965 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java @@ -11,9 +11,9 @@ package org.opensearch.security.rest; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.message.BasicHeader; @@ -21,11 +21,7 @@ import org.junit.Test; import org.junit.runner.RunWith; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.anyOf; -import static org.hamcrest.MatcherAssert.assertThat; +import org.opensearch.test.framework.AsyncActions; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; @@ -34,15 +30,14 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.zip.GZIPOutputStream; -import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; import static org.opensearch.test.framework.cluster.TestRestClientConfiguration.getBasicAuthHeader; @@ -60,7 +55,7 @@ public class CompressionTests { .build(); @Test - public void testAuthenticatedGzippedRequests() throws Exception { + public void testAuthenticatedGzippedRequests() { final String requestPath = "/*/_search"; final int parallelism = 10; final int totalNumberOfRequests = 100; @@ -69,31 +64,13 @@ public void testAuthenticatedGzippedRequests() throws Exception { final byte[] compressedRequestBody = createCompressedRequestBody(rawBody); try (final TestRestClient client = cluster.getRestClient(ADMIN_USER, new BasicHeader("Content-Encoding", "gzip"))) { + final var requests = AsyncActions.generate(() -> { + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parallelism, totalNumberOfRequests); - final ForkJoinPool forkJoinPool = new ForkJoinPool(parallelism); - - final List> waitingOn = IntStream.rangeClosed(1, totalNumberOfRequests) - .boxed() - .map(i -> CompletableFuture.supplyAsync(() -> { - final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); - post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); - return client.executeRequest(post); - }, forkJoinPool)) - .collect(Collectors.toList()); - - final CompletableFuture allOfThem = CompletableFuture.allOf(waitingOn.toArray(new CompletableFuture[0])); - - allOfThem.get(30, TimeUnit.SECONDS); - - waitingOn.stream().forEach(future -> { - try { - final HttpResponse response = future.get(); - response.assertStatusCode(HttpStatus.SC_OK); - } catch (final Exception ex) { - throw new RuntimeException(ex); - } - }); - ; + AsyncActions.getAll(requests, 30, TimeUnit.SECONDS).forEach((response) -> { response.assertStatusCode(HttpStatus.SC_OK); }); } } @@ -101,40 +78,40 @@ public void testAuthenticatedGzippedRequests() throws Exception { public void testMixOfAuthenticatedAndUnauthenticatedGzippedRequests() throws Exception { final String requestPath = "/*/_search"; final int parallelism = 10; - final int totalNumberOfRequests = 100; + final int totalNumberOfRequests = 50; final String rawBody = "{ \"query\": { \"match\": { \"foo\": \"bar\" }}}"; final byte[] compressedRequestBody = createCompressedRequestBody(rawBody); try (final TestRestClient client = cluster.getRestClient(new BasicHeader("Content-Encoding", "gzip"))) { - - final ForkJoinPool forkJoinPool = new ForkJoinPool(parallelism); - - final Header basicAuthHeader = getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword()); - - final List> waitingOn = IntStream.rangeClosed(1, totalNumberOfRequests) - .boxed() - .map(i -> CompletableFuture.supplyAsync(() -> { - final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); - post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); - return i % 2 == 0 ? client.executeRequest(post) : client.executeRequest(post, basicAuthHeader); - }, forkJoinPool)) - .collect(Collectors.toList()); - - final CompletableFuture allOfThem = CompletableFuture.allOf(waitingOn.toArray(new CompletableFuture[0])); - - allOfThem.get(30, TimeUnit.SECONDS); - - waitingOn.stream().forEach(future -> { - try { - final HttpResponse response = future.get(); - assertThat(response.getBody(), not(containsString("json_parse_exception"))); - assertThat(response.getStatusCode(), anyOf(equalTo(HttpStatus.SC_UNAUTHORIZED), equalTo(HttpStatus.SC_OK))); - } catch (final Exception ex) { - throw new RuntimeException(ex); - } + final CountDownLatch countDownLatch = new CountDownLatch(1); + + final var authorizedRequests = AsyncActions.generate(() -> { + countDownLatch.await(); + System.err.println("Generation triggerd authorizedRequests"); + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword())); + }, parallelism, totalNumberOfRequests); + + final var unauthorizedRequests = AsyncActions.generate(() -> { + countDownLatch.await(); + System.err.println("Generation triggerd unauthorizedRequests"); + final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); + post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); + return client.executeRequest(post); + }, parallelism, totalNumberOfRequests); + + // Make sure all requests start at the same time + countDownLatch.countDown(); + + AsyncActions.getAll(authorizedRequests, 30, TimeUnit.SECONDS).forEach((response) -> { + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK)); + }); + AsyncActions.getAll(unauthorizedRequests, 30, TimeUnit.SECONDS).forEach((response) -> { + assertThat(response.getBody(), not(containsString("json_parse_exception"))); + assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED)); }); - ; } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java new file mode 100644 index 0000000000..c053017a82 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.test.framework; + +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class AsyncActions { + + /** + * Using the provided generator create a list of completable futures. + * @param parrallelism How many calls to the generator should be done at the same time. + * @param generationCount The total number of calls to the generator to conduct. + * @return The list of completable futures running on the fork join thread pool. + */ + public static List> generate(final Callable generator, final int parrallelism, final int generationCount) { + final ForkJoinPool forkJoinPool = new ForkJoinPool(parrallelism); + return IntStream.rangeClosed(1, generationCount).boxed().map(i -> CompletableFuture.supplyAsync(() -> { + try { + return generator.call(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }, forkJoinPool)).collect(Collectors.toList()); + } + + /** + * Waits for futures for a time period and then returns them a list + * @param futures Futures to wait for completion with a result + * @param n Amount of time to wait + * @param unit Time associated with those units + * @return Completed results from the futures + */ + public static List getAll(final List> futures, final int n, final TimeUnit unit) { + final CompletableFuture futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + try { + futuresCompleted.get(n, unit); + } catch (final Exception ex) { + final long completedFutures = futures.stream().filter(CompletableFuture::isDone).count(); + throw new RuntimeException("Unable to wait for all futures to compete, " + completedFutures + " have finished.", ex); + } + + return futures.stream().map(future -> { + try { + return future.get(); + } catch (final Exception ex) { + throw new RuntimeException(ex); + } + }).collect(Collectors.toList()); + } +} From 67d7a586b8328c72ceef9efdef531bfbf4ef65c8 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 26 Oct 2023 18:10:15 +0000 Subject: [PATCH 2/6] Tweak test values + better per future logging for future tweaking Signed-off-by: Peter Nied --- .../security/ResourceFocusedTests.java | 2 +- .../test/framework/AsyncActions.java | 39 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java index d68d6a23d6..05973c119f 100644 --- a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -108,7 +108,7 @@ public void testUnauthenticatedTonsSmall() { final RequestBodySize size = RequestBodySize.Small; final String requestPath = "/*/_search"; final int parrallelism = 100; - final int totalNumberOfRequests = 1_000_000; + final int totalNumberOfRequests = 15_000; runResourceTest(size, requestPath, parrallelism, totalNumberOfRequests); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java index c053017a82..99f6888f05 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java +++ b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java @@ -18,7 +18,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + public class AsyncActions { + private final static Logger LOG = LogManager.getLogger(AsyncActions.class); /** * Using the provided generator create a list of completable futures. @@ -45,12 +49,31 @@ public static List> generate(final Callable generato * @return Completed results from the futures */ public static List getAll(final List> futures, final int n, final TimeUnit unit) { + LOG.info("Starting to wait for " + futures.size() + " futures to complete in for " + unit.toSeconds(n) + " seconds."); + final long startTimeMs = System.currentTimeMillis(); final CompletableFuture futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); try { futuresCompleted.get(n, unit); } catch (final Exception ex) { - final long completedFutures = futures.stream().filter(CompletableFuture::isDone).count(); - throw new RuntimeException("Unable to wait for all futures to compete, " + completedFutures + " have finished.", ex); + final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count(); + final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount); + throw new RuntimeException( + "Unable to wait for all futures to compete, of " + + futures.size() + + " futures " + + completedFuturesCount + + " have finished." + + perfReport + ); + } + final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count(); + final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount); + LOG.info(perfReport); + + final long elapsedTimeMs = System.currentTimeMillis() - startTimeMs; + final long expectedMs = unit.toMillis(n); + if (elapsedTimeMs > .75 * expectedMs) { + LOG.warn("Completion time was within 25% of the expected time, more than this threshold is recommended."); } return futures.stream().map(future -> { @@ -61,4 +84,16 @@ public static List getAll(final List> futures, final } }).collect(Collectors.toList()); } + + private static String calculatePerfReport(final long startTimeMs, final long completedFuturesCount) { + final long elapsedTimeMs = System.currentTimeMillis() - startTimeMs; + final double avgTimePerFutureMs = (double) elapsedTimeMs / completedFuturesCount; + final double futuresPerSecond = 1000 / avgTimePerFutureMs; + return String.format( + "Waited for %d seconds, completion speed was on average %.2fms per future %.2fx per second.", + TimeUnit.MILLISECONDS.toSeconds(elapsedTimeMs), + avgTimePerFutureMs, + futuresPerSecond + ); + } } From b09e59382576fff97821f37f40ddef0e76eeb308 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 26 Oct 2023 18:10:43 +0000 Subject: [PATCH 3/6] Prevent DEBUG loggers from printing to the console, should cleanup report output very much Signed-off-by: Peter Nied --- src/integrationTest/resources/log4j2-test.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties index 8d9cf87666..0b865b46b3 100644 --- a/src/integrationTest/resources/log4j2-test.properties +++ b/src/integrationTest/resources/log4j2-test.properties @@ -28,6 +28,7 @@ logger.auditlogs.level = info # Logger required by test org.opensearch.security.http.JwtAuthenticationTests logger.httpjwtauthenticator.name = com.amazon.dlic.auth.http.jwt.HTTPJwtAuthenticator logger.httpjwtauthenticator.level = debug +logger.backendreg.additivity = false logger.httpjwtauthenticator.appenderRef.capturing.ref = logCapturingAppender #Required by tests: @@ -35,10 +36,12 @@ logger.httpjwtauthenticator.appenderRef.capturing.ref = logCapturingAppender # org.opensearch.security.UserBruteForceAttacksPreventionTests logger.backendreg.name = org.opensearch.security.auth.BackendRegistry logger.backendreg.level = debug +logger.backendreg.additivity = false logger.backendreg.appenderRef.capturing.ref = logCapturingAppender #com.amazon.dlic.auth.ldap #logger.ldap.name=com.amazon.dlic.auth.ldap.backend.LDAPAuthenticationBackend logger.ldap.name=com.amazon.dlic.auth.ldap.backend logger.ldap.level=TRACE +logger.backendreg.additivity = false logger.ldap.appenderRef.capturing.ref = logCapturingAppender From 5309a6e98eba9dcf8d257deed96dc1fd10491936 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 26 Oct 2023 18:19:16 +0000 Subject: [PATCH 4/6] Allow for more time for tests to run, switch to log4j logger Signed-off-by: Peter Nied --- .../org/opensearch/security/ResourceFocusedTests.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java index 05973c119f..ae58cbfd69 100644 --- a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -28,6 +28,8 @@ import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; @@ -46,6 +48,7 @@ @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) public class ResourceFocusedTests { + private final static Logger LOG = LogManager.getLogger(AsyncActions.class); private static final User ADMIN_USER = new User("admin").roles(ALL_ACCESS); private static final User LIMITED_USER = new User("limited_user").roles( new TestSecurityConfig.Role("limited-role").clusterPermissions( @@ -127,7 +130,7 @@ private void runResourceTest( return client.executeRequest(post); }, parrallelism, totalNumberOfRequests); - AsyncActions.getAll(requests, 30, TimeUnit.SECONDS) + AsyncActions.getAll(requests, 2, TimeUnit.MINUTES) .forEach((response) -> { response.assertStatusCode(HttpStatus.SC_UNAUTHORIZED); }); } } @@ -174,9 +177,7 @@ private byte[] createCompressedRequestBody(final RequestBodySize size) { gzipOutputStream.finish(); final byte[] compressedRequestBody = byteArrayOutputStream.toByteArray(); - System.err.println( - "^^^" - + String.format( + LOG.info(String.format( "Original size was %,d bytes, compressed to %,d bytes, ratio %,.2f", uncompressedBytesSize, compressedRequestBody.length, From 9ad33146e7b5156c4eb57e2ed6c11fda266979f6 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 26 Oct 2023 18:55:17 +0000 Subject: [PATCH 5/6] Fix spotless Signed-off-by: Peter Nied --- .../opensearch/security/ResourceFocusedTests.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java index ae58cbfd69..5d441d0063 100644 --- a/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java +++ b/src/integrationTest/java/org/opensearch/security/ResourceFocusedTests.java @@ -177,12 +177,13 @@ private byte[] createCompressedRequestBody(final RequestBodySize size) { gzipOutputStream.finish(); final byte[] compressedRequestBody = byteArrayOutputStream.toByteArray(); - LOG.info(String.format( - "Original size was %,d bytes, compressed to %,d bytes, ratio %,.2f", - uncompressedBytesSize, - compressedRequestBody.length, - ((double) uncompressedBytesSize / compressedRequestBody.length) - ) + LOG.info( + String.format( + "Original size was %,d bytes, compressed to %,d bytes, ratio %,.2f", + uncompressedBytesSize, + compressedRequestBody.length, + ((double) uncompressedBytesSize / compressedRequestBody.length) + ) ); return compressedRequestBody; } catch (final IOException ioe) { From 27969f56ffad83d50a9368c0fd8614534d276572 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 27 Oct 2023 16:09:31 +0000 Subject: [PATCH 6/6] Fix spelling issues Signed-off-by: Peter Nied --- .../java/org/opensearch/security/rest/CompressionTests.java | 4 ++-- .../java/org/opensearch/test/framework/AsyncActions.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java index 9283ac1965..aa747e2586 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/CompressionTests.java @@ -88,7 +88,7 @@ public void testMixOfAuthenticatedAndUnauthenticatedGzippedRequests() throws Exc final var authorizedRequests = AsyncActions.generate(() -> { countDownLatch.await(); - System.err.println("Generation triggerd authorizedRequests"); + System.err.println("Generation triggered authorizedRequests"); final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); return client.executeRequest(post, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword())); @@ -96,7 +96,7 @@ public void testMixOfAuthenticatedAndUnauthenticatedGzippedRequests() throws Exc final var unauthorizedRequests = AsyncActions.generate(() -> { countDownLatch.await(); - System.err.println("Generation triggerd unauthorizedRequests"); + System.err.println("Generation triggered unauthorizedRequests"); final HttpPost post = new HttpPost(client.getHttpServerUri() + requestPath); post.setEntity(new ByteArrayEntity(compressedRequestBody, ContentType.APPLICATION_JSON)); return client.executeRequest(post); diff --git a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java index 99f6888f05..409aa5a416 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java +++ b/src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java @@ -49,7 +49,7 @@ public static List> generate(final Callable generato * @return Completed results from the futures */ public static List getAll(final List> futures, final int n, final TimeUnit unit) { - LOG.info("Starting to wait for " + futures.size() + " futures to complete in for " + unit.toSeconds(n) + " seconds."); + LOG.info("Starting to wait for " + futures.size() + " futures to complete in " + unit.toSeconds(n) + " seconds."); final long startTimeMs = System.currentTimeMillis(); final CompletableFuture futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); try { @@ -58,7 +58,7 @@ public static List getAll(final List> futures, final final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count(); final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount); throw new RuntimeException( - "Unable to wait for all futures to compete, of " + "Unable to wait for all futures to complete, of " + futures.size() + " futures " + completedFuturesCount