From 3d508c470976b116c5d7f4613ef4c5212a476024 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 30 Oct 2023 09:15:17 -0400 Subject: [PATCH 1/4] Update bundled JDK to JDK-21.0.1 (#10576) Signed-off-by: Andriy Redko (cherry picked from commit 0d7d1e9db7ace8f6e90ab98ade88d719efaf37e1) Signed-off-by: Andriy Redko --- .../gradle/test/DistroTestPlugin.java | 4 +-- buildSrc/version.properties | 2 +- distribution/src/config/jvm.options | 2 +- libs/core/build.gradle | 27 ++++++++++++------- server/build.gradle | 13 ++++++++- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index a420c8b63b02c..1ad7e056b6ae6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -77,9 +77,9 @@ import java.util.stream.Stream; public class DistroTestPlugin implements Plugin { - private static final String SYSTEM_JDK_VERSION = "11.0.20+8"; + private static final String SYSTEM_JDK_VERSION = "17.0.9+9"; private static final String SYSTEM_JDK_VENDOR = "adoptium"; - private static final String GRADLE_JDK_VERSION = "17.0.8+7"; + private static final String GRADLE_JDK_VERSION = "17.0.9+9"; private static final String GRADLE_JDK_VENDOR = "adoptium"; // all distributions used by distro tests. this is temporary until tests are per distribution diff --git a/buildSrc/version.properties b/buildSrc/version.properties index d9266c6008c98..38f6d6af08ca1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -2,7 +2,7 @@ opensearch = 2.12.0 lucene = 9.8.0 bundled_jdk_vendor = adoptium -bundled_jdk = 17.0.8+7 +bundled_jdk = 21.0.1+12 # optional dependencies spatial4j = 0.7 diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 952110c6c0289..1a0abcbaf9c88 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -81,7 +81,7 @@ ${error.file} # JDK 20+ Incubating Vector Module for SIMD optimizations; # disabling may reduce performance on vector optimized lucene -20:--add-modules=jdk.incubator.vector +20-:--add-modules=jdk.incubator.vector # HDFS ForkJoinPool.common() support by SecurityManager -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory diff --git a/libs/core/build.gradle b/libs/core/build.gradle index 1ab2073759788..46b6f4471731f 100644 --- a/libs/core/build.gradle +++ b/libs/core/build.gradle @@ -104,16 +104,23 @@ tasks.named('forbiddenApisMain').configure { tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( // from log4j - 'org.osgi.framework.Bundle', - 'org.osgi.framework.BundleActivator', - 'org.osgi.framework.BundleContext', - 'org.osgi.framework.BundleEvent', - 'org.osgi.framework.FrameworkUtil', - 'org.osgi.framework.ServiceReference', - 'org.osgi.framework.ServiceRegistration', - 'org.osgi.framework.SynchronousBundleListener', - 'org.osgi.framework.wiring.BundleWire', - 'org.osgi.framework.wiring.BundleWiring' + *[ + 'org.osgi.framework.Bundle', + 'org.osgi.framework.BundleActivator', + 'org.osgi.framework.BundleContext', + 'org.osgi.framework.BundleEvent', + 'org.osgi.framework.SynchronousBundleListener', + 'org.osgi.framework.wiring.BundleWire', + 'org.osgi.framework.wiring.BundleWiring', + ] + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [ + 'jdk.incubator.vector.ByteVector', + 'jdk.incubator.vector.FloatVector', + 'jdk.incubator.vector.IntVector', + 'jdk.incubator.vector.ShortVector', + 'jdk.incubator.vector.Vector', + 'jdk.incubator.vector.VectorOperators', + 'jdk.incubator.vector.VectorSpecies' + ] ) } diff --git a/server/build.gradle b/server/build.gradle index 1c2fe50834ca2..192729888c4a8 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -257,6 +257,7 @@ tasks.named("processResources").configure { tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( + *[ // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml) 'com.fasterxml.jackson.databind.ObjectMapper', @@ -385,7 +386,17 @@ tasks.named("thirdPartyAudit").configure { 'org.slf4j.Logger', 'org.slf4j.LoggerFactory', 'reactor.blockhound.BlockHound$Builder', - 'reactor.blockhound.integration.BlockHoundIntegration' + 'reactor.blockhound.integration.BlockHoundIntegration', + 'com.google.common.geometry.S2LatLng' + ] + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [ + 'jdk.incubator.vector.ByteVector', + 'jdk.incubator.vector.FloatVector', + 'jdk.incubator.vector.IntVector', + 'jdk.incubator.vector.ShortVector', + 'jdk.incubator.vector.Vector', + 'jdk.incubator.vector.VectorOperators', + 'jdk.incubator.vector.VectorSpecies' + ] ) ignoreViolations( 'com.google.protobuf.MessageSchema', From 492a17e4f26421bd7b34410423eb4d7c417171d3 Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Wed, 16 Nov 2022 10:43:00 -0800 Subject: [PATCH 2/4] Add condition to check JVM version for illegal-access switch (#5279) This switch is removed in JDK version 17. https://openjdk.org/jeps/403 Signed-off-by: Rabi Panda --- .../java/org/opensearch/gradle/OpenSearchTestBasePlugin.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java index 549e7f5c6d4df..ca2f657b65443 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java @@ -111,7 +111,9 @@ public void execute(Task t) { test.systemProperty("java.locale.providers", "SPI,JRE"); } else { test.systemProperty("java.locale.providers", "SPI,COMPAT"); - test.jvmArgs("--illegal-access=warn"); + if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) < 0) { + test.jvmArgs("--illegal-access=warn"); + } } } }); From 4a56213dff14f95ab050a49f9f869fd6e8c46044 Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Thu, 10 Nov 2022 13:48:41 -0800 Subject: [PATCH 3/4] Check for JDK version while setting java.security.manager option (#5205) Signed-off-by: Rabi Panda --- .../opensearch/gradle/OpenSearchTestBasePlugin.java | 3 +++ .../opensearch/tools/launchers/SystemJvmOptions.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java index ca2f657b65443..2ea8c2d015ecc 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java @@ -115,6 +115,9 @@ public void execute(Task t) { test.jvmArgs("--illegal-access=warn"); } } + if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) { + test.jvmArgs("-Djava.security.manager=allow"); + } } }); test.getJvmArgumentProviders().add(nonInputProperties); diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java index 62b2ded100cda..726c381db09f6 100644 --- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java @@ -77,12 +77,21 @@ static List systemJvmOptions() { // log4j 2 "-Dlog4j.shutdownHookEnabled=false", "-Dlog4j2.disable.jmx=true", - + // security manager + allowSecurityManagerOption(), javaLocaleProviders() ) ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } + private static String allowSecurityManagerOption() { + if (Runtime.version().feature() > 17) { + return "-Djava.security.manager=allow"; + } else { + return ""; + } + } + private static String maybeShowCodeDetailsInExceptionMessages() { if (Runtime.version().feature() >= 14) { return "-XX:+ShowCodeDetailsInExceptionMessages"; From cef267e3be99bfe242f49c1e37edc4f31611b32b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 30 Oct 2023 14:54:50 -0400 Subject: [PATCH 4/4] Fix tests and test skaffolding Signed-off-by: Andriy Redko --- .../discovery/ClusterManagerDisruptionIT.java | 3 ++ .../StableClusterManagerDisruptionIT.java | 4 ++ .../common/util/concurrent/BaseFuture.java | 4 +- .../MockSinglePrioritizingExecutor.java | 39 +++++++++++-------- .../disruption/LongGCDisruptionTests.java | 8 ++++ 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java index 9aee6f7f7a192..9bd924de5d9c7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java @@ -58,7 +58,9 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; +import static org.junit.Assume.assumeThat; /** * Tests relating to the loss of the cluster-manager. @@ -71,6 +73,7 @@ public class ClusterManagerDisruptionIT extends AbstractDisruptionTestCase { */ public void testClusterManagerNodeGCs() throws Exception { List nodes = startCluster(3); + assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20"))); String oldClusterManagerNode = internalCluster().getClusterManagerName(); // a very long GC, but it's OK as we remove the disruption when it has had an effect diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java index c12718704e194..f6f2b2dbfd096 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java @@ -71,6 +71,8 @@ import static java.util.Collections.singleton; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assume.assumeThat; /** * Tests relating to the loss of the cluster-manager, but which work with the default fault detection settings which are rather lenient and will @@ -195,6 +197,8 @@ private void testFollowerCheckerAfterClusterManagerReelection(NetworkLinkDisrupt * following another elected cluster-manager node. These nodes should reject this cluster state and prevent them from following the stale cluster-manager. */ public void testStaleClusterManagerNotHijackingMajority() throws Exception { + assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20"))); + final List nodes = internalCluster().startNodes( 3, Settings.builder() diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java b/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java index 47fc4fc33bd74..a423e5572d981 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java @@ -74,7 +74,7 @@ public abstract class BaseFuture implements Future { * * @throws InterruptedException if the current thread was interrupted before * or during the call (optional but recommended). - * @throws CancellationException {@inheritDoc} + * @throws CancellationException if the computation was cancelled */ @Override public V get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException, ExecutionException { @@ -96,7 +96,7 @@ public V get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutEx * * @throws InterruptedException if the current thread was interrupted before * or during the call (optional but recommended). - * @throws CancellationException {@inheritDoc} + * @throws CancellationException if the computation was cancelled */ @Override public V get() throws InterruptedException, ExecutionException { diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 10a6ec791ec66..f9b78b26c170e 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -44,25 +44,30 @@ public class MockSinglePrioritizingExecutor extends PrioritizedOpenSearchThreadPoolExecutor { public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) { - super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> new Thread() { - @Override - public void start() { - deterministicTaskQueue.scheduleNow(new Runnable() { - @Override - public void run() { - try { - r.run(); - } catch (KillWorkerError kwe) { - // hacks everywhere - } + super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> { + // This executor used to override Thread::start method so the actual runnable is + // being scheduled in the scope of current thread of execution. In JDK-19, the Thread::start + // is not called anymore (https://bugs.openjdk.org/browse/JDK-8292027) and there is no + // suitable option to alter the executor's behavior in the similar way. The closest we + // could get to is to schedule the runnable once the ThreadFactory is being asked to + // allocate the new thread. + deterministicTaskQueue.scheduleNow(new Runnable() { + @Override + public void run() { + try { + r.run(); + } catch (KillWorkerError kwe) { + // hacks everywhere } + } - @Override - public String toString() { - return r.toString(); - } - }); - } + @Override + public String toString() { + return r.toString(); + } + }); + + return new Thread(() -> {}); }, threadPool.getThreadContext(), threadPool.scheduler()); } diff --git a/test/framework/src/test/java/org/opensearch/test/disruption/LongGCDisruptionTests.java b/test/framework/src/test/java/org/opensearch/test/disruption/LongGCDisruptionTests.java index f545577ec5cc4..81292b22345a0 100644 --- a/test/framework/src/test/java/org/opensearch/test/disruption/LongGCDisruptionTests.java +++ b/test/framework/src/test/java/org/opensearch/test/disruption/LongGCDisruptionTests.java @@ -48,6 +48,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assume.assumeThat; public class LongGCDisruptionTests extends OpenSearchTestCase { @@ -65,6 +67,8 @@ public void executeLocked(Runnable r) { } public void testBlockingTimeout() throws Exception { + assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20"))); + final String nodeName = "test_node"; LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) { @Override @@ -125,6 +129,8 @@ protected long getSuspendingTimeoutInMillis() { * but does keep retrying until all threads can be safely paused */ public void testNotBlockingUnsafeStackTraces() throws Exception { + assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20"))); + final String nodeName = "test_node"; LongGCDisruption disruption = new LongGCDisruption(random(), nodeName) { @Override @@ -179,6 +185,8 @@ protected Pattern[] getUnsafeClasses() { } public void testBlockDetection() throws Exception { + assumeThat("Thread::resume / Thread::suspend are not supported anymore", Runtime.version(), lessThan(Runtime.Version.parse("20"))); + final String disruptedNodeName = "disrupted_node"; final String blockedNodeName = "blocked_node"; CountDownLatch waitForBlockDetectionResult = new CountDownLatch(1);