diff --git a/changelog/@unreleased/pr-1510.v2.yml b/changelog/@unreleased/pr-1510.v2.yml new file mode 100644 index 000000000..f3332ae9e --- /dev/null +++ b/changelog/@unreleased/pr-1510.v2.yml @@ -0,0 +1,6 @@ +type: feature +feature: + description: Add `jdks` property to `distribution` extension to help include JDKs + into dists. + links: + - https://github.com/palantir/sls-packaging/pull/1510 diff --git a/gradle-sls-packaging/build.gradle b/gradle-sls-packaging/build.gradle index a1095c8a5..49253c17c 100644 --- a/gradle-sls-packaging/build.gradle +++ b/gradle-sls-packaging/build.gradle @@ -31,6 +31,7 @@ dependencies { testImplementation 'com.netflix.nebula:nebula-test' testImplementation 'org.awaitility:awaitility' testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.rauschig:jarchivelib' testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' } diff --git a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/SlsBaseDistPlugin.java b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/SlsBaseDistPlugin.java index e7d599462..f58c21c50 100644 --- a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/SlsBaseDistPlugin.java +++ b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/SlsBaseDistPlugin.java @@ -31,7 +31,7 @@ public class SlsBaseDistPlugin implements Plugin { public static final String SLS_DIST_USAGE = "sls-dist"; - public static final GradleVersion MINIMUM_GRADLE = GradleVersion.version("6.1"); + public static final GradleVersion MINIMUM_GRADLE = GradleVersion.version("7.6"); @Override public final void apply(Project project) { diff --git a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/DistTarTask.java b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/DistTarTask.java index 09566949c..c0607f43f 100644 --- a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/DistTarTask.java +++ b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/DistTarTask.java @@ -16,7 +16,9 @@ package com.palantir.gradle.dist.service; +import java.util.Arrays; import java.util.concurrent.Callable; +import org.gradle.api.JavaVersion; import org.gradle.api.Project; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.provider.Provider; @@ -53,6 +55,21 @@ static void configure( t.setFileMode(0755); }); + // We do this trick of iterating through every java version and making a from with a lazy value to be lazy + // enough to handle the case where another plugin has set the value of the jdks property based on the result + // of resolving a configuration. Unfortunately, lots of our internal plugins/build.gradle force the value + // of the distTar task at configuration time, so this would cause a Configuration to resolved at + // configuration time (which is disallowed) with the naive getting the value from the property and looping + // over it. Reading the code below, you might be concerned that it would create empty directories for unset + // java versions, but Gradle does not appear to do this for empty file collections. + Arrays.stream(JavaVersion.values()).forEach(javaVersion -> { + root.from( + distributionExtension.getJdks().getting(javaVersion).orElse(project.provider(project::files)), + t -> { + t.into(distributionExtension.jdkPathInDist(javaVersion)); + }); + }); + root.into("service/lib", t -> { t.from(jarTask); t.from(project.getConfigurations().named("runtimeClasspath")); diff --git a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionExtension.java b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionExtension.java index fd28280d9..969060439 100644 --- a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionExtension.java +++ b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionExtension.java @@ -23,6 +23,7 @@ import groovy.lang.DelegatesTo; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.annotation.Nullable; import javax.inject.Inject; import org.gradle.api.Action; @@ -48,6 +49,7 @@ public class JavaServiceDistributionExtension extends BaseDistributionExtension private final ListProperty defaultJvmOpts; private final ListProperty excludeFromVar; private final MapProperty env; + private final MapProperty jdks; private final ObjectFactory objectFactory; @@ -63,7 +65,16 @@ public JavaServiceDistributionExtension(Project project) { .getTargetCompatibility())); mainClass = objectFactory.property(String.class); + jdks = objectFactory.mapProperty(JavaVersion.class, Object.class).empty(); + javaHome = objectFactory.property(String.class).value(javaVersion.map(javaVersionValue -> { + Optional possibleIncludedJdk = + Optional.ofNullable(jdks.getting(javaVersionValue).getOrNull()); + + if (possibleIncludedJdk.isPresent()) { + return jdkPathInDist(javaVersionValue); + } + boolean javaVersionLessThanOrEqualTo8 = javaVersionValue.compareTo(JavaVersion.VERSION_1_8) <= 0; if (javaVersionLessThanOrEqualTo8) { return ""; @@ -85,7 +96,7 @@ public JavaServiceDistributionExtension(Project project) { excludeFromVar = objectFactory.listProperty(String.class); excludeFromVar.addAll("log", "run"); - env = objectFactory.mapProperty(String.class, String.class).empty(); + env = objectFactory.mapProperty(String.class, String.class); setProductType(ProductType.SERVICE_V1); } @@ -213,6 +224,10 @@ public final void setEnv(Map env) { this.env.set(env); } + public final MapProperty getJdks() { + return jdks; + } + public final Provider getGc() { return gc; } @@ -249,4 +264,20 @@ private static GcProfile getDefaultGcProfile(JavaVersion javaVersion) { } return new GcProfile.Throughput(); } + + final String jdkPathInDist(JavaVersion javaVersionValue) { + // We put the JDK in a directory that contains the name and version of service. This is because in our cloud + // environments (and some customer environments), there is a third party security scanning tool that will report + // vulnerabilities in the JDK by printing a path, but does not display symlinks. This means it's hard to tell + // from a scan report which service is actually vulnerable, as our internal deployment infra uses symlinks, + // and you end up with a report like so: + // Path: /opt/palantir/services/.24710105/service/jdk17 + // rather than more useful: + // Path: /opt/palantir/services/.24710105/service/multipass-2.1.3-jdks/jdk17 + // which is implemented below. + + return String.format( + "service/%s-%s-jdks/jdk%s", + getDistributionServiceName().get(), project.getVersion(), javaVersionValue.getMajorVersion()); + } } diff --git a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionPlugin.java b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionPlugin.java index 2ea336d5c..9384b4c42 100644 --- a/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionPlugin.java +++ b/gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionPlugin.java @@ -35,6 +35,8 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.stream.Collectors; import org.gradle.api.Action; import org.gradle.api.InvalidUserCodeException; @@ -222,7 +224,7 @@ public void execute(Task _task) { task.getAddJava8GcLogging().set(distributionExtension.getAddJava8GcLogging()); task.getJavaHome().set(distributionExtension.getJavaHome()); task.getJavaVersion().set(distributionExtension.getJavaVersion()); - task.getEnv().set(distributionExtension.getEnv()); + task.getEnv().set(userConfiguredEnvWithJdkEnvVars(distributionExtension)); }); TaskProvider initScript = project.getTasks() @@ -323,6 +325,22 @@ public void execute(Task _task) { project.getArtifacts().add(SlsBaseDistPlugin.SLS_CONFIGURATION_NAME, distTar); } + private static Provider> userConfiguredEnvWithJdkEnvVars( + JavaServiceDistributionExtension distributionExtension) { + + return distributionExtension.getEnv().zip(distributionExtension.getJdks(), (userConfiguredEnv, jdks) -> { + Map actualEnv = new LinkedHashMap<>(userConfiguredEnv); + + jdks.keySet().stream().sorted().forEach(javaVersion -> { + actualEnv.put( + "JAVA_" + javaVersion.getMajorVersion() + "_HOME", + distributionExtension.jdkPathInDist(javaVersion)); + }); + + return Collections.unmodifiableMap(actualEnv); + }); + } + private static void replaceManifestClasspath(Path windowsScript, String manifestClassPathArchiveFileName) throws IOException { // Replace standard classpath with pathing jar in order to circumnavigate length limits: diff --git a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/Versions.java b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/Versions.java index 7a56bcf68..c5fcd52f6 100644 --- a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/Versions.java +++ b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/Versions.java @@ -18,7 +18,7 @@ public final class Versions { - public static final String GRADLE_CONSISTENT_VERSIONS = "1.27.0"; + public static final String GRADLE_CONSISTENT_VERSIONS = "2.15.0"; private Versions() {} } diff --git a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JavaServiceDistributionPluginTests.groovy b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JavaServiceDistributionPluginTests.groovy index 75d2609a6..105229125 100644 --- a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JavaServiceDistributionPluginTests.groovy +++ b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JavaServiceDistributionPluginTests.groovy @@ -179,7 +179,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } @@ -219,7 +218,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } @@ -599,7 +597,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } distribution { @@ -633,7 +630,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } version '0.0.1' @@ -678,7 +674,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } version '0.0.1' @@ -797,7 +792,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { args "hello" } repositories { - jcenter() mavenCentral() } dependencies { @@ -813,7 +807,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { id 'java-library' } repositories { - jcenter() mavenCentral() } dependencies { @@ -932,7 +925,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { enableManifestClasspath true } repositories { - jcenter() mavenCentral() } dependencies { @@ -948,7 +940,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { id 'java-library' } repositories { - jcenter() mavenCentral() } dependencies { @@ -1031,7 +1022,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } @@ -1066,7 +1056,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } @@ -1101,7 +1090,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } @@ -1385,7 +1373,6 @@ class JavaServiceDistributionPluginTests extends GradleIntegrationSpec { project.group = 'service-group' repositories { - jcenter() mavenCentral() } diff --git a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JdksInDistsIntegrationSpec.groovy b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JdksInDistsIntegrationSpec.groovy new file mode 100644 index 000000000..c69a50554 --- /dev/null +++ b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JdksInDistsIntegrationSpec.groovy @@ -0,0 +1,185 @@ +/* + * (c) Copyright 2023 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.gradle.dist.service + +import nebula.test.IntegrationSpec +import org.rauschig.jarchivelib.ArchiveFormat +import org.rauschig.jarchivelib.ArchiverFactory +import org.rauschig.jarchivelib.CompressionType + + +class JdksInDistsIntegrationSpec extends IntegrationSpec { + def setup() { + // language=gradle + settingsFile << ''' + rootProject.name = 'myService' + '''.stripIndent(true) + + // language=gradle + buildFile << ''' + apply plugin: 'java' + apply plugin: 'com.palantir.sls-java-service-distribution' + + repositories { + mavenCentral() + } + + group 'group' + version '1.0.0' + '''.stripIndent(true) + + file('versions.lock') + + // language=java + writeJavaSourceFile ''' + package app; + + import java.nio.file.Files; + import java.nio.file.Paths; + + public class Main { + public static void main(String... args) {} + } + '''.stripIndent(true) + + file('build/fake-jdk/release') << 'its a jdk trust me' + } + + def 'puts jdk in dist'() { + // language=gradle + buildFile << ''' + distribution { + javaVersion JavaVersion.VERSION_17 + jdks.put(JavaVersion.VERSION_17, fileTree('build/fake-jdk')) + } + '''.stripIndent(true) + + when: + runTasksSuccessfully('distTar') + + then: + def rootDir = extractDist() + def jdkDir = new File(rootDir, "service/myService-1.0.0-jdks/jdk17") + jdkDir.exists() + + def releaseFileText = new File(jdkDir, "release").text + + releaseFileText.contains 'its a jdk trust me' + + def launcherStatic = new File(rootDir, "service/bin/launcher-static.yml").text + launcherStatic.contains 'javaHome: "service/myService-1.0.0-jdks/jdk17"' + launcherStatic.contains ' JAVA_17_HOME: "service/myService-1.0.0-jdks/jdk17"' + } + + def 'multiple jdks can exist in the dist'() { + // language=gradle + buildFile << ''' + distribution { + javaVersion JavaVersion.VERSION_17 + jdks.put(JavaVersion.VERSION_11, fileTree('build/fake-jdk')) + jdks.put(JavaVersion.VERSION_13, fileTree('build/fake-jdk')) + jdks.put(JavaVersion.VERSION_17, fileTree('build/fake-jdk')) + } + '''.stripIndent(true) + + when: + runTasksSuccessfully('distTar') + + then: + def rootDir = extractDist() + + def launcherStatic = new File(rootDir, "service/bin/launcher-static.yml").text + launcherStatic.contains 'javaHome: "service/myService-1.0.0-jdks/jdk17"' + + for (version in [11, 13, 17]) { + def jdkDir = new File(rootDir, "service/myService-1.0.0-jdks/jdk" + version) + assert jdkDir.exists() + + def releaseFileText = new File(jdkDir, "release").text + + assert releaseFileText.contains('its a jdk trust me') + + def envVarLine = " JAVA_${version}_HOME: \"service/myService-1.0.0-jdks/jdk${version}\"" + assert launcherStatic.contains(envVarLine) + } + } + + def 'does not force value of jdks at configuration time when task is evaluated'() { + // language=gradle + buildFile << ''' + distribution { + javaVersion JavaVersion.VERSION_17 + jdks.putAll(provider { + println('hello ' + state.isConfiguring()) + if (state.isConfiguring()) { + throw new RuntimeException("Should not be called when configuring") + } + return Map.of(JavaVersion.VERSION_17, fileTree('build/fake-jdk')) + }) + } + + // Quite a lot of internal plugins/build.gradles unfortunately get the distTar task non-lazily. An internal + // piece of infra sets the jdks property by resolving a configuration, which cannot happen at configuration + // time. + tasks.getByName('distTar') + '''.stripIndent(true) + + when: + runTasksSuccessfully('distTar') + + then: + def rootDir = extractDist() + + // A way of fixing this tests seems to open up the possibility of making extra unnecessary JDK repos - ensure + // this does not happen. + !new File(rootDir, "service/myService-1.0.0-jdks/jdk11").exists() + } + + def 'even a user clearing env does not get rid of JAVA_XX_HOME env vars'() { + // language=gradle + buildFile << ''' + distribution { + javaVersion JavaVersion.VERSION_17 + jdks.put(JavaVersion.VERSION_17, fileTree('build/fake-jdk')) + jdks.put(JavaVersion.VERSION_11, fileTree('build/fake-jdk')) + + env.empty() + } + '''.stripIndent(true) + + when: + runTasksSuccessfully('distTar') + + then: + def rootDir = extractDist() + + def launcherStatic = new File(rootDir, "service/bin/launcher-static.yml").text + + launcherStatic.contains 'JAVA_11_HOME: "service/myService-1.0.0-jdks/jdk11"' + launcherStatic.contains 'JAVA_17_HOME: "service/myService-1.0.0-jdks/jdk17"' + } + + private File extractDist() { + def slsTgz = new File(projectDir, "build/distributions/myService-1.0.0.sls.tgz") + def extracted = new File(slsTgz.getParent(), "extracted") + + ArchiverFactory.createArchiver(ArchiveFormat.TAR, CompressionType.GZIP) + .extract(slsTgz, extracted) + + return new File(extracted, "myService-1.0.0") + } +} diff --git a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/MainClassInferenceIntegrationSpec.groovy b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/MainClassInferenceIntegrationSpec.groovy index 5fc19204f..e5a958fdd 100644 --- a/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/MainClassInferenceIntegrationSpec.groovy +++ b/gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/MainClassInferenceIntegrationSpec.groovy @@ -34,7 +34,6 @@ class MainClassInferenceIntegrationSpec extends GradleIntegrationSpec { } repositories { - jcenter() mavenCentral() } diff --git a/versions.lock b/versions.lock index 9c03567bf..e1a85c5c1 100644 --- a/versions.lock +++ b/versions.lock @@ -44,6 +44,7 @@ org.yaml:snakeyaml:2.0 (1 constraints: 3a178910) cglib:cglib-nodep:3.2.2 (1 constraints: 490ded24) com.netflix.nebula:nebula-test:10.0.0 (1 constraints: 3305273b) junit:junit:4.13.2 (3 constraints: a523497e) +org.apache.commons:commons-compress:1.21 (1 constraints: f70aebc8) org.apiguardian:apiguardian-api:1.1.2 (6 constraints: 896455cc) org.awaitility:awaitility:4.2.0 (1 constraints: 08050536) org.codehaus.groovy:groovy:3.0.6 (2 constraints: 1e1b476d) @@ -58,5 +59,6 @@ org.junit.platform:junit-platform-engine:1.10.0 (3 constraints: ac2e4dc4) org.junit.vintage:junit-vintage-engine:5.10.0 (1 constraints: 3805413b) org.objenesis:objenesis:2.4 (1 constraints: ea0c8c0a) org.opentest4j:opentest4j:1.3.0 (2 constraints: cf209249) +org.rauschig:jarchivelib:1.2.0 (1 constraints: 0505f635) org.spockframework:spock-core:2.0-M4-groovy-3.0 (2 constraints: e822d65a) org.spockframework:spock-junit4:2.0-M4-groovy-3.0 (1 constraints: 25115ddf) diff --git a/versions.props b/versions.props index cfe876614..c4f2c304d 100644 --- a/versions.props +++ b/versions.props @@ -9,6 +9,7 @@ org.apache.commons:commons-lang3 = 3.13.0 org.immutables:value = 2.9.3 org.junit.jupiter:* = 5.10.0 org.junit.vintage:* = 5.10.0 +org.rauschig:jarchivelib = 1.2.0 com.netflix.nebula:nebula-test = 10.0.0 junit:junit = 4.13.2