Include JDKs in Dists

changelog/@unreleased/pr-1510.v2.yml

@@ -0,0 +1,6 @@
+type: feature
+feature:
+  description: Add `jdks` property to `distribution` extension to help include JDKs
+    into dists.
+  links:
+  - https://github.com/palantir/sls-packaging/pull/1510

gradle-sls-packaging/build.gradle

@@ -31,6 +31,7 @@ dependencies {
     testImplementation 'com.netflix.nebula:nebula-test'
     testImplementation 'org.awaitility:awaitility'
     testImplementation 'org.junit.jupiter:junit-jupiter'
+    testImplementation 'org.rauschig:jarchivelib'
     testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
 }
 

gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/DistTarTask.java

@@ -16,7 +16,9 @@
 
 package com.palantir.gradle.dist.service;
 
+import java.util.Arrays;
 import java.util.concurrent.Callable;
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
 import org.gradle.api.file.DuplicatesStrategy;
 import org.gradle.api.provider.Provider;
@@ -53,6 +55,21 @@ static void configure(
                 t.setFileMode(0755);
             });
 
+            // We do this trick of iterating through every java version and making a from with a lazy value to be lazy
+            // enough to handle the case where another plugin has set the value of the jdks property based on the result
+            // of resolving a configuration. Unfortunately, lots of our internal plugins/build.gradle force the value
+            // of the distTar task at configuration time, so this would cause a Configuration to resolved at
+            // configuration time (which is disallowed) with the naive getting the value from the property and looping
+            // over it. Reading the code below, you might be concerned that it would create empty directories for unset
+            // java versions, but Gradle does not appear to do this for empty file collections.
+            Arrays.stream(JavaVersion.values()).forEach(javaVersion -> {
+                root.from(
+                        distributionExtension.getJdks().getting(javaVersion).orElse(project.provider(project::files)),
+                        t -> {
+                            t.into(distributionExtension.jdkPathInDist(javaVersion));
+                        });
+            });
+
             root.into("service/lib", t -> {
                 t.from(jarTask);
                 t.from(project.getConfigurations().named("runtimeClasspath"));

gradle-sls-packaging/src/main/java/com/palantir/gradle/dist/service/JavaServiceDistributionExtension.java

@@ -23,6 +23,8 @@
 import groovy.lang.DelegatesTo;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import org.gradle.api.Action;
@@ -48,6 +50,7 @@ public class JavaServiceDistributionExtension extends BaseDistributionExtension
     private final ListProperty<String> defaultJvmOpts;
     private final ListProperty<String> excludeFromVar;
     private final MapProperty<String, String> env;
+    private final MapProperty<JavaVersion, Object> jdks;
 
     private final ObjectFactory objectFactory;
 
@@ -63,7 +66,16 @@ public JavaServiceDistributionExtension(Project project) {
                 .getTargetCompatibility()));
         mainClass = objectFactory.property(String.class);
 
+        jdks = objectFactory.mapProperty(JavaVersion.class, Object.class).empty();
+
         javaHome = objectFactory.property(String.class).value(javaVersion.map(javaVersionValue -> {
+            Optional<Object> possibleIncludedJdk =
+                    Optional.ofNullable(jdks.getting(javaVersionValue).getOrNull());
+
+            if (possibleIncludedJdk.isPresent()) {
+                return jdkPathInDist(javaVersionValue);
+            }
+
             boolean javaVersionLessThanOrEqualTo8 = javaVersionValue.compareTo(JavaVersion.VERSION_1_8) <= 0;
             if (javaVersionLessThanOrEqualTo8) {
                 return "";
@@ -85,7 +97,13 @@ public JavaServiceDistributionExtension(Project project) {
         excludeFromVar = objectFactory.listProperty(String.class);
         excludeFromVar.addAll("log", "run");
 
-        env = objectFactory.mapProperty(String.class, String.class).empty();
+        env = objectFactory.mapProperty(String.class, String.class);
+        env.putAll(project.provider(() -> {
+            return jdks.get().entrySet().stream()
+                    .collect(Collectors.toMap(
+                            entry -> "JAVA_" + entry.getKey().getMajorVersion() + "_HOME",
+                            entry -> jdkPathInDist(entry.getKey())));
+        }));
         setProductType(ProductType.SERVICE_V1);
     }
 
@@ -213,6 +231,10 @@ public final void setEnv(Map<String, String> env) {
         this.env.set(env);
     }
 
+    public final MapProperty<JavaVersion, Object> getJdks() {
+        return jdks;
+    }
+
     public final Provider<GcProfile> getGc() {
         return gc;
     }
@@ -249,4 +271,20 @@ private static GcProfile getDefaultGcProfile(JavaVersion javaVersion) {
         }
         return new GcProfile.Throughput();
     }
+
+    final String jdkPathInDist(JavaVersion javaVersionValue) {
+        // We put the JDK in a directory that contains the name and version of service. This is because in our cloud
+        // environments (and some customer environments), there is a third party security scanning tool that will report
+        // vulnerabilities in the JDK by printing a path, but does not display symlinks. This means it's hard to tell
+        // from a scan report which service is actually vulnerable, as our internal deployment infra uses symlinks,
+        // and you end up with a report like so:
+        //      Path: /opt/palantir/services/.24710105/service/jdk17
+        // rather than more useful:
+        //      Path: /opt/palantir/services/.24710105/service/multipass-2.1.3-jdks/jdk17
+        // which is implemented below.
+
+        return String.format(
+                "service/%s-%s-jdks/jdk%s",
+                getDistributionServiceName().get(), project.getVersion(), javaVersionValue.getMajorVersion());
+    }
 }

gradle-sls-packaging/src/test/groovy/com/palantir/gradle/dist/service/JdksInDistsIntegrationSpec.groovy

@@ -0,0 +1,161 @@
+/*
+ * (c) Copyright 2023 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.gradle.dist.service
+
+import nebula.test.IntegrationSpec
+import org.rauschig.jarchivelib.ArchiveFormat
+import org.rauschig.jarchivelib.ArchiverFactory
+import org.rauschig.jarchivelib.CompressionType
+
+
+class JdksInDistsIntegrationSpec extends IntegrationSpec {
+    def setup() {
+        // language=gradle
+        settingsFile << '''
+            rootProject.name = 'myService'
+        '''.stripIndent(true)
+
+        // language=gradle
+        buildFile << '''
+            apply plugin: 'java'
+            apply plugin: 'com.palantir.sls-java-service-distribution'
+
+            repositories {
+                mavenCentral()
+            }
+
+            group 'group'
+            version '1.0.0'
+        '''.stripIndent(true)
+
+        file('versions.lock')
+
+        // language=java
+        writeJavaSourceFile '''
+            package app;
+
+            import java.nio.file.Files;
+            import java.nio.file.Paths;
+
+            public class Main {
+                public static void main(String... args) {}
+            }
+        '''.stripIndent(true)
+
+        file('build/fake-jdk/release') << 'its a jdk trust me'
+    }
+
+    def 'puts jdk in dist'() {
+        // language=gradle
+        buildFile << '''
+            distribution {
+                javaVersion JavaVersion.VERSION_17
+                jdks.put(JavaVersion.VERSION_17, fileTree('build/fake-jdk'))
+            }
+        '''.stripIndent(true)
+
+        when:
+        runTasksSuccessfully('distTar')
+
+        then:
+        def rootDir = extractDist()
+        def jdkDir = new File(rootDir, "service/myService-1.0.0-jdks/jdk17")
+        jdkDir.exists()
+
+        def releaseFileText = new File(jdkDir, "release").text
+
+        releaseFileText.contains 'its a jdk trust me'
+
+        def launcherStatic = new File(rootDir, "service/bin/launcher-static.yml").text
+        launcherStatic.contains 'javaHome: "service/myService-1.0.0-jdks/jdk17"'
+        launcherStatic.contains '  JAVA_17_HOME: "service/myService-1.0.0-jdks/jdk17"'
+    }
+
+    def 'multiple jdks can exist in the dist'() {
+        // language=gradle
+        buildFile << '''
+            distribution {
+                javaVersion JavaVersion.VERSION_17
+                jdks.put(JavaVersion.VERSION_11, fileTree('build/fake-jdk'))
+                jdks.put(JavaVersion.VERSION_13, fileTree('build/fake-jdk'))
+                jdks.put(JavaVersion.VERSION_17, fileTree('build/fake-jdk'))
+            }
+        '''.stripIndent(true)
+
+        when:
+        runTasksSuccessfully('distTar')
+
+        then:
+        def rootDir = extractDist()
+
+        def launcherStatic = new File(rootDir, "service/bin/launcher-static.yml").text
+        launcherStatic.contains 'javaHome: "service/myService-1.0.0-jdks/jdk17"'
+
+        for (version in [11, 13, 17]) {
+            def jdkDir = new File(rootDir, "service/myService-1.0.0-jdks/jdk" + version)
+            assert jdkDir.exists()
+
+            def releaseFileText = new File(jdkDir, "release").text
+
+            assert releaseFileText.contains('its a jdk trust me')
+
+            def envVarLine = "  JAVA_${version}_HOME: \"service/myService-1.0.0-jdks/jdk${version}\""
+            assert launcherStatic.contains(envVarLine)
+        }
+    }
+
+    def 'does not force value of jdks at configuration time when task is evaluated'() {
+        // language=gradle
+        buildFile << '''
+            distribution {
+                javaVersion JavaVersion.VERSION_17
+                jdks.putAll(provider {
+                    println('hello ' + state.isConfiguring())
+                    if (state.isConfiguring()) {
+                        throw new RuntimeException("Should not be called when configuring")
+                    }
+                    return Map.of(JavaVersion.VERSION_17, fileTree('build/fake-jdk'))
+                })
+            }
+
+            // Quite a lot of internal plugins/build.gradles unfortunately get the distTar task non-lazily. An internal
+            // piece of infra sets the jks property by resolving a configuration, which cannot happen at configuration
+            // time.
+            tasks.getByName('distTar')
+        '''.stripIndent(true)
+
+        when:
+        runTasksSuccessfully('distTar')
+
+        then:
+        def rootDir = extractDist()
+
+        // A way of fixing this tests seems to open up the possibility of making extra unnecessary JDK repos - ensure
+        // this does not happen.
+        !new File(rootDir, "service/myService-1.0.0-jdks/jdk11").exists()
+    }
+
+    private File extractDist() {
+        def slsTgz = new File(projectDir, "build/distributions/myService-1.0.0.sls.tgz")
+        def extracted = new File(slsTgz.getParent(), "extracted")
+
+        ArchiverFactory.createArchiver(ArchiveFormat.TAR, CompressionType.GZIP)
+                .extract(slsTgz, extracted)
+
+        return new File(extracted, "myService-1.0.0")
+    }
+}

versions.lock

@@ -44,6 +44,7 @@ org.yaml:snakeyaml:2.0 (1 constraints: 3a178910)
 cglib:cglib-nodep:3.2.2 (1 constraints: 490ded24)
 com.netflix.nebula:nebula-test:10.0.0 (1 constraints: 3305273b)
 junit:junit:4.13.2 (3 constraints: a523497e)
+org.apache.commons:commons-compress:1.21 (1 constraints: f70aebc8)
 org.apiguardian:apiguardian-api:1.1.2 (6 constraints: 896455cc)
 org.awaitility:awaitility:4.2.0 (1 constraints: 08050536)
 org.codehaus.groovy:groovy:3.0.6 (2 constraints: 1e1b476d)
@@ -58,5 +59,6 @@ org.junit.platform:junit-platform-engine:1.10.0 (3 constraints: ac2e4dc4)
 org.junit.vintage:junit-vintage-engine:5.10.0 (1 constraints: 3805413b)
 org.objenesis:objenesis:2.4 (1 constraints: ea0c8c0a)
 org.opentest4j:opentest4j:1.3.0 (2 constraints: cf209249)
+org.rauschig:jarchivelib:1.2.0 (1 constraints: 0505f635)
 org.spockframework:spock-core:2.0-M4-groovy-3.0 (2 constraints: e822d65a)
 org.spockframework:spock-junit4:2.0-M4-groovy-3.0 (1 constraints: 25115ddf)

versions.props

@@ -9,6 +9,7 @@ org.apache.commons:commons-lang3 = 3.13.0
 org.immutables:value = 2.9.3
 org.junit.jupiter:* = 5.10.0
 org.junit.vintage:* = 5.10.0
+org.rauschig:jarchivelib = 1.2.0
 
 com.netflix.nebula:nebula-test = 10.0.0
 junit:junit = 4.13.2