Skip to content

Commit

Permalink
Parallelise some tasks (#1434)
Browse files Browse the repository at this point in the history
`createLaunchConfig`, `mergeDiagnosticsJson`, `createInitScript` and `createCheckScript` can all now run in parallel within the same project.
  • Loading branch information
CRogers authored Feb 13, 2023
1 parent 8fe6ae6 commit 85749ad
Show file tree
Hide file tree
Showing 18 changed files with 533 additions and 494 deletions.
6 changes: 6 additions & 0 deletions changelog/@unreleased/pr-1434.v2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
type: fix
fix:
description: '`createLaunchConfig`, `mergeDiagnosticsJson`, `createInitScript` and
`createCheckScript` can all now run in parallel within the same project.'
links:
- https://github.com/palantir/sls-packaging/pull/1434
2 changes: 2 additions & 0 deletions gradle-sls-packaging/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ dependencies {

compileOnly 'org.immutables:value::annotations'
annotationProcessor 'org.immutables:value'
compileOnly 'com.palantir.gradle.auto-parallelizable:auto-parallelizable-annotations'
annotationProcessor 'com.palantir.gradle.auto-parallelizable:auto-parallelizable'

testImplementation gradleTestKit()
testImplementation 'com.netflix.nebula:nebula-test'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,34 +21,24 @@
import com.palantir.gradle.dist.ObjectMappers;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public final class Diagnostics {
private static final Logger log = LoggerFactory.getLogger(Diagnostics.class);
private static final String EXAMPLE =
"[{\"type\":\"foo.v1\", \"docs\":\"...\"}, \"{\"type\":\"bar.v1\", " + "\"docs\":\"...\"}]";

public static List<ObjectNode> parse(Project proj, File file) {
Path relativePath = proj.getRootDir().toPath().relativize(file.toPath());
public static List<ObjectNode> parse(File file) {
String string = null;
try {
string = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8).trim();
List<ObjectNode> value =
ObjectMappers.jsonMapper.readValue(string, new TypeReference<List<ObjectNode>>() {});
log.info("Deserialized '{}': '{}'", relativePath, value);
return value;
string = Files.readString(file.toPath()).trim();
return ObjectMappers.jsonMapper.readValue(string, new TypeReference<>() {});
} catch (IOException e) {
throw new GradleException(
String.format(
"Failed to deserialize '%s', expecting something like '%s' but was '%s'",
relativePath, EXAMPLE, string),
file.getAbsolutePath(), EXAMPLE, string),
e);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void apply(Project project) {
"diagnostics",
mergeDiagnosticsTask
.flatMap(MergeDiagnosticsJsonTask::getOutputJsonFile)
.map(file -> Diagnostics.parse(project, file.getAsFile())));
.map(file -> Diagnostics.parse(file.getAsFile())));
});
project.getTasks().named("createManifest", CreateManifestTask.class).configure(createManifestTask -> {
createManifestTask.dependsOn(mergeDiagnosticsJson);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* (c) Copyright 2020 Palantir Technologies Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.gradle.dist.service;

import com.fasterxml.jackson.databind.node.ObjectNode;
import com.palantir.gradle.autoparallelizable.AutoParallelizable;
import com.palantir.gradle.dist.ObjectMappers;
import java.io.File;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.gradle.api.GradleException;
import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.PathSensitive;
import org.gradle.api.tasks.PathSensitivity;

@AutoParallelizable
final class MergeDiagnosticsJson {
interface Params {
@InputFiles
@PathSensitive(PathSensitivity.NONE)
ConfigurableFileCollection getClasspath();

@OutputFile
RegularFileProperty getOutputJsonFile();
}

static void action(Params params) {
List<ObjectNode> aggregated = params.getClasspath().getFiles().stream()
.flatMap(file -> Diagnostics.parse(file).stream())
.distinct()
.sorted(Comparator.comparing(node -> node.get("type").asText()))
.collect(Collectors.toList());

File out = params.getOutputJsonFile().getAsFile().get();
try {
ObjectMappers.jsonMapper.writeValue(out, aggregated);
} catch (IOException e) {
throw new GradleException("Failed to write " + out, e);
}
}

private MergeDiagnosticsJson() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,47 +16,7 @@

package com.palantir.gradle.dist.service;

import com.fasterxml.jackson.databind.node.ObjectNode;
import com.palantir.gradle.dist.ObjectMappers;
import java.io.File;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.tasks.CacheableTask;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.PathSensitive;
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.TaskAction;

@CacheableTask
public abstract class MergeDiagnosticsJsonTask extends DefaultTask {

@InputFiles
@PathSensitive(PathSensitivity.NONE)
public abstract ConfigurableFileCollection getClasspath();

@OutputFile
public abstract RegularFileProperty getOutputJsonFile();

@TaskAction
public final void taskAction() {
List<ObjectNode> aggregated = getClasspath().getFiles().stream()
.flatMap(file -> Diagnostics.parse(getProject(), file).stream())
.distinct()
.sorted(Comparator.comparing(node -> node.get("type").asText()))
.collect(Collectors.toList());

File out = getOutputJsonFile().getAsFile().get();
try {
ObjectMappers.jsonMapper.writeValue(out, aggregated);
} catch (IOException e) {
throw new GradleException("Failed to write " + out, e);
}
}
}
public abstract class MergeDiagnosticsJsonTask extends MergeDiagnosticsJsonTaskImpl {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* (c) Copyright 2016 Palantir Technologies Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.gradle.dist.service.tasks;

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.palantir.gradle.autoparallelizable.AutoParallelizable;
import com.palantir.gradle.dist.service.JavaServiceDistributionPlugin;
import com.palantir.gradle.dist.service.util.EmitFiles;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.ListProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFile;

@AutoParallelizable
public final class CreateCheckScript {
interface Params {
@Input
Property<String> getServiceName();

@Input
ListProperty<String> getCheckArgs();

@OutputFile
RegularFileProperty getOutputFile();
}

static void action(Params params) {
if (!params.getCheckArgs().get().isEmpty()) {
EmitFiles.replaceVars(
JavaServiceDistributionPlugin.class.getResourceAsStream("/check.sh"),
params.getOutputFile().get().getAsFile().toPath(),
ImmutableMap.of(
"@serviceName@", params.getServiceName().get(),
"@checkArgs@",
Joiner.on(" ")
.join(params.getCheckArgs().get())))
.toFile()
.setExecutable(true);
}
}

private CreateCheckScript() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,54 +16,8 @@

package com.palantir.gradle.dist.service.tasks;

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.palantir.gradle.dist.service.JavaServiceDistributionPlugin;
import com.palantir.gradle.dist.service.util.EmitFiles;
import java.io.IOException;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.ListProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;

public class CreateCheckScriptTask extends DefaultTask {
private final Property<String> serviceName = getProject().getObjects().property(String.class);
private final ListProperty<String> checkArgs = getProject().getObjects().listProperty(String.class);
private final RegularFileProperty outputFile = getProject().getObjects().fileProperty();

public abstract class CreateCheckScriptTask extends CreateCheckScriptTaskImpl {
public CreateCheckScriptTask() {
outputFile.set(getProject().getLayout().getBuildDirectory().file("monitoring/check.sh"));
}

@Input
public final Property<String> getServiceName() {
return serviceName;
}

@Input
public final ListProperty<String> getCheckArgs() {
return checkArgs;
}

@OutputFile
public final RegularFileProperty getOutputFile() {
return outputFile;
}

@TaskAction
final void createInitScript() throws IOException {
if (!checkArgs.get().isEmpty()) {
EmitFiles.replaceVars(
JavaServiceDistributionPlugin.class.getResourceAsStream("/check.sh"),
getOutputFile().get().getAsFile().toPath(),
ImmutableMap.of(
"@serviceName@", serviceName.get(),
"@checkArgs@", Joiner.on(" ").join(checkArgs.get())))
.toFile()
.setExecutable(true);
}
getOutputFile().set(getProject().getLayout().getBuildDirectory().file("monitoring/check.sh"));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* (c) Copyright 2016 Palantir Technologies Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.gradle.dist.service.tasks;

import com.google.common.collect.ImmutableMap;
import com.palantir.gradle.autoparallelizable.AutoParallelizable;
import com.palantir.gradle.dist.service.JavaServiceDistributionPlugin;
import com.palantir.gradle.dist.service.util.EmitFiles;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFile;

@AutoParallelizable
public final class CreateInitScript {
interface Params {
@Input
Property<String> getServiceName();

@OutputFile
RegularFileProperty getOutputFile();
}

static void action(Params params) {
EmitFiles.replaceVars(
JavaServiceDistributionPlugin.class.getResourceAsStream("/init.sh"),
params.getOutputFile().get().getAsFile().toPath(),
ImmutableMap.of("@serviceName@", params.getServiceName().get()))
.toFile()
.setExecutable(true);
}

private CreateInitScript() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,42 +16,8 @@

package com.palantir.gradle.dist.service.tasks;

import com.google.common.collect.ImmutableMap;
import com.palantir.gradle.dist.service.JavaServiceDistributionPlugin;
import com.palantir.gradle.dist.service.util.EmitFiles;
import java.io.IOException;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;

public class CreateInitScriptTask extends DefaultTask {
private final Property<String> serviceName = getProject().getObjects().property(String.class);
private final RegularFileProperty outputFile = getProject().getObjects().fileProperty();

public abstract class CreateInitScriptTask extends CreateInitScriptTaskImpl {
public CreateInitScriptTask() {
outputFile.set(getProject().getLayout().getBuildDirectory().file("scripts/init.sh"));
}

@Input
public final Property<String> getServiceName() {
return serviceName;
}

@OutputFile
public final RegularFileProperty getOutputFile() {
return outputFile;
}

@TaskAction
final void createInitScript() throws IOException {
EmitFiles.replaceVars(
JavaServiceDistributionPlugin.class.getResourceAsStream("/init.sh"),
getOutputFile().get().getAsFile().toPath(),
ImmutableMap.of("@serviceName@", serviceName.get()))
.toFile()
.setExecutable(true);
getOutputFile().set(getProject().getLayout().getBuildDirectory().file("scripts/init.sh"));
}
}
Loading

0 comments on commit 85749ad

Please sign in to comment.