Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement streaming for source file parsing #576

Merged
merged 1 commit into from
Jun 6, 2023
Merged

Implement streaming for source file parsing #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 75 additions & 76 deletions src/main/java/org/openrewrite/maven/AbstractRewriteMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
package org.openrewrite.maven;

import com.puppycrawl.tools.checkstyle.Checker;
import io.micrometer.core.instrument.Metrics;
import org.apache.maven.artifact.DependencyResolutionRequiredException;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.model.Plugin;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Component;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import org.apache.maven.rtinfo.RuntimeInformation;
import org.apache.maven.settings.crypto.SettingsDecrypter;
import org.codehaus.plexus.classworlds.realm.ClassRealm;
import org.codehaus.plexus.util.xml.Xpp3Dom;
import org.eclipse.aether.RepositorySystem;
import org.eclipse.aether.artifact.Artifact;
import org.openrewrite.*;
Expand All @@ -21,10 +18,11 @@
import org.openrewrite.config.RecipeDescriptor;
import org.openrewrite.config.YamlResourceLoader;
import org.openrewrite.internal.InMemoryLargeSourceSet;
import org.openrewrite.internal.ListUtils;
import org.openrewrite.internal.lang.Nullable;
import org.openrewrite.ipc.http.HttpSender;
import org.openrewrite.ipc.http.HttpUrlConnectionSender;
import org.openrewrite.java.style.CheckstyleConfigLoader;
import org.openrewrite.java.tree.JavaSourceFile;
import org.openrewrite.marker.*;
import org.openrewrite.style.NamedStyles;
import org.openrewrite.xml.tree.Xml;
Expand All @@ -44,7 +42,6 @@
import java.util.stream.Stream;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;

Expand Down Expand Up @@ -208,37 +205,6 @@ protected ResultsContainer listResults() throws MojoExecutionException {
}
Environment env = environment(recipeArtifactCoordinatesClassloader);

List<NamedStyles> styles;
styles = env.activateStyles(getActiveStyles());
try {
Plugin checkstylePlugin = project.getPlugin("org.apache.maven.plugins:maven-checkstyle-plugin");
if (checkstyleConfigFile != null && !checkstyleConfigFile.isEmpty()) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(Paths.get(checkstyleConfigFile), emptyMap()));
} else if (checkstyleDetectionEnabled && checkstylePlugin != null) {
Object checkstyleConfRaw = checkstylePlugin.getConfiguration();
if (checkstyleConfRaw instanceof Xpp3Dom) {
Xpp3Dom xmlCheckstyleConf = (Xpp3Dom) checkstyleConfRaw;
Xpp3Dom xmlConfigLocation = xmlCheckstyleConf.getChild("configLocation");

if (xmlConfigLocation == null) {
// When no config location is specified, the maven-checkstyle-plugin falls back on sun_checks.xml
try (InputStream is = Checker.class.getResourceAsStream("/sun_checks.xml")) {
if (is != null) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(is, emptyMap()));
}
}
} else {
Path configPath = Paths.get(xmlConfigLocation.getValue());
if (configPath.toFile().exists()) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(configPath, emptyMap()));
}
}
}
}
} catch (Exception e) {
getLog().warn("Unable to parse checkstyle configuration. Checkstyle will not inform rewrite execution.", e);
}

Recipe recipe = env.activateRecipes(getActiveRecipes());
if (recipe.getRecipeList().isEmpty()) {
getLog().warn("No recipes were activated. " +
Expand All @@ -261,37 +227,11 @@ protected ResultsContainer listResults() throws MojoExecutionException {
getLog().error("Recipe validation errors detected as part of one or more activeRecipe(s). Execution will continue regardless.");
}
}
ExecutionContext ctx = executionContext();

//Parse and collect source files from each project in the maven session.
MavenMojoProjectParser projectParser = new MavenMojoProjectParser(getLog(), repositoryRoot, pomCacheEnabled, pomCacheDirectory, runtime, skipMavenParsing, getExclusions(), getPlainTextMasks(), sizeThresholdMb, mavenSession, settingsDecrypter);

List<SourceFile> sourceFiles = new ArrayList<>();
if (runPerSubmodule) {
//If running per submodule, parse the source files for only the current project.
projectParser.resetTypeCache();
sourceFiles.addAll(projectParser.listSourceFiles(project, styles, ctx));
} else {
//If running across all project, iterate and parse source files from each project
Map<MavenProject, List<Marker>> projectProvenances = mavenSession.getProjects().stream()
.collect(Collectors.toMap(Function.identity(), projectParser::generateProvenance));
Map<MavenProject, Xml.Document> projectMap = projectParser.parseMaven(mavenSession.getProjects(), projectProvenances, ctx);
for (MavenProject mavenProject : mavenSession.getProjects()) {
List<Marker> projectProvenance = projectProvenances.get(mavenProject);
sourceFiles.addAll(projectParser.listSourceFiles(mavenProject, projectMap.get(mavenProject), projectProvenance, styles, ctx));
}
}
ExecutionContext ctx = executionContext();
LargeSourceSet sourceSet = loadSourceSet(repositoryRoot, env, ctx);

getLog().info("Running recipe(s)...");
List<Result> results = recipe.run(new InMemoryLargeSourceSet(sourceFiles), ctx).getChangeset().getAllResults().stream()
.filter(source -> {
// Remove ASTs originating from generated files
if (source.getBefore() != null) {
return !source.getBefore().getMarkers().findFirst(Generated.class).isPresent();
}
return true;
})
.collect(toList());
List<Result> results = runRecipe(recipe, sourceSet, ctx);

Metrics.removeRegistry(meterRegistryProvider.registry());

Expand All @@ -301,6 +241,68 @@ protected ResultsContainer listResults() throws MojoExecutionException {
}
}

protected LargeSourceSet loadSourceSet(Path repositoryRoot, Environment env, ExecutionContext ctx) throws DependencyResolutionRequiredException, MojoExecutionException {
List<NamedStyles> styles = loadStyles(project, env);

//Parse and collect source files from each project in the maven session.
MavenMojoProjectParser projectParser = new MavenMojoProjectParser(getLog(), repositoryRoot, pomCacheEnabled, pomCacheDirectory, runtime, skipMavenParsing, getExclusions(), getPlainTextMasks(), sizeThresholdMb, mavenSession, settingsDecrypter);

Stream<SourceFile> sourceFiles = Stream.empty();
if (runPerSubmodule) {
//If running per submodule, parse the source files for only the current project.
projectParser.resetTypeCache();
sourceFiles = Stream.concat(sourceFiles, projectParser.listSourceFiles(project, styles, ctx));
} else {
//If running across all project, iterate and parse source files from each project
Map<MavenProject, List<Marker>> projectProvenances = mavenSession.getProjects().stream()
.collect(Collectors.toMap(Function.identity(), projectParser::generateProvenance));
Map<MavenProject, Xml.Document> projectMap = projectParser.parseMaven(mavenSession.getProjects(), projectProvenances, ctx);
for (MavenProject mavenProject : mavenSession.getProjects()) {
List<Marker> projectProvenance = projectProvenances.get(mavenProject);
sourceFiles = Stream.concat(sourceFiles, projectParser.listSourceFiles(mavenProject, projectMap.get(mavenProject), projectProvenance, styles, ctx));
}
}

List<SourceFile> sourceFileList = sourcesWithAutoDetectedStyles(sourceFiles);
return new InMemoryLargeSourceSet(sourceFileList);
}

protected List<Result> runRecipe(Recipe recipe, LargeSourceSet sourceSet, ExecutionContext ctx) {
getLog().info("Running recipe(s)...");
return recipe.run(sourceSet, ctx).getChangeset().getAllResults().stream()
.filter(source -> {
// Remove ASTs originating from generated files
if (source.getBefore() != null) {
return !source.getBefore().getMarkers().findFirst(Generated.class).isPresent();
}
return true;
})
.collect(toList());
}

private List<SourceFile> sourcesWithAutoDetectedStyles(Stream<SourceFile> sourceFiles) {
org.openrewrite.java.style.Autodetect.Detector javaDetector = org.openrewrite.java.style.Autodetect.detect(sourceFiles);
org.openrewrite.xml.style.Autodetect.Detector xmlDetector = org.openrewrite.xml.style.Autodetect.detect(javaDetector);
List<SourceFile> sourceFileList = xmlDetector.collect(toList());

Map<Class<? extends Tree>, NamedStyles> stylesByType = new HashMap<>();
stylesByType.put(JavaSourceFile.class, javaDetector.build());
stylesByType.put(Xml.Document.class, xmlDetector.build());

return ListUtils.map(sourceFileList, applyAutodetectedStyle(stylesByType));
}

private UnaryOperator<SourceFile> applyAutodetectedStyle(Map<Class<? extends Tree>, NamedStyles> stylesByType) {
return before -> {
for (Map.Entry<Class<? extends Tree>, NamedStyles> styleTypeEntry : stylesByType.entrySet()) {
if (styleTypeEntry.getKey().isAssignableFrom(before.getClass())) {
before = before.withMarkers(before.getMarkers().add(styleTypeEntry.getValue()));
}
}
return before;
};
}

@Nullable
protected URLClassLoader getRecipeArtifactCoordinatesClassloader() throws MojoExecutionException {
if (getRecipeArtifactCoordinates().isEmpty()) {
Expand Down Expand Up @@ -411,18 +413,15 @@ public RuntimeException getFirstException() {
private List<RuntimeException> getRecipeErrors(Result result) {
List<RuntimeException> exceptions = new ArrayList<>();
new TreeVisitor<Tree, Integer>() {
@Nullable
@Override
public Tree visit(@Nullable Tree tree, Integer p) {
if (tree != null) {
Markers markers = tree.getMarkers();
markers.findFirst(Markup.Error.class).ifPresent(e -> {
Optional<SourceFile> sourceFile = Optional.ofNullable(getCursor().firstEnclosing(SourceFile.class));
String sourcePath = sourceFile.map(SourceFile::getSourcePath).map(Path::toString).orElse("<unknown>");
exceptions.add(new RuntimeException("Error while visiting " + sourcePath + ": " + e.getMessage()));
});
}
return super.visit(tree, p);
public Tree preVisit(Tree tree, Integer integer) {
Markers markers = tree.getMarkers();
markers.findFirst(Markup.Error.class).ifPresent(e -> {
Optional<SourceFile> sourceFile = Optional.ofNullable(getCursor().firstEnclosing(SourceFile.class));
String sourcePath = sourceFile.map(SourceFile::getSourcePath).map(Path::toString).orElse("<unknown>");
exceptions.add(new RuntimeException("Error while visiting " + sourcePath + ": " + e.getDetail()));
});
return tree;
}
}.visit(result.getAfter(), 0);
return exceptions;
Expand Down
47 changes: 47 additions & 0 deletions src/main/java/org/openrewrite/maven/ConfigurableRewriteMojo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,24 @@
package org.openrewrite.maven;

import com.puppycrawl.tools.checkstyle.Checker;
import org.apache.maven.model.Plugin;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import org.codehaus.plexus.util.xml.Xpp3Dom;
import org.openrewrite.config.Environment;
import org.openrewrite.internal.lang.Nullable;
import org.openrewrite.java.style.CheckstyleConfigLoader;
import org.openrewrite.style.NamedStyles;

import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;

import static java.util.Collections.emptyMap;

@SuppressWarnings("FieldMayBeFinal")
public abstract class ConfigurableRewriteMojo extends AbstractMojo {

Expand Down Expand Up @@ -185,6 +197,41 @@ protected Set<String> getActiveStyles() {
return computedStyles;
}

protected List<NamedStyles> loadStyles(MavenProject project, Environment env) {
List<NamedStyles> styles = env.activateStyles(getActiveStyles());
try {
Plugin checkstylePlugin = project.getPlugin("org.apache.maven.plugins:maven-checkstyle-plugin");
if (checkstyleConfigFile != null && !checkstyleConfigFile.isEmpty()) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(Paths.get(checkstyleConfigFile), emptyMap()));
} else if (checkstyleDetectionEnabled && checkstylePlugin != null) {
Object checkstyleConfRaw = checkstylePlugin.getConfiguration();
if (checkstyleConfRaw instanceof Xpp3Dom) {
Xpp3Dom xmlCheckstyleConf = (Xpp3Dom) checkstyleConfRaw;
Xpp3Dom xmlConfigLocation = xmlCheckstyleConf.getChild("configLocation");

if (xmlConfigLocation == null) {
// When no config location is specified, the maven-checkstyle-plugin falls back on sun_checks.xml
try (InputStream is = Checker.class.getResourceAsStream("/sun_checks.xml")) {
if (is != null) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(is, emptyMap()));
}
}
} else {
// resolve location against plugin location (could be in parent pom)
Path configPath = Paths.get(checkstylePlugin.getLocation("").getSource().getLocation())
.resolveSibling(xmlConfigLocation.getValue());
if (configPath.toFile().exists()) {
styles.add(CheckstyleConfigLoader.loadCheckstyleConfig(configPath, emptyMap()));
}
}
}
}
} catch (Exception e) {
getLog().warn("Unable to parse checkstyle configuration. Checkstyle will not inform rewrite execution.", e);
}
return styles;
}

protected Set<String> getRecipeArtifactCoordinates() {
if (computedRecipeArtifactCoordinates == null) {
synchronized (this) {
Expand Down
Loading