Skip to content

Commit

Permalink
Scope ResourceParse to the current maven project.
Browse files Browse the repository at this point in the history 
fixes #352.
  • Loading branch information
@traceyyoshima
traceyyoshima committed May 6, 2022
1 parent 580784b commit 6315922
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 60 deletions.
18 changes: 14 additions & 4 deletions src/main/java/org/openrewrite/maven/MavenMojoProjectParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,16 @@ private static RawRepositories buildRawRepositories(@Nullable List<Repository> r
return rawRepositories;
}

/**
* Used to scope `Files.walkFileTree` to the current maven project by skipping the subtrees of other MavenProjects.
*/
private Set<Path> pathsToOtherMavenProjects() {
return mavenSession.getProjects().stream()
.filter(o -> o != mavenSession.getCurrentProject())
.map(o -> o.getBasedir().toPath())
.collect(Collectors.toSet());
}

public List<SourceFile> listSourceFiles(Iterable<NamedStyles> styles,
ExecutionContext ctx) throws DependencyResolutionRequiredException, MojoExecutionException {

Expand Down Expand Up @@ -342,11 +352,11 @@ public List<SourceFile> listSourceFiles(Iterable<NamedStyles> styles,
sourceFiles.addAll(ListUtils.map(maybeAutodetectStyles(javaParser.parse(mainJavaSources, baseDir, ctx), styles),
addProvenance(baseDir, projectProvenance, generatedSourcePaths)));

ResourceParser rp = new ResourceParser(logger, exclusions, sizeThresholdMb);
ResourceParser rp = new ResourceParser(baseDir, logger, exclusions, sizeThresholdMb, pathsToOtherMavenProjects());

// Any resources parsed from "main/resources" should also have the main source set added to them.
sourceFiles.addAll(ListUtils.map(
rp.parse(baseDir, mavenProject.getBasedir().toPath().resolve("src/main/resources"), alreadyParsed),
rp.parse(mavenProject.getBasedir().toPath().resolve("src/main/resources"), alreadyParsed),
addProvenance(baseDir, ListUtils.concat(projectProvenance, javaParser.getSourceSet(ctx)), null)));

logger.info("Parsing Java test files...");
Expand All @@ -366,12 +376,12 @@ public List<SourceFile> listSourceFiles(Iterable<NamedStyles> styles,

// Any resources parsed from "test/resources" should also have the test source set added to them.
sourceFiles.addAll(ListUtils.map(
rp.parse(baseDir, mavenProject.getBasedir().toPath().resolve("src/test/resources"), alreadyParsed),
rp.parse(mavenProject.getBasedir().toPath().resolve("src/test/resources"), alreadyParsed),
addProvenance(baseDir, ListUtils.concat(projectProvenance, javaParser.getSourceSet(ctx)), null)));

// Parse non-java, non-resource files
sourceFiles.addAll(ListUtils.map(
rp.parse(baseDir, mavenProject.getBasedir().toPath(), alreadyParsed),
rp.parse(mavenProject.getBasedir().toPath(), alreadyParsed),
addProvenance(baseDir, projectProvenance, null)
));

Expand Down
172 changes: 116 additions & 56 deletions src/main/java/org/openrewrite/maven/ResourceParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import org.apache.maven.plugin.logging.Log;
import org.openrewrite.ExecutionContext;
import org.openrewrite.InMemoryExecutionContext;
import org.openrewrite.Parser;
import org.openrewrite.SourceFile;
import org.openrewrite.hcl.HclParser;
import org.openrewrite.json.JsonParser;
Expand All @@ -14,96 +13,157 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class ResourceParser {
private static final Set<String> DEFAULT_EXCLUSIONS = new HashSet<>(Arrays.asList("build", "target", "out", ".gradle", ".idea", ".project", "node_modules", ".git", ".metadata", ".DS_Store"));

private final Path baseDir;
private final Log logger;
private final Collection<String> exclusions;
private final Collection<PathMatcher> exclusions;
private final int sizeThresholdMb;
private final Collection<Path> excludedDirectories;

public ResourceParser(Log logger, Collection<String> exclusions, int thresholdMb) {
public ResourceParser(Path baseDir, Log logger, Collection<String> exclusions, int sizeThresholdMb, Collection<Path> excludedDirectories) {
this.baseDir = baseDir;
this.logger = logger;
this.exclusions = exclusions;
sizeThresholdMb = thresholdMb;
this.exclusions = exclusionMatchers(exclusions);
this.sizeThresholdMb = sizeThresholdMb;
this.excludedDirectories = excludedDirectories;
}

private Collection<PathMatcher> exclusionMatchers(Collection<String> exclusions) {
return exclusions.stream()
.map(o -> baseDir.getFileSystem().getPathMatcher("glob:" + o))
.collect(Collectors.toList());
}

public List<SourceFile> parse(Path baseDir, Path searchDir, Collection<Path> alreadyParsed) {
public List<SourceFile> parse(Path searchDir, Collection<Path> alreadyParsed) {
List<SourceFile> sourceFiles = new ArrayList<>();
if (!searchDir.toFile().exists()) {
return sourceFiles;
}
Consumer<Throwable> errorConsumer = t -> logger.error("Error parsing", t);
InMemoryExecutionContext ctx = new InMemoryExecutionContext(errorConsumer);

sourceFiles.addAll(parseSourceFiles(baseDir, new JsonParser(), searchDir, alreadyParsed, ctx));
sourceFiles.addAll(parseSourceFiles(baseDir, new XmlParser(), searchDir, alreadyParsed, ctx));
sourceFiles.addAll(parseSourceFiles(baseDir, new YamlParser(), searchDir, alreadyParsed, ctx));
sourceFiles.addAll(parseSourceFiles(baseDir, new PropertiesParser(), searchDir, alreadyParsed, ctx));
sourceFiles.addAll(parseSourceFiles(baseDir, new ProtoParser(), searchDir, alreadyParsed, ctx));
sourceFiles.addAll(parseSourceFiles(baseDir, HclParser.builder().build(), searchDir, alreadyParsed, ctx));
try {
sourceFiles.addAll(parseSourceFiles(searchDir, alreadyParsed, ctx));
} catch (IOException e) {
logger.error(e.getMessage(), e);
throw new UncheckedIOException(e);
}

return sourceFiles;
}

@SuppressWarnings({"DuplicatedCode", "unchecked"})
public <S extends SourceFile> List<S> parseSourceFiles(
Path baseDir,
Parser<S> parser,
Path searchDir,
Collection<Path> alreadyParsed,
ExecutionContext ctx) {
try (Stream<Path> resources = Files.find(searchDir, 16, (path, attrs) -> {
if (!parser.accept(path)) {
return false;
}
ExecutionContext ctx) throws IOException {

for(Path pathSegment : searchDir.relativize(path)) {
String pathStr = pathSegment.toString();
if("target".equals(pathStr) || "build".equals(pathStr) || "out".equals(pathStr) ||
".gradle".equals(pathStr) || "node_modules".equals(pathStr) || ".metadata".equals(pathStr)) {
return false;
List<Path> resources = new ArrayList<>();
Files.walkFileTree(searchDir, Collections.emptySet(), 16, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
if (isExcluded(dir) || excludedDirectories.contains(dir)) {
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}

if (attrs.isDirectory() || attrs.size() == 0) {
return false;
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (!attrs.isOther() && !alreadyParsed.contains(file) && !isExcluded(file)) {
if (isOverSizeThreshold(attrs.size())) {
logger.info("Skipping parsing " + file + " as its size + " + attrs.size() / (1024L * 1024L) +
"Mb exceeds size threshold " + sizeThresholdMb + "Mb");
alreadyParsed.add(file);
} else {
resources.add(file);
}
}
return FileVisitResult.CONTINUE;
}
});

if (alreadyParsed.contains(path)) {
return false;
}
List<S> sourceFiles = new ArrayList<>(resources.size());

for (String exclusion : exclusions) {
PathMatcher matcher = baseDir.getFileSystem().getPathMatcher("glob:" + exclusion);
if (matcher.matches(baseDir.relativize(path))) {
alreadyParsed.add(path);
return false;
}
JsonParser jsonParser = new JsonParser();
List<Path> jsonPaths = new ArrayList<>();

XmlParser xmlParser = new XmlParser();
List<Path> xmlPaths = new ArrayList<>();

YamlParser yamlParser = new YamlParser();
List<Path> yamlPaths = new ArrayList<>();

PropertiesParser propertiesParser = new PropertiesParser();
List<Path> propertiesPaths = new ArrayList<>();

ProtoParser protoParser = new ProtoParser();
List<Path> protoPaths = new ArrayList<>();

HclParser hclParser = HclParser.builder().build();
List<Path> hclPaths = new ArrayList<>();

resources.forEach(path -> {
if (jsonParser.accept(path)) {
jsonPaths.add(path);
} else if (xmlParser.accept(path)) {
xmlPaths.add(path);
} else if (yamlParser.accept(path)) {
yamlPaths.add(path);
} else if (propertiesParser.accept(path)) {
propertiesPaths.add(path);
} else if (protoParser.accept(path)) {
protoPaths.add(path);
} else if (hclParser.accept(path)) {
hclPaths.add(path);
}
});

long fileSize = attrs.size();
if ((sizeThresholdMb > 0 && fileSize > sizeThresholdMb * 1024L * 1024L)) {
alreadyParsed.add(path);
logger.info("Skipping parsing " + path + " as its size + " + fileSize / (1024L * 1024L) +
"Mb exceeds size threshold " + sizeThresholdMb + "Mb");
return false;
sourceFiles.addAll((List<S>) jsonParser.parse(jsonPaths, baseDir, ctx));
alreadyParsed.addAll(jsonPaths);

sourceFiles.addAll((List<S>) xmlParser.parse(xmlPaths, baseDir, ctx));
alreadyParsed.addAll(xmlPaths);

sourceFiles.addAll((List<S>) yamlParser.parse(yamlPaths, baseDir, ctx));
alreadyParsed.addAll(yamlPaths);

sourceFiles.addAll((List<S>) propertiesParser.parse(propertiesPaths, baseDir, ctx));
alreadyParsed.addAll(propertiesPaths);

sourceFiles.addAll((List<S>) protoParser.parse(protoPaths, baseDir, ctx));
alreadyParsed.addAll(protoPaths);

sourceFiles.addAll((List<S>) hclParser.parse(hclPaths, baseDir, ctx));
alreadyParsed.addAll(hclPaths);

return sourceFiles;
}

private boolean isOverSizeThreshold(long fileSize) {
return (sizeThresholdMb > 0 && fileSize > sizeThresholdMb * 1024L * 1024L);
}

private boolean isExcluded(Path path) {
for (PathMatcher excluded : exclusions) {
if (excluded.matches(baseDir.relativize(path))) {
return true;
}
}

return true;
})) {
List<Path> resourceFiles = resources.collect(Collectors.toList());
alreadyParsed.addAll(resourceFiles);
return parser.parse(resourceFiles, baseDir, ctx);
} catch (IOException e) {
logger.error(e.getMessage(), e);
throw new UncheckedIOException(e);
for (Path pathSegment : baseDir.relativize(path)) {
if (DEFAULT_EXCLUSIONS.contains(pathSegment.toString())) {
return true;
}
}
return false;
}
}

0 comments on commit 6315922

Please sign in to comment.