diff --git a/src/main/java/org/openrewrite/maven/MavenMojoProjectParser.java b/src/main/java/org/openrewrite/maven/MavenMojoProjectParser.java index bdfbee33..42314dc9 100644 --- a/src/main/java/org/openrewrite/maven/MavenMojoProjectParser.java +++ b/src/main/java/org/openrewrite/maven/MavenMojoProjectParser.java @@ -306,6 +306,16 @@ private static RawRepositories buildRawRepositories(@Nullable List r return rawRepositories; } + /** + * Used to scope `Files.walkFileTree` to the current maven project by skipping the subtrees of other MavenProjects. + */ + private Set pathsToOtherMavenProjects() { + return mavenSession.getProjects().stream() + .filter(o -> o != mavenSession.getCurrentProject()) + .map(o -> o.getBasedir().toPath()) + .collect(Collectors.toSet()); + } + public List listSourceFiles(Iterable styles, ExecutionContext ctx) throws DependencyResolutionRequiredException, MojoExecutionException { @@ -342,11 +352,11 @@ public List listSourceFiles(Iterable styles, sourceFiles.addAll(ListUtils.map(maybeAutodetectStyles(javaParser.parse(mainJavaSources, baseDir, ctx), styles), addProvenance(baseDir, projectProvenance, generatedSourcePaths))); - ResourceParser rp = new ResourceParser(logger, exclusions, sizeThresholdMb); + ResourceParser rp = new ResourceParser(baseDir, logger, exclusions, sizeThresholdMb, pathsToOtherMavenProjects()); // Any resources parsed from "main/resources" should also have the main source set added to them. sourceFiles.addAll(ListUtils.map( - rp.parse(baseDir, mavenProject.getBasedir().toPath().resolve("src/main/resources"), alreadyParsed), + rp.parse(mavenProject.getBasedir().toPath().resolve("src/main/resources"), alreadyParsed), addProvenance(baseDir, ListUtils.concat(projectProvenance, javaParser.getSourceSet(ctx)), null))); logger.info("Parsing Java test files..."); @@ -366,12 +376,12 @@ public List listSourceFiles(Iterable styles, // Any resources parsed from "test/resources" should also have the test source set added to them. sourceFiles.addAll(ListUtils.map( - rp.parse(baseDir, mavenProject.getBasedir().toPath().resolve("src/test/resources"), alreadyParsed), + rp.parse(mavenProject.getBasedir().toPath().resolve("src/test/resources"), alreadyParsed), addProvenance(baseDir, ListUtils.concat(projectProvenance, javaParser.getSourceSet(ctx)), null))); // Parse non-java, non-resource files sourceFiles.addAll(ListUtils.map( - rp.parse(baseDir, mavenProject.getBasedir().toPath(), alreadyParsed), + rp.parse(mavenProject.getBasedir().toPath(), alreadyParsed), addProvenance(baseDir, projectProvenance, null) )); diff --git a/src/main/java/org/openrewrite/maven/ResourceParser.java b/src/main/java/org/openrewrite/maven/ResourceParser.java index d0a00094..b35f3ab5 100644 --- a/src/main/java/org/openrewrite/maven/ResourceParser.java +++ b/src/main/java/org/openrewrite/maven/ResourceParser.java @@ -3,7 +3,6 @@ import org.apache.maven.plugin.logging.Log; import org.openrewrite.ExecutionContext; import org.openrewrite.InMemoryExecutionContext; -import org.openrewrite.Parser; import org.openrewrite.SourceFile; import org.openrewrite.hcl.HclParser; import org.openrewrite.json.JsonParser; @@ -14,28 +13,36 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; public class ResourceParser { + private static final Set DEFAULT_EXCLUSIONS = new HashSet<>(Arrays.asList("build", "target", "out", ".gradle", ".idea", ".project", "node_modules", ".git", ".metadata", ".DS_Store")); + + private final Path baseDir; private final Log logger; - private final Collection exclusions; + private final Collection exclusions; private final int sizeThresholdMb; + private final Collection excludedDirectories; - public ResourceParser(Log logger, Collection exclusions, int thresholdMb) { + public ResourceParser(Path baseDir, Log logger, Collection exclusions, int sizeThresholdMb, Collection excludedDirectories) { + this.baseDir = baseDir; this.logger = logger; - this.exclusions = exclusions; - sizeThresholdMb = thresholdMb; + this.exclusions = exclusionMatchers(exclusions); + this.sizeThresholdMb = sizeThresholdMb; + this.excludedDirectories = excludedDirectories; + } + + private Collection exclusionMatchers(Collection exclusions) { + return exclusions.stream() + .map(o -> baseDir.getFileSystem().getPathMatcher("glob:" + o)) + .collect(Collectors.toList()); } - public List parse(Path baseDir, Path searchDir, Collection alreadyParsed) { + public List parse(Path searchDir, Collection alreadyParsed) { List sourceFiles = new ArrayList<>(); if (!searchDir.toFile().exists()) { return sourceFiles; @@ -43,67 +50,120 @@ public List parse(Path baseDir, Path searchDir, Collection alr Consumer errorConsumer = t -> logger.error("Error parsing", t); InMemoryExecutionContext ctx = new InMemoryExecutionContext(errorConsumer); - sourceFiles.addAll(parseSourceFiles(baseDir, new JsonParser(), searchDir, alreadyParsed, ctx)); - sourceFiles.addAll(parseSourceFiles(baseDir, new XmlParser(), searchDir, alreadyParsed, ctx)); - sourceFiles.addAll(parseSourceFiles(baseDir, new YamlParser(), searchDir, alreadyParsed, ctx)); - sourceFiles.addAll(parseSourceFiles(baseDir, new PropertiesParser(), searchDir, alreadyParsed, ctx)); - sourceFiles.addAll(parseSourceFiles(baseDir, new ProtoParser(), searchDir, alreadyParsed, ctx)); - sourceFiles.addAll(parseSourceFiles(baseDir, HclParser.builder().build(), searchDir, alreadyParsed, ctx)); + try { + sourceFiles.addAll(parseSourceFiles(searchDir, alreadyParsed, ctx)); + } catch (IOException e) { + logger.error(e.getMessage(), e); + throw new UncheckedIOException(e); + } return sourceFiles; } + @SuppressWarnings({"DuplicatedCode", "unchecked"}) public List parseSourceFiles( - Path baseDir, - Parser parser, Path searchDir, Collection alreadyParsed, - ExecutionContext ctx) { - try (Stream resources = Files.find(searchDir, 16, (path, attrs) -> { - if (!parser.accept(path)) { - return false; - } + ExecutionContext ctx) throws IOException { - for(Path pathSegment : searchDir.relativize(path)) { - String pathStr = pathSegment.toString(); - if("target".equals(pathStr) || "build".equals(pathStr) || "out".equals(pathStr) || - ".gradle".equals(pathStr) || "node_modules".equals(pathStr) || ".metadata".equals(pathStr)) { - return false; + List resources = new ArrayList<>(); + Files.walkFileTree(searchDir, Collections.emptySet(), 16, new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + if (isExcluded(dir) || excludedDirectories.contains(dir)) { + return FileVisitResult.SKIP_SUBTREE; } + return FileVisitResult.CONTINUE; } - if (attrs.isDirectory() || attrs.size() == 0) { - return false; + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (!attrs.isOther() && !alreadyParsed.contains(file) && !isExcluded(file)) { + if (isOverSizeThreshold(attrs.size())) { + logger.info("Skipping parsing " + file + " as its size + " + attrs.size() / (1024L * 1024L) + + "Mb exceeds size threshold " + sizeThresholdMb + "Mb"); + alreadyParsed.add(file); + } else { + resources.add(file); + } + } + return FileVisitResult.CONTINUE; } + }); - if (alreadyParsed.contains(path)) { - return false; - } + List sourceFiles = new ArrayList<>(resources.size()); - for (String exclusion : exclusions) { - PathMatcher matcher = baseDir.getFileSystem().getPathMatcher("glob:" + exclusion); - if (matcher.matches(baseDir.relativize(path))) { - alreadyParsed.add(path); - return false; - } + JsonParser jsonParser = new JsonParser(); + List jsonPaths = new ArrayList<>(); + + XmlParser xmlParser = new XmlParser(); + List xmlPaths = new ArrayList<>(); + + YamlParser yamlParser = new YamlParser(); + List yamlPaths = new ArrayList<>(); + + PropertiesParser propertiesParser = new PropertiesParser(); + List propertiesPaths = new ArrayList<>(); + + ProtoParser protoParser = new ProtoParser(); + List protoPaths = new ArrayList<>(); + + HclParser hclParser = HclParser.builder().build(); + List hclPaths = new ArrayList<>(); + + resources.forEach(path -> { + if (jsonParser.accept(path)) { + jsonPaths.add(path); + } else if (xmlParser.accept(path)) { + xmlPaths.add(path); + } else if (yamlParser.accept(path)) { + yamlPaths.add(path); + } else if (propertiesParser.accept(path)) { + propertiesPaths.add(path); + } else if (protoParser.accept(path)) { + protoPaths.add(path); + } else if (hclParser.accept(path)) { + hclPaths.add(path); } + }); - long fileSize = attrs.size(); - if ((sizeThresholdMb > 0 && fileSize > sizeThresholdMb * 1024L * 1024L)) { - alreadyParsed.add(path); - logger.info("Skipping parsing " + path + " as its size + " + fileSize / (1024L * 1024L) + - "Mb exceeds size threshold " + sizeThresholdMb + "Mb"); - return false; + sourceFiles.addAll((List) jsonParser.parse(jsonPaths, baseDir, ctx)); + alreadyParsed.addAll(jsonPaths); + + sourceFiles.addAll((List) xmlParser.parse(xmlPaths, baseDir, ctx)); + alreadyParsed.addAll(xmlPaths); + + sourceFiles.addAll((List) yamlParser.parse(yamlPaths, baseDir, ctx)); + alreadyParsed.addAll(yamlPaths); + + sourceFiles.addAll((List) propertiesParser.parse(propertiesPaths, baseDir, ctx)); + alreadyParsed.addAll(propertiesPaths); + + sourceFiles.addAll((List) protoParser.parse(protoPaths, baseDir, ctx)); + alreadyParsed.addAll(protoPaths); + + sourceFiles.addAll((List) hclParser.parse(hclPaths, baseDir, ctx)); + alreadyParsed.addAll(hclPaths); + + return sourceFiles; + } + + private boolean isOverSizeThreshold(long fileSize) { + return (sizeThresholdMb > 0 && fileSize > sizeThresholdMb * 1024L * 1024L); + } + + private boolean isExcluded(Path path) { + for (PathMatcher excluded : exclusions) { + if (excluded.matches(baseDir.relativize(path))) { + return true; } + } - return true; - })) { - List resourceFiles = resources.collect(Collectors.toList()); - alreadyParsed.addAll(resourceFiles); - return parser.parse(resourceFiles, baseDir, ctx); - } catch (IOException e) { - logger.error(e.getMessage(), e); - throw new UncheckedIOException(e); + for (Path pathSegment : baseDir.relativize(path)) { + if (DEFAULT_EXCLUSIONS.contains(pathSegment.toString())) { + return true; + } } + return false; } }