diff --git a/f8a_worker/workers/dependency_parser.py b/f8a_worker/workers/dependency_parser.py index 20d013f6a..f7cebb91f 100644 --- a/f8a_worker/workers/dependency_parser.py +++ b/f8a_worker/workers/dependency_parser.py @@ -2,6 +2,7 @@ import re import anymarkup +import itertools from pathlib import Path from tempfile import TemporaryDirectory @@ -33,11 +34,13 @@ def execute(self, arguments=None): "github_sha": github_sha, "email_ids": arguments.get('email_ids')} @staticmethod - def extract_dependencies(github_repo, github_sha=None): + def extract_dependencies(github_repo, github_sha=None, + user_flow=False): """Extract the dependencies information. Currently assuming repository is maven/npm/python repository. + :param user_flow: to indicate if user flow is invoked :param github_repo: repository url :param github_sha: commit hash :return: set of direct (and indirect) dependencies @@ -52,8 +55,9 @@ def extract_dependencies(github_repo, github_sha=None): # First change the package-lock.json to npm-shrinkwrap.json GithubDependencyTreeTask.change_package_lock_to_shrinkwrap() + # Since user flow is only called for maven, we pass this flag only to maven if peek(Path.cwd().glob("pom.xml")): - return GithubDependencyTreeTask.get_maven_dependencies() + return GithubDependencyTreeTask.get_maven_dependencies(user_flow) elif peek(Path.cwd().glob("npm-shrinkwrap.json")) \ or peek(Path.cwd().glob("package.json")): return GithubDependencyTreeTask.get_npm_dependencies(repo.repo_path) @@ -68,24 +72,50 @@ def extract_dependencies(github_repo, github_sha=None): "python or Go repository for scanning!") @staticmethod - def get_maven_dependencies(): + def get_maven_dependencies(user_flow): """Get direct and indirect dependencies from pom.xml by using maven dependency tree plugin. :return: set of direct and indirect dependencies """ output_file = Path.cwd() / "dependency-tree.txt" - cmd = ["mvn", "org.apache.maven.plugins:maven-dependency-plugin:3.0.2:tree", - "-DoutputType=dot", - "-DoutputFile={filename}".format(filename=output_file), - "-DappendOutput=true"] + if user_flow: + cmd = ["mvn", "org.apache.maven.plugins:maven-dependency-plugin:3.1.1:resolve", + "-DoutputFile={filename}".format(filename=output_file), + "-DincludeScope=compile", + "-DexcludeTransitive=true"] + else: + cmd = ["mvn", "org.apache.maven.plugins:maven-dependency-plugin:3.0.2:tree", + "-DoutputType=dot", + "-DoutputFile={filename}".format(filename=output_file), + "-DappendOutput=true"] + timed_cmd = TimedCommand(cmd) status, output, _ = timed_cmd.run(timeout=3600) if status != 0 or not output_file.is_file(): # all errors are in stdout, not stderr raise TaskError(output) with output_file.open() as f: + if user_flow: + return GithubDependencyTreeTask.parse_maven_dependency_resolve(f) + return GithubDependencyTreeTask.parse_maven_dependency_tree(f.readlines()) + @staticmethod + def parse_maven_dependency_resolve(file): + """Parse the output of mvn dependency:resolve command. + + :param file: file containing the output of mvn dependency:resolve command + :return: set of direct dependencies + """ + set_package_names = set() + for line in itertools.islice(file, 2, None): + package_name = line.strip() + if package_name: + # Remove scope from package name + package_name = package_name.rsplit(':', 1)[0] + add_maven_coords_to_set(package_name, set_package_names) + return set_package_names + @staticmethod def parse_maven_dependency_tree(dependency_tree): """Parse the dot representation of maven dependency tree. diff --git a/f8a_worker/workers/repo_dependency_finder.py b/f8a_worker/workers/repo_dependency_finder.py index 978779eb7..21c7bd7d7 100644 --- a/f8a_worker/workers/repo_dependency_finder.py +++ b/f8a_worker/workers/repo_dependency_finder.py @@ -40,7 +40,8 @@ def execute(self, arguments=None): except TaskError as e: raise TaskError('Failed to get CVEs') else: - dependencies = list(GithubDependencyTreeTask.extract_dependencies(github_repo)) + dependencies = list(GithubDependencyTreeTask.extract_dependencies(github_repo, + user_flow=True)) self.log.debug('Retrieved dependencies list %r' % dependencies) try: # forward only the available dependencies in the system. Unknown