From 4f502b01b2783183e5c88ead586fffe49b47520e Mon Sep 17 00:00:00 2001 From: Martin Nonnenmacher Date: Mon, 4 Oct 2021 23:49:24 +0200 Subject: [PATCH] Conan: Fix building the dependency tree Fix two issues with building the dependency tree: * The algorithm wrongly used all nodes in the output of "conan info" as root nodes of the dependency, but only the first node represents the root. * The dependencies for different scopes were assigned to separate `PackageReference` objects with the same id and afterwards added to a set. As a result only one of them was contained in the set. The result was that the root level of the dependency tree contained too many dependencies, while transitive dependencies were missing. While at it, rename some variables for better readability. Signed-off-by: Martin Nonnenmacher --- .../synthetic/conan-expected-output-py.yml | 1 - .../synthetic/conan-expected-output-txt.yml | 11 ++- analyzer/src/main/kotlin/managers/Conan.kt | 75 +++++++++---------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml index 13b0b38c55e6a..7f89d41172fdb 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml @@ -28,7 +28,6 @@ project: - id: "Conan::openssl:3.0.0" dependencies: - id: "Conan::zlib:1.2.11" - - id: "Conan::zlib:1.2.11" packages: - id: "Conan::openssl:3.0.0" purl: "pkg:conan/openssl@3.0.0" diff --git a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml index 2c40b350d7070..8e57464dc902d 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml @@ -18,19 +18,18 @@ project: scopes: - name: "build_requires" dependencies: - - id: "Conan::libiconv:1.16" - - id: "Conan::libxml2:2.9.12" - dependencies: - - id: "Conan::zlib:1.2.11" - id: "Conan::libxslt:1.1.34" + dependencies: + - id: "Conan::libxml2:2.9.12" + dependencies: + - id: "Conan::libiconv:1.16" + - id: "Conan::zlib:1.2.11" - name: "requires" dependencies: - id: "Conan::libcurl:7.79.1" dependencies: - id: "Conan::openssl:1.1.1l" - id: "Conan::zlib:1.2.11" - - id: "Conan::openssl:1.1.1l" - - id: "Conan::zlib:1.2.11" packages: - id: "Conan::libcurl:7.79.1" purl: "pkg:conan/libcurl@7.79.1" diff --git a/analyzer/src/main/kotlin/managers/Conan.kt b/analyzer/src/main/kotlin/managers/Conan.kt index d54245108a408..869e1df2e1b5b 100644 --- a/analyzer/src/main/kotlin/managers/Conan.kt +++ b/analyzer/src/main/kotlin/managers/Conan.kt @@ -176,20 +176,20 @@ class Conan( installDependencies(workingDir) val dependenciesJson = run(workingDir, "info", ".", "-j").stdout - val rootNode = jsonMapper.readTree(dependenciesJson) - val packageList = removeProjectPackage(rootNode, definitionFile) + val pkgInfos = jsonMapper.readTree(dependenciesJson) + val packageList = removeProjectPackage(pkgInfos, definitionFile.name) val packages = parsePackages(packageList, workingDir) val dependenciesScope = Scope( name = SCOPE_NAME_DEPENDENCIES, - dependencies = parseDependencies(rootNode, SCOPE_NAME_DEPENDENCIES, workingDir) + dependencies = parseDependencies(pkgInfos, definitionFile.name, SCOPE_NAME_DEPENDENCIES, workingDir) ) val devDependenciesScope = Scope( name = SCOPE_NAME_DEV_DEPENDENCIES, - dependencies = parseDependencies(rootNode, SCOPE_NAME_DEV_DEPENDENCIES, workingDir) + dependencies = parseDependencies(pkgInfos, definitionFile.name, SCOPE_NAME_DEV_DEPENDENCIES, workingDir) ) - val projectPackage = parseProjectPackage(rootNode, definitionFile, workingDir) + val projectPackage = parseProjectPackage(pkgInfos, definitionFile, workingDir) return listOf( ProjectAnalyzerResult( @@ -214,37 +214,29 @@ class Conan( } /** - * Return the dependency tree starting from a [rootNode] for given [scopeName]. + * Return the dependency tree for [pkg] for the given [scopeName]. */ private fun parseDependencyTree( - rootNode: JsonNode, - workingDir: File, + pkgInfos: JsonNode, pkg: JsonNode, - scopeName: String + scopeName: String, + workingDir: File ): SortedSet { val result = mutableSetOf() - pkg[scopeName]?.forEach { - val childRef = it.textValueOrEmpty() - rootNode.forEach { child -> - if (child["reference"].textValueOrEmpty() == childRef) { - log.debug { "Found child '$childRef'." } - - val packageReference = PackageReference( - id = parsePackageId(child, workingDir), - dependencies = parseDependencyTree(rootNode, workingDir, child, SCOPE_NAME_DEPENDENCIES) - ) - result += packageReference + pkg[scopeName]?.forEach { childNode -> + val childRef = childNode.textValueOrEmpty() + pkgInfos.find { it["reference"].textValueOrEmpty() == childRef }?.let { pkgInfo -> + log.debug { "Found child '$childRef'." } - val packageDevReference = PackageReference( - id = parsePackageId(child, workingDir), - dependencies = parseDependencyTree(rootNode, workingDir, child, SCOPE_NAME_DEV_DEPENDENCIES) - ) + val id = parsePackageId(pkgInfo, workingDir) + val dependencies = parseDependencyTree(pkgInfos, pkgInfo, SCOPE_NAME_DEPENDENCIES, workingDir) + + parseDependencyTree(pkgInfos, pkgInfo, SCOPE_NAME_DEV_DEPENDENCIES, workingDir) - result += packageDevReference - } + result += PackageReference(id, dependencies = dependencies.toSortedSet()) } } + return result.toSortedSet() } @@ -252,11 +244,12 @@ class Conan( * Run through each package and parse the list of its dependencies (also transitive ones). */ private fun parseDependencies( - rootNode: JsonNode, + pkgInfos: JsonNode, + definitionFileName: String, scopeName: String, workingDir: File - ): SortedSet = - rootNode.flatMapTo(sortedSetOf()) { pkg -> parseDependencyTree(rootNode, workingDir, pkg, scopeName) } + ): SortedSet = + parseDependencyTree(pkgInfos, findProjectNode(pkgInfos, definitionFileName), scopeName, workingDir) /** * Return the map of packages and their identifiers which are contained in [nodes]. @@ -292,16 +285,20 @@ class Conan( jsonMapper.readTree(jsonFile).also { jsonFile.parentFile.safeDeleteRecursively(force = true) } }.get(field).textValueOrEmpty() + /** + * Find the node that represents the project defined in the definition file. + */ + private fun findProjectNode(pkgInfos: JsonNode, definitionFileName: String): JsonNode = + pkgInfos.first { + // Use "in" because conanfile.py's reference string often includes other data. + definitionFileName in it["reference"].textValueOrEmpty() + } + /** * Return the full list of packages, excluding the project level information. */ - private fun removeProjectPackage(rootNode: JsonNode, definitionFile: File): List = - rootNode.find { - // Contains because conanfile.py's reference string often includes other data. - it["reference"].textValueOrEmpty().contains(definitionFile.name) - }?.let { projectPackage -> - rootNode.minusElement(projectPackage) - } ?: rootNode.toList() + private fun removeProjectPackage(pkgInfos: JsonNode, definitionFileName: String): List = + pkgInfos.minusElement(findProjectNode(pkgInfos, definitionFileName)) /** * Return the set of declared licenses contained in [node]. @@ -346,10 +343,8 @@ class Conan( * TODO: The format of `conan info` output for a conanfile.txt file may be such that we can get project metadata * from the `requires` field. Need to investigate whether this is a sure thing before implementing. */ - private fun parseProjectPackage(rootNode: JsonNode, definitionFile: File, workingDir: File): Package { - val projectPackageJson = requireNotNull(rootNode.find { - it["reference"].textValue().contains(definitionFile.name) - }) + private fun parseProjectPackage(pkgInfos: JsonNode, definitionFile: File, workingDir: File): Package { + val projectPackageJson = findProjectNode(pkgInfos, definitionFile.name) return if (definitionFile.name == "conanfile.py") { generateProjectPackageFromConanfilePy(projectPackageJson, definitionFile, workingDir)