Conan: Fix building the dependency tree

Fix two issues with building the dependency tree: * The algorithm wrongly used all nodes in the output of "conan info" as root nodes of the dependency, but only the first node represents the root. * The dependencies for different scopes were assigned to separate `PackageReference` objects with the same id and afterwards added to a set. As a result only one of them was contained in the set. The result was that the root level of the dependency tree contained too many dependencies, while transitive dependencies were missing. While at it, rename some variables for better readability. Signed-off-by: Martin Nonnenmacher <martin.nonnenmacher@bosch.io>
oss-review-toolkit · Oct 5, 2021 · 4f502b0 · 4f502b0
1 parent 7535ddb
commit 4f502b0
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 47 deletions.
diff --git a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-py.yml
@@ -28,7 +28,6 @@ project:
     - id: "Conan::openssl:3.0.0"
       dependencies:
       - id: "Conan::zlib:1.2.11"
-    - id: "Conan::zlib:1.2.11"
 packages:
 - id: "Conan::openssl:3.0.0"
   purl: "pkg:conan/openssl@3.0.0"

diff --git a/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml b/analyzer/src/funTest/assets/projects/synthetic/conan-expected-output-txt.yml
@@ -18,19 +18,18 @@ project:
   scopes:
   - name: "build_requires"
     dependencies:
-    - id: "Conan::libiconv:1.16"
-    - id: "Conan::libxml2:2.9.12"
-      dependencies:
-      - id: "Conan::zlib:1.2.11"
     - id: "Conan::libxslt:1.1.34"
+      dependencies:
+      - id: "Conan::libxml2:2.9.12"
+        dependencies:
+        - id: "Conan::libiconv:1.16"
+        - id: "Conan::zlib:1.2.11"
   - name: "requires"
     dependencies:
     - id: "Conan::libcurl:7.79.1"
       dependencies:
       - id: "Conan::openssl:1.1.1l"
       - id: "Conan::zlib:1.2.11"
-    - id: "Conan::openssl:1.1.1l"
-    - id: "Conan::zlib:1.2.11"
 packages:
 - id: "Conan::libcurl:7.79.1"
   purl: "pkg:conan/libcurl@7.79.1"

diff --git a/analyzer/src/main/kotlin/managers/Conan.kt b/analyzer/src/main/kotlin/managers/Conan.kt
@@ -176,20 +176,20 @@ class Conan(
             installDependencies(workingDir)
 
             val dependenciesJson = run(workingDir, "info", ".", "-j").stdout
-            val rootNode = jsonMapper.readTree(dependenciesJson)
-            val packageList = removeProjectPackage(rootNode, definitionFile)
+            val pkgInfos = jsonMapper.readTree(dependenciesJson)
+            val packageList = removeProjectPackage(pkgInfos, definitionFile.name)
             val packages = parsePackages(packageList, workingDir)
 
             val dependenciesScope = Scope(
                 name = SCOPE_NAME_DEPENDENCIES,
-                dependencies = parseDependencies(rootNode, SCOPE_NAME_DEPENDENCIES, workingDir)
+                dependencies = parseDependencies(pkgInfos, definitionFile.name, SCOPE_NAME_DEPENDENCIES, workingDir)
             )
             val devDependenciesScope = Scope(
                 name = SCOPE_NAME_DEV_DEPENDENCIES,
-                dependencies = parseDependencies(rootNode, SCOPE_NAME_DEV_DEPENDENCIES, workingDir)
+                dependencies = parseDependencies(pkgInfos, definitionFile.name, SCOPE_NAME_DEV_DEPENDENCIES, workingDir)
             )
 
-            val projectPackage = parseProjectPackage(rootNode, definitionFile, workingDir)
+            val projectPackage = parseProjectPackage(pkgInfos, definitionFile, workingDir)
 
             return listOf(
                 ProjectAnalyzerResult(
@@ -214,49 +214,42 @@ class Conan(
     }
 
     /**
-     * Return the dependency tree starting from a [rootNode] for given [scopeName].
+     * Return the dependency tree for [pkg] for the given [scopeName].
      */
     private fun parseDependencyTree(
-        rootNode: JsonNode,
-        workingDir: File,
+        pkgInfos: JsonNode,
         pkg: JsonNode,
-        scopeName: String
+        scopeName: String,
+        workingDir: File
     ): SortedSet<PackageReference> {
         val result = mutableSetOf<PackageReference>()
 
-        pkg[scopeName]?.forEach {
-            val childRef = it.textValueOrEmpty()
-            rootNode.forEach { child ->
-                if (child["reference"].textValueOrEmpty() == childRef) {
-                    log.debug { "Found child '$childRef'." }
-
-                    val packageReference = PackageReference(
-                        id = parsePackageId(child, workingDir),
-                        dependencies = parseDependencyTree(rootNode, workingDir, child, SCOPE_NAME_DEPENDENCIES)
-                    )
-                    result += packageReference
+        pkg[scopeName]?.forEach { childNode ->
+            val childRef = childNode.textValueOrEmpty()
+            pkgInfos.find { it["reference"].textValueOrEmpty() == childRef }?.let { pkgInfo ->
+                log.debug { "Found child '$childRef'." }
 
-                    val packageDevReference = PackageReference(
-                        id = parsePackageId(child, workingDir),
-                        dependencies = parseDependencyTree(rootNode, workingDir, child, SCOPE_NAME_DEV_DEPENDENCIES)
-                    )
+                val id = parsePackageId(pkgInfo, workingDir)
+                val dependencies = parseDependencyTree(pkgInfos, pkgInfo, SCOPE_NAME_DEPENDENCIES, workingDir) +
+                        parseDependencyTree(pkgInfos, pkgInfo, SCOPE_NAME_DEV_DEPENDENCIES, workingDir)
 
-                    result += packageDevReference
-                }
+                result += PackageReference(id, dependencies = dependencies.toSortedSet())
             }
         }
+
         return result.toSortedSet()
     }
 
     /**
      * Run through each package and parse the list of its dependencies (also transitive ones).
      */
     private fun parseDependencies(
-        rootNode: JsonNode,
+        pkgInfos: JsonNode,
+        definitionFileName: String,
         scopeName: String,
         workingDir: File
-    ): SortedSet<PackageReference>  =
-        rootNode.flatMapTo(sortedSetOf()) { pkg -> parseDependencyTree(rootNode, workingDir, pkg, scopeName) }
+    ): SortedSet<PackageReference> =
+        parseDependencyTree(pkgInfos, findProjectNode(pkgInfos, definitionFileName), scopeName, workingDir)
 
     /**
      * Return the map of packages and their identifiers which are contained in [nodes].
@@ -292,16 +285,20 @@ class Conan(
             jsonMapper.readTree(jsonFile).also { jsonFile.parentFile.safeDeleteRecursively(force = true) }
         }.get(field).textValueOrEmpty()
 
+    /**
+     * Find the node that represents the project defined in the definition file.
+     */
+    private fun findProjectNode(pkgInfos: JsonNode, definitionFileName: String): JsonNode =
+        pkgInfos.first {
+            // Use "in" because conanfile.py's reference string often includes other data.
+            definitionFileName in it["reference"].textValueOrEmpty()
+        }
+
     /**
      * Return the full list of packages, excluding the project level information.
      */
-    private fun removeProjectPackage(rootNode: JsonNode, definitionFile: File): List<JsonNode> =
-        rootNode.find {
-            // Contains because conanfile.py's reference string often includes other data.
-            it["reference"].textValueOrEmpty().contains(definitionFile.name)
-        }?.let { projectPackage ->
-            rootNode.minusElement(projectPackage)
-        } ?: rootNode.toList<JsonNode>()
+    private fun removeProjectPackage(pkgInfos: JsonNode, definitionFileName: String): List<JsonNode> =
+        pkgInfos.minusElement(findProjectNode(pkgInfos, definitionFileName))
 
     /**
      * Return the set of declared licenses contained in [node].
@@ -346,10 +343,8 @@ class Conan(
      * TODO: The format of `conan info` output for a conanfile.txt file may be such that we can get project metadata
      *       from the `requires` field. Need to investigate whether this is a sure thing before implementing.
      */
-    private fun parseProjectPackage(rootNode: JsonNode, definitionFile: File, workingDir: File): Package {
-        val projectPackageJson = requireNotNull(rootNode.find {
-            it["reference"].textValue().contains(definitionFile.name)
-        })
+    private fun parseProjectPackage(pkgInfos: JsonNode, definitionFile: File, workingDir: File): Package {
+        val projectPackageJson = findProjectNode(pkgInfos, definitionFile.name)
 
         return if (definitionFile.name == "conanfile.py") {
             generateProjectPackageFromConanfilePy(projectPackageJson, definitionFile, workingDir)