Python reachables improvements

Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>

build.sbt

@@ -1,9 +1,9 @@
 name                     := "atom"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "1.5.3"
+ThisBuild / version      := "1.5.4"
 ThisBuild / scalaVersion := "3.3.1"
 
-val chenVersion      = "0.5.3"
+val chenVersion      = "0.5.4"
 
 lazy val atom = Projects.atom
 

project/build.properties

@@ -1 +1 @@
-sbt.version=1.9.6
+sbt.version=1.9.7

src/main/scala/io/appthreat/atom/Atom.scala

@@ -414,7 +414,7 @@ object Atom {
               .withOutputPath(outputAtomFile)
               .withDefaultIgnoredFilesRegex(List("\\..*".r))
               .withIgnoredFilesRegex(
-                ".*(samples|examples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials).*"
+                ".*(samples|examples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials|noxfile).*"
               )
           )
           .map { ag =>

src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala

@@ -25,7 +25,7 @@ object PythonDependencyParser extends XDependencyParser {
 
   private def parseSetupPy(cpg: Cpg): Set[ModuleWithVersion] = {
     val dataFlowEnabled     = cpg.metaData.overlays.contains(OssDataFlow.overlayName)
-    val requirementsPattern = "([/.\\w_-]+)((=>|<=|==|>=|=<|<|>|!=).*)".r
+    val requirementsPattern = """([\[\]/.\w_-]+)[\s]?((=>|<=|==|>=|=<|<|>|!=).*)""".r
 
     def dataSourcesToRequires = (cpg.literal ++ cpg.identifier)
       .where(_.file.name(".*setup.py"))
@@ -70,14 +70,9 @@ object PythonDependencyParser extends XDependencyParser {
     val localModuleNames = cpg.file.name
       .filterNot(_ == "N/A")
       .map(x => ScalaFile(x))
-      .flatMap(_.parentOption.map(_.pathAsString))
-      .filterNot(_ == root)
-      .flatMap(_.stripPrefix(s"$root${JFile.separatorChar}").split(JFile.separatorChar).headOption)
+      .map(_.pathAsString)
+      .map(_.stripPrefix(s"$root${JFile.separatorChar}").split(JFile.separatorChar).head.replaceFirst("\\.py", ""))
       .toSet
-    val fileList = cpg.file.name
-      .filterNot(_ == "N/A")
-      .map(x => ScalaFile(x))
-      .l
     cpg.imports
       .whereNot(_.call.file.name(".*setup.py"))
       .filterNot {

src/main/scala/io/appthreat/atom/parsedeps/package.scala

@@ -55,7 +55,7 @@ package object parsedeps {
 
     def merge(x: ModuleWithVersion): ModuleWithVersion = {
       val vs = this.versions ++ x.versions
-      val is = this.importedSymbols + "," + x.importedSymbols
+      val is = if (x.importedSymbols.nonEmpty) this.importedSymbols + "," + x.importedSymbols else this.importedSymbols
       vs.find(_.startsWith("==")) match
         case Some(exactVersion) =>
           ModuleWithVersion(

src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala

@@ -25,12 +25,8 @@ object ReachableSlicing {
     val language  = atom.metaData.language.head
     def sourceP   = atom.tag.name(config.sourceTag).parameter
     def sourceI   = atom.tag.name(config.sourceTag).identifier
-    def sink      = atom.ret.where(_.tag.name(config.sinkTag))
-    var flowsList = sink.reachableByFlows(sourceP).map(toSlice).toList
-    flowsList ++= sink
-      .reachableByFlows(sourceI, sourceP)
-      .map(toSlice)
-      .toList
+    def sink      = atom.ret.where(_.method.tag.name(config.sourceTag))
+    var flowsList = sink.reachableByFlows(sourceP, sourceI).map(toSlice).toList
     flowsList ++=
       atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList
     // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares
@@ -57,7 +53,7 @@ object ReachableSlicing {
         .call
         .argument
         .isIdentifier
-        .reachableByFlows(sourceI)
+        .reachableByFlows(sourceI, dynFrameworkIdentifier)
         .map(toSlice)
         .toList
     }

src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala

@@ -63,7 +63,11 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa
         |    "PackageC==1.2.0.dev1+hg.5.b11e5e6f0b0b",
         |    "typing-extensions==3.10.0.2",
         |    "re-wx>=0.0.2",
-        |    "zope.interface>=5.1.0"
+        |    "zope.interface>=5.1.0",
+        |    "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
+        |    "proto-plus >= 1.22.0, <2.0.0dev",
+        |    "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'",
+        |    "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5"
         |]
         |test_requirements = [
         |    "pytest-httpbin==2.0.0",
@@ -197,18 +201,21 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa
     "have the modules scanned successfully" in {
       val scanResult = PythonDependencyParser.parse(cpg)
       scanResult.modules shouldBe List(
-        ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b", ""),
+        ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b", "", ""),
         ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0", ""),
         ModuleWithVersion("certifi", "", ">=2017.4.17", ""),
         ModuleWithVersion("charset_normalizer", "", ">=2,<4", ""),
+        ModuleWithVersion("google-api-core[grpc]", "", ">= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", ""),
         ModuleWithVersion("idna", "", ">=2.5,<4", ""),
         ModuleWithVersion("os", "", "", "os.path"),
         ModuleWithVersion("packageA", "", ">=1.4.2,<1.9,!=1.5.*,!=1.6.*", ""),
         ModuleWithVersion("packageB", "", ">=0.5.0,< 0.7.0", ""),
+        ModuleWithVersion("proto-plus", "", ">= 1.22.2, <2.0.0dev; python_version>='3.11',>= 1.22.0, <2.0.0dev", ""),
+        ModuleWithVersion("protobuf", "", ">=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", ""),
         ModuleWithVersion("re-wx", "", ">=0.0.2", ""),
         ModuleWithVersion("socket", "", "", "socket"),
         ModuleWithVersion("typing-extensions", "3.10.0.2", "", ""),
-        ModuleWithVersion("urllib3", "", ">=1.21.1,<3", "urllib3.poolmanager.proxy_from_url,urllib3.util.Timeout,urllib3.exceptions.LocationValueError,urllib3.contrib.socks.SOCKSProxyManager,urllib3.exceptions.HTTPError,urllib3.exceptions.SSLError,urllib3.exceptions.ProxyError,urllib3.exceptions.InvalidHeader,urllib3.exceptions.MaxRetryError,urllib3.exceptions.ConnectTimeoutError,urllib3.exceptions.ClosedPoolError,urllib3.exceptions.ProtocolError,urllib3.util.retry.Retry,urllib3.exceptions.ResponseError,,urllib3.exceptions.ReadTimeoutError,urllib3.exceptions.NewConnectionError,urllib3.util.parse_url,urllib3.poolmanager.PoolManager"),
+        ModuleWithVersion("urllib3", "", ">=1.21.1,<3", "urllib3.poolmanager.proxy_from_url,urllib3.util.Timeout,urllib3.exceptions.LocationValueError,urllib3.contrib.socks.SOCKSProxyManager,urllib3.exceptions.HTTPError,urllib3.exceptions.SSLError,urllib3.exceptions.ProxyError,urllib3.exceptions.InvalidHeader,urllib3.exceptions.MaxRetryError,urllib3.exceptions.ConnectTimeoutError,urllib3.exceptions.ClosedPoolError,urllib3.exceptions.ProtocolError,urllib3.util.retry.Retry,urllib3.exceptions.ResponseError,urllib3.exceptions.ReadTimeoutError,urllib3.exceptions.NewConnectionError,urllib3.util.parse_url,urllib3.poolmanager.PoolManager"),
         ModuleWithVersion("zope.interface", "", ">=5.1.0", "")
       )
     }

wrapper/nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "1.5.3",
+  "version": "1.5.4",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",