diff --git a/build.sbt b/build.sbt index 805bebf..db86a18 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,9 @@ name := "atom" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "1.5.3" +ThisBuild / version := "1.5.4" ThisBuild / scalaVersion := "3.3.1" -val chenVersion = "0.5.3" +val chenVersion = "0.5.4" lazy val atom = Projects.atom diff --git a/project/build.properties b/project/build.properties index 2743082..e8a1e24 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.6 +sbt.version=1.9.7 diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala index f1e19b7..eceb5dd 100644 --- a/src/main/scala/io/appthreat/atom/Atom.scala +++ b/src/main/scala/io/appthreat/atom/Atom.scala @@ -414,7 +414,7 @@ object Atom { .withOutputPath(outputAtomFile) .withDefaultIgnoredFilesRegex(List("\\..*".r)) .withIgnoredFilesRegex( - ".*(samples|examples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials).*" + ".*(samples|examples|test|tests|unittests|docs|virtualenvs|venv|benchmarks|tutorials|noxfile).*" ) ) .map { ag => diff --git a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala index a64ba53..23e8f1c 100644 --- a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala +++ b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala @@ -25,7 +25,7 @@ object PythonDependencyParser extends XDependencyParser { private def parseSetupPy(cpg: Cpg): Set[ModuleWithVersion] = { val dataFlowEnabled = cpg.metaData.overlays.contains(OssDataFlow.overlayName) - val requirementsPattern = "([/.\\w_-]+)((=>|<=|==|>=|=<|<|>|!=).*)".r + val requirementsPattern = """([\[\]/.\w_-]+)[\s]?((=>|<=|==|>=|=<|<|>|!=).*)""".r def dataSourcesToRequires = (cpg.literal ++ cpg.identifier) .where(_.file.name(".*setup.py")) @@ -70,14 +70,9 @@ object PythonDependencyParser extends XDependencyParser { val localModuleNames = cpg.file.name .filterNot(_ == "N/A") .map(x => ScalaFile(x)) - .flatMap(_.parentOption.map(_.pathAsString)) - .filterNot(_ == root) - .flatMap(_.stripPrefix(s"$root${JFile.separatorChar}").split(JFile.separatorChar).headOption) + .map(_.pathAsString) + .map(_.stripPrefix(s"$root${JFile.separatorChar}").split(JFile.separatorChar).head.replaceFirst("\\.py", "")) .toSet - val fileList = cpg.file.name - .filterNot(_ == "N/A") - .map(x => ScalaFile(x)) - .l cpg.imports .whereNot(_.call.file.name(".*setup.py")) .filterNot { diff --git a/src/main/scala/io/appthreat/atom/parsedeps/package.scala b/src/main/scala/io/appthreat/atom/parsedeps/package.scala index 273c21d..14c5eea 100644 --- a/src/main/scala/io/appthreat/atom/parsedeps/package.scala +++ b/src/main/scala/io/appthreat/atom/parsedeps/package.scala @@ -55,7 +55,7 @@ package object parsedeps { def merge(x: ModuleWithVersion): ModuleWithVersion = { val vs = this.versions ++ x.versions - val is = this.importedSymbols + "," + x.importedSymbols + val is = if (x.importedSymbols.nonEmpty) this.importedSymbols + "," + x.importedSymbols else this.importedSymbols vs.find(_.startsWith("==")) match case Some(exactVersion) => ModuleWithVersion( diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index e7a6dcb..8f66971 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -25,12 +25,8 @@ object ReachableSlicing { val language = atom.metaData.language.head def sourceP = atom.tag.name(config.sourceTag).parameter def sourceI = atom.tag.name(config.sourceTag).identifier - def sink = atom.ret.where(_.tag.name(config.sinkTag)) - var flowsList = sink.reachableByFlows(sourceP).map(toSlice).toList - flowsList ++= sink - .reachableByFlows(sourceI, sourceP) - .map(toSlice) - .toList + def sink = atom.ret.where(_.method.tag.name(config.sourceTag)) + var flowsList = sink.reachableByFlows(sourceP, sourceI).map(toSlice).toList flowsList ++= atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares @@ -57,7 +53,7 @@ object ReachableSlicing { .call .argument .isIdentifier - .reachableByFlows(sourceI) + .reachableByFlows(sourceI, dynFrameworkIdentifier) .map(toSlice) .toList } diff --git a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala index aea7e85..ffd8e41 100644 --- a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala +++ b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala @@ -63,7 +63,11 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa | "PackageC==1.2.0.dev1+hg.5.b11e5e6f0b0b", | "typing-extensions==3.10.0.2", | "re-wx>=0.0.2", - | "zope.interface>=5.1.0" + | "zope.interface>=5.1.0", + | "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + | "proto-plus >= 1.22.0, <2.0.0dev", + | "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'", + | "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5" |] |test_requirements = [ | "pytest-httpbin==2.0.0", @@ -197,18 +201,21 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa "have the modules scanned successfully" in { val scanResult = PythonDependencyParser.parse(cpg) scanResult.modules shouldBe List( - ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b", ""), + ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b", "", ""), ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0", ""), ModuleWithVersion("certifi", "", ">=2017.4.17", ""), ModuleWithVersion("charset_normalizer", "", ">=2,<4", ""), + ModuleWithVersion("google-api-core[grpc]", "", ">= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", ""), ModuleWithVersion("idna", "", ">=2.5,<4", ""), ModuleWithVersion("os", "", "", "os.path"), ModuleWithVersion("packageA", "", ">=1.4.2,<1.9,!=1.5.*,!=1.6.*", ""), ModuleWithVersion("packageB", "", ">=0.5.0,< 0.7.0", ""), + ModuleWithVersion("proto-plus", "", ">= 1.22.2, <2.0.0dev; python_version>='3.11',>= 1.22.0, <2.0.0dev", ""), + ModuleWithVersion("protobuf", "", ">=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", ""), ModuleWithVersion("re-wx", "", ">=0.0.2", ""), ModuleWithVersion("socket", "", "", "socket"), ModuleWithVersion("typing-extensions", "3.10.0.2", "", ""), - ModuleWithVersion("urllib3", "", ">=1.21.1,<3", "urllib3.poolmanager.proxy_from_url,urllib3.util.Timeout,urllib3.exceptions.LocationValueError,urllib3.contrib.socks.SOCKSProxyManager,urllib3.exceptions.HTTPError,urllib3.exceptions.SSLError,urllib3.exceptions.ProxyError,urllib3.exceptions.InvalidHeader,urllib3.exceptions.MaxRetryError,urllib3.exceptions.ConnectTimeoutError,urllib3.exceptions.ClosedPoolError,urllib3.exceptions.ProtocolError,urllib3.util.retry.Retry,urllib3.exceptions.ResponseError,,urllib3.exceptions.ReadTimeoutError,urllib3.exceptions.NewConnectionError,urllib3.util.parse_url,urllib3.poolmanager.PoolManager"), + ModuleWithVersion("urllib3", "", ">=1.21.1,<3", "urllib3.poolmanager.proxy_from_url,urllib3.util.Timeout,urllib3.exceptions.LocationValueError,urllib3.contrib.socks.SOCKSProxyManager,urllib3.exceptions.HTTPError,urllib3.exceptions.SSLError,urllib3.exceptions.ProxyError,urllib3.exceptions.InvalidHeader,urllib3.exceptions.MaxRetryError,urllib3.exceptions.ConnectTimeoutError,urllib3.exceptions.ClosedPoolError,urllib3.exceptions.ProtocolError,urllib3.util.retry.Retry,urllib3.exceptions.ResponseError,urllib3.exceptions.ReadTimeoutError,urllib3.exceptions.NewConnectionError,urllib3.util.parse_url,urllib3.poolmanager.PoolManager"), ModuleWithVersion("zope.interface", "", ">=5.1.0", "") ) } diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json index 7efeb65..b6af4c8 100644 --- a/wrapper/nodejs/package-lock.json +++ b/wrapper/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@appthreat/atom", - "version": "1.5.3", + "version": "1.5.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@appthreat/atom", - "version": "1.5.3", + "version": "1.5.4", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.0", diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json index 8b43fb4..93bddb8 100644 --- a/wrapper/nodejs/package.json +++ b/wrapper/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@appthreat/atom", - "version": "1.5.3", + "version": "1.5.4", "description": "Create atom (⚛) representation for your application, packages and libraries", "exports": "./index.js", "type": "module",