diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8de2b36..aed3f2f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -29,6 +29,10 @@ jobs: with: repository: 'HooliCorp/DjanGoat' path: 'repotests/DjanGoat' + - uses: actions/checkout@v3 + with: + repository: 'DefectDojo/django-DefectDojo' + path: 'repotests/django-DefectDojo' - uses: coursier/cache-action@v6 - name: Set up JDK uses: actions/setup-java@v3 @@ -41,6 +45,7 @@ jobs: ./atom.sh -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml + ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json @@ -54,7 +59,15 @@ jobs: ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json + ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json ls -lh /tmp/*.atom /tmp/*.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: | + npm install -g @cyclonedx/cdxgen --omit=optional + cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo + ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json + env: + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + if: runner.os != 'Windows' diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index 7b15afd..3ec4b43 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -40,6 +40,10 @@ jobs: with: repository: 'HooliCorp/DjanGoat' path: 'repotests/DjanGoat' + - uses: actions/checkout@v3 + with: + repository: 'DefectDojo/django-DefectDojo' + path: 'repotests/django-DefectDojo' - uses: coursier/cache-action@v6 - name: Set up JDK uses: actions/setup-java@v3 @@ -57,6 +61,7 @@ jobs: ./atom.sh -o /tmp/ts.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml ./atom.sh parsedeps -o /tmp/py.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml + ./atom.sh -o /tmp/py2.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml ./atom.sh -o /tmp/c.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml ./atom.sh data-flow -o /tmp/java2.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java.slices.json @@ -69,6 +74,7 @@ jobs: # ./atom.sh usages -o /tmp/juice3.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/juice.usages.json ./atom.sh usages -o /tmp/ts3.atom -l js $GITHUB_WORKSPACE/repotests/shiftleft-ts-example -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/ts.usages.json ./atom.sh usages -o /tmp/py3.atom -l python $GITHUB_WORKSPACE/repotests/DjanGoat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py.usages.json + ./atom.sh usages -o /tmp/py4.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/py4.usages.json ./atom.sh usages -o /tmp/c3.atom -l c $GITHUB_WORKSPACE/repotests/libexpat -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/c.usages.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -77,6 +83,9 @@ jobs: npm install -g @cyclonedx/cdxgen --omit=optional cdxgen -t java --deep -o $GITHUB_WORKSPACE/repotests/java-sec-code/bom.json $GITHUB_WORKSPACE/repotests/java-sec-code ./atom.sh reachables -o /tmp/java-sec-code.atom -l java $GITHUB_WORKSPACE/repotests/java-sec-code -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/java-sec-code.reachables.json + cdxgen -t python --deep -o $GITHUB_WORKSPACE/repotests/django-DefectDojo/bom.json $GITHUB_WORKSPACE/repotests/django-DefectDojo + ./atom.sh reachables -o /tmp/django-DefectDojo.atom -l python $GITHUB_WORKSPACE/repotests/django-DefectDojo -Dlog4j.configurationFile=log4j2.xml --slice-outfile /tmp/django-DefectDojo.reachables.json + if: runner.os != 'Windows' env: JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" - run: | diff --git a/build.sbt b/build.sbt index f245abd..f61c7c6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,9 +1,9 @@ name := "atom" ThisBuild / organization := "io.appthreat" -ThisBuild / version := "1.5.2" +ThisBuild / version := "1.5.3" ThisBuild / scalaVersion := "3.3.1" -val chenVersion = "0.0.20" +val chenVersion = "0.5.2" lazy val atom = Projects.atom diff --git a/lib/README.md b/lib/README.md index 64118d5..937016e 100644 --- a/lib/README.md +++ b/lib/README.md @@ -1,3 +1,3 @@ org.eclipse.cdt jars were downloaded from -https://download.eclipse.org/tools/cdt/releases/11.2/cdt-11.2.0/plugins/ +https://download.eclipse.org/tools/cdt/releases/11.3/cdt-11.3.1/plugins/ diff --git a/lib/org.eclipse.cdt.core_8.2.0.202305101618.jar b/lib/org.eclipse.cdt.core_8.3.1.202309150117.jar similarity index 75% rename from lib/org.eclipse.cdt.core_8.2.0.202305101618.jar rename to lib/org.eclipse.cdt.core_8.3.1.202309150117.jar index a706497..56103ac 100644 Binary files a/lib/org.eclipse.cdt.core_8.2.0.202305101618.jar and b/lib/org.eclipse.cdt.core_8.3.1.202309150117.jar differ diff --git a/src/main/scala/io/appthreat/atom/Atom.scala b/src/main/scala/io/appthreat/atom/Atom.scala index fd14740..f1e19b7 100644 --- a/src/main/scala/io/appthreat/atom/Atom.scala +++ b/src/main/scala/io/appthreat/atom/Atom.scala @@ -268,6 +268,7 @@ object Atom { new DataFlowSlicing().calculateDataFlowSlice(cpg, dataFlowConfig.asInstanceOf[DataFlowConfig]) case x: AtomUsagesConfig => println("Slicing the atom for usages. This might take a few minutes ...") + new ChennaiTagsPass(cpg).createAndApply() val usagesConfig = migrateAtomConfigToSliceConfig(x) Option(UsageSlicing.calculateUsageSlice(cpg, usagesConfig.asInstanceOf[UsagesConfig])) case x: AtomReachablesConfig => diff --git a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala index f852a3d..f3798d7 100644 --- a/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala +++ b/src/main/scala/io/appthreat/atom/parsedeps/PythonDependencyParser.scala @@ -80,7 +80,6 @@ object PythonDependencyParser extends XDependencyParser { .filterNot(_ == "N/A") .map(x => ScalaFile(x)) .l - val parentList = fileList.flatMap(_.parentOption.map(_.pathAsString)) cpg.imports .whereNot(_.call.file.name(".*setup.py")) .filterNot { @@ -92,8 +91,7 @@ object PythonDependencyParser extends XDependencyParser { } .dedup .importedEntity - .flatMap(_.split('.').headOption) - .map(x => ModuleWithVersion(x)) + .map(x => ModuleWithVersion(name = x.split('.').head, importedSymbols = x)) .toSet } diff --git a/src/main/scala/io/appthreat/atom/parsedeps/package.scala b/src/main/scala/io/appthreat/atom/parsedeps/package.scala index 1afb58b..273c21d 100644 --- a/src/main/scala/io/appthreat/atom/parsedeps/package.scala +++ b/src/main/scala/io/appthreat/atom/parsedeps/package.scala @@ -36,22 +36,35 @@ package object parsedeps { } implicit val moduleWithVersionEncoder: Encoder[ModuleWithVersion] = - Encoder.forProduct3("name", "version", "versionSpecifiers")(x => (x.name, x.version, x.versionSpecifiers)) + Encoder.forProduct4("name", "version", "versionSpecifiers", "importedSymbols")(x => + (x.name, x.version, x.versionSpecifiers, x.importedSymbols) + ) implicit val moduleWithVersionDecoder: Decoder[ModuleWithVersion] = - Decoder.forProduct3("name", "version", "versionSpecifiers")(ModuleWithVersion.apply) + Decoder.forProduct4("name", "version", "versionSpecifiers", "importedSymbols")(ModuleWithVersion.apply) case class DependencySlice(modules: Seq[ModuleWithVersion]) extends AtomSlice { override def toJson: String = this.asJson.spaces2 } - case class ModuleWithVersion(name: String, version: String = "", versionSpecifiers: String = "") { + case class ModuleWithVersion( + name: String, + version: String = "", + versionSpecifiers: String = "", + importedSymbols: String = "" + ) { def merge(x: ModuleWithVersion): ModuleWithVersion = { val vs = this.versions ++ x.versions + val is = this.importedSymbols + "," + x.importedSymbols vs.find(_.startsWith("==")) match case Some(exactVersion) => - ModuleWithVersion(name, exactVersion.stripPrefix("=="), (vs diff Set(exactVersion)).mkString(",")) - case None => ModuleWithVersion(name, versionSpecifiers = vs.mkString(",")) + ModuleWithVersion( + name, + exactVersion.stripPrefix("=="), + (vs diff Set(exactVersion)).mkString(","), + importedSymbols = is + ) + case None => ModuleWithVersion(name, versionSpecifiers = vs.mkString(","), importedSymbols = is) } diff --git a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala index 7b070f1..ed0c4ff 100644 --- a/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/ReachableSlicing.scala @@ -36,21 +36,23 @@ object ReachableSlicing { .toList flowsList ++= atom.tag.name(API_TAG).parameter.reachableByFlows(atom.tag.name(API_TAG).parameter).map(toSlice).toList - // For JavaScript, we need flows between arguments of call nodes to track callbacks and middlewares - if (language == Languages.JSSRC || language == Languages.JAVASCRIPT) { - def jsCallSource = atom.tag.name(config.sourceTag).call.argument.isIdentifier - def jsFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier - def jsFrameworkParameter = atom.tag.name(FRAMEWORK_TAG).parameter - def jsSink = atom.tag.name(config.sinkTag).call.argument.isIdentifier - flowsList ++= jsSink - .reachableByFlows(jsCallSource, jsFrameworkIdentifier, jsFrameworkParameter) + // For JavaScript and Python, we need flows between arguments of call nodes to track callbacks and middlewares + if ( + language == Languages.JSSRC || language == Languages.JAVASCRIPT || language == Languages.PYTHON || language == Languages.PYTHONSRC + ) { + def dynCallSource = atom.tag.name(config.sourceTag).call.argument.isIdentifier + def dynFrameworkIdentifier = atom.tag.name(FRAMEWORK_TAG).identifier + def dynFrameworkParameter = atom.tag.name(FRAMEWORK_TAG).parameter + def dynSink = atom.tag.name(config.sinkTag).call.argument.isIdentifier + flowsList ++= dynSink + .reachableByFlows(dynCallSource, dynFrameworkIdentifier, dynFrameworkParameter) .map(toSlice) .toList flowsList ++= atom.tag .name(FRAMEWORK_TAG) .call .argument - .reachableByFlows(jsFrameworkParameter) + .reachableByFlows(dynFrameworkParameter) .map(toSlice) .toList } diff --git a/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala b/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala index 812435f..368c7cc 100644 --- a/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala +++ b/src/main/scala/io/appthreat/atom/slicing/UsageSlicing.scala @@ -4,6 +4,7 @@ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.codepropertygraph.generated.{Languages, Operators, PropertyNames} import io.shiftleft.semanticcpg.language.* +import overflowdb.PropertyKey import java.util.concurrent.* import java.util.concurrent.atomic.AtomicBoolean @@ -21,6 +22,7 @@ object UsageSlicing { val exec: ExecutorService = Executors.newWorkStealingPool(Runtime.getRuntime.availableProcessors() / 2) private val constructorTypeMatcher = Pattern.compile(".*new (\\w+)\\(.*") private val excludeOperatorCalls = new AtomicBoolean(true) + private val FRAMEWORK_ROUTE = "framework-route" /** Generates object slices from the given CPG. * @@ -44,6 +46,8 @@ object UsageSlicing { val userDefTypes = userDefinedTypes(atom) if (language.get == Languages.NEWC || language.get == Languages.C) ProgramUsageSlice(slices ++ importsAsSlices(atom), userDefTypes) + else if (language.get == Languages.PYTHON || language.get == Languages.PYTHONSRC) + ProgramUsageSlice(slices, userDefTypes ++ routesAsUDT(atom)) else ProgramUsageSlice(slices, userDefTypes) } @@ -101,6 +105,58 @@ object UsageSlicing { }) } + /** Discovers internally defined routes. + * + * @param atom + * the CPG to query for types. + * @return + * a list of user defined types. + */ + def routesAsUDT(atom: Cpg): List[UserDefinedType] = { + + def generateUDT(call: Call): UserDefinedType = { + UserDefinedType( + call.name, + call.argument.isLiteral + .map(m => + LocalDef( + name = m.code, + typeFullName = m.typeFullName, + lineNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.LINE_NUMBER))).map(_.toInt), + columnNumber = Option(m.property(new PropertyKey[Integer](PropertyNames.COLUMN_NUMBER))).map(_.toInt) + ) + ) + .collectAll[LocalDef] + .l, + call + .callee(NoResolve) + .method + .filterNot(m => m.name.startsWith("")) + .map(m => + ObservedCall( + m.name, + Option(m.fullName), + m.parameter.map(_.typeFullName).toList, + m.methodReturn.typeFullName, + Option(m.isExternal), + m.lineNumber.map(_.intValue()), + m.columnNumber.map(_.intValue()) + ) + ) + .l, + call.location.filename, + call.lineNumber.map(_.intValue()), + call.columnNumber.map(_.intValue()) + ) + } + + atom.call + .where(_.argument.tag.nameExact(FRAMEWORK_ROUTE)) + .map(generateUDT) + .filter(udt => udt.fields.nonEmpty || udt.procedures.nonEmpty) + .l + } + private def TimedGet(dsf: Future[Option[(Method, ObjectUsageSlice)]]) = { try { dsf.get(5, TimeUnit.SECONDS) diff --git a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala index 0a026f1..aea7e85 100644 --- a/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala +++ b/src/test/scala/io/appthreat/atom/PythonDependencyScannerTests.scala @@ -197,19 +197,19 @@ class PythonDependencyScannerTests extends PySrc2CpgFixture(withOssDataflow = fa "have the modules scanned successfully" in { val scanResult = PythonDependencyParser.parse(cpg) scanResult.modules shouldBe List( - ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b"), - ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0"), - ModuleWithVersion("certifi", "", ">=2017.4.17"), - ModuleWithVersion("charset_normalizer", "", ">=2,<4"), - ModuleWithVersion("idna", "", ">=2.5,<4"), - ModuleWithVersion("os", "", ""), - ModuleWithVersion("packageA", "", ">=1.4.2,<1.9,!=1.5.*,!=1.6.*"), - ModuleWithVersion("packageB", "", ">=0.5.0,< 0.7.0"), - ModuleWithVersion("re-wx", "", ">=0.0.2"), - ModuleWithVersion("socket", "", ""), - ModuleWithVersion("typing-extensions", "3.10.0.2", ""), - ModuleWithVersion("urllib3", "", ">=1.21.1,<3"), - ModuleWithVersion("zope.interface", "", ">=5.1.0") + ModuleWithVersion("PackageC", "1.2.0.dev1+hg.5.b11e5e6f0b0b", ""), + ModuleWithVersion("PickyThing", "2.4c1", "<1.6,>1.9,!=1.9.6,<2.0a0", ""), + ModuleWithVersion("certifi", "", ">=2017.4.17", ""), + ModuleWithVersion("charset_normalizer", "", ">=2,<4", ""), + ModuleWithVersion("idna", "", ">=2.5,<4", ""), + ModuleWithVersion("os", "", "", "os.path"), + ModuleWithVersion("packageA", "", ">=1.4.2,<1.9,!=1.5.*,!=1.6.*", ""), + ModuleWithVersion("packageB", "", ">=0.5.0,< 0.7.0", ""), + ModuleWithVersion("re-wx", "", ">=0.0.2", ""), + ModuleWithVersion("socket", "", "", "socket"), + ModuleWithVersion("typing-extensions", "3.10.0.2", "", ""), + ModuleWithVersion("urllib3", "", ">=1.21.1,<3", "urllib3.poolmanager.proxy_from_url,urllib3.util.Timeout,urllib3.exceptions.LocationValueError,urllib3.contrib.socks.SOCKSProxyManager,urllib3.exceptions.HTTPError,urllib3.exceptions.SSLError,urllib3.exceptions.ProxyError,urllib3.exceptions.InvalidHeader,urllib3.exceptions.MaxRetryError,urllib3.exceptions.ConnectTimeoutError,urllib3.exceptions.ClosedPoolError,urllib3.exceptions.ProtocolError,urllib3.util.retry.Retry,urllib3.exceptions.ResponseError,,urllib3.exceptions.ReadTimeoutError,urllib3.exceptions.NewConnectionError,urllib3.util.parse_url,urllib3.poolmanager.PoolManager"), + ModuleWithVersion("zope.interface", "", ">=5.1.0", "") ) } } diff --git a/wrapper/nodejs/index.js b/wrapper/nodejs/index.js index b59b3cb..c265717 100755 --- a/wrapper/nodejs/index.js +++ b/wrapper/nodejs/index.js @@ -2,6 +2,8 @@ import { freemem, platform as _platform } from "node:os"; import { dirname, join, delimiter } from "node:path"; +import { readFileSync } from "node:fs"; + import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { detectJava } from "./utils.mjs"; @@ -12,7 +14,8 @@ if (!url.startsWith("file://")) { url = new URL(`file://${import.meta.url}`).toString(); } const dirName = import.meta ? dirname(fileURLToPath(url)) : __dirname; - +const selfPJson = JSON.parse(readFileSync(join(dirName, "package.json"))); +const _version = selfPJson.version; export const LOG4J_CONFIG = join(dirName, "plugins", "log4j2.xml"); export const ATOM_HOME = join(dirName, "plugins"); export const APP_LIB_DIR = join(ATOM_HOME, "lib"); @@ -22,7 +25,7 @@ export const JAVA_OPTS = `${ process.env.JAVA_OPTS || "" } -Xmx${freeMemoryGB}G ${JVM_ARGS}`; export const APP_MAIN_CLASS = "io.appthreat.atom.Atom"; -export const ATOM_VERSION = "1.5.2"; +export const ATOM_VERSION = _version; export const APP_CLASSPATH = join( APP_LIB_DIR, `io.appthreat.atom-${ATOM_VERSION}-classpath.jar` diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json index 1503f57..7efeb65 100644 --- a/wrapper/nodejs/package-lock.json +++ b/wrapper/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@appthreat/atom", - "version": "1.5.2", + "version": "1.5.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@appthreat/atom", - "version": "1.5.2", + "version": "1.5.3", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.0", diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json index e9af5b3..8b43fb4 100644 --- a/wrapper/nodejs/package.json +++ b/wrapper/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "@appthreat/atom", - "version": "1.5.2", + "version": "1.5.3", "description": "Create atom (⚛) representation for your application, packages and libraries", "exports": "./index.js", "type": "module",