Release 11 July 2023: Scala 3 Upgrade

.github/workflows/code-quality.yml

@@ -18,7 +18,7 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Check formatting
-        run: sbt ++2.13.8 scalafmtCheck test:scalafmtCheck
+        run: sbt scalafmtCheck test:scalafmtCheck
       - run: echo "Previous step failed because code is not formatted. Run 'sbt scalafmt Test/scalafmt'"
         if: ${{ failure() }}
 
@@ -35,6 +35,6 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: Run unit test
-        run: sbt ++2.13.8 test test:test
+        run: sbt test test:test
       - run: echo "Previous step failed because unit test failed."
         if: ${{ failure() }}

.scalafmt.conf

@@ -1,5 +1,5 @@
 version = 3.5.1
-runner.dialect = scala213
+runner.dialect = scala3
 preset = IntelliJ
 maxColumn = 120
 align.preset = true

build.sbt

@@ -1,60 +1,58 @@
 import sbt.Credentials
 name                     := "privado-core"
 ThisBuild / organization := "ai.privado"
-ThisBuild / scalaVersion := "2.13.8"
+ThisBuild / scalaVersion := "3.3.0"
 ThisBuild / version      := sys.env.getOrElse("BUILD_VERSION", "dev-SNAPSHOT")
 // parsed by project/Versions.scala, updated by updateDependencies.sh
 
-val cpgVersion        = "1.3.612"
-val joernVersion      = "1.2.36"
+val cpgVersion        = "1.4.1"
+val joernVersion      = "2.0.5"
 val overflowdbVersion = "1.179"
 
 //External dependency versions
-val circeVersion   = "0.14.1"
-val jacksonVersion = "2.14.3"
-val mockitoVersion = "1.17.12"
+val circeVersion   = "0.14.2"
+val jacksonVersion = "2.15.2"
+val mockitoVersion = "1.17.14"
 
 lazy val schema         = Projects.schema
 lazy val domainClasses  = Projects.domainClasses
 lazy val schemaExtender = Projects.schemaExtender
 
 dependsOn(domainClasses)
 libraryDependencies ++= Seq(
-  "com.github.pathikrit"            %% "better-files"            % "3.9.1",
-  "com.github.scopt"                %% "scopt"                   % "3.7.1",
+  "com.github.pathikrit"            %% "better-files"            % "3.9.2",
+  "com.github.scopt"                %% "scopt"                   % "4.1.0",
   "io.joern"                        %% "x2cpg"                   % Versions.joern,
   "io.joern"                        %% "javasrc2cpg"             % Versions.joern,
   "io.joern"                        %% "pysrc2cpg"               % Versions.joern,
   "io.joern"                        %% "rubysrc2cpg"             % Versions.joern,
   "io.joern"                        %% "joern-cli"               % Versions.joern,
   "io.joern"                        %% "semanticcpg"             % Versions.joern,
   "io.joern"                        %% "semanticcpg"             % Versions.joern % Test classifier "tests",
-  "org.scalatest"                   %% "scalatest"               % "3.1.1"        % Test,
-  "org.mockito"                     %% "mockito-scala"           % mockitoVersion % Test,
-  "org.mockito"                     %% "mockito-scala-scalatest" % mockitoVersion % Test,
-  "org.mockito"                     %% "mockito-scala-specs2"    % mockitoVersion % Test,
+  "org.scalatest"                   %% "scalatest"               % "3.2.16"       % Test,
   "io.circe"                        %% "circe-core"              % circeVersion,
   "io.circe"                        %% "circe-generic"           % circeVersion,
   "io.circe"                        %% "circe-parser"            % circeVersion,
-  "io.circe"                        %% "circe-yaml"              % circeVersion,
+  // NOTE: circe-yaml currently only goes until 0.14.2 (Last checked 06/07/2023)
+  "io.circe"                        %% "circe-yaml"              % circeVersion exclude("org.yaml", "snakeyaml"),
   "com.lihaoyi"                     %% "upickle"                 % "2.0.0",
   "com.lihaoyi"                     %% "requests"                % "0.7.0",
   "org.scala-lang.modules"          %% "scala-xml"               % "2.1.0",
   "commons-io"                       % "commons-io"              % "2.11.0",
   "com.networknt"                    % "json-schema-validator"   % "1.0.72",
   "com.fasterxml.jackson.module"    %% "jackson-module-scala"    % jacksonVersion,
-  "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion,
+  "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion exclude("org.yaml", "snakeyaml"),
   "com.github.wnameless.json"        % "json-flattener"          % "0.14.0",
   "org.apache.logging.log4j"         % "log4j-core"              % "2.19.0",
   "org.apache.logging.log4j"         % "log4j-slf4j2-impl"       % "2.19.0"       % Runtime,
   "org.apache.poi"                   % "poi-ooxml"               % "5.2.2",
   "com.github.jsqlparser"            % "jsqlparser"              % "4.6",
   "org.apache.maven"                 % "maven-model"             % "3.9.0",
   "net.sourceforge.htmlunit"         % "htmlunit"                % "2.70.0",
-  "org.yaml"                         % "snakeyaml"               % "1.29",
+  "org.yaml"                         % "snakeyaml"               % "1.33",
   "org.scala-lang"                   % "scala-reflect"           % "2.13.8",
   "org.scala-lang"                   % "scala-compiler"          % "2.13.8",
-  "com.iheart"                      %% "ficus"                   % "1.4.7" exclude ("com.typesafe", "config")
+  "com.iheart"                      %% "ficus"                   % "1.5.2" exclude ("com.typesafe", "config")
 )
 
 ThisBuild / Compile / scalacOptions ++= Seq("-feature", "-deprecation", "-language:implicitConversions")
@@ -70,7 +68,7 @@ ThisBuild / resolvers ++= Seq(
   "Sonatype OSS" at "https://oss.sonatype.org/content/repositories/public",
   "Gradle Releases" at "https://repo.gradle.org/gradle/libs-releases"
 ) ++ Resolver.sonatypeOssRepos("snapshots")
-lazy val astGenDlUrl       = "https://github.com/joernio/astgen/releases/download/v2.14.0/"
+lazy val astGenDlUrl       = "https://github.com/joernio/astgen/releases/download/v3.1.0/"
 lazy val astGenBinaryNames = Seq("astgen-linux", "astgen-macos", "astgen-win.exe", "astgen-macos-arm")
 
 lazy val astGenDlTask = taskKey[Unit](s"Download astgen binaries")

src/main/scala/ai/privado/audit/AuditReportConstants.scala

@@ -114,7 +114,7 @@ object AuditReportConstants {
   val JS_ELEMENT_DISCOVERY_OBJECT_INCLUDE_REGEX = ".*__ecma\\.(String|Number|Boolean|Object).*"
 
   val JS_ELEMENT_DISCOVERY_EXCLUDE_PARAMS_REGEX =
-    "(?i)(this|request|response|req|error|req|res|state|e|i|formik)|param.*"
+    "(?i)(this|request|response|req|error|req|res|state|e|i|formik)|param.*|_tmp.*"
 
   val JS_ELEMENT_DISCOVERY_TYPE_EXCLUDE_REGEX = ".*(?i)(anonymous|props|response|request).*"
 }

src/main/scala/ai/privado/audit/DataElementDiscovery.scala

@@ -2,10 +2,9 @@ package ai.privado.audit
 
 import ai.privado.audit.DataElementDiscovery.{CollectionMethodInfo, getClass, getFileScore, getSourceUsingRules, logger}
 import ai.privado.cache.TaggerCache
-import ai.privado.languageEngine.java.language.module.{NodeStarters, StepsForModule}
 import ai.privado.model.{CatLevelOne, Constants, InternalTag}
 import io.shiftleft.codepropertygraph.generated.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes.{Member, MethodParameterIn, TypeDecl}
+import io.shiftleft.codepropertygraph.generated.nodes.{File, Identifier, Local, Member, MethodParameterIn, TypeDecl}
 import io.shiftleft.semanticcpg.language._
 import org.slf4j.LoggerFactory
 import ai.privado.dataflow.Dataflow
@@ -429,15 +428,39 @@ object DataElementDiscoveryJS {
     methodParameterMap.toMap
   }
 
-  def processDataElementDiscovery(xtocpg: Try[Cpg], taggerCache: TaggerCache): List[List[String]] = {
-    val classNameRuleList   = getSourceUsingRules(xtocpg)
-    val memberInfo          = DataElementDiscovery.getMemberUsingClassName(xtocpg, classNameRuleList.toSet)
-    val workbookResult      = new ListBuffer[List[String]]()
-    val typeDeclMemberCache = taggerCache.typeDeclMemberCache
+  def getIdentifiers(xtocpg: Try[Cpg]): List[Identifier] = {
+    xtocpg match {
+      case Success(cpg) => {
+        cpg.identifier.l
+      }
+      case Failure(ex) => {
+        ex.printStackTrace()
+        List[Identifier]()
+      }
+    }
+  }
 
-    val methodParametersFromTypes = getMethodParametersFromTypes(xtocpg, classNameRuleList.toSet)
+  def getLocals(xtocpg: Try[Cpg]): List[Local] = {
+    xtocpg match {
+      case Success(cpg) => {
+        cpg.local.l
+      }
+      case Failure(ex) => {
+        ex.printStackTrace()
+        List[Local]()
+      }
+    }
+  }
 
-    val elementInfo = mutable.HashMap[TypeDecl, ListBuffer[Any]]()
+  def processDataElementDiscovery(xtocpg: Try[Cpg], taggerCache: TaggerCache): List[List[String]] = {
+    val classNameRuleList         = getSourceUsingRules(xtocpg)
+    val memberInfo                = DataElementDiscovery.getMemberUsingClassName(xtocpg, classNameRuleList.toSet)
+    val workbookResult            = new ListBuffer[List[String]]()
+    val typeDeclMemberCache       = taggerCache.typeDeclMemberCache
+    val methodParametersFromTypes = getMethodParametersFromTypes(xtocpg, classNameRuleList.toSet)
+    val identifiers               = getIdentifiers(xtocpg)
+    val locals                    = getLocals(xtocpg)
+    val elementInfo               = mutable.HashMap[TypeDecl, ListBuffer[Any]]()
 
     methodParametersFromTypes.foreach {
       case (typeDecl, paramList) => {
@@ -585,11 +608,53 @@ object DataElementDiscoveryJS {
         }
       }
 
+      identifiers.foreach(identifier => {
+        if (
+          identifier.name.nonEmpty && !identifier.name
+            .matches(AuditReportConstants.JS_ELEMENT_DISCOVERY_TYPE_EXCLUDE_REGEX)
+          && !identifier.name
+            .matches(AuditReportConstants.JS_ELEMENT_DISCOVERY_EXCLUDE_PARAMS_REGEX)
+        )
+          workbookResult += List(
+            identifier.typeFullName,
+            identifier.file.name.headOption.getOrElse(AuditReportConstants.AUDIT_EMPTY_CELL_VALUE),
+            getFileScore(identifier.file.name.headOption.getOrElse(Constants.EMPTY), xtocpg),
+            identifier.name,
+            identifier.typeFullName,
+            AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
+          )
+      })
+
+      locals.foreach(local => {
+        if (
+          local.name.nonEmpty && !local.name
+            .matches(AuditReportConstants.JS_ELEMENT_DISCOVERY_TYPE_EXCLUDE_REGEX) && !local.name
+            .matches(AuditReportConstants.JS_ELEMENT_DISCOVERY_EXCLUDE_PARAMS_REGEX)
+        )
+          workbookResult += List(
+            local.typeFullName,
+            local.file.head.name,
+            getFileScore(local.file.name.headOption.getOrElse(Constants.EMPTY), xtocpg),
+            local.name,
+            local.typeFullName,
+            AuditReportConstants.AUDIT_NOT_CHECKED_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE,
+            AuditReportConstants.AUDIT_EMPTY_CELL_VALUE
+          )
+      })
+
       logger.info("Shutting down audit engine")
     } catch {
       case ex: Exception =>
         println("Failed to process Data Element Discovery report")
         logger.debug("Failed to process Data Element Discovery report", ex)
+        ex.printStackTrace()
     }
     workbookResult.toList
   }

src/main/scala/ai/privado/dataflow/Dataflow.scala

@@ -70,13 +70,11 @@ class Dataflow(cpg: Cpg) {
     println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --no of source nodes - ${sources.size}")
     println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --no of sinks nodes - ${sinks.size}")
 
-    if (sources.isEmpty || sinks.isEmpty)
-      Map[String, Path]()
+    if (sources.isEmpty || sinks.isEmpty) Map[String, Path]()
     else {
       println(s"${TimeMetric.getNewTimeAndSetItToStageLast()} - --Finding flows invoked...")
       val dataflowPathsUnfiltered = {
-        if (privadoScanConfig.disable2ndLevelClosure)
-          sinks.reachableByFlows(sources).l
+        if (privadoScanConfig.disable2ndLevelClosure) sinks.reachableByFlows(sources).l
         else {
           val firstLevelSources =
             sources.or(

src/main/scala/ai/privado/entrypoint/CommandParser.scala

@@ -255,7 +255,7 @@ object CommandParser {
             println(OParser.usage(parser))
             exit(1)
         }
-        commandProcessor.config = config
+        commandProcessor.withConfig(config)
         Some(commandProcessor)
       case _ =>
         println(OParser.usage(parser))

src/main/scala/ai/privado/entrypoint/CommandProcessor.scala

@@ -23,6 +23,13 @@
 package ai.privado.entrypoint
 
 trait CommandProcessor {
-  var config: PrivadoInput
+
+  var config: PrivadoInput = null
+
+  def withConfig(value: PrivadoInput): CommandProcessor = {
+    this.config = value
+    this
+  }
+
   def process(): Either[String, Unit]
 }

src/main/scala/ai/privado/entrypoint/RuleValidator.scala

@@ -9,8 +9,6 @@ object RuleValidator extends CommandProcessor {
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
-  override var config: PrivadoInput = _
-
   override def process(): Either[String, Unit] = {
     println("Starting rule validations ...")
     validateRules() match {

src/main/scala/ai/privado/entrypoint/ScanProcessor.scala

@@ -378,5 +378,4 @@ object ScanProcessor extends CommandProcessor {
     sourceLocation.listRecursively.count(f => f.extension(toLowerCase = true).toString.contains(".java")) > 0
   }
 
-  override var config: PrivadoInput = _
 }

src/main/scala/ai/privado/entrypoint/UploadProcessor.scala

@@ -50,5 +50,4 @@ object UploadProcessor extends CommandProcessor {
         Left("Output file does not exist.")
     }
   }
-  override var config: PrivadoInput = _
 }

src/main/scala/ai/privado/exporter/ExporterUtility.scala

@@ -122,8 +122,7 @@ object ExporterUtility {
                 currentNodeModel
             }
         }
-      } else
-        currentNodeModel
+      } else currentNodeModel
     }
   }
 
@@ -168,8 +167,7 @@ object ExporterUtility {
       }
     }
 
-    if (fileName.equals(Constants.EMPTY) || sample.equals(Constants.EMPTY))
-      None
+    if (fileName.equals(Constants.EMPTY) || sample.equals(Constants.EMPTY)) None
     else {
       val message = {
         if (Iterator(node).isCall.nonEmpty) {

src/main/scala/ai/privado/exporter/SinkExporter.scala

@@ -142,8 +142,7 @@ class SinkExporter(cpg: Cpg, ruleCache: RuleCache) {
                   .toArray
               }
               apiurls
-            } else
-              Array[String]()
+            } else Array[String]()
           }
           val databaseDetails = DatabaseDetailsCache.getDatabaseDetails(rule.id)
           Some(

src/main/scala/ai/privado/languageEngine/java/passes/read/DatabaseReadUtility.scala

@@ -77,8 +77,7 @@ object DatabaseReadUtility {
             .l
         } else
           List()
-      } else
-        List(node)
+      } else List(node)
     }
 
     val sensitiveClassesWithMatchedRules = taggerCache.typeDeclMemberCache

src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala

@@ -161,8 +161,7 @@ class FeignAPI(cpg: Cpg, ruleCache: RuleCache) {
             .getOrElse(feignFlows.head.elements.head.code.split(" ").last)
       }
       (cpg.typeDecl.name(firstArgument).l, apiLiteral)
-    } else
-      (List[TypeDecl](), "")
+    } else (List[TypeDecl](), "")
   }
 
   /** Tag all the feign api calls which have some url like thing associated with them

src/main/scala/ai/privado/languageEngine/javascript/tagger/sink/JSAPITagger.scala

@@ -41,8 +41,7 @@ class JSAPITagger(cpg: Cpg, ruleCache: RuleCache) extends APITagger(cpg, ruleCac
     scriptTags.foreach(scriptTag => {
       var newRuleIdToUse = ruleInfo.id
       val domain         = getDomainFromTemplates(scriptTag.code)
-      if (ruleInfo.id.equals(Constants.internalAPIRuleId))
-        addRuleTags(builder, scriptTag, ruleInfo, ruleCache)
+      if (ruleInfo.id.equals(Constants.internalAPIRuleId)) addRuleTags(builder, scriptTag, ruleInfo, ruleCache)
       else {
         newRuleIdToUse = ruleInfo.id + "." + domain._2
         ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain._2))
@@ -77,8 +76,7 @@ class JSAPITagger(cpg: Cpg, ruleCache: RuleCache) extends APITagger(cpg, ruleCac
       var newRuleIdToUse = ruleInfo.id
       val domain         = getDomainFromTemplates(link.code)
       val callTag        = link.astParent
-      if (ruleInfo.id.equals(Constants.internalAPIRuleId))
-        addRuleTags(builder, callTag, ruleInfo, ruleCache)
+      if (ruleInfo.id.equals(Constants.internalAPIRuleId)) addRuleTags(builder, callTag, ruleInfo, ruleCache)
       else {
         newRuleIdToUse = ruleInfo.id + "." + domain._2
         ruleCache.setRuleInfo(ruleInfo.copy(id = newRuleIdToUse, name = ruleInfo.name + " " + domain._2))

src/main/scala/ai/privado/languageEngine/python/semantic/PythonSemanticGenerator.scala

@@ -68,8 +68,7 @@ object PythonSemanticGenerator extends SemanticGenerator {
             arg.order - 1
         }
         Some(index.toString + s"${Constants.semanticDelimeter}\"${arg.argumentName.get}\"")
-      } else
-        None
+      } else None
     }.l
     val parameterSemantic = mutable.HashSet[String]()