diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala index c213b6e8fe71..aa47f4c8d423 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/ExternalCommand.scala @@ -1,33 +1,24 @@ package io.joern.c2cpg.utils -import java.util.concurrent.ConcurrentLinkedQueue -import scala.sys.process.{Process, ProcessLogger} import scala.util.{Failure, Success, Try} -import scala.jdk.CollectionConverters.* -object ExternalCommand extends io.joern.x2cpg.utils.ExternalCommand { +object ExternalCommand { - override def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - result match { - case Success(0) => + import io.joern.x2cpg.utils.ExternalCommand.ExternalCommandResult + + private val IsWin = scala.util.Properties.isWin + + def run(command: Seq[String], cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { + io.joern.x2cpg.utils.ExternalCommand.run(command, cwd, mergeStdErrInStdOut = true, extraEnv) match { + case ExternalCommandResult(0, stdOut, _) => Success(stdOut) - case Success(1) if IsWin && IncludeAutoDiscovery.gccAvailable() => + case ExternalCommandResult(1, stdOut, _) if IsWin && IncludeAutoDiscovery.gccAvailable() => // the command to query the system header file locations within a Windows // environment always returns Success(1) for whatever reason... Success(stdOut) - case _ => + case ExternalCommandResult(_, stdOut, _) => Failure(new RuntimeException(stdOut.mkString(System.lineSeparator()))) } } - override def run(command: String, cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { - val stdOutOutput = new ConcurrentLinkedQueue[String] - val processLogger = ProcessLogger(stdOutOutput.add, stdOutOutput.add) - val process = shellPrefix match { - case Nil => Process(command, new java.io.File(cwd), extraEnv.toList*) - case _ => Process(shellPrefix :+ command, new java.io.File(cwd), extraEnv.toList*) - } - handleRunResult(Try(process.!(processLogger)), stdOutOutput.asScala.toSeq, Nil) - } - } diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala index 131434e29564..286f67d94cf4 100644 --- a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala +++ b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/utils/IncludeAutoDiscovery.scala @@ -13,13 +13,15 @@ object IncludeAutoDiscovery { private val IS_WIN = scala.util.Properties.isWin - val GCC_VERSION_COMMAND = "gcc --version" + val GCC_VERSION_COMMAND = Seq("gcc", "--version") private val CPP_INCLUDE_COMMAND = - if (IS_WIN) "gcc -xc++ -E -v . -o nul" else "gcc -xc++ -E -v /dev/null -o /dev/null" + if (IS_WIN) Seq("gcc", "-xc++", "-E", "-v", ".", "-o", "nul") + else Seq("gcc", "-xc++", "-E", "-v", "/dev/null", "-o", "/dev/null") private val C_INCLUDE_COMMAND = - if (IS_WIN) "gcc -xc -E -v . -o nul" else "gcc -xc -E -v /dev/null -o /dev/null" + if (IS_WIN) Seq("gcc", "-xc", "-E", "-v", ".", "-o", "nul") + else Seq("gcc", "-xc", "-E", "-v", "/dev/null", "-o", "/dev/null") // Only check once private var isGccAvailable: Option[Boolean] = None @@ -57,7 +59,7 @@ object IncludeAutoDiscovery { output.slice(startIndex, endIndex).map(p => Paths.get(p.trim).toRealPath()).toSet } - private def discoverPaths(command: String): Set[Path] = ExternalCommand.run(command, ".") match { + private def discoverPaths(command: Seq[String]): Set[Path] = ExternalCommand.run(command, ".") match { case Success(output) => extractPaths(output) case Failure(exception) => logger.warn(s"Unable to discover system include paths. Running '$command' failed.", exception) diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala index e5ac1153b08c..8a86765291f3 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DotNetAstGenRunner.scala @@ -63,8 +63,8 @@ class DotNetAstGenRunner(config: Config) extends AstGenRunnerBase(config) { override def runAstGenNative(in: String, out: File, exclude: String, include: String)(implicit metaData: AstGenProgramMetaData ): Try[Seq[String]] = { - val excludeCommand = if (exclude.isEmpty) "" else s"-e \"$exclude\"" - ExternalCommand.run(s"$astGenCommand -o ${out.toString()} -i \"$in\" $excludeCommand", ".") + val excludeCommand = if (exclude.isEmpty) Seq.empty else Seq("-e", exclude) + ExternalCommand.run(Seq(astGenCommand, "-o", out.toString(), "-i", in) ++ excludeCommand, ".").toTry } } diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala index 744d42b93185..d3234b5bc1db 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/passes/DownloadDependenciesPass.scala @@ -27,13 +27,13 @@ class DownloadDependenciesPass(cpg: Cpg, parentGoMod: GoModHelper, goGlobal: GoG parentGoMod .getModMetaData() .foreach(mod => { - ExternalCommand.run("go mod init joern.io/temp", projDir) match { + ExternalCommand.run(Seq("go", "mod", "init", "joern.io/temp"), projDir).toTry match { case Success(_) => mod.dependencies .filter(dep => dep.beingUsed) .map(dependency => { - val cmd = s"go get ${dependency.dependencyStr()}" - val results = ExternalCommand.run(cmd, projDir) + val cmd = Seq("go", "get", dependency.dependencyStr()) + val results = ExternalCommand.run(cmd, projDir).toTry results match { case Success(_) => print(". ") diff --git a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala index c7d107ce76cc..48fa174edbeb 100644 --- a/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/gosrc2cpg/src/main/scala/io/joern/gosrc2cpg/utils/AstGenRunner.scala @@ -75,16 +75,18 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen override def runAstGenNative(in: String, out: File, exclude: String, include: String)(implicit metaData: AstGenProgramMetaData ): Try[Seq[String]] = { - val excludeCommand = if (exclude.isEmpty) "" else s"-exclude \"$exclude\"" - val includeCommand = if (include.isEmpty) "" else s"-include-packages \"$include\"" - ExternalCommand.run(s"$astGenCommand $excludeCommand $includeCommand -out ${out.toString()} $in", ".") + val excludeCommand = if (exclude.isEmpty) Seq.empty else Seq("-exclude", exclude) + val includeCommand = if (include.isEmpty) Seq.empty else Seq("-include-packages", include) + ExternalCommand + .run((astGenCommand +: excludeCommand) ++ includeCommand ++ Seq("-out", out.toString(), in), ".") + .toTry } def executeForGo(out: File): List[GoAstGenRunnerResult] = { implicit val metaData: AstGenProgramMetaData = config.astGenMetaData val in = File(config.inputPath) logger.info(s"Running goastgen in '$config.inputPath' ...") - runAstGenNative(config.inputPath, out, config.ignoredFilesRegex.toString(), includeFileRegex.toString()) match { + runAstGenNative(config.inputPath, out, config.ignoredFilesRegex.toString(), includeFileRegex) match { case Success(result) => val srcFiles = SourceFiles.determine( out.toString(), @@ -114,7 +116,7 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen ): List[GoAstGenRunnerResult] = { val moduleMeta: ModuleMeta = ModuleMeta(inputPath, outPath, None, ListBuffer[String](), ListBuffer[String](), ListBuffer[ModuleMeta]()) - if (parsedModFiles.size > 0) { + if (parsedModFiles.nonEmpty) { parsedModFiles .sortBy(_.split(UtilityConstants.fileSeparateorPattern).length) .foreach(modFile => { @@ -122,11 +124,11 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen }) parsedFiles.foreach(moduleMeta.addParsedFile) skippedFiles.foreach(moduleMeta.addSkippedFile) - moduleMeta.getOnlyChilds() + moduleMeta.getOnlyChildren } else { parsedFiles.foreach(moduleMeta.addParsedFile) skippedFiles.foreach(moduleMeta.addSkippedFile) - moduleMeta.getAllChilds() + moduleMeta.getAllChildren } } @@ -184,12 +186,12 @@ class AstGenRunner(config: Config, includeFileRegex: String = "") extends AstGen } } - def getOnlyChilds(): List[GoAstGenRunnerResult] = { - childModules.flatMap(_.getAllChilds()).toList + def getOnlyChildren: List[GoAstGenRunnerResult] = { + childModules.flatMap(_.getAllChildren).toList } - def getAllChilds(): List[GoAstGenRunnerResult] = { - getOnlyChilds() ++ List( + def getAllChildren: List[GoAstGenRunnerResult] = { + getOnlyChildren ++ List( GoAstGenRunnerResult( modulePath = modulePath, parsedModFile = modFilePath, diff --git a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala index 1d1ab11b932a..69b1354d07be 100644 --- a/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala +++ b/joern-cli/frontends/javasrc2cpg/src/main/scala/io/joern/javasrc2cpg/util/Delombok.scala @@ -1,14 +1,14 @@ package io.joern.javasrc2cpg.util import better.files.File -import io.joern.x2cpg.utils.ExternalCommand import io.joern.javasrc2cpg.util.Delombok.DelombokMode.* +import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory -import java.nio.file.{Path, Paths} -import scala.collection.mutable -import scala.util.matching.Regex -import scala.util.{Failure, Success, Try} +import java.nio.file.Path +import scala.util.Failure +import scala.util.Success +import scala.util.Try object Delombok { @@ -53,8 +53,17 @@ object Delombok { System.getProperty("java.class.path") } val command = - s"$javaPath -cp $classPathArg lombok.launch.Main delombok ${inputPath.toAbsolutePath.toString} -d ${outputDir.canonicalPath}" - logger.debug(s"Executing delombok with command $command") + Seq( + javaPath, + "-cp", + classPathArg, + "lombok.launch.Main", + "delombok", + inputPath.toAbsolutePath.toString, + "-d", + outputDir.canonicalPath + ) + logger.debug(s"Executing delombok with command ${command.mkString(" ")}") command } @@ -72,6 +81,7 @@ object Delombok { Try(delombokTempDir.createChild(relativeOutputPath, asDirectory = true)).flatMap { packageOutputDir => ExternalCommand .run(delombokToTempDirCommand(inputDir, packageOutputDir, analysisJavaHome), ".") + .toTry .map(_ => delombokTempDir.path.toAbsolutePath.toString) } } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala index 17f636c75958..be065b36625f 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala @@ -126,7 +126,7 @@ object AstGenRunner { val astGenCommand = path.getOrElse("astgen") val localPath = path.flatMap(File(_).parentOption.map(_.pathAsString)).getOrElse(".") val debugMsgPath = path.getOrElse("PATH") - ExternalCommand.run(s"$astGenCommand --version", localPath).toOption.map(_.mkString.strip()) match { + ExternalCommand.run(Seq(astGenCommand, "--version"), localPath).successOption.map(_.mkString.strip()) match { case Some(installedVersion) if installedVersion != "unknown" && Try(VersionHelper.compare(installedVersion, astGenVersion)).toOption.getOrElse(-1) >= 0 => @@ -175,7 +175,7 @@ class AstGenRunner(config: Config) { import io.joern.jssrc2cpg.utils.AstGenRunner._ - private val executableArgs = if (!config.tsTypes) " --no-tsTypes" else "" + private val executableArgs = if (!config.tsTypes) Seq("--no-tsTypes") else Seq.empty private def skippedFiles(astGenOut: List[String]): List[String] = { val skipped = astGenOut.collect { @@ -297,7 +297,11 @@ class AstGenRunner(config: Config) { } val result = - ExternalCommand.run(s"$astGenCommand$executableArgs -t ts -o $out", out.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand.run( + (astGenCommand +: executableArgs) ++ Seq("-t", "ts", "-o", out.toString), + out.toString(), + extraEnv = NODE_OPTIONS + ) val jsons = SourceFiles.determine(out.toString(), Set(".json")) jsons.foreach { jsonPath => @@ -312,7 +316,7 @@ class AstGenRunner(config: Config) { } tmpJsFiles.foreach(_.delete()) - result + result.toTry } private def ejsFiles(in: File, out: File): Try[Seq[String]] = { @@ -337,12 +341,24 @@ class AstGenRunner(config: Config) { ignoredFilesPath = Some(config.ignoredFiles) ) if (files.nonEmpty) - ExternalCommand.run(s"$astGenCommand$executableArgs -t vue -o $out", in.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand + .run( + (astGenCommand +: executableArgs) ++ Seq("-t", "vue", "-o", out.toString), + in.toString(), + extraEnv = NODE_OPTIONS + ) + .toTry else Success(Seq.empty) } private def jsFiles(in: File, out: File): Try[Seq[String]] = - ExternalCommand.run(s"$astGenCommand$executableArgs -t ts -o $out", in.toString(), extraEnv = NODE_OPTIONS) + ExternalCommand + .run( + (astGenCommand +: executableArgs) ++ Seq("-t", "ts", "-o", out.toString), + in.toString(), + extraEnv = NODE_OPTIONS + ) + .toTry private def runAstGenNative(in: File, out: File): Try[Seq[String]] = for { ejsResult <- ejsFiles(in, out) diff --git a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala index d7c78defaa52..e8dd77a86bb6 100644 --- a/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala +++ b/joern-cli/frontends/kotlin2cpg/src/test/scala/io/joern/kotlin2cpg/compiler/CompilerAPITests.scala @@ -77,8 +77,9 @@ class CompilerAPITests extends AnyFreeSpec with Matchers { "should not contain methods with unresolved types/namespaces" in { val command = - if (scala.util.Properties.isWin) "cmd.exe /C gradlew.bat gatherDependencies" else "./gradlew gatherDependencies" - ExternalCommand.run(command, projectDirPath) shouldBe Symbol("success") + if (scala.util.Properties.isWin) Seq("cmd.exe", "/C", "gradlew.bat", "gatherDependencies") + else Seq("./gradlew", "gatherDependencies") + ExternalCommand.run(command, projectDirPath).toTry shouldBe Symbol("success") val config = Config(classpath = Set(projectDependenciesPath.toString)) val cpg = new Kotlin2Cpg().createCpg(projectDirPath)(config).getOrElse { fail("Could not create a CPG!") diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala index d0230fd616d5..c2b0b759be25 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/Php2Cpg.scala @@ -22,7 +22,7 @@ class Php2Cpg extends X2CpgFrontend[Config] { private val PhpVersionRegex = new Regex("^PHP ([78]\\.[1-9]\\.[0-9]|[9-9]\\d\\.\\d\\.\\d)") private def isPhpVersionSupported: Boolean = { - val result = ExternalCommand.run("php --version", ".") + val result = ExternalCommand.run(Seq("php", "--version"), ".").toTry result match { case Success(listString) => val phpVersionStr = listString.headOption.getOrElse("") diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala index 0ac7a485d9b0..7110da272d71 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/ClassParser.scala @@ -3,7 +3,6 @@ import better.files.File import io.joern.x2cpg.utils.ExternalCommand import org.slf4j.LoggerFactory -import scala.collection.immutable.LazyList.from import scala.io.Source import scala.util.{Failure, Success, Try, Using} import upickle.default.* @@ -26,11 +25,12 @@ class ClassParser(targetDir: File) { f } - private lazy val phpClassParseCommand: String = s"php ${classParserScript.pathAsString} ${targetDir.pathAsString}" + private lazy val phpClassParseCommand: Seq[String] = + Seq("php", classParserScript.pathAsString, targetDir.pathAsString) def parse(): Try[List[ClassParserClass]] = Try { val inputDirectory = targetDir.parent.canonicalPath - ExternalCommand.run(phpClassParseCommand, inputDirectory).map(_.reverse) match { + ExternalCommand.run(phpClassParseCommand, inputDirectory).toTry.map(_.reverse) match { case Success(output) => read[List[ClassParserClass]](output.mkString("\n")) case Failure(exception) => diff --git a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala index 0b8d704d806c..583000ec8fed 100644 --- a/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala +++ b/joern-cli/frontends/php2cpg/src/main/scala/io/joern/php2cpg/parser/PhpParser.scala @@ -17,10 +17,9 @@ class PhpParser private (phpParserPath: String, phpIniPath: String, disableFileC private val logger = LoggerFactory.getLogger(this.getClass) - private def phpParseCommand(filenames: collection.Seq[String]): String = { - val phpParserCommands = "--with-recovery --resolve-names --json-dump" - val filenamesString = filenames.mkString(" ") - s"php --php-ini $phpIniPath $phpParserPath $phpParserCommands $filenamesString" + private def phpParseCommand(filenames: collection.Seq[String]): Seq[String] = { + val phpParserCommands = Seq("--with-recovery", "--resolve-names", "--json-dump") + Seq("php", "--php-ini", phpIniPath, phpParserPath) ++ phpParserCommands ++ filenames } def parseFiles(inputPaths: collection.Seq[String]): collection.Seq[(String, Option[PhpFile], String)] = { @@ -37,10 +36,10 @@ class PhpParser private (phpParserPath: String, phpIniPath: String, disableFileC val command = phpParseCommand(inputPaths) - val (returnValue, output) = ExternalCommand.runWithMergeStdoutAndStderr(command, ".") - returnValue match { - case 0 => - val asJson = linesToJsonValues(output.lines().toArray(size => new Array[String](size))) + val result = ExternalCommand.run(command, ".", mergeStdErrInStdOut = true) + result match { + case ExternalCommand.ExternalCommandResult(0, stdOut, _) => + val asJson = linesToJsonValues(stdOut) val asPhpFile = asJson.map { case (filename, jsonObjectOption, infoLines) => (filename, jsonToPhpFile(jsonObjectOption, filename), infoLines) } @@ -48,8 +47,8 @@ class PhpParser private (phpParserPath: String, phpIniPath: String, disableFileC (canonicalToInputPath.apply(filename), phpFileOption, infoLines) } withRemappedFileName - case exitCode => - logger.error(s"Failure running php-parser with $command, exit code $exitCode") + case ExternalCommand.ExternalCommandResult(exitCode, _, _) => + logger.error(s"Failure running php-parser with ${command.mkString(" ")}, exit code $exitCode") Nil } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala index 7d04f4d21adf..7c9301ed293d 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/RubySrc2Cpg.scala @@ -91,14 +91,13 @@ class RubySrc2Cpg extends X2CpgFrontend[Config] { private def downloadDependency(inputPath: String, tempPath: String): Unit = { if (Files.isRegularFile(Paths.get(s"${inputPath}${java.io.File.separator}Gemfile"))) { - ExternalCommand.run(s"bundle config set --local path ${tempPath}", inputPath) match { + ExternalCommand.run(Seq("bundle", "config", "set", "--local", "path", tempPath), inputPath).toTry match { case Success(configOutput) => logger.info(s"Gem config successfully done: $configOutput") case Failure(exception) => logger.error(s"Error while configuring Gem Path: ${exception.getMessage}") } - val command = s"bundle install" - ExternalCommand.run(command, inputPath) match { + ExternalCommand.run(Seq("bundle", "install"), inputPath).toTry match { case Success(bundleOutput) => logger.info(s"Dependency installed successfully: $bundleOutput") case Failure(exception) => diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala index 1b2ef98e9a94..88b07c32917d 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/AstGenRunner.scala @@ -65,7 +65,7 @@ object AstGenRunner { val astGenCommand = path.getOrElse("SwiftAstGen") val localPath = path.flatMap(File(_).parentOption.map(_.pathAsString)).getOrElse(".") val debugMsgPath = path.getOrElse("PATH") - ExternalCommand.run(s"$astGenCommand -h", localPath).toOption match { + ExternalCommand.run(Seq(astGenCommand, "-h"), localPath).toOption match { case Some(_) => logger.debug(s"Using SwiftAstGen from $debugMsgPath") true @@ -140,7 +140,7 @@ class AstGenRunner(config: Config) { } private def runAstGenNative(in: File, out: File): Try[Seq[String]] = - ExternalCommand.run(s"$astGenCommand -o $out", in.toString()) + ExternalCommand.run(Seq(astGenCommand, "-o", out.toString), in.toString()) private def checkParsedFiles(files: List[String], in: File): List[String] = { val numOfParsedFiles = files.size diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala index b2ba122f8a5d..84ff1422e78c 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/utils/ExternalCommand.scala @@ -4,19 +4,20 @@ import scala.util.Failure import scala.util.Success import scala.util.Try -object ExternalCommand extends io.joern.x2cpg.utils.ExternalCommand { +object ExternalCommand { - override def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - result match { - case Success(0) => + import io.joern.x2cpg.utils.ExternalCommand.ExternalCommandResult + + def run(command: Seq[String], cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { + io.joern.x2cpg.utils.ExternalCommand.run(command, cwd, mergeStdErrInStdOut = true, extraEnv) match { + case ExternalCommandResult(0, stdOut, _) => Success(stdOut) - case Success(_) if stdErr.isEmpty && stdOut.nonEmpty => + case ExternalCommandResult(_, stdOut, stdErr) if stdErr.isEmpty && stdOut.nonEmpty => // SwiftAstGen exits with exit code != 0 on Windows. // To catch with we specifically handle the empty stdErr here. Success(stdOut) - case _ => - val allOutput = stdOut ++ stdErr - Failure(new RuntimeException(allOutput.mkString(System.lineSeparator()))) + case ExternalCommandResult(_, stdOut, _) => + Failure(new RuntimeException(stdOut.mkString(System.lineSeparator()))) } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala index ae3ebc43c33f..6852d6c611ce 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/astgen/AstGenRunner.scala @@ -53,7 +53,7 @@ object AstGenRunner { .toString def hasCompatibleAstGenVersion(compatibleVersion: String)(implicit metaData: AstGenProgramMetaData): Boolean = { - ExternalCommand.run(s"$metaData.name -version", ".").toOption.map(_.mkString.strip()) match { + ExternalCommand.run(Seq(metaData.name, "-version"), ".").successOption.map(_.mkString.strip()) match { case Some(installedVersion) if installedVersion != "unknown" && Try(VersionHelper.compare(installedVersion, compatibleVersion)).toOption.getOrElse(-1) >= 0 => @@ -64,7 +64,8 @@ object AstGenRunner { s"Found local ${metaData.name} v$installedVersion in systems PATH but ${metaData.name} requires at least v$compatibleVersion" ) false - case _ => false + case _ => + false } } diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala index 5e75bece13ed..30b0d5e569a5 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/ExternalCommand.scala @@ -2,70 +2,84 @@ package io.joern.x2cpg.utils import org.slf4j.LoggerFactory +import java.io.BufferedReader import java.io.File -import java.nio.file.{Path, Paths} -import java.util.concurrent.ConcurrentLinkedQueue -import scala.sys.process.{Process, ProcessLogger} -import scala.util.{Failure, Success, Try} +import java.io.InputStreamReader +import java.nio.file.Path +import java.nio.file.Paths import scala.jdk.CollectionConverters.* -import System.lineSeparator +import scala.util.control.NonFatal +import scala.util.Failure +import scala.util.Success +import scala.util.Try -trait ExternalCommand { +object ExternalCommand { - private val logger = LoggerFactory.getLogger(this.getClass) + private val logger = LoggerFactory.getLogger(ExternalCommand.getClass) - protected val IsWin: Boolean = scala.util.Properties.isWin - - // do not prepend any shell layer by default - // individual frontends may override this - protected val shellPrefix: Seq[String] = Nil - - protected def handleRunResult(result: Try[Int], stdOut: Seq[String], stdErr: Seq[String]): Try[Seq[String]] = { - if (stdErr.nonEmpty) logger.warn(s"subprocess stderr: ${stdErr.mkString(lineSeparator)}") - - result match { - case Success(0) => Success(stdOut) - case Failure(error) => Failure(error) - case Success(nonZeroExitCode) => + case class ExternalCommandResult(exitCode: Int, stdOut: Seq[String], stdErr: Seq[String]) { + def successOption: Option[Seq[String]] = exitCode match { + case 0 => Some(stdOut) + case _ => None + } + def toTry: Try[Seq[String]] = exitCode match { + case 0 => Success(stdOut) + case nonZeroExitCode => val allOutput = stdOut ++ stdErr - val message = - s"""Process exited with code $nonZeroExitCode. Output: - |${allOutput.mkString(lineSeparator)} - |""".stripMargin + val message = s"""Process exited with code $nonZeroExitCode. Output: + |${allOutput.mkString(System.lineSeparator())} + |""".stripMargin Failure(new RuntimeException(message)) } } - def run(command: String, cwd: String, extraEnv: Map[String, String] = Map.empty): Try[Seq[String]] = { - val stdOutOutput = new ConcurrentLinkedQueue[String] - val stdErrOutput = new ConcurrentLinkedQueue[String] - val processLogger = ProcessLogger(stdOutOutput.add, stdErrOutput.add) - val process = shellPrefix match { - case Nil => Process(command, new java.io.File(cwd), extraEnv.toList*) - case _ => Process(shellPrefix :+ command, new java.io.File(cwd), extraEnv.toList*) - } - handleRunResult(Try(process.!(processLogger)), stdOutOutput.asScala.toSeq, stdErrOutput.asScala.toSeq) - } - - // We use the java ProcessBuilder API instead of the Scala version because it - // offers the possibility to merge stdout and stderr into one stream of output. - // Maybe the Scala version also offers this but since there is no documentation - // I was not able to figure it out. - def runWithMergeStdoutAndStderr(command: String, cwd: String): (Int, String) = { + def run( + command: Seq[String], + cwd: String, + mergeStdErrInStdOut: Boolean = false, + extraEnv: Map[String, String] = Map.empty + ): ExternalCommandResult = { val builder = new ProcessBuilder() - builder.command(command.split(' ')*) - builder.directory(new File(cwd)) - builder.redirectErrorStream(true) + .command(command.toArray*) + .directory(new File(cwd)) + .redirectErrorStream(mergeStdErrInStdOut) + builder.environment().putAll(extraEnv.asJava) - val process = builder.start() - val outputBytes = process.getInputStream.readAllBytes() - val returnValue = process.waitFor() + val stdOut = scala.collection.mutable.ArrayBuffer.empty[String] + val stdErr = scala.collection.mutable.ArrayBuffer.empty[String] - (returnValue, new String(outputBytes)) - } -} + try { + val process = builder.start() + + val outputReaderThread = new Thread(() => { + val outputReader = new BufferedReader(new InputStreamReader(process.getInputStream)) + outputReader.lines.iterator.forEachRemaining(stdOut.addOne) + }) + + val errorReaderThread = new Thread(() => { + val errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream)) + errorReader.lines.iterator.forEachRemaining(stdErr.addOne) + }) + + outputReaderThread.start() + errorReaderThread.start() -object ExternalCommand extends ExternalCommand { + val returnValue = process.waitFor() + outputReaderThread.join() + errorReaderThread.join() + + process.getInputStream.close() + process.getOutputStream.close() + process.getErrorStream.close() + process.destroy() + + if (stdErr.nonEmpty) logger.warn(s"subprocess stderr: ${stdErr.mkString(System.lineSeparator())}") + ExternalCommandResult(returnValue, stdOut.toSeq, stdErr.toSeq) + } catch { + case NonFatal(exception) => + ExternalCommandResult(1, Seq.empty, stdErr = Seq(exception.getMessage)) + } + } /** Finds the absolute path to the executable directory (e.g. `/path/to/javasrc2cpg/bin`). Based on the package path * of a loaded classfile based on some (potentially flakey?) filename heuristics. Context: we want to be able to diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala index 51f4986e4e52..5cfdb1f50769 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala @@ -45,7 +45,9 @@ object DependencyResolver { projectDir: Path, configuration: String ): Option[collection.Seq[String]] = { - val lines = ExternalCommand.run(s"gradle dependencies --configuration $configuration", projectDir.toString) match { + val lines = ExternalCommand + .run(Seq("gradle", "dependencies", "--configuration,", configuration), projectDir.toString) + .toTry match { case Success(lines) => lines case Failure(exception) => logger.warn( diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala index 4594359e9a31..caa20f7745fd 100644 --- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala +++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/MavenDependencies.scala @@ -17,14 +17,14 @@ object MavenDependencies { private val fetchCommand = s"mvn $$$MavenCliOpts --fail-never -B dependency:build-classpath -DincludeScope=compile -Dorg.slf4j.simpleLogger.defaultLogLevel=info -Dorg.slf4j.simpleLogger.logFile=System.out" - private val fetchCommandWithOpts = { + private val fetchCommandWithOpts: Seq[String] = { // These options suppress output, so if they're provided we won't get any results. // "-q" and "--quiet" are the only ones that would realistically be used. val optionsToStrip = Set("-h", "--help", "-q", "--quiet", "-v", "--version") val mavenOpts = Option(System.getenv(MavenCliOpts)).getOrElse("") val mavenOptsStripped = mavenOpts.split(raw"\s").filterNot(optionsToStrip.contains).mkString(" ") - fetchCommand.replace(s"$$$MavenCliOpts", mavenOptsStripped) + fetchCommand.replace(s"$$$MavenCliOpts", mavenOptsStripped).split(" ").toSeq } private def logErrors(output: String): Unit = { @@ -40,7 +40,7 @@ object MavenDependencies { } private[dependency] def get(projectDir: Path): Option[collection.Seq[String]] = { - val lines = ExternalCommand.run(fetchCommandWithOpts, projectDir.toString) match { + val lines = ExternalCommand.run(fetchCommandWithOpts, projectDir.toString).toTry match { case Success(lines) => if (lines.contains("[INFO] Build failures were ignored.")) { logErrors(lines.mkString(System.lineSeparator())) diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala index 108c4806ae2e..ef89b57fd2b3 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/ExternalCommandTest.scala @@ -8,18 +8,19 @@ import scala.util.Properties.isWin import scala.util.{Failure, Success} class ExternalCommandTest extends AnyWordSpec with Matchers { - def cwd = File.currentWorkingDirectory.pathAsString + + private def cwd = File.currentWorkingDirectory.pathAsString "ExternalCommand.run" should { "be able to run `ls` successfully" in { File.usingTemporaryDirectory("sample") { sourceDir => - val cmd = "ls " + sourceDir.pathAsString - ExternalCommand.run(cmd, sourceDir.pathAsString) should be a Symbol("success") + val cmd = Seq("ls", sourceDir.pathAsString) + ExternalCommand.run(cmd, sourceDir.pathAsString).toTry should be a Symbol("success") } } "report exit code and stdout/stderr for nonzero exit code" in { - ExternalCommand.run("ls /does/not/exist", cwd) match { + ExternalCommand.run(Seq("ls", "/does/not/exist"), cwd).toTry match { case result: Success[_] => fail(s"expected failure, but got $result") case Failure(exception) => @@ -29,7 +30,7 @@ class ExternalCommandTest extends AnyWordSpec with Matchers { } "report error for io exception (e.g. for nonexisting command)" in { - ExternalCommand.run("/command/does/not/exist", cwd) match { + ExternalCommand.run(Seq("/command/does/not/exist"), cwd).toTry match { case result: Success[_] => fail(s"expected failure, but got $result") case Failure(exception) => diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala index 58bde517e1bd..0a8c79b14727 100644 --- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala +++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/utils/dependency/DependencyResolverTests.scala @@ -31,7 +31,7 @@ class DependencyResolverTests extends AnyWordSpec with Matchers { "test maven dependency resolution" ignore { // check that `mvn` is available - otherwise test will fail with only some logged warnings... withClue("`mvn` must be installed in order for this test to work...") { - ExternalCommand.run("mvn --version", ".").get.exists(_.contains("Apache Maven")) shouldBe true + ExternalCommand.run(Seq("mvn", "--version"), ".").successOption.exists(_.contains("Apache Maven")) shouldBe true } @nowarn // otherwise scalac warns that this might be an interpolated expression