From 595a0875fdc6d618b6108f0c047dde08b60fd17e Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Wed, 25 Sep 2024 09:59:30 +0200 Subject: [PATCH 1/2] [ruby] Add handling for multiple call args --- .../io/joern/rubysrc2cpg/parser/RubyParser.g4 | 5 +++-- .../parser/AntlrContextHelpers.scala | 3 ++- .../querying/SingleAssignmentTests.scala | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 index 3faeff03db83..3cd87839949f 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 @@ -249,9 +249,10 @@ splatArgList ; commandArgumentList - : associationList + : argumentList + | associationList | primaryValueList (COMMA NL* associationList)? - ; + ; primaryValueList : primaryValue (COMMA NL* primaryValue)* diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala index 79ebcdc68e5a..07bba3e6a824 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala @@ -158,7 +158,8 @@ object AntlrContextHelpers { def elements: List[ParserRuleContext] = { val primaryValues = Option(ctx.primaryValueList()).map(_.primaryValue().asScala.toList).getOrElse(List()) val associations = Option(ctx.associationList()).map(_.association().asScala.toList).getOrElse(List()) - primaryValues ++ associations + val argumentLists = Option(ctx.argumentList()).map(_.elements).getOrElse(List()) + primaryValues ++ associations ++ argumentLists } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala index f6a29543d839..c682714155ef 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/SingleAssignmentTests.scala @@ -1,7 +1,7 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal} +import io.shiftleft.codepropertygraph.generated.nodes.{Block, Call, Identifier, Literal} import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, DispatchTypes, Operators} import io.shiftleft.semanticcpg.language.* import io.joern.rubysrc2cpg.passes.Defines as RubyDefines @@ -489,4 +489,20 @@ class SingleAssignmentTests extends RubyCode2CpgFixture { case xs => fail(s"Expected one call for assignment, got ${xs.code.mkString(",")}") } } + + "MethodInvocationWithoutParentheses multiple call args" in { + val cpg = code(""" + |def gl_badge_tag(*args, &block) + | render :some_symbol, &block + |end + |""".stripMargin) + + inside(cpg.call.name("render").argument.l) { + case _ :: (blockArg: Identifier) :: (symbolArg: Literal) :: Nil => + blockArg.code shouldBe "block" + symbolArg.code shouldBe ":some_symbol" + + case xs => fail(s"Expected two args, found [${xs.code.mkString(",")}]") + } + } } From 6912b483ee9d62ff28b7826b4d920e7b494eb435 Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Wed, 25 Sep 2024 11:23:57 +0200 Subject: [PATCH 2/2] [ruby] fixed failing tests --- .../antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 | 11 ++++++++--- .../rubysrc2cpg/parser/AntlrContextHelpers.scala | 12 +++++++++++- .../io/joern/rubysrc2cpg/parser/AstPrinter.scala | 8 ++++---- .../joern/rubysrc2cpg/parser/RubyNodeCreator.scala | 10 +++++----- .../rubysrc2cpg/parser/AssignmentParserTests.scala | 2 +- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 index 3cd87839949f..70a5eddd5217 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 +++ b/joern-cli/frontends/rubysrc2cpg/src/main/antlr4/io/joern/rubysrc2cpg/parser/RubyParser.g4 @@ -143,10 +143,16 @@ command # commandTernaryOperatorExpression | primary NL? (AMPDOT | DOT | COLON2) methodName commandArgument # memberAccessCommand - | methodIdentifier commandArgument + | methodIdentifier simpleCommandArgumentList # simpleCommand ; +simpleCommandArgumentList + : associationList + | primaryValueList (COMMA NL* associationList)? + | argumentList + ; + commandArgument : commandArgumentList # commandArgumentCommandArgumentList @@ -249,8 +255,7 @@ splatArgList ; commandArgumentList - : argumentList - | associationList + : associationList | primaryValueList (COMMA NL* associationList)? ; diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala index 07bba3e6a824..2fb6347b127f 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AntlrContextHelpers.scala @@ -146,7 +146,7 @@ object AntlrContextHelpers { def parameters: List[ParserRuleContext] = Option(ctx.blockParameterList()).map(_.parameters).getOrElse(List()) } - sealed implicit class CommandArgumentContextHelper(ctx: CommandArgumentContext) { + sealed implicit class CommandArgumentContextelper(ctx: CommandArgumentContext) { def arguments: List[ParserRuleContext] = ctx match { case ctx: CommandCommandArgumentListContext => ctx.command() :: Nil case ctx: CommandArgumentCommandArgumentListContext => ctx.commandArgumentList().elements @@ -156,6 +156,14 @@ object AntlrContextHelpers { sealed implicit class CommandArgumentListContextHelper(ctx: CommandArgumentListContext) { def elements: List[ParserRuleContext] = { + val primaryValues = Option(ctx.primaryValueList()).map(_.primaryValue().asScala.toList).getOrElse(List()) + val associations = Option(ctx.associationList()).map(_.association().asScala.toList).getOrElse(List()) + primaryValues ++ associations + } + } + + sealed implicit class SimpleCommandArgumentListContextHelper(ctx: SimpleCommandArgumentListContext) { + def arguments: List[ParserRuleContext] = { val primaryValues = Option(ctx.primaryValueList()).map(_.primaryValue().asScala.toList).getOrElse(List()) val associations = Option(ctx.associationList()).map(_.association().asScala.toList).getOrElse(List()) val argumentLists = Option(ctx.argumentList()).map(_.elements).getOrElse(List()) @@ -333,6 +341,8 @@ object AntlrContextHelpers { Option(ctx.blockArgument()).toList case ctx: ArrayArgumentListContext => Option(ctx.indexingArgumentList()).toList + case ctx: SingleCommandArgumentListContext => + Option(ctx.command()).toList case ctx => logger.warn(s"ArgumentListContextHelper - Unsupported element type ${ctx.getClass.getSimpleName}") List() diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala index 8ab33db21348..ac16b39440eb 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/AstPrinter.scala @@ -601,13 +601,13 @@ class AstPrinter extends RubyParserBaseVisitor[String] { } override def visitSimpleCommand(ctx: RubyParser.SimpleCommandContext): String = { - if (Option(ctx.commandArgument()).map(_.getText).exists(_.startsWith("::"))) { - val memberName = ctx.commandArgument().getText.stripPrefix("::") + if (Option(ctx.simpleCommandArgumentList()).map(_.getText).exists(_.startsWith("::"))) { + val memberName = ctx.simpleCommandArgumentList().getText.stripPrefix("::") val methodIdentifier = visit(ctx.methodIdentifier()) s"$methodIdentifier::$memberName" } else if (!ctx.methodIdentifier().isAttrDeclaration) { val identifierCtx = ctx.methodIdentifier() - val arguments = ctx.commandArgument().arguments.map(visit) + val arguments = ctx.simpleCommandArgumentList().arguments.map(visit) (identifierCtx.getText, arguments) match { case ("require", List(argument)) => s"require ${arguments.mkString(",")}" @@ -627,7 +627,7 @@ class AstPrinter extends RubyParserBaseVisitor[String] { s"${visit(identifierCtx)} ${arguments.mkString(",")}" } } else { - s"${ctx.commandArgument.arguments.map(visit).mkString(",")}" + s"${ctx.simpleCommandArgumentList.arguments.map(visit).mkString(",")}" } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala index 6d534953b6d6..cddc74784dae 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyNodeCreator.scala @@ -681,8 +681,8 @@ class RubyNodeCreator( } override def visitSimpleCommand(ctx: RubyParser.SimpleCommandContext): RubyExpression = { - if (Option(ctx.commandArgument()).map(_.getText).exists(_.startsWith("::"))) { - val memberName = ctx.commandArgument().getText.stripPrefix("::") + if (Option(ctx.simpleCommandArgumentList()).map(_.getText).exists(_.startsWith("::"))) { + val memberName = ctx.simpleCommandArgumentList().getText.stripPrefix("::") if (memberName.headOption.exists(_.isUpper)) { // Constant accesses are upper-case 1st letter MemberAccess(visit(ctx.methodIdentifier()), "::", memberName)(ctx.toTextSpan) } else { @@ -690,7 +690,7 @@ class RubyNodeCreator( } } else if (!ctx.methodIdentifier().isAttrDeclaration) { val identifierCtx = ctx.methodIdentifier() - val arguments = ctx.commandArgument().arguments.map(visit) + val arguments = ctx.simpleCommandArgumentList().arguments.map(visit) (identifierCtx.getText, arguments) match { case (requireLike, List(argument)) if ImportsPass.ImportCallNames.contains(requireLike) => val isRelative = requireLike == "require_relative" || requireLike == "require_all" @@ -713,14 +713,14 @@ class RubyNodeCreator( val lhsIdentifier = SimpleIdentifier(None)(identifierCtx.toTextSpan.spanStart(idAssign.stripSuffix("="))) val argNode = arguments match { case arg :: Nil => arg - case xs => ArrayLiteral(xs)(ctx.commandArgument().toTextSpan) + case xs => ArrayLiteral(xs)(ctx.simpleCommandArgumentList().toTextSpan) } SingleAssignment(lhsIdentifier, "=", argNode)(ctx.toTextSpan) case _ => SimpleCall(visit(identifierCtx), arguments)(ctx.toTextSpan) } } else { - FieldsDeclaration(ctx.commandArgument().arguments.map(visit))(ctx.toTextSpan) + FieldsDeclaration(ctx.simpleCommandArgumentList().arguments.map(visit))(ctx.toTextSpan) } } diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala index 9b6a3953d0e5..4b48440153df 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/parser/AssignmentParserTests.scala @@ -50,7 +50,7 @@ class AssignmentParserTests extends RubyParserFixture with Matchers { test("*a, b, c = 1, 2, 3, 4") test("a, b, c = 1, 2, *list") test("a, b, c = 1, *list") - test("a = b, *c, d") + test("a = *c, b, d") } "Class Constant Assign" in {