From c1621d339faf1f7227acf43066ec32c697564ba2 Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Thu, 9 May 2024 11:36:05 +0200 Subject: [PATCH 1/5] [ruby] Added lifting of method and type refs to directly under :program --- .../rubysrc2cpg/astcreation/AstCreator.scala | 67 +++++++++++++- .../astcreation/AstCreatorHelper.scala | 20 ++++- .../rubysrc2cpg/querying/MethodTests.scala | 90 ++++++++++++++++++- 3 files changed, 172 insertions(+), 5 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index e329b69b034c..51725f0332da 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -6,7 +6,7 @@ import io.joern.rubysrc2cpg.parser.{RubyNodeCreator, RubyParser} import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.utils.NodeBuilders.newModifierNode import io.joern.x2cpg.{Ast, AstCreatorBase, AstNodeBuilder, ValidationMode} -import io.shiftleft.codepropertygraph.generated.ModifierTypes +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, ModifierTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.* import io.shiftleft.semanticcpg.language.types.structure.NamespaceTraversal import org.slf4j.{Logger, LoggerFactory} @@ -44,6 +44,8 @@ class AstCreator( .map(_.stripPrefix(java.io.File.separator)) .getOrElse(fileName) + private val internalLineAndColNum: Option[Integer] = Option(1) + /** The relative file name, in a unix path delimited format. */ private def relativeUnixStyleFileName = @@ -94,14 +96,73 @@ class AstCreator( scope.pushNewScope(moduleScope) val block = blockNode(rootNode) scope.pushNewScope(BlockScope(block)) - val statementAsts = rootNode.statements.flatMap(astsForStatement) + val statementAsts = rootNode.statements.flatMap(astsForStatement) + val internalMethodRefAsts = methodRefNodesForInternalMethods() + val internalTypeRefAsts = typeRefNodesForInternalDecls() scope.popScope() - val bodyAst = blockAst(block, statementAsts) + val bodyAst = blockAst(block, internalTypeRefAsts ++ internalMethodRefAsts ++ statementAsts) scope.popScope() methodAst(methodNode_, Seq.empty, bodyAst, methodReturn, newModifierNode(ModifierTypes.MODULE) :: Nil) } .getOrElse(Ast()) } + + private def methodRefNodesForInternalMethods(): List[Ast] = { + val typeNameForMethods = scope.surroundingTypeFullName + .map { x => + x.split("[:]{2}").dropRight(1).mkString("") + } + .getOrElse(Defines.Undefined) + + programSummary.namespaceToType + .filter(_._1 == typeNameForMethods) + .flatMap(_._2) + .filter(x => x.name.contains(":program")) + .flatMap(_.methods) + .filterNot(x => x.name.contains("") || x.name.contains("")) + .map { method => + val methodRefNode = NewMethodRef() + .code(s"def ${method.name} (...)") + .methodFullName(scope.surroundingTypeFullName.map { x => s"$x:${method.name}" }.getOrElse(method.name)) + .typeFullName(Defines.Any) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + val methodRefIdent = NewIdentifier() + .code(method.name) + .name(method.name) + .typeFullName(Defines.Any) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + astForAssignment(methodRefIdent, methodRefNode, internalLineAndColNum, internalLineAndColNum) + } + .toList + } + + private def typeRefNodesForInternalDecls(): List[Ast] = { + programSummary.namespaceToType + .filter(_._1.contains(scope.surroundingTypeFullName.getOrElse(Defines.Undefined))) + .flatMap(_._2) + .map { x => + val typeRefName = x.name.split("[.]").takeRight(1).head + val typeRefNode = NewTypeRef() + .code(s"class ${x.name} (...)") + .typeFullName(x.name) + + val typeRefIdent = NewIdentifier() + .code(typeRefName) + .name(typeRefName) + .typeFullName(x.name) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + astForAssignment(typeRefIdent, typeRefNode, internalLineAndColNum, internalLineAndColNum) + } + .toList + } + + private def generateAssignmentNode(): Unit = {} } /** Determines till what depth the AST creator will parse until. diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala index 6616e0cd4f04..92b04a95d6be 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreatorHelper.scala @@ -10,7 +10,7 @@ import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.{ } import io.joern.rubysrc2cpg.datastructures.{BlockScope, FieldDecl} import io.joern.x2cpg.{Ast, ValidationMode} -import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Operators} +import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, Operators} import io.shiftleft.codepropertygraph.generated.nodes.* import io.joern.rubysrc2cpg.passes.Defines import io.joern.rubysrc2cpg.passes.Defines.RubyOperators @@ -81,6 +81,24 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As } + protected def astForAssignment( + lhs: NewNode, + rhs: NewNode, + lineNumber: Option[Integer], + columnNumber: Option[Integer] + ): Ast = { + val code = Seq(lhs, rhs).collect { case x: AstNodeNew => x.code }.mkString(" = ") + val assignment = NewCall() + .name(Operators.assignment) + .methodFullName(Operators.assignment) + .code(code) + .dispatchType(DispatchTypes.STATIC_DISPATCH) + .lineNumber(lineNumber) + .columnNumber(columnNumber) + + callAst(assignment, Seq(Ast(lhs), Ast(rhs))) + } + protected val UnaryOperatorNames: Map[String, String] = Map( "!" -> Operators.logicalNot, "not" -> Operators.logicalNot, diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala index 38f007d3f761..192cb513ccc9 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/MethodTests.scala @@ -2,8 +2,9 @@ package io.joern.rubysrc2cpg.querying import io.joern.rubysrc2cpg.testfixtures.RubyCode2CpgFixture import io.joern.x2cpg.Defines +import io.joern.rubysrc2cpg.passes.Defines as RDefines import io.shiftleft.codepropertygraph.generated.{ControlStructureTypes, Operators} -import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, Return} +import io.shiftleft.codepropertygraph.generated.nodes.{Call, Identifier, Literal, MethodRef, Return, TypeRef} import io.shiftleft.semanticcpg.language.* class MethodTests extends RubyCode2CpgFixture { @@ -538,4 +539,91 @@ class MethodTests extends RubyCode2CpgFixture { } } } + + "METHOD_REF and TYPE_REF nodes" should { + val cpg = code( + """ + |module A + | def foo + | end + |end + | + |class B + |end + | + |def c + |end + |""".stripMargin, + "t1.rb" + ) + .moreCode( + """ + |require 't1' + |class D + |end + | + |def e + |end + |""".stripMargin, + "t2.rb" + ) + + "be directly under :program" in { + inside(cpg.method.name(RDefines.Program).filename("t1.rb").assignment.l) { + case moduleAssignment :: classAssignment :: methodAssignment :: Nil => + moduleAssignment.code shouldBe "A = class t1.rb:::program.A (...)" + classAssignment.code shouldBe "B = class t1.rb:::program.B (...)" + methodAssignment.code shouldBe "c = def c (...)" + + inside(moduleAssignment.argument.l) { + case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => + lhs.name shouldBe "A" + rhs.typeFullName shouldBe "t1.rb:::program.A" + case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") + } + + inside(classAssignment.argument.l) { + case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => + lhs.name shouldBe "B" + rhs.typeFullName shouldBe "t1.rb:::program.B" + case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") + } + + inside(methodAssignment.argument.l) { + case (lhs: Identifier) :: (rhs: MethodRef) :: Nil => + lhs.name shouldBe "c" + rhs.methodFullName shouldBe "t1.rb:::program:c" + rhs.typeFullName shouldBe RDefines.Any + case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") + } + + case xs => fail(s"Expected three assignments, got [${xs.code.mkString(",")}]") + } + } + + "not be present in other files" in { + inside(cpg.method.name(RDefines.Program).filename("t2.rb").assignment.l) { + case classAssignment :: methodAssignment :: Nil => + classAssignment.code shouldBe "D = class t2.rb:::program.D (...)" + methodAssignment.code shouldBe "e = def e (...)" + + inside(classAssignment.argument.l) { + case (lhs: Identifier) :: (rhs: TypeRef) :: Nil => + lhs.name shouldBe "D" + rhs.typeFullName shouldBe "t2.rb:::program.D" + case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") + } + + inside(methodAssignment.argument.l) { + case (lhs: Identifier) :: (rhs: MethodRef) :: Nil => + lhs.name shouldBe "e" + rhs.methodFullName shouldBe "t2.rb:::program:e" + rhs.typeFullName shouldBe RDefines.Any + case xs => fail(s"Expected lhs and rhs, instead got ${xs.code.mkString(",")}") + } + + case xs => fail(s"Expected two assignments, got [${xs.code.mkString(",")}]") + } + } + } } From 6685355352c7b095742811625421ebbce7cdea5c Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Thu, 9 May 2024 11:37:33 +0200 Subject: [PATCH 2/5] [ruby] remove method --- .../scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 51725f0332da..09fc3296ca4b 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -161,8 +161,6 @@ class AstCreator( } .toList } - - private def generateAssignmentNode(): Unit = {} } /** Determines till what depth the AST creator will parse until. From 8792a3eaa82393892f3f2ba5a6d20b378c1b89ca Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Thu, 9 May 2024 11:43:02 +0200 Subject: [PATCH 3/5] [ruby] remove unnecessary filterNot --- .../main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 09fc3296ca4b..d13dffd4c178 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -119,7 +119,6 @@ class AstCreator( .flatMap(_._2) .filter(x => x.name.contains(":program")) .flatMap(_.methods) - .filterNot(x => x.name.contains("") || x.name.contains("")) .map { method => val methodRefNode = NewMethodRef() .code(s"def ${method.name} (...)") From 14be3e5077dc1260cb5c23e6047b2a37ee3c6270 Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Thu, 9 May 2024 11:58:35 +0200 Subject: [PATCH 4/5] review comments --- .../rubysrc2cpg/astcreation/AstCreator.scala | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index d13dffd4c178..6bba6e14da39 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -1,7 +1,7 @@ package io.joern.rubysrc2cpg.astcreation import io.joern.rubysrc2cpg.astcreation.RubyIntermediateAst.* -import io.joern.rubysrc2cpg.datastructures.{BlockScope, NamespaceScope, RubyProgramSummary, RubyScope} +import io.joern.rubysrc2cpg.datastructures.{BlockScope, NamespaceScope, RubyProgramSummary, RubyScope, RubyStubbedType} import io.joern.rubysrc2cpg.parser.{RubyNodeCreator, RubyParser} import io.joern.rubysrc2cpg.passes.Defines import io.joern.x2cpg.utils.NodeBuilders.newModifierNode @@ -44,7 +44,7 @@ class AstCreator( .map(_.stripPrefix(java.io.File.separator)) .getOrElse(fileName) - private val internalLineAndColNum: Option[Integer] = Option(1) + private def internalLineAndColNum: Option[Integer] = Option(1) /** The relative file name, in a unix path delimited format. */ @@ -110,14 +110,14 @@ class AstCreator( private def methodRefNodesForInternalMethods(): List[Ast] = { val typeNameForMethods = scope.surroundingTypeFullName .map { x => - x.split("[:]{2}").dropRight(1).mkString("") + x.stripSuffix(s":${Defines.Program}") } .getOrElse(Defines.Undefined) programSummary.namespaceToType .filter(_._1 == typeNameForMethods) .flatMap(_._2) - .filter(x => x.name.contains(":program")) + .filter(!_.isInstanceOf[RubyStubbedType]) .flatMap(_.methods) .map { method => val methodRefNode = NewMethodRef() @@ -140,25 +140,30 @@ class AstCreator( } private def typeRefNodesForInternalDecls(): List[Ast] = { - programSummary.namespaceToType - .filter(_._1.contains(scope.surroundingTypeFullName.getOrElse(Defines.Undefined))) - .flatMap(_._2) - .map { x => - val typeRefName = x.name.split("[.]").takeRight(1).head - val typeRefNode = NewTypeRef() - .code(s"class ${x.name} (...)") - .typeFullName(x.name) - - val typeRefIdent = NewIdentifier() - .code(typeRefName) - .name(typeRefName) - .typeFullName(x.name) - .lineNumber(internalLineAndColNum) - .columnNumber(internalLineAndColNum) - - astForAssignment(typeRefIdent, typeRefNode, internalLineAndColNum, internalLineAndColNum) + scope.surroundingTypeFullName + .map { surroundingTypeFullName => + programSummary.namespaceToType + .filter(_._1.contains(surroundingTypeFullName)) + .flatMap(_._2) + .map { x => + val typeRefName = x.name.split("[.]").takeRight(1).head + val typeRefNode = NewTypeRef() + .code(s"class ${x.name} (...)") + .typeFullName(x.name) + + val typeRefIdent = NewIdentifier() + .code(typeRefName) + .name(typeRefName) + .typeFullName(x.name) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + astForAssignment(typeRefIdent, typeRefNode, internalLineAndColNum, internalLineAndColNum) + } + .toList } - .toList + .getOrElse(List.empty) + } } From 41f7ec0faad274e06298e9f2b14df5cb8deea2ed Mon Sep 17 00:00:00 2001 From: Andrei Dreyer Date: Thu, 9 May 2024 12:14:05 +0200 Subject: [PATCH 5/5] [ruby] Added mapping of surroundingTypeFullName to method ref generation as well --- .../rubysrc2cpg/astcreation/AstCreator.scala | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala index 6bba6e14da39..060097193bb3 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/astcreation/AstCreator.scala @@ -114,29 +114,35 @@ class AstCreator( } .getOrElse(Defines.Undefined) - programSummary.namespaceToType - .filter(_._1 == typeNameForMethods) - .flatMap(_._2) - .filter(!_.isInstanceOf[RubyStubbedType]) - .flatMap(_.methods) - .map { method => - val methodRefNode = NewMethodRef() - .code(s"def ${method.name} (...)") - .methodFullName(scope.surroundingTypeFullName.map { x => s"$x:${method.name}" }.getOrElse(method.name)) - .typeFullName(Defines.Any) - .lineNumber(internalLineAndColNum) - .columnNumber(internalLineAndColNum) - - val methodRefIdent = NewIdentifier() - .code(method.name) - .name(method.name) - .typeFullName(Defines.Any) - .lineNumber(internalLineAndColNum) - .columnNumber(internalLineAndColNum) - - astForAssignment(methodRefIdent, methodRefNode, internalLineAndColNum, internalLineAndColNum) + scope.surroundingTypeFullName + .map { x => + val typeNameForMethods = x.stripSuffix(s":${Defines.Program}") + programSummary.namespaceToType + .filter(_._1 == typeNameForMethods) + .flatMap(_._2) + .filter(!_.isInstanceOf[RubyStubbedType]) + .flatMap(_.methods) + .map { method => + val methodRefNode = NewMethodRef() + .code(s"def ${method.name} (...)") + .methodFullName(scope.surroundingTypeFullName.map { x => s"$x:${method.name}" }.getOrElse(method.name)) + .typeFullName(Defines.Any) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + val methodRefIdent = NewIdentifier() + .code(method.name) + .name(method.name) + .typeFullName(Defines.Any) + .lineNumber(internalLineAndColNum) + .columnNumber(internalLineAndColNum) + + astForAssignment(methodRefIdent, methodRefNode, internalLineAndColNum, internalLineAndColNum) + } + .toList } - .toList + .getOrElse(List.empty) + } private def typeRefNodesForInternalDecls(): List[Ast] = {