diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 29a4712225..45685a2cdf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,8 +6,8 @@ cpg-language-go @oxisto *.ts @oxisto cpg-language-typescript @oxisto -*.py @maximiliankaul -cpg-language-python @maximiliankaul +*.py @maximiliankaul @lshala +cpg-language-python @maximiliankaul @lshala *.c @peckto *.cpp @peckto diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74cc51f4cf..77a59f1fa3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,11 +17,9 @@ on: jobs: build: - runs-on: [self-hosted, linux, x64, faster] + runs-on: self-hosted steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of SonarQube analysis - run: | cp gradle.properties.example gradle.properties - uses: actions/setup-java@v4 @@ -41,11 +39,6 @@ jobs: - name: Setup neo4j run: | docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true - - name: Cache SonarCloud packages - uses: actions/cache@v4 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - name: Determine Version run: | # determine version from tag @@ -67,23 +60,21 @@ jobs: if [ -d "/opt/hostedtoolcache/Python" ]; then find /opt/hostedtoolcache/Python/ -name libjep.so -exec sudo cp '{}' /usr/lib/ \; fi + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: Build ${{ env.version }} run: | - if [ "$SONAR_TOKEN" != "" ] - then - ./gradlew --parallel -Pversion=$VERSION spotlessCheck -x spotlessApply build -x distZip -x distTar sonar performanceTest integrationTest \ - -Dsonar.projectKey=Fraunhofer-AISEC_cpg \ - -Dsonar.organization=fraunhofer-aisec \ - -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.token=$SONAR_TOKEN - else - ./gradlew --parallel -Pversion=$VERSION spotlessCheck -x spotlessApply build -x distZip -x distTar performanceTest integrationTest - fi + ./gradlew --parallel -Pversion=$VERSION spotlessCheck -x spotlessApply build -x distZip -x distTar koverXmlReport koverHtmlReport performanceTest integrationTest id: build env: VERSION: ${{ env.version }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Code Coverage + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + files: ./cpg-all/build/reports/kover/report.xml + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true - name: Prepare test and coverage reports if: ${{ always() }} run: | diff --git a/.gitignore b/.gitignore index 8dcaac4922..99d9338d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ out .data/ logs /lsp/*.log -*.class *.dylib *.so diff --git a/README.md b/README.md index 88a62bbd45..126102c450 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Code Property Graph [![Actions Status](https://github.com/Fraunhofer-AISEC/cpg/workflows/build/badge.svg)](https://github.com/Fraunhofer-AISEC/cpg/actions) - [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Fraunhofer-AISEC_cpg&metric=alert_status)](https://sonarcloud.io/dashboard?id=Fraunhofer-AISEC_cpg) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=Fraunhofer-AISEC_cpg&metric=security_rating)](https://sonarcloud.io/dashboard?id=Fraunhofer-AISEC_cpg) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=Fraunhofer-AISEC_cpg&metric=coverage)](https://sonarcloud.io/dashboard?id=Fraunhofer-AISEC_cpg) [![](https://jitpack.io/v/Fraunhofer-AISEC/cpg.svg)](https://jitpack.io/#Fraunhofer-AISEC/cpg) + [![codecov](https://codecov.io/gh/Fraunhofer-AISEC/cpg/graph/badge.svg?token=XBXZZOQIID)](https://codecov.io/gh/Fraunhofer-AISEC/cpg) [![](https://jitpack.io/v/Fraunhofer-AISEC/cpg.svg)](https://jitpack.io/#Fraunhofer-AISEC/cpg) A simple library to extract a *code property graph* out of source code. It has support for multiple passes that can extend the analysis after the graph is constructed. It currently supports C/C++ (C17), Java (Java 13) and has experimental support for Golang, Python and TypeScript. Furthermore, it has support for the [LLVM IR](http://llvm.org/docs/LangRef.html) and thus, theoretically support for all languages that compile using LLVM. @@ -128,15 +128,16 @@ Languages are maintained to different degrees, and are noted in the table below The current state of languages is: -| Language | Module | Branch | State | -|---|---|---|---| -| Java | cpg-language-java | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | -| C++ | cpg-language-cxx | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | -| Python | cpg-language-python | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | -| Go | cpg-language-go | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | -| LLVM | cpg-language-llvm | [main](https://github.com/Fraunhofer-AISEC/cpg) | `incubating` | -| TypeScript/JavaScript | cpg-language-typescript | [main](https://github.com/Fraunhofer-AISEC/cpg) | `experimental` | -| Ruby | cpg-language-ruby | [main](https://github.com/Fraunhofer-AISEC/cpg) | `experimental` | +| Language | Module | Branch | State | +|--------------------------|---------------------------------------|-------------------------------------------------------------------------|----------------| +| Java (Source) | cpg-language-java | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | +| C++ | cpg-language-cxx | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | +| Python | cpg-language-python | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | +| Go | cpg-language-go | [main](https://github.com/Fraunhofer-AISEC/cpg) | `maintained` | +| JVM (Bytecode) | cpg-language-jvm | [main](https://github.com/Fraunhofer-AISEC/cpg) | `incubating` | +| LLVM | cpg-language-llvm | [main](https://github.com/Fraunhofer-AISEC/cpg) | `incubating` | +| TypeScript/JavaScript | cpg-language-typescript | [main](https://github.com/Fraunhofer-AISEC/cpg) | `experimental` | +| Ruby | cpg-language-ruby | [main](https://github.com/Fraunhofer-AISEC/cpg) | `experimental` | | {OpenQASM,Python-Qiskit} | cpg-language-{openqasm,python-qiskit} | [quantum-cpg](https://github.com/Fraunhofer-AISEC/cpg/tree/quantum-cpg) | `experimental` | ### Languages and Configuration diff --git a/build.gradle.kts b/build.gradle.kts index 205e0059fa..368ea647b4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -29,7 +29,6 @@ // plugins { id("org.jetbrains.dokka") - id("org.sonarqube") id("io.github.gradle-nexus.publish-plugin") } @@ -77,18 +76,6 @@ fun generateDokkaWithVersionTag(dokkaMultiModuleTask: org.jetbrains.dokka.gradle } -// -// Configure sonarqube for the whole cpg project -// -sonarqube { - properties { - property("sonar.sourceEncoding", "UTF-8") - // The report part is either relative to the submodules or the main module. We want to specify our - // aggregated jacoco report here - property("sonar.coverage.jacoco.xmlReportPaths", "../cpg-all/build/reports/kover/report.xml,cpg-all/build/reports/kover/report.xml") - } -} - /** * Publishing to maven central */ @@ -150,3 +137,9 @@ val enableRubyFrontend: Boolean by extra { enableRubyFrontend.toBoolean() } project.logger.lifecycle("Ruby frontend is ${if (enableRubyFrontend) "enabled" else "disabled"}") + +val enableJVMFrontend: Boolean by extra { + val enableJVMFrontend: String? by project + enableJVMFrontend.toBoolean() +} +project.logger.lifecycle("JVM frontend is ${if (enableJVMFrontend) "enabled" else "disabled"}") diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index d58e968189..6418a1982a 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -10,7 +10,6 @@ dependencies { implementation(libs.kotlin.gradle) implementation(libs.dokka.gradle) implementation(libs.kover.gradle) - implementation(libs.sonarqube.gradle) implementation(libs.spotless.gradle) implementation(libs.nexus.publish.gradle) implementation(files(libs.javaClass.superclass.protectionDomain.codeSource.location)) // this is only there to be able to import 'LibrariesForLibs' in the convention plugins to access the version catalog in buildSrc diff --git a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts index 7f6c09c47c..42ddb6e7d9 100644 --- a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts @@ -107,7 +107,7 @@ kotlin { tasks.withType { compilerOptions { - freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-Xcontext-receivers") + freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-opt-in=kotlin.uuid.ExperimentalUuidApi", "-Xcontext-receivers") } } @@ -148,8 +148,6 @@ val performanceTest = tasks.register("performanceTest") { maxParallelForks = 1 // make sure that several performance tests (e.g. in different frontends) also do NOT run in parallel usesService(serialExecutionService) - - mustRunAfter(tasks.getByPath(":sonar")) } // A build service that ensures serial execution of a group of tasks diff --git a/buildSrc/src/main/kotlin/cpg.frontend-dependency-conventions.gradle.kts b/buildSrc/src/main/kotlin/cpg.frontend-dependency-conventions.gradle.kts index 7325c19481..10fef182f3 100644 --- a/buildSrc/src/main/kotlin/cpg.frontend-dependency-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/cpg.frontend-dependency-conventions.gradle.kts @@ -11,12 +11,17 @@ val enablePythonFrontend: Boolean by rootProject.extra val enableLLVMFrontend: Boolean by rootProject.extra val enableTypeScriptFrontend: Boolean by rootProject.extra val enableRubyFrontend: Boolean by rootProject.extra +val enableJVMFrontend: Boolean by rootProject.extra dependencies { if (enableJavaFrontend) { api(project(":cpg-language-java")) kover(project(":cpg-language-java")) } + if (enableJVMFrontend) { + api(project(":cpg-language-jvm")) + kover(project(":cpg-language-jvm")) + } if (enableCXXFrontend) { api(project(":cpg-language-cxx")) kover(project(":cpg-language-cxx")) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000..c49059603a --- /dev/null +++ b/codecov.yml @@ -0,0 +1,12 @@ +coverage: + range: "70...95" + status: + project: + default: + target: auto + threshold: 0.5% + patch: + default: + target: 75% +comment: + layout: "condensed_header, files, condensed_footer" diff --git a/configure_frontends.sh b/configure_frontends.sh index 7304b0b04c..49e3233752 100755 --- a/configure_frontends.sh +++ b/configure_frontends.sh @@ -58,3 +58,5 @@ answerTypescript=$(ask "Do you want to enable the TypeScript frontend? (currentl setProperty "enableTypeScriptFrontend" $answerTypescript answerRuby=$(ask "Do you want to enable the Ruby frontend? (currently $(getProperty "enableRubyFrontend"))") setProperty "enableRubyFrontend" $answerRuby +answerJVM=$(ask "Do you want to enable the JVM frontend? (currently $(getProperty "enableJVMFrontend"))") +setProperty "enableJVMFrontend" $answerJVM diff --git a/cpg-all/build.gradle.kts b/cpg-all/build.gradle.kts index dca1533085..18bd65c219 100644 --- a/cpg-all/build.gradle.kts +++ b/cpg-all/build.gradle.kts @@ -16,6 +16,12 @@ publishing { } } +repositories { + maven { + setUrl("https://jitpack.io") + } +} + dependencies { // this exposes all of our (published) modules as dependency api(projects.cpgConsole) @@ -28,7 +34,3 @@ dependencies { kover(projects.cpgAnalysis) kover(projects.cpgNeo4j) } - -val sonar = tasks.getByPath(":sonar") -sonar.dependsOn(tasks.named("koverHtmlReport")) -sonar.dependsOn(tasks.named("koverXmlReport")) diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt index 56fb4c9348..471bb78259 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt @@ -32,16 +32,10 @@ import de.fraunhofer.aisec.cpg.graph.invoke import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass -import de.fraunhofer.aisec.cpg.passes.astParent import org.slf4j.Logger import org.slf4j.LoggerFactory -/** - * This [ValueEvaluator] can resolve multiple possible values of a node. - * - * It requires running the [EdgeCachePass] after the translation to add all necessary edges. - */ +/** This [ValueEvaluator] can resolve multiple possible values of a node. */ class MultiValueEvaluator : ValueEvaluator() { companion object { const val MAX_DEPTH: Int = 20 @@ -268,7 +262,8 @@ class MultiValueEvaluator : ValueEvaluator() { forStatement.initializerStatement == node || // The node is the initialization (initializerDecl != null && initializerDecl == - node.astParent) || // The parent of the node is the initializer of the loop + node.astParent) || // The parent of the node is the initializer of the + // loop // variable forStatement.iterationStatement == node || // The node or its parent are the iteration statement of the loop diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt index 83ec63be30..4e807603a7 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt @@ -180,7 +180,7 @@ open class ValueEvaluator( "<=" -> handleLEq(lhsValue, rhsValue, expr) "==" -> handleEq(lhsValue, rhsValue, expr) "!=" -> handleNEq(lhsValue, rhsValue, expr) - else -> cannotEvaluate(expr as Node, this) + else -> cannotEvaluate(expr, this) } } diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt index a5467d553c..a369fd78ac 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/fsm/DFAOrderEvaluator.kt @@ -28,14 +28,11 @@ package de.fraunhofer.aisec.cpg.analysis.fsm import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.passes.astParent import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -435,14 +432,12 @@ open class DFAOrderEvaluator( val outNodes = mutableListOf() outNodes += if (eliminateUnreachableCode) { - PropertyEdge.unwrap( - node.nextEOGEdges.filter { e -> e.getProperty(Properties.UNREACHABLE) != true } - ) + node.nextEOGEdges.filter { e -> e.unreachable != true }.map { it.end } } else { node.nextEOG } - if (outNodes.size == 1 && node.nextEOG.size == 1) { + if (outNodes.size == 1 && node.nextEOGEdges.size == 1) { // We only have one node following this node, so we // simply propagate the current eogPath to the next node. outNodes[0].addEogPath(eogPath) diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPass.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPass.kt index dfc0c45084..01ab3025e4 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPass.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPass.kt @@ -30,8 +30,8 @@ import de.fraunhofer.aisec.cpg.analysis.ValueEvaluator import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.WhileStatement import de.fraunhofer.aisec.cpg.helpers.* @@ -39,7 +39,7 @@ import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn /** * A [Pass] which uses a simple logic to determine constant values and mark unreachable code regions - * by setting the [Properties.UNREACHABLE] property of an eog-edge to true. + * by setting the [EvaluationOrder.unreachable] property to true. */ @DependsOn(ControlFlowSensitiveDFGPass::class) class UnreachableEOGPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { @@ -68,7 +68,7 @@ class UnreachableEOGPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { for ((key, value) in finalState) { if (value.elements == Reachability.UNREACHABLE) { - key.addProperty(Properties.UNREACHABLE, true) + (key as? EvaluationOrder)?.unreachable = true } } } @@ -90,9 +90,9 @@ class UnreachableEOGPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { * Returns the updated state and true because we always expect an update of the state. */ fun transfer( - currentEdge: PropertyEdge, - currentState: State, Reachability> -): State, Reachability> { + currentEdge: Edge, + currentState: State, Reachability> +): State, Reachability> { when (val currentNode = currentEdge.end) { is IfStatement -> { handleIfStatement(currentEdge, currentNode, currentState) @@ -117,22 +117,24 @@ fun transfer( * All other cases simply copy the state which led us here. */ private fun handleIfStatement( - enteringEdge: PropertyEdge, + enteringEdge: Edge, n: IfStatement, - state: State, Reachability> + state: State, Reachability> ) { val evalResult = ValueEvaluator().evaluate(n.condition) val (unreachableEdge, remainingEdges) = - if (evalResult is Boolean && evalResult == true) { + if (evalResult == true) { + // If the condition is always true, the "false" branch is always unreachable Pair( - n.nextEOGEdges.firstOrNull { e -> e.getProperty(Properties.INDEX) == 1 }, - n.nextEOGEdges.filter { e -> e.getProperty(Properties.INDEX) != 1 } + n.nextEOGEdges.firstOrNull { e -> e.branch == false }, + n.nextEOGEdges.filter { e -> e.branch != false } ) - } else if (evalResult is Boolean && evalResult == false) { + } else if (evalResult == false) { + // If the condition is always false, the "true" branch is always unreachable Pair( - n.nextEOGEdges.firstOrNull { e -> e.getProperty(Properties.INDEX) == 0 }, - n.nextEOGEdges.filter { e -> e.getProperty(Properties.INDEX) != 0 } + n.nextEOGEdges.firstOrNull { e -> e.branch == true }, + n.nextEOGEdges.filter { e -> e.branch != true } ) } else { Pair(null, n.nextEOGEdges) @@ -156,9 +158,9 @@ private fun handleIfStatement( * us here. */ private fun handleWhileStatement( - enteringEdge: PropertyEdge, + enteringEdge: Edge, n: WhileStatement, - state: State, Reachability> + state: State, Reachability> ) { /* * Note: It does not understand that code like @@ -173,13 +175,13 @@ private fun handleWhileStatement( val (unreachableEdge, remainingEdges) = if (evalResult is Boolean && evalResult == true) { Pair( - n.nextEOGEdges.firstOrNull { e -> e.getProperty(Properties.INDEX) == 1 }, - n.nextEOGEdges.filter { e -> e.getProperty(Properties.INDEX) != 1 } + n.nextEOGEdges.firstOrNull { e -> e.index == 1 }, + n.nextEOGEdges.filter { e -> e.index != 1 } ) } else if (evalResult is Boolean && evalResult == false) { Pair( - n.nextEOGEdges.firstOrNull { e -> e.getProperty(Properties.INDEX) == 0 }, - n.nextEOGEdges.filter { e -> e.getProperty(Properties.INDEX) != 0 } + n.nextEOGEdges.firstOrNull { e -> e.index == 0 }, + n.nextEOGEdges.filter { e -> e.index != 0 } ) } else { Pair(null, n.nextEOGEdges) @@ -221,7 +223,7 @@ enum class Reachability { } /** - * A state which actually holds a state for all [PropertyEdge]s, one only for declarations and one - * for ReturnStatements. + * A state which actually holds a state for all [Edge]s, one only for declarations and one for + * ReturnStatements. */ -class UnreachabilityState : State, Reachability>() +class UnreachabilityState : State, Reachability>() diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPassTest.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPassTest.kt index 2b0f0128fa..3519e55a39 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPassTest.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/UnreachableEOGPassTest.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.testcases.Passes import kotlin.test.* import org.junit.jupiter.api.BeforeAll @@ -51,7 +50,7 @@ class UnreachableEOGPassTest { assertNotNull(ifStatement) for (edge in ifStatement.nextEOGEdges) { - assertFalse(edge.getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(edge.unreachable) } } @@ -66,43 +65,43 @@ class UnreachableEOGPassTest { // Check if the then-branch is set as reachable including all the edges until reaching the // print val thenDecl = ifStatement.nextEOGEdges[0] - assertFalse(thenDecl.getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(thenDecl.unreachable) assertEquals(1, thenDecl.end.nextEOGEdges.size) // The "++" val incOp = thenDecl.end.nextEOGEdges[0] - assertFalse(incOp.getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(incOp.unreachable) assertEquals(1, incOp.end.nextEOGEdges.size) // The block val thenCompound = incOp.end.nextEOGEdges[0] - assertFalse(thenCompound.getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(thenCompound.unreachable) assertEquals(1, thenCompound.end.nextEOGEdges.size) // There's the outgoing EOG edge to the statement after the branching val thenExit = thenCompound.end.nextEOGEdges[0] - assertFalse(thenExit.getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(thenExit.unreachable) // Check if the else-branch is set as unreachable including all the edges until reaching the // print val elseDecl = ifStatement.nextEOGEdges[1] - assertTrue(elseDecl.getProperty(Properties.UNREACHABLE) as Boolean) + assertTrue(elseDecl.unreachable) assertEquals(1, elseDecl.end.nextEOGEdges.size) // The "--" val decOp = elseDecl.end.nextEOGEdges[0] - assertTrue(decOp.getProperty(Properties.UNREACHABLE) as Boolean) + assertTrue(decOp.unreachable) assertEquals(1, decOp.end.nextEOGEdges.size) // The block val elseCompound = decOp.end.nextEOGEdges[0] - assertTrue(elseCompound.getProperty(Properties.UNREACHABLE) as Boolean) + assertTrue(elseCompound.unreachable) assertEquals(1, elseCompound.end.nextEOGEdges.size) // There's the outgoing EOG edge to the statement after the branching val elseExit = elseCompound.end.nextEOGEdges[0] - assertTrue(elseExit.getProperty(Properties.UNREACHABLE) as Boolean) + assertTrue(elseExit.unreachable) // After the branching, it's reachable again. Check that we found the merge node and that we // continue with reachable edges. assertEquals(thenExit.end, elseExit.end) val mergeNode = thenExit.end assertEquals(1, mergeNode.nextEOGEdges.size) - assertFalse(mergeNode.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(mergeNode.nextEOGEdges[0].unreachable) } @Test @@ -113,8 +112,8 @@ class UnreachableEOGPassTest { val ifStatement = method.ifs.firstOrNull() assertNotNull(ifStatement) - assertFalse(ifStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(ifStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(ifStatement.nextEOGEdges[1].unreachable) + assertTrue(ifStatement.nextEOGEdges[0].unreachable) } @Test @@ -125,8 +124,8 @@ class UnreachableEOGPassTest { val ifStatement = method.ifs.firstOrNull() assertNotNull(ifStatement) - assertFalse(ifStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(ifStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(ifStatement.nextEOGEdges[0].unreachable) + assertTrue(ifStatement.nextEOGEdges[1].unreachable) } @Test @@ -137,8 +136,8 @@ class UnreachableEOGPassTest { val ifStatement = method.ifs.firstOrNull() assertNotNull(ifStatement) - assertFalse(ifStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(ifStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(ifStatement.nextEOGEdges[1].unreachable) + assertTrue(ifStatement.nextEOGEdges[0].unreachable) } @Test @@ -149,8 +148,8 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[0].unreachable) + assertTrue(whileStatement.nextEOGEdges[1].unreachable) } @Test @@ -161,8 +160,8 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) - assertFalse(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[0].unreachable) + assertFalse(whileStatement.nextEOGEdges[1].unreachable) } @Test @@ -173,8 +172,8 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[0].unreachable) + assertTrue(whileStatement.nextEOGEdges[1].unreachable) } @Test @@ -185,8 +184,8 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[1].unreachable) + assertTrue(whileStatement.nextEOGEdges[0].unreachable) } @Test @@ -197,8 +196,8 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) - assertTrue(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[1].unreachable) + assertTrue(whileStatement.nextEOGEdges[0].unreachable) } @Test @@ -209,7 +208,7 @@ class UnreachableEOGPassTest { val whileStatement = method.whileLoops.firstOrNull() assertNotNull(whileStatement) - assertFalse(whileStatement.nextEOGEdges[1].getProperty(Properties.UNREACHABLE) as Boolean) - assertFalse(whileStatement.nextEOGEdges[0].getProperty(Properties.UNREACHABLE) as Boolean) + assertFalse(whileStatement.nextEOGEdges[1].unreachable) + assertFalse(whileStatement.nextEOGEdges[0].unreachable) } } diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/query/QueryTest.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/query/QueryTest.kt index 7753272530..e8630c6f84 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/query/QueryTest.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/query/QueryTest.kt @@ -598,7 +598,7 @@ class QueryTest { result .all( { it.name.localName == "print" }, - { n2 -> dataFlow(n1 as Node, n2.parameters[0]).value } + { n2 -> dataFlow(n1, n2.parameters[0]).value } ) .first } @@ -613,7 +613,7 @@ class QueryTest { { n1 -> result.allExtended( { it.name.localName == "print" }, - { n2 -> dataFlow(n1 as Node, n2.parameters[0]) } + { n2 -> dataFlow(n1, n2.parameters[0]) } ) } ) diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/OrderingTests.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/OrderingTests.kt index 06753748bb..a89ed36095 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/OrderingTests.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/OrderingTests.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.graph.array import de.fraunhofer.aisec.cpg.graph.builder.* -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass class GraphExamples { @@ -45,7 +44,6 @@ class GraphExamples { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { @@ -244,7 +242,6 @@ class GraphExamples { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/Query.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/Query.kt index 8f5ef62437..2c79d4ec8c 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/Query.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/Query.kt @@ -32,7 +32,6 @@ import de.fraunhofer.aisec.cpg.graph.array import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.newNewArrayExpression import de.fraunhofer.aisec.cpg.graph.pointer -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass class Query { companion object { @@ -99,7 +98,6 @@ class Query { TranslationConfiguration.builder() .defaultPasses() .registerLanguage(TestLanguage(".")) - .registerPass() .inferenceConfiguration(InferenceConfiguration.builder().enabled(false).build()) .build() ) = @@ -176,7 +174,6 @@ class Query { TranslationConfiguration.builder() .defaultPasses() .registerLanguage(TestLanguage(".")) - .registerPass() .inferenceConfiguration( InferenceConfiguration.builder().inferFunctions(false).build() ) @@ -255,7 +252,6 @@ class Query { TranslationConfiguration.builder() .defaultPasses() .registerLanguage(TestLanguage(".")) - .registerPass() .build() ) = testFrontend(config).build { @@ -392,7 +388,6 @@ class Query { config: TranslationConfiguration = TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) = @@ -438,7 +433,6 @@ class Query { config: TranslationConfiguration = TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) = @@ -498,7 +492,6 @@ class Query { config: TranslationConfiguration = TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) = diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt index 46dd31222b..2eca132843 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/testcases/ValueEvaluationTests.kt @@ -29,7 +29,6 @@ import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import de.fraunhofer.aisec.cpg.passes.UnreachableEOGPass class ValueEvaluationTests { @@ -40,7 +39,6 @@ class ValueEvaluationTests { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { @@ -123,7 +121,6 @@ class ValueEvaluationTests { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { @@ -166,7 +163,6 @@ class ValueEvaluationTests { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { @@ -272,7 +268,6 @@ class ValueEvaluationTests { .defaultPasses() .registerLanguage(TestLanguage(".")) .registerPass() - .registerPass() .build() ) = testFrontend(config).build { diff --git a/cpg-console/README.md b/cpg-console/README.md index 23b4a82611..153b02c182 100644 --- a/cpg-console/README.md +++ b/cpg-console/README.md @@ -15,8 +15,8 @@ build/install/cpg-console/bin/cpg-console The following example snippet can be used: ```kotlin -:tr src/test/resources/array.cpp -var main = tu.byName("main") +:tr cpg-console/bin/test/array.cpp +var main = tu.functions["main"] :code main? var decl = main?.body(0) var v = decl?.singleDeclaration as? VariableDeclaration diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CompilationDatabase.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CompilationDatabase.kt index 127b3422d5..0a84988d33 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CompilationDatabase.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CompilationDatabase.kt @@ -59,7 +59,6 @@ class CompilationDatabase : Plugin { "import de.fraunhofer.aisec.cpg.graph.allChildren", "import de.fraunhofer.aisec.cpg.graph.ast", "import de.fraunhofer.aisec.cpg.graph.dfgFrom", - "import de.fraunhofer.aisec.cpg.graph.byName", "import de.fraunhofer.aisec.cpg.graph.body", "import de.fraunhofer.aisec.cpg.graph.capacity", "import de.fraunhofer.aisec.cpg.console.printCode", diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index a0ca77f2c6..dd9230af42 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -59,7 +59,6 @@ class TranslatePlugin : Plugin { "import de.fraunhofer.aisec.cpg.graph.allChildren", "import de.fraunhofer.aisec.cpg.graph.ast", "import de.fraunhofer.aisec.cpg.graph.dfgFrom", - "import de.fraunhofer.aisec.cpg.graph.byName", "import de.fraunhofer.aisec.cpg.graph.body", "import de.fraunhofer.aisec.cpg.graph.capacity", "import de.fraunhofer.aisec.cpg.console.printCode", diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/TranslatePluginTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/TranslatePluginTest.kt index 7513f23e29..398cd7ff71 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/TranslatePluginTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/TranslatePluginTest.kt @@ -41,6 +41,6 @@ class TranslatePluginTest { val result = plugin.execute(":tr") assertTrue(result is Command.Result.RunSnippets) - assertEquals(19, result.snippetsToRun.toList().size) + assertEquals(18, result.snippetsToRun.toList().size) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt new file mode 100644 index 0000000000..61f4d325f8 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.passes.hardDependencies +import de.fraunhofer.aisec.cpg.passes.hardExecuteBefore +import de.fraunhofer.aisec.cpg.passes.isFirstPass +import de.fraunhofer.aisec.cpg.passes.isLastPass +import de.fraunhofer.aisec.cpg.passes.softDependencies +import de.fraunhofer.aisec.cpg.passes.softExecuteBefore +import kotlin.reflect.KClass + +private const val UNKNOWN_PASS = "UnknownPass" +private const val FIRST_PASSES_SUBGRAPH_IDENTIFIER = "FirstPassesSubgraph" +private const val NORMAL_PASSES_SUBGRAPH_IDENTIFIER = "NormalPassesSubgraph" +private const val LAST_PASSES_SUBGRAPH_IDENTIFIER = "LastPassesSubgraph" +private const val FIRST_PASS_IDENTIFIER = "FirstPass" +private const val LAST_PASS_IDENTIFIER = "LastPass" + +/** Helper function to replace the first and last passes names by their identifier. */ +private fun mermaidPassName(pass: KClass>): String { + return when { + pass.isFirstPass -> FIRST_PASS_IDENTIFIER + pass.isLastPass -> LAST_PASS_IDENTIFIER + else -> pass.simpleName ?: UNKNOWN_PASS + } +} +/** + * Builds a markdown representation of a pass dependency graph, based on + * [Mermaid](https://mermaid.js.org) syntax. + */ +internal fun buildMermaid(passes: List>>): String { + var s = "```mermaid\n" + s += "flowchart TD;\n" + + s += " subgraph $FIRST_PASSES_SUBGRAPH_IDENTIFIER [\"First Passes\"];\n" + passes + .filter { it.isFirstPass } + .forEach { s += " $FIRST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" } + s += " end;\n" + s += " subgraph $LAST_PASSES_SUBGRAPH_IDENTIFIER [\"Last Passes\"];\n" + passes + .filter { it.isLastPass } + .forEach { s += " $LAST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" } + s += " end;\n" + + s += " $FIRST_PASSES_SUBGRAPH_IDENTIFIER~~~$NORMAL_PASSES_SUBGRAPH_IDENTIFIER;\n" + s += " subgraph $NORMAL_PASSES_SUBGRAPH_IDENTIFIER [\"Normal Passes\"];\n" + for ((pass, deps) in passes.associateWith { it.softDependencies }.entries) { + for (dep in deps) { + s += " ${mermaidPassName(dep)}-.->${mermaidPassName(pass)};\n" + } + } + for ((pass, deps) in passes.associateWith { it.hardDependencies }.entries) { + for (dep in deps) { + s += " ${mermaidPassName(dep)}-->${mermaidPassName(pass)};\n" + } + } + for ((pass, before) in passes.associateWith { it.softExecuteBefore }.entries) { + for (execBefore in before) { + s += " ${mermaidPassName(pass)}-.->${mermaidPassName(execBefore)};\n" + } + } + for ((pass, beforeList) in passes.associateWith { it.hardExecuteBefore }.entries) { + for (execBefore in beforeList) { + s += " ${mermaidPassName(pass)}-->${mermaidPassName(execBefore)};\n" + } + } + s += " end;\n" + s += " $NORMAL_PASSES_SUBGRAPH_IDENTIFIER~~~$LAST_PASSES_SUBGRAPH_IDENTIFIER;\n" + s += "```" + return s +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 161e24788a..f45809b3a5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -596,8 +596,6 @@ class ScopeManager : ScopeProvider { * * @param ref * @return - * - * TODO: We should merge this function with [.resolveFunction] */ fun resolveReference(ref: Reference): ValueDeclaration? { val startScope = ref.scope @@ -614,7 +612,10 @@ class ScopeManager : ScopeProvider { return pair.second } - val (scope, name) = extractScope(ref, startScope) + var (scope, name) = extractScope(ref, startScope) + if (scope == null) { + scope = startScope + } // Try to resolve value declarations according to our criteria val decl = @@ -657,111 +658,8 @@ class ScopeManager : ScopeProvider { } /** - * Tries to resolve a function in a call expression. - * - * @param call the call expression - * @return a list of possible functions - */ - @JvmOverloads - fun resolveFunctionLegacy( - call: CallExpression, - startScope: Scope? = currentScope - ): List { - val (scope, name) = extractScope(call, startScope) - - val func = - resolve(scope) { - it.name.lastPartsMatch(name) && - it.matchesSignature(call.signature) != IncompatibleSignature - } - - return func - } - - /** - * This function tries to resolve a [CallExpression] into its matching [FunctionDeclaration] (or - * multiple functions, if applicable). The result is returned in the form of a - * [CallResolutionResult] which holds detail information about intermediate results as well as - * the kind of success the resolution had. - * - * Note: The [CallExpression.callee] needs to be resolved first, otherwise the call resolution - * fails. - */ - fun resolveCall(call: CallExpression, startScope: Scope? = currentScope): CallResolutionResult { - val result = - CallResolutionResult( - call, - setOf(), - setOf(), - mapOf(), - setOf(), - CallResolutionResult.SuccessKind.UNRESOLVED, - startScope, - ) - val language = call.language - - if (language == null) { - result.success = CallResolutionResult.SuccessKind.PROBLEMATIC - return result - } - - // We can only resolve non-dynamic function calls here that have a reference node to our - // function - val callee = call.callee as? Reference ?: return result - - val (scope, _) = extractScope(callee, startScope) - result.actualStartScope = scope - - // Retrieve a list of possible functions with a matching name - result.candidateFunctions = - callee.candidates.filterIsInstance().toSet() - - if (call.language !is HasFunctionOverloading) { - // If the function does not allow function overloading, and we have multiple candidate - // symbols, the - // result is "problematic" - if (result.candidateFunctions.size > 1) { - result.success = CallResolutionResult.SuccessKind.PROBLEMATIC - } - } - - // Filter functions that match the signature of our call, either directly or with casts; - // those functions are "viable". Take default arguments into account if the language has - // them. - result.signatureResults = - result.candidateFunctions - .map { - Pair( - it, - it.matchesSignature( - call.signature, - call.language is HasDefaultArguments, - call - ) - ) - } - .filter { it.second is SignatureMatches } - .associate { it } - result.viableFunctions = result.signatureResults.keys - - // If we have a "problematic" result, we can stop here. In this case we cannot really - // determine anything more. - if (result.success == CallResolutionResult.SuccessKind.PROBLEMATIC) { - result.bestViable = result.viableFunctions - return result - } - - // Otherwise, give the language a chance to narrow down the result (ideally to one) and set - // the success kind. - val pair = language.bestViableResolution(result) - result.bestViable = pair.first - result.success = pair.second - - return result - } - - /** - * This function extracts a scope for the [Name] in node, e.g. if the name is fully qualified. + * This function extracts a scope for the [Name], e.g. if the name is fully qualified. `null` is + * returned, if no scope can be extracted. * * The pair returns the extracted scope and a name that is adjusted by possible import aliases. * The extracted scope is "responsible" for the name (e.g. declares the parent namespace) and @@ -776,7 +674,7 @@ class ScopeManager : ScopeProvider { * @param scope the current scope relevant for the name resolution, e.g. parent of node * @return a pair with the scope of node.name and the alias-adjusted name */ - fun extractScope(node: Node, scope: Scope? = currentScope): Pair { + fun extractScope(node: HasNameAndLocation, scope: Scope? = currentScope): Pair { return extractScope(node.name, node.location, scope) } @@ -803,7 +701,7 @@ class ScopeManager : ScopeProvider { scope: Scope? = currentScope, ): Pair { var n = name - var s = scope + var s: Scope? = null // First, we need to check, whether we have some kind of scoping. if (n.isQualified()) { @@ -910,12 +808,6 @@ class ScopeManager : ScopeProvider { return ret } - fun resolveFunctionStopScopeTraversalOnDefinition( - call: CallExpression - ): List { - return resolve(currentScope, true) { f -> f.name.lastPartsMatch(call.name) } - } - /** * Traverses the scope upwards and looks for declarations of type [T] which matches the * condition [predicate]. @@ -928,7 +820,7 @@ class ScopeManager : ScopeProvider { * @param predicate predicate the element must match to * @param */ - inline fun resolve( + internal inline fun resolve( searchScope: Scope?, stopIfFound: Boolean = false, noinline predicate: (T) -> Boolean @@ -936,7 +828,7 @@ class ScopeManager : ScopeProvider { return resolve(T::class.java, searchScope, stopIfFound, predicate) } - fun resolve( + internal fun resolve( klass: Class, searchScope: Scope?, stopIfFound: Boolean = false, @@ -1007,7 +899,7 @@ class ScopeManager : ScopeProvider { * @return the declaration, or null if it does not exist */ fun getRecordForName(name: Name): RecordDeclaration? { - return findSymbols(name).filterIsInstance().singleOrNull() + return lookupSymbolByName(name).filterIsInstance().singleOrNull() } fun typedefFor(alias: Name, scope: Scope? = currentScope): Type? { @@ -1068,25 +960,39 @@ class ScopeManager : ScopeProvider { get() = currentScope /** - * This function tries to resolve a [Node.name] to a list of symbols (a symbol represented by a - * [Declaration]) starting with [startScope]. This function can return a list of multiple - * symbols in order to check for things like function overloading. but it will only return list - * of symbols within the same scope; the list cannot be spread across different scopes. + * This function tries to convert a [Node.name] into a [Symbol] and then performs a lookup of + * this symbol. This can either be an "unqualified lookup" if [name] is not qualified or a + * "qualified lookup" if [Name.isQualified] is true. In the unqualified case the lookup starts + * in [startScope], in the qualified case we use [extractScope] to find the appropriate scope + * and need to restrict our search to this particular scope. * - * This means that as soon one or more symbols are found in a "local" scope, these shadow all - * other occurrences of the same / symbol in a "higher" scope and only the ones from the lower - * ones will be returned. + * This function can return a list of multiple declarations in order to check for things like + * function overloading. But it will only return list of declarations within the same scope; the + * list cannot be spread across different scopes. + * + * This means that as soon one or more declarations for the symbol are found in a "local" scope, + * these shadow all other occurrences of the same / symbol in a "higher" scope and only the ones + * from the lower ones will be returned. */ - fun findSymbols( + fun lookupSymbolByName( name: Name, location: PhysicalLocation? = null, startScope: Scope? = currentScope, predicate: ((Declaration) -> Boolean)? = null, ): List { val (scope, n) = extractScope(name, location, startScope) + + // We need to differentiate between a qualified and unqualified lookup. We have a qualified + // lookup, if the scope is not null. In this case we need to stay within the specified scope val list = - scope?.lookupSymbol(n.localName, predicate = predicate)?.toMutableList() - ?: mutableListOf() + if (scope != null) { + scope.lookupSymbol(n.localName, thisScopeOnly = true, predicate = predicate) + } else { + // Otherwise, we can look up the symbol alone (without any FQN) starting from + // the startScope + startScope?.lookupSymbol(n.localName, predicate = predicate) + } + ?.toMutableList() ?: return listOf() // If we have both the definition and the declaration of a function declaration in our list, // we chose only the definition @@ -1132,8 +1038,8 @@ data class SignatureMatches(override val casts: List) : SignatureRes fun FunctionDeclaration.matchesSignature( signature: List, + arguments: List? = null, useDefaultArguments: Boolean = false, - call: CallExpression? = null, ): SignatureResult { val casts = mutableListOf() @@ -1155,7 +1061,7 @@ fun FunctionDeclaration.matchesSignature( // Check, if we can cast the arg into our target type; and if, yes, what is // the "distance" to the base type. We need this to narrow down the type during // resolving - val match = type.tryCast(param.type, call?.arguments?.getOrNull(i), param) + val match = type.tryCast(param.type, arguments?.getOrNull(i), param) if (match == CastNotPossible) { return IncompatibleSignature } @@ -1198,17 +1104,20 @@ fun FunctionDeclaration.matchesSignature( } /** - * This is the result of [ScopeManager.resolveCall]. It holds all necessary intermediate results - * (such as [candidateFunctions], [viableFunctions]) as well as the final result (see [bestViable]) - * of the call resolution. + * This is the result of [SymbolResolver.resolveWithArguments]. It holds all necessary intermediate + * results (such as [candidateFunctions], [viableFunctions]) as well as the final result (see + * [bestViable]) of the call resolution. */ data class CallResolutionResult( - /** The original call expression. */ - val call: CallExpression, + /** The original expression that triggered the resolution. Most likely a [CallExpression]. */ + val source: Expression, + + /** The arguments that were supplied to the expression. */ + val arguments: List, /** * A set of candidate symbols we discovered based on the [CallExpression.callee] (using - * [ScopeManager.findSymbols]), more specifically a list of [FunctionDeclaration] nodes. + * [ScopeManager.lookupSymbolByName]), more specifically a list of [FunctionDeclaration] nodes. */ var candidateFunctions: Set, @@ -1235,7 +1144,7 @@ data class CallResolutionResult( /** * The actual start scope of the resolution, after [ScopeManager.extractScope] is called on the * callee. This can differ from the original start scope parameter handed to - * [ScopeManager.resolveCall] if the callee contains an FQN. + * [SymbolResolver.resolveWithArguments] if the callee contains an FQN. */ var actualStartScope: Scope? ) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index a8daa6b0c9..73d7a20d22 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -507,7 +507,7 @@ private constructor( registerPass() registerPass() registerPass() - registerPass() + registerPass() useDefaultPasses = true return this } @@ -651,171 +651,16 @@ private constructor( ) } - /** - * Collects the requested passes stored in [registeredPasses] and generates a - * [PassWithDepsContainer] consisting of pairs of passes and their dependencies. - * - * @return A populated [PassWithDepsContainer] derived from [registeredPasses]. - */ - private fun collectInitialPasses(): PassWithDepsContainer { - val workingList = PassWithDepsContainer() - - val softDependencies = - mutableMapOf>, MutableSet>>>() - val hardDependencies = - mutableMapOf>, MutableSet>>>() - - // Add the "execute before" dependencies. - for (p in passes) { - val executeBefore = mutableListOf>>() - - val depAnn = p.findAnnotations() - // collect all dependencies added by [DependsOn] annotations. - for (d in depAnn) { - val deps = - if (d.softDependency) { - softDependencies.computeIfAbsent(p) { mutableSetOf() } - } else { - hardDependencies.computeIfAbsent(p) { mutableSetOf() } - } - deps += d.value - } - - val execBeforeAnn = p.findAnnotations() - for (d in execBeforeAnn) { - executeBefore.add(d.other) - } - - for (eb in executeBefore) { - passes - .filter { eb == it } - .forEach { - val deps = softDependencies.computeIfAbsent(it) { mutableSetOf() } - deps += p - } - } - } - - log.info( - "The following mermaid graph represents the pass dependencies: \n ${buildMermaid(softDependencies, hardDependencies)}" - ) - - for (p in passes) { - var passFound = false - for ((pass) in workingList.getWorkingList()) { - if (pass == p) { - passFound = true - break - } - } - if (!passFound) { - workingList.addToWorkingList( - PassWithDependencies( - p, - softDependencies[p] ?: mutableSetOf(), - hardDependencies[p] ?: mutableSetOf() - ) - ) - } - } - return workingList - } - - /** - * Builds a markdown representation of a pass dependency graph, based on - * [Mermaid](https://mermaid.js.org) syntax. - */ - private fun buildMermaid( - softDependencies: MutableMap>, MutableSet>>>, - hardDependencies: MutableMap>, MutableSet>>> - ): String { - var s = "```mermaid\n" - s += "flowchart TD;\n" - for ((pass, deps) in softDependencies.entries) { - for (dep in deps) { - s += " ${dep.simpleName}-->${pass.simpleName};\n" - } - } - for ((pass, deps) in hardDependencies.entries) { - for (dep in deps) { - s += " ${dep.simpleName}-->${pass.simpleName};\n" - } - } - s += "```" - return s - } - - /** - * This function reorders passes in order to meet their dependency requirements. - * * soft dependencies [DependsOn] with `softDependency == true`: all passes registered as - * soft dependency will be executed before the current pass if they are registered - * * hard dependencies [DependsOn] with `softDependency == false (default)`: all passes - * registered as hard dependency will be executed before the current pass (hard - * dependencies will be registered even if the user did not register them) - * * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the - * beginning - * * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end - * - * This function uses a very simple (and inefficient) logic to meet the requirements above: - * 1. A list of all registered passes and their dependencies is build - * [PassWithDepsContainer.workingList] - * 1. All missing hard dependencies [DependsOn] are added to the - * [PassWithDepsContainer.workingList] - * 1. The first pass [ExecuteFirst] is added to the result and removed from the other passes - * dependencies - * 1. A list of passes in the workingList without dependencies are added to the result, and - * removed from the other passes dependencies - * 1. The above step is repeated until all passes are added to the result - * - * @return a sorted list of passes, with passes that can be run in parallel together in a - * nested list. - */ + /** This function reorders passes in order to meet their dependency requirements. */ @Throws(ConfigurationException::class) private fun orderPasses(): List>>> { log.info("Passes before enforcing order: {}", passes.map { it.simpleName }) - val result = mutableListOf>>>() - - // Create a local copy of all passes and their "current" dependencies without possible - // duplicates - val workingList = collectInitialPasses() - log.debug("Working list after initial scan: {}", workingList) - workingList.addMissingDependencies() - log.debug("Working list after adding missing dependencies: {}", workingList) - if (workingList.getFirstPasses().size > 1) { - log.error( - "Too many passes require to be executed as first pass: {}", - workingList.getWorkingList() - ) - throw ConfigurationException( - "Too many passes require to be executed as first pass." - ) - } - if (workingList.getLastPasses().size > 1) { - log.error( - "Too many passes require to be executed as last pass: {}", - workingList.getLastPasses() - ) - throw ConfigurationException("Too many passes require to be executed as last pass.") - } - val firstPass = workingList.getAndRemoveFirstPass() - if (firstPass != null) { - result.add(listOf(firstPass)) - } - while (!workingList.isEmpty) { - val p = workingList.getAndRemoveFirstPassWithoutDependencies() - if (p.isNotEmpty()) { - result.add(p) - } else { - // failed to find a pass that can be added to the result -> deadlock :( - throw ConfigurationException("Failed to satisfy ordering requirements.") - } - } + val orderingHelper = PassOrderingHelper(passes) log.info( - "Passes after enforcing order: {}", - result.map { list -> list.map { it.simpleName } } + "The following mermaid graph represents the pass dependencies: \n${buildMermaid(passes)}" ) - return result + return orderingHelper.order() } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt index 9eb4d07da5..199f03fe91 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -423,9 +423,21 @@ private constructor( } else null } + /** + * This extension function returns an appropriate [Language] for this [File] based on the + * registered file extensions of [TranslationConfiguration.languages]. It will emit a warning if + * multiple languages are registered for the same extension (and the first one that was + * registered will be returned). + */ private val File.language: Language<*>? get() { - return config.languages.firstOrNull { it.handlesFile(this) } + val languages = config.languages.filter { it.handlesFile(this) } + if (languages.size > 1) { + log.warn( + "Multiple languages match for file extension ${this.extension}, the first registered language will be used." + ) + } + return languages.firstOrNull() } class Builder { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 0a2a8a403a..05ace1a4dc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -26,16 +26,18 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.Component import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass import java.util.* import java.util.concurrent.ConcurrentHashMap +import org.neo4j.ogm.annotation.Relationship /** * The global (intermediate) result of the translation. A [LanguageFrontend] will initially populate @@ -52,11 +54,12 @@ class TranslationResult( var finalCtx: TranslationContext, ) : Node(), StatisticsHolder { + @Relationship("COMPONENTS") val componentEdges = astEdgesOf() /** * Entry points to the CPG: "SoftwareComponent" refer to programs, application, other "bundles" * of software. */ - @AST val components = mutableListOf() + val components by unwrapping(TranslationResult::componentEdges) /** * Scratch storage that can be used by passes to store additional information in this result. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 4709e6bece..8745e7944d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.CastNotPossible import de.fraunhofer.aisec.cpg.frontends.CastResult import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope @@ -140,8 +139,7 @@ class TypeManager { val node = scope.astNode // We need an additional check here, because of parsing or other errors, the AST node - // might - // not necessarily be a template declaration. + // might not necessarily be a template declaration. if (node is TemplateDeclaration) { val parameterizedType = getTypeParameter(node, name) if (parameterizedType != null) { @@ -224,12 +222,9 @@ class TypeManager { return t } - fun typeExists(name: String): Boolean { - return firstOrderTypes.any { type: Type -> type.root.name.toString() == name } - } - - fun typeExists(name: Name): Type? { - return firstOrderTypes.firstOrNull { type: Type -> type.root.name == name } + /** Checks, whether a [Type] with the given [name] exists. */ + fun typeExists(name: CharSequence): Boolean { + return firstOrderTypes.any { type: Type -> type.root.name == name } } fun resolvePossibleTypedef(alias: Type, scopeManager: ScopeManager): Type { @@ -243,7 +238,7 @@ class TypeManager { * is [Type.Origin.RESOLVED]. */ fun lookupResolvedType( - fqn: String, + fqn: CharSequence, generics: List? = null, language: Language<*>? = null ): Type? { @@ -254,7 +249,7 @@ class TypeManager { return firstOrderTypes.firstOrNull { (it.typeOrigin == Type.Origin.RESOLVED || it.typeOrigin == Type.Origin.GUESSED) && - it.name.toString() == fqn && + it.root.name == fqn && if (generics != null) { (it as? ObjectType)?.generics == generics } else { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/FrontendUtils.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/FrontendUtils.kt index c262db3654..0994f418fb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/FrontendUtils.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/FrontendUtils.kt @@ -137,13 +137,13 @@ class FrontendUtils { val smallestEnclosingNode = enclosingNodes.sortedWith(compareBy { it.code?.length ?: 10000 }).first() - val children = SubgraphWalker.getAstChildren(smallestEnclosingNode).toMutableList() + val children = smallestEnclosingNode.astChildren.toMutableList() // Because in GO we wrap all elements into a NamespaceDeclaration we have to extract the // natural children children.addAll( children.filterIsInstance().flatMap { namespace -> - SubgraphWalker.getAstChildren(namespace).filter { it !in children } + namespace.astChildren.filter { it !in children } } ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index b09a518bff..6192a7f130 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -34,9 +34,12 @@ import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.TemplateArguments import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.graph.unknownType +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.io.File import kotlin.reflect.KClass import kotlin.reflect.full.primaryConstructor @@ -118,7 +121,7 @@ abstract class Language> : Node() { * [builtInTypes] map, it returns null. The [typeString] must precisely match the key in the * map. */ - fun getSimpleTypeOf(typeString: String) = builtInTypes[typeString] + fun getSimpleTypeOf(typeString: CharSequence) = builtInTypes[typeString.toString()] /** Returns true if the [file] can be handled by the frontend of this language. */ fun handlesFile(file: File): Boolean { @@ -278,9 +281,9 @@ abstract class Language> : Node() { /** * This functions gives the language a chance to refine the results of a - * [ScopeManager.resolveCall] by choosing the best viable function(s) out of the set of viable - * functions. It can also influence the [CallResolutionResult.SuccessKind] of the resolution, - * e.g., if the result is ambiguous. + * [SymbolResolver.resolveWithArguments] by choosing the best viable function(s) out of the set + * of viable functions. It can also influence the [CallResolutionResult.SuccessKind] of the + * resolution, e.g., if the result is ambiguous. * * The default implementation will follow the following heuristic: * - If the list of [CallResolutionResult.viableFunctions] is empty, we can directly return. @@ -318,14 +321,15 @@ abstract class Language> : Node() { // We need to check, whether this language has special handling of templates. In this // case, we need to check, whether a template matches directly after we have no direct // matches - if (this is HasTemplates) { - result.call.templateParameterEdges = mutableListOf() + val source = result.source + if (this is HasTemplates && source is CallExpression) { + source.templateArgumentEdges = TemplateArguments(source) val (ok, candidates) = this.handleTemplateFunctionCalls( null, - result.call, + source, false, - result.call.ctx!!, + source.ctx!!, null, needsExactMatch = true ) @@ -333,7 +337,7 @@ abstract class Language> : Node() { return Pair(candidates.toSet(), CallResolutionResult.SuccessKind.SUCCESSFUL) } - result.call.templateParameterEdges = null + source.templateArgumentEdges = null } // If the list of viable functions is still empty at this point, the call is unresolved diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt index 3f85b97642..beb36b3661 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt @@ -27,16 +27,17 @@ package de.fraunhofer.aisec.cpg.frontends import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.graph.HasOperatorCode +import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation +import de.fraunhofer.aisec.cpg.graph.LanguageProvider +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration -import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.passes.ReplaceCallCastPass -import de.fraunhofer.aisec.cpg.passes.SymbolResolver +import de.fraunhofer.aisec.cpg.graph.scopes.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.passes.* +import kotlin.reflect.KClass /** * A language trait is a feature or trait that is common to a group of programming languages. Any @@ -85,29 +86,6 @@ interface HasTemplates : HasGenerics { */ interface HasDefaultArguments : LanguageTrait -/** - * A language trait that specifies that this language has a complex call resolution that we need to - * fine-tune in the language implementation. - */ -interface HasComplexCallResolution : LanguageTrait { - /** - * A function that can be used to fine-tune resolution of a method [call]. - * - * Note: The function itself should NOT set the [CallExpression.invokes] but rather return a - * list of possible candidates. - * - * @return a list of [FunctionDeclaration] candidates. - */ - fun refineMethodCallResolution( - curClass: RecordDeclaration?, - possibleContainingTypes: Set, - call: CallExpression, - ctx: TranslationContext, - currentTU: TranslationUnitDeclaration, - callResolver: SymbolResolver - ): List -} - /** A language trait that specifies if the language supports function pointers. */ interface HasFunctionPointers : LanguageTrait @@ -134,12 +112,12 @@ interface HasClasses : LanguageTrait interface HasSuperClasses : LanguageTrait { /** * Determines which keyword is used to access functions, etc. of the superclass of an object - * (often "super). + * (often `super`). */ val superClassKeyword: String - fun handleSuperCall( - callee: MemberExpression, + fun handleSuperExpression( + memberExpression: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager, ): Boolean @@ -217,16 +195,91 @@ interface HasAnonymousIdentifier : LanguageTrait { */ interface HasGlobalVariables : LanguageTrait +/** + * A common super-class for all language traits that arise because they are an ambiguity of a + * function call, e.g., function-style casts. This means that we cannot differentiate between a + * [CallExpression] and other expressions during the frontend and we need to invoke the + * [ResolveCallExpressionAmbiguityPass] to resolve this. + */ +sealed interface HasCallExpressionAmbiguity : LanguageTrait + /** * A language trait, that specifies that the language has so-called functional style casts, meaning * that they look like regular call expressions. Since we can therefore not distinguish between a * [CallExpression] and a [CastExpression], we need to employ an additional pass - * ([ReplaceCallCastPass]) after the initial language frontends are done. + * ([ResolveCallExpressionAmbiguityPass]) after the initial language frontends are done. + */ +interface HasFunctionStyleCasts : HasCallExpressionAmbiguity + +/** + * A language trait, that specifies that the language has functional style (object) construction, + * meaning that constructor calls look like regular call expressions (usually meaning that the + * language has no dedicated `new` keyword). + * + * Since we can therefore not distinguish between a [CallExpression] and a [ConstructExpression] in + * the frontend, we need to employ an additional pass ([ResolveCallExpressionAmbiguityPass]) after + * the initial language frontends are done. */ -interface HasFunctionalCasts : LanguageTrait +interface HasFunctionStyleConstruction : HasCallExpressionAmbiguity /** * A language trait that specifies that this language allowed overloading functions, meaning that * multiple functions can share the same name with different parameters. */ interface HasFunctionOverloading : LanguageTrait + +/** A language trait that specifies that this language allows overloading of operators. */ +interface HasOperatorOverloading : LanguageTrait { + + /** + * A map of operator codes and function names acting as overloaded operators. The key is a pair + * of the class and [HasOperatorCode.operatorCode] (ideally created by [of]) and the value is + * the name of the function. + */ + val overloadedOperatorNames: Map, String>, Symbol> + + /** + * Returns the matching operator code for [name] in [overloadedOperatorNames]. While + * [overloadedOperatorNames] can have multiple entries for a single operator code (e.g. to + * differentiate between unary and binary ops), we only ever allow one distinct operator code + * for a specific symbol. If non such distinct operator code is found, null is returned. + */ + fun operatorCodeFor(name: Symbol): String? { + return overloadedOperatorNames + .filterValues { it == name } + .keys + .map { it.second } + .distinct() + .singleOrNull() + } +} + +/** + * Creates a [Pair] of class and operator code used in + * [HasOperatorOverloading.overloadedOperatorNames]. + */ +inline infix fun KClass.of( + operatorCode: String +): Pair, String> { + return Pair(T::class, operatorCode) +} + +/** Checks whether the name for a function (as [CharSequence]) is a known operator name. */ +context(LanguageProvider) +val CharSequence.isKnownOperatorName: Boolean + get() { + val language = language + if (language !is HasOperatorOverloading) { + return false + } + + // If this is a parsed name, we only are interested in the local name + val name = + if (this is Name) { + this.localName + } else { + this + } + + return language.overloadedOperatorNames.containsValue(name) + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolder.kt index 49001da796..eafe01c47a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolder.kt @@ -53,6 +53,10 @@ interface ArgumentHolder : Holder { return false } + override fun replace(old: Expression, new: Expression): Boolean { + return replaceArgument(old, new) + } + /** * Replaces the existing argument specified in [old] with the one in [new]. Implementation how * to do that might be specific to the argument holder. @@ -68,4 +72,7 @@ interface ArgumentHolder : Holder { operator fun minusAssign(node: Expression) { removeArgument(node) } + + /** Checks, if [expression] is part of the arguments. */ + fun hasArgument(expression: Expression): Boolean } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index 2ef8f01c47..d9a32dbe22 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -26,7 +26,7 @@ package de.fraunhofer.aisec.cpg.graph import com.fasterxml.jackson.annotation.JsonIgnore -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.HasType @@ -45,4 +45,4 @@ class Assignment( /** The holder of this assignment */ @JsonIgnore val holder: AssignmentHolder -) : PropertyEdge(value, target as Node) +) : Edge(value, target as Node) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt index 3a8e368bd9..f50f6d3b78 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt @@ -26,6 +26,9 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import org.neo4j.ogm.annotation.Relationship /** * A node which presents some kind of complete piece of software, e.g., an application, a library, @@ -35,8 +38,10 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration * entry points or interactions with other software. */ open class Component : Node() { + @Relationship("TRANSLATION_UNITS") + val translationUnitEdges = astEdgesOf() /** All translation units belonging to this application. */ - @AST val translationUnits: MutableList = mutableListOf() + val translationUnits by unwrapping(Component::translationUnitEdges) @Synchronized fun addTranslationUnit(tu: TranslationUnitDeclaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt index 5d6fcb260f..c2e429edfd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationBuilder.kt @@ -95,6 +95,29 @@ fun MetadataProvider.newMethodDeclaration( return node } +/** + * Creates a new [OperatorDeclaration]. The [MetadataProvider] receiver will be used to fill + * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin + * requires an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional + * prepended argument. + */ +@JvmOverloads +fun MetadataProvider.newOperatorDeclaration( + name: CharSequence, + operatorCode: String, + recordDeclaration: RecordDeclaration? = null, + rawNode: Any? = null +): OperatorDeclaration { + val node = OperatorDeclaration() + node.applyMetadata(this, name, rawNode, defaultNamespace = recordDeclaration?.name) + + node.operatorCode = operatorCode + node.recordDeclaration = recordDeclaration + + log(node) + return node +} + /** * Creates a new [ConstructorDeclaration]. The [MetadataProvider] receiver will be used to fill * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin @@ -118,10 +141,10 @@ fun MetadataProvider.newConstructorDeclaration( } /** - * Creates a new [MethodDeclaration]. The [MetadataProvider] receiver will be used to fill different - * meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin requires - * an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional prepended - * argument. + * Creates a new [ParameterDeclaration]. The [MetadataProvider] receiver will be used to fill + * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin + * requires an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional + * prepended argument. */ @JvmOverloads fun MetadataProvider.newParameterDeclaration( @@ -183,7 +206,7 @@ fun LanguageProvider.newTupleDeclaration( // Also all our elements need to have an auto-type elements.forEach { it.type = autoType() } - node.elements = elements + node.elements = elements.toMutableList() node.initializer = initializer diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.kt index 129111ef4a..58a37471b8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.kt @@ -26,8 +26,10 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList interface DeclarationHolder { /** @@ -44,10 +46,11 @@ interface DeclarationHolder { } } - fun addIfNotContains( - collection: MutableCollection>, - declaration: T - ) { + fun > addIfNotContains(collection: AstEdges, declaration: T) { + addIfNotContains(collection, declaration, true) + } + + fun > addIfNotContains(collection: EdgeList, declaration: T) { addIfNotContains(collection, declaration, true) } @@ -59,28 +62,31 @@ interface DeclarationHolder { * @param the type of the declaration * @param outgoing whether the property is outgoing */ - fun addIfNotContains( - collection: MutableCollection>, + fun > addIfNotContains( + collection: EdgeList, declaration: T, outgoing: Boolean ) { - // create a new property edge - val propertyEdge = - if (outgoing) PropertyEdge(this as Node, declaration) - else PropertyEdge(declaration, this as T) - - // set the index property - propertyEdge.addProperty(Properties.INDEX, collection.size) var contains = false for (element in collection) { - if (element.end == propertyEdge.end) { - contains = true - break + if (outgoing) { + if (element.end == declaration) { + contains = true + break + } + } else { + if (element.start == declaration) { + contains = true + break + } } } - if (!contains) { - collection.add(propertyEdge) + + if (contains) { + return } + + collection.add(declaration) } val declarations: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index 6a72ab6484..af68b0f6e8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -25,12 +25,10 @@ */ package de.fraunhofer.aisec.cpg.graph -import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators -import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log -import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.ContextSensitiveDataflow import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.types.ProblemType @@ -129,8 +127,8 @@ fun MetadataProvider.newAssignExpression( val node = AssignExpression() node.applyMetadata(this, operatorCode, rawNode, true) node.operatorCode = operatorCode - node.lhs = lhs - node.rhs = rhs + node.lhs = lhs.toMutableList() + node.rhs = rhs.toMutableList() log(node) @@ -184,8 +182,8 @@ fun MetadataProvider.newConstructExpression( @JvmOverloads fun MetadataProvider.newConditionalExpression( condition: Expression, - thenExpression: Expression?, - elseExpression: Expression?, + thenExpression: Expression? = null, + elseExpression: Expression? = null, type: Type = unknownType(), rawNode: Any? = null ): ConditionalExpression { @@ -209,8 +207,8 @@ fun MetadataProvider.newConditionalExpression( */ @JvmOverloads fun MetadataProvider.newKeyValueExpression( - key: Expression? = null, - value: Expression? = null, + key: Expression, + value: Expression, rawNode: Any? = null ): KeyValueExpression { val node = KeyValueExpression() @@ -273,13 +271,39 @@ fun MetadataProvider.newCallExpression( callee.resolutionHelper = node } - node.callee = callee + if (callee != null) { + node.callee = callee + } node.template = template log(node) return node } +/** + * Creates a new [MemberCallExpression]. The [MetadataProvider] receiver will be used to fill + * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin + * requires an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional + * prepended argument. + */ +@JvmOverloads +fun MetadataProvider.newOperatorCallExpression( + operatorCode: String, + callee: Expression?, + rawNode: Any? = null +): OperatorCallExpression { + val node = OperatorCallExpression() + node.applyMetadata(this, operatorCode, rawNode) + + node.operatorCode = operatorCode + if (callee != null) { + node.callee = callee + } + + log(node) + return node +} + /** * Creates a new [MemberCallExpression]. The [MetadataProvider] receiver will be used to fill * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin @@ -304,7 +328,9 @@ fun MetadataProvider.newMemberCallExpression( callee.resolutionHelper = node } - node.callee = callee + if (callee != null) { + node.callee = callee + } node.isStatic = isStatic log(node) @@ -562,23 +588,34 @@ fun Literal.duplicate(implicit: Boolean): Literal { duplicate.comment = this.comment duplicate.file = this.file duplicate.name = this.name.clone() - for (next in this.nextDFGEdges) { - duplicate.addNextDFG( - next.end, - next.granularity, - (next as? ContextSensitiveDataflow)?.callingContext - ) + for (edge in this.nextDFGEdges) { + if (edge is ContextSensitiveDataflow) { + duplicate.nextDFGEdges.addContextSensitive( + edge.end, + edge.granularity, + edge.callingContext + ) + } else { + duplicate.nextDFGEdges += edge + } + } + for (edge in this.prevDFGEdges) { + if (edge is ContextSensitiveDataflow) { + duplicate.prevDFGEdges.addContextSensitive( + edge.start, + edge.granularity, + edge.callingContext + ) + } else { + duplicate.prevDFGEdges += edge + } + } + for (edge in this.prevEOGEdges) { + duplicate.prevEOGEdges += edge } - for (next in this.prevDFGEdges) { - duplicate.addPrevDFG( - next.start, - next.granularity, - (next as? ContextSensitiveDataflow)?.callingContext - ) + for (edge in this.nextEOGEdges) { + duplicate.nextEOGEdges += edge } - // TODO: This loses the properties of the edges. - duplicate.nextEOG = this.nextEOG - duplicate.prevEOG = this.prevEOG duplicate.isImplicit = implicit return duplicate } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index 88d65ab47a..a76bd4b027 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -27,8 +27,7 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement import de.fraunhofer.aisec.cpg.graph.statements.IfStatement @@ -41,7 +40,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.WhileStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.astParent import kotlin.math.absoluteValue /** @@ -330,20 +328,14 @@ fun Node.followNextEOGEdgesUntilHit(predicate: (Node) -> Boolean): FulfilledAndF val currentPath = worklist.removeFirst() // The last node of the path is where we continue. We get all of its outgoing DFG edges and // follow them - if ( - currentPath.last().nextEOGEdges.none { it.getProperty(Properties.UNREACHABLE) != true } - ) { + if (currentPath.last().nextEOGEdges.none { it.unreachable != true }) { // No further nodes in the path and the path criteria are not satisfied. failedPaths.add(currentPath) continue // Don't add this path anymore. The requirement is satisfied. } for (next in - currentPath - .last() - .nextEOGEdges - .filter { it.getProperty(Properties.UNREACHABLE) != true } - .map { it.end }) { + currentPath.last().nextEOGEdges.filter { it.unreachable != true }.map { it.end }) { // Copy the path for each outgoing DFG edge and add the next node val nextPath = mutableListOf() nextPath.addAll(currentPath) @@ -388,20 +380,14 @@ fun Node.followPrevEOGEdgesUntilHit(predicate: (Node) -> Boolean): FulfilledAndF val currentPath = worklist.removeFirst() // The last node of the path is where we continue. We get all of its outgoing DFG edges and // follow them - if ( - currentPath.last().prevEOGEdges.none { it.getProperty(Properties.UNREACHABLE) != true } - ) { + if (currentPath.last().prevEOGEdges.none { it.unreachable != true }) { // No further nodes in the path and the path criteria are not satisfied. failedPaths.add(currentPath) continue // Don't add this path anymore. The requirement is satisfied. } for (next in - currentPath - .last() - .prevEOGEdges - .filter { it.getProperty(Properties.UNREACHABLE) != true } - .map { it.start }) { + currentPath.last().prevEOGEdges.filter { it.unreachable != true }.map { it.start }) { // Copy the path for each outgoing DFG edge and add the next node val nextPath = mutableListOf() nextPath.addAll(currentPath) @@ -429,10 +415,10 @@ fun Node.followPrevEOGEdgesUntilHit(predicate: (Node) -> Boolean): FulfilledAndF * * It returns only a single possible path even if multiple paths are possible. */ -fun Node.followNextEOG(predicate: (PropertyEdge<*>) -> Boolean): List>? { - val path = mutableListOf>() +fun Node.followNextEOG(predicate: (Edge<*>) -> Boolean): List>? { + val path = mutableListOf>() - for (edge in this.nextEOGEdges.filter { it.getProperty(Properties.UNREACHABLE) != true }) { + for (edge in this.nextEOGEdges.filter { it.unreachable != true }) { val target = edge.end path.add(edge) @@ -459,10 +445,10 @@ fun Node.followNextEOG(predicate: (PropertyEdge<*>) -> Boolean): List) -> Boolean): List>? { - val path = mutableListOf>() +fun Node.followPrevEOG(predicate: (Edge<*>) -> Boolean): List>? { + val path = mutableListOf>() - for (edge in this.prevEOGEdges.filter { it.getProperty(Properties.UNREACHABLE) != true }) { + for (edge in this.prevEOGEdges.filter { it.unreachable != true }) { val source = edge.start path.add(edge) @@ -518,6 +504,10 @@ val Node?.nodes: List val Node?.calls: List get() = this.allChildren() +/** Returns all [OperatorCallExpression] children in this graph, starting with this [Node]. */ +val Node?.operatorCalls: List + get() = this.allChildren() + /** Returns all [MemberCallExpression] children in this graph, starting with this [Node]. */ val Node?.mcalls: List get() = this.allChildren() @@ -530,6 +520,10 @@ val Node?.casts: List val Node?.methods: List get() = this.allChildren() +/** Returns all [OperatorDeclaration] children in this graph, starting with this [Node]. */ +val Node?.operators: List + get() = this.allChildren() + /** Returns all [FieldDeclaration] children in this graph, starting with this [Node]. */ val Node?.fields: List get() = this.allChildren() @@ -614,6 +608,29 @@ val Node?.returns: List val Node?.assigns: List get() = this.allChildren() +/** + * Return all [ProblemNode] children in this graph (either stored directly or in + * [Node.additionalProblems]), starting with this [Node]. + */ +val Node?.problems: List + get() { + val relevantNodes = + this.allChildren { it is ProblemNode || it.additionalProblems.isNotEmpty() } + + val result = mutableListOf() + + relevantNodes.forEach { + if (it.additionalProblems.isNotEmpty()) { + result += it.additionalProblems + } + if (it is ProblemNode) { + result += it + } + } + + return result + } + /** Returns all [Assignment] child edges in this graph, starting with this [Node]. */ val Node?.assignments: List get() { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt deleted file mode 100644 index ff1c5963cf..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.graph - -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.types.HasType - -/** - * Specifies that a certain node has an initializer. It is a special case of [ArgumentHolder], in - * which the initializer is treated as the first (and only) argument. - */ -interface HasInitializer : HasType, ArgumentHolder, AssignmentHolder { - - var initializer: Expression? - - override fun addArgument(expression: Expression) { - this.initializer = expression - } - - override fun removeArgument(expression: Expression): Boolean { - return if (this.initializer == expression) { - this.initializer = null - true - } else { - false - } - } - - override fun replaceArgument(old: Expression, new: Expression): Boolean { - this.initializer = new - return true - } - - override val assignments: List - get() { - return initializer?.let { listOf(Assignment(it, this, this)) } ?: listOf() - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Holder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Holder.kt index ad44450b1c..7192071242 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Holder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Holder.kt @@ -37,6 +37,15 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression * [CallExpression]). */ interface Holder { + + /** + * Replaces the existing node specified in [old] with the one in [new]. Implementation how to do + * that might be specific to the holder. + * + * An indication whether this operation was successful needs to be returned. + */ + fun replace(old: NodeTypeToHold, new: NodeTypeToHold): Boolean + /** Adds a [Node] to the list of "held" nodes. */ operator fun plusAssign(node: NodeTypeToHold) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Interfaces.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Interfaces.kt new file mode 100644 index 0000000000..103a71d9ad --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Interfaces.kt @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.graph.declarations.OperatorDeclaration +import de.fraunhofer.aisec.cpg.graph.scopes.Scope +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.passes.SymbolResolver +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation + +/** A simple interface that a node has [language]. */ +interface HasLanguage { + + var language: Language<*>? +} + +/** A simple interface that a node has [name] and [location]. */ +interface HasNameAndLocation : HasLanguage { + + val name: Name + + /** Location of the finding in source code. */ + val location: PhysicalLocation? +} + +/** A simple interface that a node has [scope]. */ +interface HasScope : HasNameAndLocation { + + /** The scope this node lives in. */ + val scope: Scope? +} + +/** A simple interface to denote that the implementing class has some kind of [operatorCode]. */ +interface HasOperatorCode : HasScope { + + /** The operator code, identifying an operation executed on one or more [Expression]s */ + val operatorCode: String? +} + +/** Specifies that a certain node has a base on which it executes an operation. */ +interface HasBase : HasOperatorCode { + + /** The base. If there is no actual base, it can be null. */ + val base: Expression? + + /** + * The operator that is used to access the [base]. Usually either `.` or `->`, but some + * languages offer additional operator codes. If the [base] is null, the [operatorCode] should + * also be null. + */ + override val operatorCode: String? +} + +/** + * Interface that allows us to mark nodes that contain a default value + * + * @param type of the default node + */ +interface HasDefault : HasScope { + var default: T +} + +/** + * Specifies that a certain node has an initializer. It is a special case of [ArgumentHolder], in + * which the initializer is treated as the first (and only) argument. + */ +interface HasInitializer : HasScope, HasType, ArgumentHolder, AssignmentHolder { + + var initializer: Expression? + + override fun addArgument(expression: Expression) { + this.initializer = expression + } + + override fun removeArgument(expression: Expression): Boolean { + return if (this.initializer == expression) { + this.initializer = null + true + } else { + false + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + this.initializer = new + return true + } + + override fun hasArgument(expression: Expression): Boolean { + return initializer == expression + } + + override val assignments: List + get() { + return initializer?.let { listOf(Assignment(it, this, this)) } ?: listOf() + } +} + +/** + * Some nodes have aliases, i.e., it potentially references another variable. This means that + * writing to this node, also writes to its [aliases] and vice-versa. + */ +interface HasAliases : HasScope { + /** The aliases which this node has. */ + var aliases: MutableSet +} + +/** + * Specifies that this node (e.g. a [BinaryOperator] contains an operation that can be overloaded by + * an [OperatorDeclaration]. + */ +interface HasOverloadedOperation : HasOperatorCode { + + /** + * Arguments forwarded to the operator. This might not necessarily be all of the regular + * "arguments", since often the the first argument is part of the [operatorBase]. + */ + val operatorArguments: List + + /** + * The base expression this operator works on. The [Type] of this is also the source where the + * [SymbolResolver] is looking for an overloaded [OperatorDeclaration]. + */ + val operatorBase: Expression +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt index 58bbd68617..6f04d5dd1b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt @@ -25,9 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph -import de.fraunhofer.aisec.cpg.graph.edge.Dataflow -import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.PartialDataflowGranularity import de.fraunhofer.aisec.cpg.helpers.identitySetOf import kotlin.reflect.KProperty1 @@ -51,11 +51,11 @@ fun Node.printEOG(maxConnections: Int = 25): String { * https://github.blog/2022-02-14-include-diagrams-markdown-files-mermaid/). * * The edge type can be specified with the [nextEdgeGetter] and [prevEdgeGetter] functions, that - * need to return a list of edges (as a [PropertyEdge]) beginning from this node. + * need to return a list of edges (as a [Edge]) beginning from this node. */ -fun > Node.printGraph( - nextEdgeGetter: KProperty1>, - prevEdgeGetter: KProperty1>, +fun > Node.printGraph( + nextEdgeGetter: KProperty1>, + prevEdgeGetter: KProperty1>, maxConnections: Int = 25 ): String { val builder = StringBuilder() @@ -65,8 +65,8 @@ fun > Node.printGraph( // We use a set with a defined ordering to hold our work-list to have a somewhat consistent // ordering of statements in the mermaid file. - val worklist = LinkedHashSet>() - val alreadySeen = identitySetOf>() + val worklist = LinkedHashSet>() + val alreadySeen = identitySetOf>() var conns = 0 worklist.addAll(nextEdgeGetter.get(this)) @@ -106,14 +106,15 @@ fun > Node.printGraph( return builder.toString() } -private fun PropertyEdge.label(): String { +private fun Edge.label(): String { val builder = StringBuilder() builder.append("\"") builder.append(this.label) if (this is Dataflow) { - if (this.granularity is PartialDataflowGranularity) { - builder.append(" (partial, ${this.granularity.partialTarget?.name})") + var granularity = this.granularity + if (granularity is PartialDataflowGranularity) { + builder.append(" (partial, ${granularity.partialTarget?.name})") } else { builder.append(" (full)") } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt index 1cf9d66218..c8cf38536a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import java.util.* +import kotlin.uuid.Uuid /** * This class represents anything that can have a "Name". In the simplest case it only represents a @@ -49,6 +50,18 @@ class Name( language: Language<*>? ) : this(localName, parent, language?.namespaceDelimiter ?: ".") + companion object { + /** + * Creates a random name starting with a prefix plus a random UUID (version 4). The Name is + * prefixed by [prefix], followed by a separator character [separatorChar] and finalized by + * a random UUID ("-" separators also replaced with [separatorChar]). + */ + fun random(prefix: String, separatorChar: Char = '_'): Name { + val randomPart = Uuid.random().toString().replace('-', separatorChar) + return Name(localName = prefix + separatorChar + randomPart) + } + } + /** * The full string representation of this name. Since [localName] and [parent] are immutable, * this is basically a cache for [toString]. Otherwise, we would need to call [toString] a lot @@ -70,11 +83,13 @@ class Name( override fun equals(other: Any?): Boolean { if (this === other) return true - if (other !is Name) return false + if (other is String) return this.fullName == other + if (other is Name) + return localName == other.localName && + parent == other.parent && + delimiter == other.delimiter - return localName == other.localName && - parent == other.parent && - delimiter == other.delimiter + return false } override fun get(index: Int) = fullName[index] @@ -153,9 +168,9 @@ internal fun parseName(fqn: CharSequence, delimiter: String, vararg splitDelimit } /** Returns a new [Name] based on the [localName] and the current name as parent. */ -fun Name?.fqn(localName: String) = +fun Name?.fqn(localName: String, delimiter: String = this?.delimiter ?: ".") = if (this == null) { - Name(localName) + Name(localName, null, delimiter) } else { - Name(localName, this, this.delimiter) + Name(localName, this, delimiter) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 073c837764..a9b65a315b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -29,14 +29,18 @@ import com.fasterxml.jackson.annotation.JsonBackReference import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope -import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope -import de.fraunhofer.aisec.cpg.graph.scopes.Scope +import de.fraunhofer.aisec.cpg.graph.edges.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.flows.ControlDependences +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflows +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrders +import de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.edges.flows.ProgramDependences +import de.fraunhofer.aisec.cpg.graph.scopes.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter @@ -52,7 +56,14 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory /** The base class for all graph objects that are going to be persisted in the database. */ -open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider, ContextProvider { +abstract class Node : + IVisitable, + Persistable, + LanguageProvider, + ScopeProvider, + ContextProvider, + HasNameAndLocation, + HasScope { /** * Because we are updating type information in the properties of the node, we need a reference * to managers such as the [TypeManager] instance which is responsible for this particular node. @@ -61,11 +72,8 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider */ @get:JsonIgnore @Transient override var ctx: TranslationContext? = null - /** - * This property holds the full name using our new [Name] class. It is currently not persisted - * in the graph database. - */ - @Convert(NameConverter::class) open var name: Name = Name(EMPTY_NAME) + /** This property holds the full name using our new [Name] class. */ + @Convert(NameConverter::class) override var name: Name = Name(EMPTY_NAME) /** * Original code snippet of this node. Most nodes will have a corresponding "code", but in cases @@ -97,8 +105,7 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider /** Optional comment of this node. */ var comment: String? = null - /** Location of the finding in source code. */ - @Convert(LocationConverter::class) var location: PhysicalLocation? = null + @Convert(LocationConverter::class) override var location: PhysicalLocation? = null /** * Name of the containing file. It can be null for artificially created nodes or if just @@ -109,13 +116,15 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider /** Incoming control flow edges. */ @Relationship(value = "EOG", direction = Relationship.Direction.INCOMING) @PopulatedByPass(EvaluationOrderGraphPass::class) - var prevEOGEdges: MutableList> = ArrayList() + var prevEOGEdges: EvaluationOrders = + EvaluationOrders(this, mirrorProperty = Node::nextEOGEdges, outgoing = false) protected set /** Outgoing control flow edges. */ @Relationship(value = "EOG", direction = Relationship.Direction.OUTGOING) @PopulatedByPass(EvaluationOrderGraphPass::class) - var nextEOGEdges: MutableList> = ArrayList() + var nextEOGEdges: EvaluationOrders = + EvaluationOrders(this, mirrorProperty = Node::prevEOGEdges, outgoing = true) protected set /** @@ -124,10 +133,11 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider */ @PopulatedByPass(ControlDependenceGraphPass::class) @Relationship(value = "CDG", direction = Relationship.Direction.OUTGOING) - var nextCDGEdges: MutableList> = ArrayList() + var nextCDGEdges: ControlDependences = + ControlDependences(this, mirrorProperty = Node::prevCDGEdges, outgoing = true) protected set - var nextCDG by PropertyEdgeDelegate(Node::nextCDGEdges, true) + var nextCDG by unwrapping(Node::nextCDGEdges) /** * The nodes which dominate this node via the control-flow, i.e., the parents of the Control @@ -135,10 +145,11 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider */ @PopulatedByPass(ControlDependenceGraphPass::class) @Relationship(value = "CDG", direction = Relationship.Direction.INCOMING) - var prevCDGEdges: MutableList> = ArrayList() + var prevCDGEdges: ControlDependences = + ControlDependences(this, mirrorProperty = Node::nextCDGEdges, outgoing = false) protected set - var prevCDG by PropertyEdgeDelegate(Node::prevCDGEdges, false) + var prevCDG by unwrapping(Node::prevCDGEdges) /** * Virtual property to return a list of the node's children. Uses the [SubgraphWalker] to @@ -151,28 +162,33 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider * persistence. Therefore, this property is a `var` and not a `val`. */ @Relationship("AST") + @JsonIgnore var astChildren: List = listOf() get() = SubgraphWalker.getAstChildren(this) + @Transient var astParent: Node? = null + /** Virtual property for accessing [prevEOGEdges] without property edges. */ - @PopulatedByPass(EvaluationOrderGraphPass::class) - var prevEOG: List by PropertyEdgeDelegate(Node::prevEOGEdges, false) + @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG by unwrapping(Node::prevEOGEdges) /** Virtual property for accessing [nextEOGEdges] without property edges. */ - @PopulatedByPass(EvaluationOrderGraphPass::class) - var nextEOG: List by PropertyEdgeDelegate(Node::nextEOGEdges) + @PopulatedByPass(EvaluationOrderGraphPass::class) var nextEOG by unwrapping(Node::nextEOGEdges) /** Incoming data flow edges */ @Relationship(value = "DFG", direction = Relationship.Direction.INCOMING) @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) - var prevDFGEdges: MutableList = mutableListOf() + var prevDFGEdges: Dataflows = + Dataflows(this, mirrorProperty = Node::nextDFGEdges, outgoing = false) protected set /** Virtual property for accessing [prevDFGEdges] without property edges. */ @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) - var prevDFG: MutableSet by PropertyEdgeSetDelegate(Node::prevDFGEdges, false) + var prevDFG by unwrapping(Node::prevDFGEdges) - /** Virtual property for accessing [nextDFGEdges] that have a [FullDataflowGranularity]. */ + /** + * Virtual property for accessing [nextDFGEdges] that have a + * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. + */ @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val prevFullDFG: List get() { @@ -184,14 +200,18 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider /** Outgoing data flow edges */ @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) @Relationship(value = "DFG", direction = Relationship.Direction.OUTGOING) - var nextDFGEdges: MutableList = mutableListOf() + var nextDFGEdges: Dataflows = + Dataflows(this, mirrorProperty = Node::prevDFGEdges, outgoing = true) protected set /** Virtual property for accessing [nextDFGEdges] without property edges. */ @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) - var nextDFG: MutableSet by PropertyEdgeSetDelegate(Node::nextDFGEdges, true) + var nextDFG by unwrapping(Node::nextDFGEdges) - /** Virtual property for accessing [nextDFGEdges] that have a [FullDataflowGranularity]. */ + /** + * Virtual property for accessing [nextDFGEdges] that have a + * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. + */ @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val nextFullDFG: List get() { @@ -201,21 +221,17 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider /** Outgoing Program Dependence Edges. */ @PopulatedByPass(ProgramDependenceGraphPass::class) @Relationship(value = "PDG", direction = Relationship.Direction.OUTGOING) - var nextPDGEdges: MutableSet> = mutableSetOf() + var nextPDGEdges: ProgramDependences = + ProgramDependences(this, mirrorProperty = Node::prevPDGEdges, outgoing = false) protected set - /** Virtual property for accessing the children of the Program Dependence Graph (PDG). */ - var nextPDG: MutableSet by PropertyEdgeSetDelegate(Node::nextPDGEdges, true) - /** Incoming Program Dependence Edges. */ @PopulatedByPass(ProgramDependenceGraphPass::class) @Relationship(value = "PDG", direction = Relationship.Direction.INCOMING) - var prevPDGEdges: MutableSet> = mutableSetOf() + var prevPDGEdges: ProgramDependences = + ProgramDependences(this, mirrorProperty = Node::nextPDGEdges, outgoing = false) protected set - /** Virtual property for accessing the parents of the Program Dependence Graph (PDG). */ - var prevPDG: MutableSet by PropertyEdgeSetDelegate(Node::prevPDGEdges, false) - /** * If a node is marked as being inferred, it means that it was created artificially and does not * necessarily have a real counterpart in the scanned source code. However, the nodes @@ -238,139 +254,14 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider var argumentIndex = 0 /** List of annotations associated with that node. */ - @AST var annotations: MutableList = ArrayList() - - fun removePrevEOGEntry(eog: Node) { - removePrevEOGEntries(listOf(eog)) - } - - private fun removePrevEOGEntries(prevEOGs: List) { - for (n in prevEOGs) { - val remove = PropertyEdge.findPropertyEdgesByPredicate(prevEOGEdges) { it.start === n } - prevEOGEdges.removeAll(remove) - } - } - - fun addPrevEOG(propertyEdge: PropertyEdge) { - prevEOGEdges.add(propertyEdge) - } - - fun addNextEOG(propertyEdge: PropertyEdge) { - nextEOGEdges.add(propertyEdge) - } - - fun clearNextEOG() { - nextEOGEdges.clear() - } - - /** Adds a [Dataflow] edge from this node to [next], with the given [Granularity]. */ - fun addNextDFG( - next: Node, - granularity: Granularity = default(), - callingContext: CallingContext? = null, - ) { - val edge = - if (callingContext != null) { - ContextSensitiveDataflow(this, next, callingContext, granularity) - } else { - Dataflow(this, next, granularity) - } - nextDFGEdges.add(edge) - next.prevDFGEdges.add(edge) - } - - fun removeNextDFG(next: Node?) { - if (next != null) { - val thisRemove = - PropertyEdge.findPropertyEdgesByPredicate(nextDFGEdges) { it.end === next } - nextDFGEdges.removeAll(thisRemove) - - val nextRemove = - PropertyEdge.findPropertyEdgesByPredicate(next.prevDFGEdges) { it.start == this } - next.prevDFGEdges.removeAll(nextRemove) - } - } - - /** - * Adds a [Dataflow] edge from [prev] node to this node, with the given [Granularity] and - * [CallingContext]. - */ - open fun addPrevDFG( - prev: Node, - granularity: Granularity = default(), - callingContext: CallingContext? = null, - ) { - val edge = - if (callingContext != null) { - ContextSensitiveDataflow(prev, this, callingContext, granularity) - } else { - Dataflow(prev, this, granularity) - } - prevDFGEdges.add(edge) - prev.nextDFGEdges.add(edge) - } - - fun addPrevCDG( - prev: Node, - properties: MutableMap = EnumMap(Properties::class.java) - ) { - val edge = PropertyEdge(prev, this, properties) - prevCDGEdges.add(edge) - prev.nextCDGEdges.add(edge) - } + @Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf() + var annotations by unwrapping(Node::annotationEdges) /** - * Adds a [Dataflow] edge from all [prev] nodes to this node, with the given [Granularity] and - * [CallingContext] if applicable. + * Additional problem nodes. These nodes represent problems which occurred during processing of + * a node (i.e. only partially processed). */ - fun addAllPrevDFG( - prev: Collection, - granularity: Granularity = full(), - callingContext: CallingContext? = null, - ) { - prev.forEach { addPrevDFG(it, granularity, callingContext) } - } - - fun addAllPrevPDG(prev: Collection, dependenceType: DependenceType) { - addAllPrevPDGEdges(prev.map { PropertyEdge(it, this) }, dependenceType) - } - - fun addAllPrevPDGEdges(prev: Collection>, dependenceType: DependenceType) { - prev.forEach { - val edge = PropertyEdge(it).apply { addProperty(Properties.DEPENDENCE, dependenceType) } - this.prevPDGEdges.add(edge) - val other = if (it.start != this) it.start else it.end - other.nextPDGEdges.add(edge) - } - } - - fun removePrevDFG(prev: Node?) { - if (prev != null) { - val thisRemove = - PropertyEdge.findPropertyEdgesByPredicate(prevDFGEdges) { it.start === prev } - prevDFGEdges.removeAll(thisRemove) - - val prevRemove = - PropertyEdge.findPropertyEdgesByPredicate(prev.nextDFGEdges) { it.end === this } - prev.nextDFGEdges.removeAll(prevRemove) - } - } - - fun clearPrevDFG() { - for (prev in ArrayList(prevDFG)) { - removePrevDFG(prev) - } - } - - fun clearNextDFG() { - for (prev in ArrayList(nextDFG)) { - removeNextDFG(prev) - } - } - - fun addAnnotations(annotations: Collection) { - this.annotations.addAll(annotations) - } + val additionalProblems: MutableSet = mutableSetOf() /** * If a node should be removed from the graph, just removing it from the AST is not enough (see @@ -382,32 +273,13 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider * further children that have no alternative connection paths to the rest of the graph. */ fun disconnectFromGraph() { - for (n in nextDFGEdges) { - val remove = - PropertyEdge.findPropertyEdgesByPredicate(n.end.prevDFGEdges) { it.start == this } - n.end.prevDFGEdges.removeAll(remove) - } nextDFGEdges.clear() - - for (n in prevDFGEdges) { - val remove = - PropertyEdge.findPropertyEdgesByPredicate(n.start.nextDFGEdges) { it.end == this } - n.start.nextDFGEdges.removeAll(remove) - } prevDFGEdges.clear() - - for (n in nextEOGEdges) { - val remove = - PropertyEdge.findPropertyEdgesByPredicate(n.end.prevEOGEdges) { it.start == this } - n.end.prevEOGEdges.removeAll(remove) - } + prevCDGEdges.clear() + nextCDGEdges.clear() + prevPDGEdges.clear() + nextPDGEdges.clear() nextEOGEdges.clear() - - for (n in prevEOGEdges) { - val remove = - PropertyEdge.findPropertyEdgesByPredicate(n.start.nextEOGEdges) { it.end == this } - n.start.nextEOGEdges.removeAll(remove) - } prevEOGEdges.clear() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilder.kt index c4157ff013..b01b6ce5c1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilder.kt @@ -28,6 +28,8 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node.Companion.EMPTY_NAME import de.fraunhofer.aisec.cpg.graph.NodeBuilder.log +import de.fraunhofer.aisec.cpg.graph.scopes.Scope +import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.* /** @@ -329,3 +331,28 @@ fun MetadataProvider.newDefaultStatement(rawNode: Any? = null): DefaultStatement log(node) return node } + +/** + * Creates a new [LookupScopeStatement]. The [MetadataProvider] receiver will be used to fill + * different meta-data using [Node.applyMetadata]. Calling this extension function outside of Kotlin + * requires an appropriate [MetadataProvider], such as a [LanguageFrontend] as an additional + * prepended argument. + */ +@JvmOverloads +fun MetadataProvider.newLookupScopeStatement( + symbols: List, + targetScope: Scope?, + rawNode: Any? = null +): LookupScopeStatement { + val node = LookupScopeStatement() + node.targetScope = targetScope + node.applyMetadata(this, EMPTY_NAME, rawNode, true) + + // Add it to our scope + for (symbol in symbols) { + node.scope?.predefinedLookupScopes[symbol] = node + } + + log(node) + return node +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt index 39d9492be5..2a79238571 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/StatementHolder.kt @@ -25,11 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.wrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.edges.collections.UnwrappedEdgeList.Delegate import de.fraunhofer.aisec.cpg.graph.statements.Statement /** @@ -43,47 +41,20 @@ import de.fraunhofer.aisec.cpg.graph.statements.Statement */ interface StatementHolder : Holder { /** List of statements as property edges. */ - var statementEdges: MutableList> + var statementEdges: AstEdges> /** * Virtual property to access [statementEdges] without property edges. * - * Note: We cannot use [PropertyEdgeDelegate] because delegates are not allowed in interfaces. + * Note: We cannot use [Delegate] because delegates are not allowed in interfaces. */ - var statements: List - get() { - return unwrap(statementEdges) - } - set(value) { - statementEdges = wrap(value, this as Node) - } + var statements: MutableList - /** - * Adds the specified statement to this statement holder. The statements have to be stored as a - * list of statements as we try to avoid adding new AST-nodes that do not exist, e.g. a code - * body to hold statements - * - * @param s the statement - */ - fun addStatement(s: Statement) { - val propertyEdge = PropertyEdge((this as Node), s) - propertyEdge.addProperty(Properties.INDEX, statementEdges.size) - statementEdges.add(propertyEdge) - } - - /** Inserts the statement [s] before the statement specified in [before]. */ - fun insertStatementBefore(s: Statement, beforeStmt: Statement) { - val statements = this.statements - val idx = statements.indexOf(beforeStmt) - if (idx != -1) { - val before = statements.subList(0, idx) - val after = statements.subList(idx, statements.size) - - this.statements = listOf(*before.toTypedArray(), s, *after.toTypedArray()) - } + override fun replace(old: Statement, new: Statement): Boolean { + return statementEdges.replace(old, new) } override operator fun plusAssign(node: Statement) { - addStatement(node) + statementEdges += node } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt index 78ce712c10..f4443b780e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt @@ -263,7 +263,7 @@ fun LanguageFrontend<*, *>.block(needsScope: Boolean = true, init: Block.() -> U val node = newBlock() scopeIfNecessary(needsScope, node, init) - (this@StatementHolder).addStatement(node) + (this@StatementHolder).statementEdges += node return node } @@ -375,7 +375,28 @@ fun LanguageFrontend<*, *>.variable( val node = newVariableDeclaration(name, type) if (init != null) init(node) - addToPropertyEdgeDeclaration(node) + declarationEdges += node + + scopeManager.addDeclaration(node) + + return node +} + +/** + * Creates a new [ProblemDeclaration] in the Fluent Node DSL and adds it to the + * [DeclarationStatement.declarations] of the nearest enclosing [DeclarationStatement]. The [init] + * block can be used to create further sub-nodes as well as configuring the created node itself. + */ +context(DeclarationStatement) +fun LanguageFrontend<*, *>.problemDecl( + description: String, + type: ProblemNode.ProblemType = ProblemNode.ProblemType.TRANSLATION, + init: (ProblemDeclaration.() -> Unit)? = null +): ProblemDeclaration { + val node = newProblemDeclaration(problem = description, problemType = type) + if (init != null) init(node) + + declarationEdges += node scopeManager.addDeclaration(node) @@ -785,8 +806,8 @@ fun LanguageFrontend<*, *>.elseStmt(needsScope: Boolean = true, init: Block.() - } /** - * Creates a new [LabelStatement] in the Fluent Node DSL and invokes [StatementHolder.addStatement] - * of the nearest enclosing [Holder], but only if it is an [StatementHolder]. + * Creates a new [LabelStatement] in the Fluent Node DSL and adds it to the nearest enclosing + * [StatementHolder]. */ context(Holder) fun LanguageFrontend<*, *>.label( @@ -809,8 +830,8 @@ fun LanguageFrontend<*, *>.label( } /** - * Creates a new [ContinueStatement] in the Fluent Node DSL and invokes - * [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * Creates a new [ContinueStatement] in the Fluent Node DSL and adds it to the nearest enclosing + * [StatementHolder]. */ context(StatementHolder) fun LanguageFrontend<*, *>.continueStmt(label: String? = null): ContinueStatement { @@ -823,8 +844,8 @@ fun LanguageFrontend<*, *>.continueStmt(label: String? = null): ContinueStatemen } /** - * Creates a new [BreakStatement] in the Fluent Node DSL and invokes [StatementHolder.addStatement] - * of the nearest enclosing [Holder], but only if it is an [StatementHolder]. + * Creates a new [BreakStatement] in the Fluent Node DSL and adds it to the nearest enclosing + * [Holder], but only if it is an [StatementHolder]. */ context(Holder) fun LanguageFrontend<*, *>.breakStmt(label: String? = null): BreakStatement { @@ -841,8 +862,8 @@ fun LanguageFrontend<*, *>.breakStmt(label: String? = null): BreakStatement { } /** - * Creates a new [CaseStatement] in the Fluent Node DSL and invokes [StatementHolder.addStatement] - * of the nearest enclosing [Holder], but only if it is an [StatementHolder]. + * Creates a new [CaseStatement] in the Fluent Node DSL and adds it to the nearest enclosing + * [Holder], but only if it is an [StatementHolder]. */ context(Holder) fun LanguageFrontend<*, *>.case(caseExpression: Expression? = null): CaseStatement { @@ -858,8 +879,7 @@ fun LanguageFrontend<*, *>.case(caseExpression: Expression? = null): CaseStateme return node } /** - * Creates a new [DefaultStatement] in the Fluent Node DSL and invokes - * [StatementHolder.addStatement] of the nearest enclosing [Holder], but only if it is an + * Creates a new [DefaultStatement] in the Fluent Node DSL and adds it to the nearest enclosing * [StatementHolder]. */ context(Holder) @@ -1084,7 +1104,7 @@ operator fun Expression.plus(rhs: Expression): BinaryOperator { /** * Creates a new [BinaryOperator] with a `+` [BinaryOperator.operatorCode] in the Fluent Node DSL - * and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, StatementHolder) operator fun Expression.plusAssign(rhs: Expression) { @@ -1145,7 +1165,7 @@ fun reference(input: Expression): UnaryOperator { /** * Creates a new [UnaryOperator] with a `--` [UnaryOperator.operatorCode] in the Fluent Node DSL and - * invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) operator fun Expression.dec(): UnaryOperator { @@ -1284,7 +1304,7 @@ infix fun Expression.le(rhs: Expression): BinaryOperator { /** * Creates a new [ConditionalExpression] with a `=` [BinaryOperator.operatorCode] in the Fluent Node - * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * DSL and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) fun Expression.conditional( @@ -1306,12 +1326,12 @@ fun Expression.conditional( /** * Creates a new [BinaryOperator] with a `=` [BinaryOperator.operatorCode] in the Fluent Node DSL - * and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, StatementHolder) infix fun Expression.assign(init: AssignExpression.() -> Expression): AssignExpression { val node = (this@LanguageFrontend).newAssignExpression("=") - node.lhs = listOf(this) + node.lhs = mutableListOf(this) init(node) // node.rhs = listOf(init(node)) @@ -1322,7 +1342,7 @@ infix fun Expression.assign(init: AssignExpression.() -> Expression): AssignExpr /** * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node - * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * DSL and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assign(rhs: Expression): AssignExpression { @@ -1337,7 +1357,7 @@ infix fun Expression.assign(rhs: Expression): AssignExpression { /** * Creates a new [AssignExpression] with a `+=` [AssignExpression.operatorCode] in the Fluent Node - * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * DSL and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assignPlus(rhs: Expression): AssignExpression { @@ -1352,7 +1372,7 @@ infix fun Expression.assignPlus(rhs: Expression): AssignExpression { /** * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node - * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * DSL and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assignAsExpr(rhs: Expression): AssignExpression { @@ -1364,7 +1384,7 @@ infix fun Expression.assignAsExpr(rhs: Expression): AssignExpression { } /** * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node - * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * DSL and adds it to the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) infix fun Expression.assignAsExpr(rhs: AssignExpression.() -> Unit): AssignExpression { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.kt index 60261034dc..62f86daefc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.kt @@ -26,9 +26,6 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.DeclarationHolder -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate -import org.neo4j.ogm.annotation.Relationship /** * This represents a sequence of one or more declaration(s). The purpose of this node is primarily @@ -37,30 +34,27 @@ import org.neo4j.ogm.annotation.Relationship * the translation unit. It should NOT end up in the final graph. */ class DeclarationSequence : Declaration(), DeclarationHolder { - @Relationship(value = "CHILDREN", direction = Relationship.Direction.OUTGOING) - val childEdges: MutableList> = mutableListOf() - - val children: List by PropertyEdgeDelegate(DeclarationSequence::childEdges) + val children = mutableListOf() override fun addDeclaration(declaration: Declaration) { if (declaration is DeclarationSequence) { for (declarationChild in declaration.children) { - addIfNotContains(childEdges, declarationChild) + addIfNotContains(children, declarationChild) } } else { - addIfNotContains(childEdges, declaration) + addIfNotContains(children, declaration) } } - fun asList(): List { + fun asMutableList(): MutableList { return children } val isSingle: Boolean - get() = childEdges.size == 1 + get() = children.size == 1 fun first(): Declaration { - return childEdges[0].end + return children.first() } operator fun plusAssign(declaration: Declaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt index 3b2b8c7848..4ef95234bc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt @@ -25,14 +25,17 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasInitializer +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import org.neo4j.ogm.annotation.Relationship /** * Represents a constant within an [EnumDeclaration]. Depending on the language, this might have an * explicit initializer value. */ class EnumConstantDeclaration : ValueDeclaration(), HasInitializer { - @AST override var initializer: Expression? = null + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + override var initializer by unwrapping(EnumConstantDeclaration::initializerEdge) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt index de9e48eff1..c0e912c32f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt @@ -25,18 +25,15 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship class EnumDeclaration : RecordDeclaration() { @Relationship(value = "ENTRIES", direction = Relationship.Direction.OUTGOING) - @AST - var entryEdges: MutableList> = ArrayList() - - var entries by PropertyEdgeDelegate(EnumDeclaration::entryEdges) + var entryEdges = astEdgesOf() + var entries by unwrapping(EnumDeclaration::entryEdges) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index a1bb9f7fa3..f20968b641 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -26,10 +26,10 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression @@ -40,25 +40,15 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { + @Relationship("BODY") var bodyEdge = astOptionalEdgeOf() /** The function body. Usually a [Block]. */ - @AST var body: Statement? = null - - /** - * Classes and Structs can be declared inside a function and are only valid within the function. - */ - @Relationship(value = "RECORDS", direction = Relationship.Direction.OUTGOING) - var recordEdges = mutableListOf>() + var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - @AST - var parameterEdges = mutableListOf>() - - /** Virtual property for accessing [parameterEdges] without property edges. */ - var parameters by PropertyEdgeDelegate(FunctionDeclaration::parameterEdges) - + var parameterEdges = astEdgesOf() /** Virtual property for accessing [parameterEdges] without property edges. */ - var records by PropertyEdgeDelegate(FunctionDeclaration::recordEdges) + var parameters by unwrapping(FunctionDeclaration::parameterEdges) @Relationship(value = "THROWS_TYPES", direction = Relationship.Direction.OUTGOING) var throwsTypes = mutableListOf() @@ -157,16 +147,6 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart val signatureTypes: List get() = parameters.map { it.type } - fun addParameter(parameterDeclaration: ParameterDeclaration) { - val propertyEdge = PropertyEdge(this, parameterDeclaration) - propertyEdge.addProperty(Properties.INDEX, parameters.size) - parameterEdges.add(propertyEdge) - } - - fun removeParameter(parameterDeclaration: ParameterDeclaration) { - parameterEdges.removeIf { it.end == parameterDeclaration } - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) @@ -197,10 +177,6 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart if (declaration is ParameterDeclaration) { addIfNotContains(parameterEdges, declaration) } - - if (declaration is RecordDeclaration) { - addIfNotContains(recordEdges, declaration) - } } override val declarations: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt index 3ebaa16e33..9b9bcdca32 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt @@ -25,11 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.* import org.neo4j.ogm.annotation.Relationship @@ -41,25 +39,12 @@ class FunctionTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the FunctionTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - @AST - private val realizationEdges: MutableList> = ArrayList() - - val realization: List by - PropertyEdgeDelegate(FunctionTemplateDeclaration::realizationEdges) + val realizationEdges = astEdgesOf() + val realization by unwrapping(FunctionTemplateDeclaration::realizationEdges) override val realizations: List get() = ArrayList(realization) - fun addRealization(realizedFunction: FunctionDeclaration) { - val propertyEdge = PropertyEdge(this, realizedFunction) - propertyEdge.addProperty(Properties.INDEX, realizationEdges.size) - realizationEdges.add(propertyEdge) - } - - fun removeRealization(realizedFunction: FunctionDeclaration?) { - realizationEdges.removeIf { it.end == realizedFunction } - } - override fun addDeclaration(declaration: Declaration) { if (declaration is TypeParameterDeclaration || declaration is ParameterDeclaration) { addIfNotContains(this.parameterEdges, declaration) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt index 9e2eb140bc..da8e3edb14 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt @@ -25,11 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -37,12 +35,12 @@ import org.neo4j.ogm.annotation.Relationship /** This declaration represents either an include or an import, depending on the language. */ class IncludeDeclaration : Declaration() { @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - @AST - private val includeEdges: MutableList> = ArrayList() + val includeEdges = astEdgesOf() + val includes by unwrapping(IncludeDeclaration::includeEdges) @Relationship(value = "PROBLEMS", direction = Relationship.Direction.OUTGOING) - @AST - private val problemEdges: MutableList> = ArrayList() + val problemEdges = astEdgesOf() + val problems by unwrapping(IncludeDeclaration::problemEdges) /** * This property refers to the file or directory or path. For example, in C this refers to an @@ -50,29 +48,6 @@ class IncludeDeclaration : Declaration() { */ var filename: String? = null - val includes: List by PropertyEdgeDelegate(IncludeDeclaration::includeEdges) - - val problems: List by PropertyEdgeDelegate(IncludeDeclaration::problemEdges) - - fun addInclude(includeDeclaration: IncludeDeclaration?) { - if (includeDeclaration == null) return - val propertyEdge = PropertyEdge(this, includeDeclaration) - propertyEdge.addProperty(Properties.INDEX, includeEdges.size) - includeEdges.add(propertyEdge) - } - - fun addProblems(c: Collection) { - for (problemDeclaration in c) { - addProblem(problemDeclaration) - } - } - - fun addProblem(problemDeclaration: ProblemDeclaration) { - val propertyEdge = PropertyEdge(this, problemDeclaration) - propertyEdge.addProperty(Properties.INDEX, problemEdges.size) - problemEdges.add(propertyEdge) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt index 11578253b8..422e69fcef 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt @@ -25,9 +25,11 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.passes.SymbolResolver +import org.neo4j.ogm.annotation.Relationship /** * A method declaration is a [FunctionDeclaration] that is part of to a specific [RecordDeclaration] @@ -41,6 +43,7 @@ open class MethodDeclaration : FunctionDeclaration() { */ open var recordDeclaration: RecordDeclaration? = null + @Relationship("RECEIVER") var receiverEdge = astOptionalEdgeOf() /** * The receiver variable of this method. In most cases, this variable is called `this`, but in * some languages, it is `self` (e.g. in Rust or Python) or can be freely named (e.g. in @@ -70,5 +73,5 @@ open class MethodDeclaration : FunctionDeclaration() { * share the same name. The [SymbolResolver] will recognize this and treat the scoping aspect of * the super-call accordingly. */ - @AST var receiver: VariableDeclaration? = null + var receiver by unwrapping(MethodDeclaration::receiverEdge) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index ecee983b6f..c2e9927eb5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -26,8 +26,8 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -48,12 +48,12 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, * Edges to nested namespaces, records, functions, fields etc. contained in the current * namespace. */ - @AST override val declarations: MutableList = ArrayList() + val declarationEdges = astEdgesOf() + override val declarations by unwrapping(NamespaceDeclaration::declarationEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - @AST - override var statementEdges: MutableList> = ArrayList() + override var statementEdges = astEdgesOf() /** * In some languages, there is a relationship between paths / directories and the package @@ -73,8 +73,7 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, addIfNotContains(declarations, declaration) } - override var statements: List by - PropertyEdgeDelegate(NamespaceDeclaration::statementEdges) + override var statements by unwrapping(NamespaceDeclaration::statementEdges) override val eogStarters: List get() { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/OperatorDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/OperatorDeclaration.kt new file mode 100644 index 0000000000..89cba6d9f5 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/OperatorDeclaration.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.declarations + +import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading +import de.fraunhofer.aisec.cpg.graph.HasOperatorCode +import de.fraunhofer.aisec.cpg.graph.types.ObjectType + +/** + * Some languages allow to either overload operators or to add custom operators to classes (see + * [HasOperatorOverloading]). In both cases, this special function class denotes that this handles + * this particular operator call. + * + * We need to derive from [MethodDeclaration] because all operators have a base class on which they + * operate on. Therefore, we need to associate them with an [ObjectType] and/or [RecordDeclaration]. + * There are some very special cases for C++, where we can have a global operator for a particular + * class. In this case we just pretend like it is a method operator. + */ +class OperatorDeclaration : MethodDeclaration(), HasOperatorCode { + /** The operator code which this operator declares. */ + override var operatorCode: String? = null + + val isPrefix: Boolean = false + val isPostfix: Boolean = false +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt index a7ca01e17b..cd820498d0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt @@ -25,8 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasDefault +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.* import org.neo4j.ogm.annotation.Relationship @@ -36,8 +37,8 @@ class ParameterDeclaration : ValueDeclaration(), HasDefault { var isVariadic = false @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - @AST - private var defaultValue: Expression? = null + var defaultValueEdge = astOptionalEdgeOf() + private var defaultValue by unwrapping(ParameterDeclaration::defaultValueEdge) var modifiers: List = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 31ead94be0..86a9ceef33 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -26,9 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.types.DeclaresType import de.fraunhofer.aisec.cpg.graph.types.ObjectType @@ -44,41 +44,29 @@ open class RecordDeclaration : var kind: String? = null @Relationship(value = "FIELDS", direction = Relationship.Direction.OUTGOING) - @AST - var fieldEdges: MutableList> = ArrayList() - - var fields by PropertyEdgeDelegate(RecordDeclaration::fieldEdges) + var fieldEdges = astEdgesOf() + var fields by unwrapping(RecordDeclaration::fieldEdges) @Relationship(value = "METHODS", direction = Relationship.Direction.OUTGOING) - @AST - var methodEdges: MutableList> = ArrayList() - - var methods by PropertyEdgeDelegate(RecordDeclaration::methodEdges) + var methodEdges = astEdgesOf() + var methods by unwrapping(RecordDeclaration::methodEdges) @Relationship(value = "CONSTRUCTORS", direction = Relationship.Direction.OUTGOING) - @AST - var constructorEdges: MutableList> = ArrayList() - - var constructors by PropertyEdgeDelegate(RecordDeclaration::constructorEdges) + var constructorEdges = astEdgesOf() + var constructors by unwrapping(RecordDeclaration::constructorEdges) @Relationship(value = "RECORDS", direction = Relationship.Direction.OUTGOING) - @AST - var recordEdges: MutableList> = ArrayList() - - var records by PropertyEdgeDelegate(RecordDeclaration::recordEdges) + var recordEdges = astEdgesOf() + var records by unwrapping(RecordDeclaration::recordEdges) @Relationship(value = "TEMPLATES", direction = Relationship.Direction.OUTGOING) - @AST - var templateEdges: MutableList> = ArrayList() - - var templates by PropertyEdgeDelegate(RecordDeclaration::templateEdges) + var templateEdges = astEdgesOf() + var templates by unwrapping(RecordDeclaration::templateEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - @AST - override var statementEdges: MutableList> = ArrayList() - - override var statements by PropertyEdgeDelegate(RecordDeclaration::statementEdges) + override var statementEdges = astEdgesOf() + override var statements by unwrapping(RecordDeclaration::statementEdges) @Transient var superClasses: MutableList = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt index 4a50f04d19..56e4b31645 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt @@ -25,13 +25,10 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.* -import kotlin.collections.ArrayList import org.neo4j.ogm.annotation.Relationship /** Node representing a declaration of a template class or struct */ @@ -42,21 +39,8 @@ class RecordTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the ClassTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - @AST - val realizationEdges: MutableList> = ArrayList() - - override val realizations: List by - PropertyEdgeDelegate(RecordTemplateDeclaration::realizationEdges) - - fun addRealization(realizedRecord: RecordDeclaration) { - val propertyEdge = PropertyEdge(this, realizedRecord) - propertyEdge.addProperty(Properties.INDEX, realizationEdges.size) - realizationEdges.add(propertyEdge) - } - - fun removeRealization(realizedRecordDeclaration: RecordDeclaration) { - realizationEdges.removeIf { it.end == realizedRecordDeclaration } - } + val realizationEdges = astEdgesOf() + override val realizations by unwrapping(RecordTemplateDeclaration::realizationEdges) override fun addDeclaration(declaration: Declaration) { if (declaration is TypeParameterDeclaration || declaration is ParameterDeclaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index 94481cf24b..2b53dc4efa 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -25,13 +25,11 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -53,10 +51,8 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { /** Parameters the Template requires for instantiation */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - @AST - var parameterEdges: MutableList> = ArrayList() - - val parameters by PropertyEdgeDelegate(TemplateDeclaration::parameterEdges) + var parameterEdges = astEdgesOf() + val parameters by unwrapping(TemplateDeclaration::parameterEdges) val parametersWithDefaults: List get() { @@ -85,18 +81,6 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { return defaults } - fun addParameter(parameterizedType: TypeParameterDeclaration) { - val propertyEdge = PropertyEdge(this, parameterizedType) - propertyEdge.addProperty(Properties.INDEX, parameterEdges.size) - parameterEdges.add(propertyEdge) - } - - fun addParameter(nonTypeTemplateParamDeclaration: ParameterDeclaration) { - val propertyEdge = PropertyEdge(this, nonTypeTemplateParamDeclaration) - propertyEdge.addProperty(Properties.INDEX, parameterEdges.size) - parameterEdges.add(propertyEdge) - } - override val declarations: List get() { val list = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index f916ce71e0..99b6f5ea16 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -26,10 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder @@ -40,35 +39,23 @@ class TranslationUnitDeclaration : Declaration(), DeclarationHolder, StatementHolder, EOGStarterHolder { /** A list of declarations within this unit. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - @AST - val declarationEdges: MutableList> = ArrayList() + val declarationEdges = astEdgesOf() + override val declarations by unwrapping(TranslationUnitDeclaration::declarationEdges) /** A list of includes within this unit. */ @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - @AST - val includeEdges: MutableList> = ArrayList() + val includeEdges = astEdgesOf() + val includes by unwrapping(TranslationUnitDeclaration::includeEdges) /** A list of namespaces within this unit. */ @Relationship(value = "NAMESPACES", direction = Relationship.Direction.OUTGOING) - @AST - val namespaceEdges: MutableList> = ArrayList() + val namespaceEdges = astEdgesOf() + val namespaces by unwrapping(TranslationUnitDeclaration::namespaceEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - @AST - override var statementEdges: MutableList> = ArrayList() - - override val declarations: List - get() = unwrap(declarationEdges) - - override var statements: List by - PropertyEdgeDelegate(TranslationUnitDeclaration::statementEdges) - - val includes: List - get() = unwrap(includeEdges) - - val namespaces: List - get() = unwrap(namespaceEdges) + override var statementEdges = astEdgesOf() + override var statements by unwrapping(TranslationUnitDeclaration::statementEdges) override fun addDeclaration(declaration: Declaration) { if (declaration is IncludeDeclaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt index 9d33d90930..acd7fe230f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt @@ -25,8 +25,9 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.newTupleDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.types.AutoType @@ -62,13 +63,12 @@ import de.fraunhofer.aisec.cpg.graph.types.TupleType */ class TupleDeclaration : VariableDeclaration() { /** The list of elements in this tuple. */ - @AST - var elements: List = mutableListOf() - set(value) { - field = value - // Make sure we inform our elements about our type changes - value.forEach { registerTypeObserver(it) } - } + var elementEdges = + astEdgesOf( + onAdd = { registerTypeObserver(it.end) }, + onRemove = { unregisterTypeObserver(it.end) } + ) + var elements by unwrapping(TupleDeclaration::elementEdges) override var name: Name get() = Name(elements.joinToString(",", "(", ")") { it.name.toString() }) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt index 75e9f05d11..2f061836a2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt @@ -25,18 +25,19 @@ */ package de.fraunhofer.aisec.cpg.graph.declarations -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasDefault +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* import org.neo4j.ogm.annotation.Relationship /** A declaration of a type template parameter */ class TypeParameterDeclaration : ValueDeclaration(), HasDefault { - /** TemplateParameters can define a default for the type parameter. */ @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - @AST - override var default: Type? = null + var defaultEdge = astOptionalEdgeOf() + /** TemplateParameters can define a default for the type parameter. */ + override var default by unwrapping(TypeParameterDeclaration::defaultEdge) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt index 8d70aaa83d..b87b0c382e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.kt @@ -28,14 +28,12 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap +import de.fraunhofer.aisec.cpg.graph.edges.flows.Usages +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.passes.SymbolResolver -import java.util.stream.Collectors import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -92,31 +90,10 @@ abstract class ValueDeclaration : Declaration(), HasType, HasAliases { */ @PopulatedByPass(SymbolResolver::class) @Relationship(value = "USAGE") - var usageEdges: MutableList> = ArrayList() + var usageEdges = Usages(this) /** All usages of the variable/field. */ - @PopulatedByPass(SymbolResolver::class) - var usages: List - get() = unwrap(usageEdges, true) - /** Set all usages of the variable/field and assembles the access properties. */ - set(usages) { - usageEdges = - usages - .stream() - .map { ref: Reference -> - val edge = PropertyEdge(this, ref) - edge.addProperty(Properties.ACCESS, ref.access) - edge - } - .collect(Collectors.toList()) - } - - /** Adds a usage of the variable/field and assembles the access property. */ - fun addUsage(reference: Reference) { - val usageEdge = PropertyEdge(this, reference) - usageEdge.addProperty(Properties.ACCESS, reference.access) - usageEdges.add(usageEdge) - } + @PopulatedByPass(SymbolResolver::class) var usages by unwrapping(ValueDeclaration::usageEdges) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE).appendSuper(super.toString()).toString() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index 964ec5abc3..83fecda3e0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -26,6 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression @@ -43,13 +46,10 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ /** * We need a way to store the templateParameters that a [VariableDeclaration] might have before * the [ConstructExpression] is created. - * - * Because templates are only used by a small subset of languages and variable declarations are - * used often, we intentionally make this a nullable list instead of an empty list. */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - @AST - var templateParameters: List? = null + var templateParameterEdges = astEdgesOf() + var templateParameters by unwrapping(VariableDeclaration::templateParameterEdges) /** Determines if this is a global variable. */ val isGlobal: Boolean @@ -66,17 +66,19 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ var isImplicitInitializerAllowed = false var isArray = false - /** The (optional) initializer of the declaration. */ - @AST - override var initializer: Expression? = null - set(value) { - field?.unregisterTypeObserver(this) - field = value - if (value is Reference) { - value.resolutionHelper = this + @Relationship("INITIALIZER") + var initializerEdge = + astOptionalEdgeOf( + onChanged = { old, new -> + val value = new?.end + exchangeTypeObserver(old, new) + if (value is Reference) { + value.resolutionHelper = this + } } - value?.registerTypeObserver(this) - } + ) + /** The (optional) initializer of the declaration. */ + override var initializer by unwrapping(VariableDeclaration::initializerEdge) fun getInitializerAs(clazz: Class): T? { return clazz.cast(initializer) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Properties.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Properties.kt deleted file mode 100644 index ed8f2063ae..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Properties.kt +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.graph.edge - -import java.util.* - -/** - * INDEX: (int) Indicates the position in a list of edges - * - * BRANCH: (boolean) If we have multiple EOG edges the branch property indicates which EOG edge - * leads to true branch (expression evaluated to true) or the false branch (e.g. with an if/else - * condition) - * - * DEFAULT: (boolean) Indicates which arguments edge of a CallExpression leads to a default argument - * - * NAME: (String) An optional name for the property edge - * - * [UNREACHABLE]:(boolean) True if the edge flows into unreachable code i.e. a branch condition - * which is always false. - * - * [DEPENDENCE]: ([DependenceType] Specifies the type of dependence the property edge might - * represent - */ -enum class Properties { - INDEX, - BRANCH, - NAME, - INSTANTIATION, - UNREACHABLE, - ACCESS, - DEPENDENCE, - DYNAMIC_INVOKE, - SENSITIVITY, - CALLING_CONTEXT_IN, - CALLING_CONTEXT_OUT, -} - -/** The types of dependencies that might be represented in the CPG */ -enum class DependenceType { - CONTROL, - DATA -} - -/** Sensitivity options (of DFG edges). */ -enum class SensitivitySpecifier { - FIELD, - CONTEXT; - - infix fun and(other: SensitivitySpecifier) = Sensitivities.of(this, other) -} - -typealias Sensitivities = EnumSet - -infix fun Sensitivities.allOf(other: Sensitivities) = this.containsAll(other) - -infix fun Sensitivities.and(other: SensitivitySpecifier) = - Sensitivities.of(other, *this.toTypedArray()) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdge.kt deleted file mode 100644 index f9fb9408f7..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdge.kt +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.graph.edge - -import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.graph.Persistable -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal -import java.lang.reflect.Field -import java.lang.reflect.InvocationTargetException -import java.lang.reflect.ParameterizedType -import java.util.* -import kotlin.reflect.KMutableProperty1 -import kotlin.reflect.KProperty -import kotlin.reflect.KProperty1 -import kotlin.reflect.jvm.isAccessible -import org.neo4j.ogm.annotation.* -import org.neo4j.ogm.annotation.typeconversion.Convert -import org.slf4j.LoggerFactory - -/** - * This class represents an edge between two [Node] objects in a Neo4J graph. It can be used to - * store additional information that relate to the relationship between the two nodes that belong to - * neither of the two nodes directly. - * - * An example would be the name (in this case `a`) of an argument between a [CallExpression] (`foo`) - * and its argument (a [Literal] of `2`) in languages that support keyword arguments, such as - * Python: - * ```python - * foo("bar", a = 2) - * ``` - */ -@RelationshipEntity -open class PropertyEdge : Persistable { - /** Required field for object graph mapping. It contains the node id. */ - @field:Id @field:GeneratedValue private val id: Long? = null - - // Node where the edge is outgoing - @field:StartNode var start: Node - - // Node where the edge is ingoing - @field:EndNode var end: T - - constructor(start: Node, end: T) { - this.start = start - this.end = end - properties = EnumMap(Properties::class.java) - } - - constructor(propertyEdge: PropertyEdge) { - start = propertyEdge.start - end = propertyEdge.end - properties = EnumMap(Properties::class.java) - properties.putAll(propertyEdge.properties) - } - - constructor(start: Node, end: T, properties: MutableMap) { - this.start = start - this.end = end - this.properties = properties - } - - open val label: String = "EDGE" - - /** Map containing all properties of an edge */ - @Convert(PropertyEdgeConverter::class) private var properties: MutableMap - - fun getProperty(property: Properties): Any? { - return properties.getOrDefault(property, null) - } - - /** - * Adds a property to a [PropertyEdge] If the object is not a built-in type you must provide a - * serializer and deserializer in the [PropertyEdgeConverter] - * - * @param property String containing the name of the property - * @param value Object containing the value of the property - */ - fun addProperty(property: Properties, value: Any?) { - properties[property] = value - } - - fun addProperties(propertyMap: Map?) { - propertyMap?.let { properties.putAll(it) } - } - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is PropertyEdge<*>) return false - - return (properties == other.properties && start == other.start && end == other.end) - } - - fun propertyEquals(obj: Any?): Boolean { - if (this === obj) return true - if (obj !is PropertyEdge<*>) return false - return properties == obj.properties - } - - /** - * Checks if the properties of the edge contain the given properties with the specified values. - */ - fun containsProperties(props: Map): Boolean { - return properties.entries.containsAll(props.entries) - } - - override fun hashCode(): Int { - return Objects.hash(end, properties) - } - - companion object { - protected val log = LoggerFactory.getLogger(PropertyEdge::class.java) - - fun ?> findPropertyEdgesByPredicate( - edges: Collection, - predicate: (S) -> Boolean - ): List { - return edges.filter(predicate) - } - - /** - * Add/Update index element of list of PropertyEdges - * - * @param propertyEdges propertyEdge list - * @return new PropertyEdge list with updated index property - */ - fun applyIndexProperty( - propertyEdges: List> - ): List> { - for ((counter, propertyEdge) in propertyEdges.withIndex()) { - propertyEdge.addProperty(Properties.INDEX, counter) - } - return propertyEdges - } - - /** - * Transforms a List of Nodes into targets of PropertyEdges. Include Index Property as Lists - * are indexed - * - * @param nodes List of nodes that should be transformed into PropertyEdges - * @param commonRelationshipNode node where all the Edges should start - * @return List of PropertyEdges with the targets of the nodes and index property. - */ - @JvmStatic - fun wrap( - nodes: List, - commonRelationshipNode: Node, - outgoing: Boolean = true - ): MutableList> { - val propertyEdges: MutableList> = ArrayList() - for (n in nodes) { - val propertyEdge = - if (outgoing) { - PropertyEdge(commonRelationshipNode, n) - } else { - PropertyEdge(n, commonRelationshipNode) - } - propertyEdge.addProperty(Properties.INDEX, propertyEdges.size) - propertyEdges.add(propertyEdge as PropertyEdge) - } - return propertyEdges - } - - /** - * Unwraps this property edge into a list of its target nodes. - * - * @param collection the collection of edges - * @param outgoing whether it is outgoing or not - * @param the type of the edges - * @return the list of target nodes - */ - @JvmStatic - @JvmOverloads - fun unwrap( - collection: List>, - outgoing: Boolean = true - ): List { - return collection.map { if (outgoing) it.end else it.start as T } - } - - /** - * @param collection is a collection that presumably holds property edges - * @param outgoing direction of the edges - * @return collection of nodes containing the targets of the edges - */ - private fun unwrapPropertyEdgeCollection( - collection: Collection<*>, - outgoing: Boolean - ): Any { - var element: Any? = null - val value = collection.stream().findAny() - if (value.isPresent) { - element = value.get() - } - if (element is PropertyEdge<*>) { - try { - val outputCollection = - collection.javaClass - .getDeclaredConstructor() - .newInstance() - .filterIsInstance() - .toMutableList() - for (obj in collection) { - if (obj is PropertyEdge<*>) { - if (outgoing) { - outputCollection.add(obj.end) - } else { - outputCollection.add(obj.start) - } - } - } - return outputCollection - } catch (e: InstantiationException) { - log.warn("PropertyEdges could not be unwrapped") - } catch (e: IllegalAccessException) { - log.warn("PropertyEdges could not be unwrapped") - } catch (e: InvocationTargetException) { - log.warn("PropertyEdges could not be unwrapped") - } catch (e: NoSuchMethodException) { - log.warn("PropertyEdges could not be unwrapped") - } - } - return collection - } - - /** - * @param obj PropertyEdge or collection of property edges that must be unwrapped - * @param outgoing direction of the edge - * @return node or collection representing target of edge - */ - @JvmStatic - fun unwrapPropertyEdge(obj: Any, outgoing: Boolean): Any { - if (obj is PropertyEdge<*>) { - return if (outgoing) { - obj.end - } else { - obj.start - } - } else if (obj is Collection<*> && obj.isNotEmpty()) { - return unwrapPropertyEdgeCollection(obj, outgoing) - } - return obj - } - - /** - * Checks if an Object is a PropertyEdge or a collection of PropertyEdges - * - * @param f Field containing the object - * @param obj object that is checked if it is a PropertyEdge - * @return true if obj is/contains a PropertyEdge - */ - @JvmStatic - fun checkForPropertyEdge(f: Field, obj: Any?): Boolean { - if (obj is PropertyEdge<*>) { - return true - } else if (obj is Collection<*>) { - val collectionTypes = - listOf(*(f.genericType as ParameterizedType).actualTypeArguments) - for (t in collectionTypes) { - if (t is ParameterizedType) { - return t.rawType == PropertyEdge::class.java - } else if (PropertyEdge::class.java == t) { - return true - } - } - } - return false - } - - fun checkForPropertyEdge(member: KProperty1, obj: Any?): Boolean { - if (obj is PropertyEdge<*>) { - return true - } else if (obj is Collection<*>) { - val returnType = member.returnType - return returnType.classifier == List::class && - returnType.arguments.any { it.type?.classifier == PropertyEdge::class } - } - return false - } - - @JvmStatic - fun removeElementFromList( - propertyEdges: List>, - element: T, - end: Boolean - ): List> { - val newPropertyEdges: MutableList> = ArrayList() - for (propertyEdge in propertyEdges) { - if (end && propertyEdge.end != element) { - newPropertyEdges.add(propertyEdge) - } - if (!end && propertyEdge.start != element) { - newPropertyEdges.add(propertyEdge) - } - } - return applyIndexProperty(newPropertyEdges) - } - - @JvmStatic - fun propertyEqualsList( - propertyEdges: List>?, - propertyEdges2: List>? - ): Boolean { - // Check, if the first edge is null - if (propertyEdges == null) { - // They can only be equal now, if the second one is also null - return propertyEdges2 == null - } - - // Otherwise, try to compare the contents of the lists with the propertyEquals (the - // second one still might be null) - if (propertyEdges.size == propertyEdges2?.size) { - for (i in propertyEdges.indices) { - if (!propertyEdges[i].propertyEquals(propertyEdges2[i])) { - return false - } - } - return true - } - return false - } - } -} - -/** - * This class can be used to implement - * [delegated properties](https://kotlinlang.org/docs/delegated-properties.html) in [Node] classes. - * The most common use case is to have a property that is a list of [PropertyEdge] objects (for - * persistence) and a second (delegated) property that allows easy access just to the connected - * nodes of the individual edges for in-memory access. - * - * For example: - * ```kotlin - * - * class MyNode { - * @Relationship(value = "EXPRESSIONS", direction = "OUTGOING") - * @field:SubGraph("AST") - * var expressionsEdges = mutableListOf>() - * var expressions by PropertyEdgeDelegate(MyNode::expressionsEdges) - * } - * ``` - * - * This class is intentionally marked with [Transient], so that the delegated properties are not - * transferred to the Neo4J OGM. Only the property that contains the property edges should be - * persisted in the graph database. - */ -@Transient -class PropertyEdgeDelegate( - val edge: KProperty1>>, - val outgoing: Boolean = true -) { - operator fun getValue(thisRef: S, property: KProperty<*>): List { - return PropertyEdge.unwrap(edge.get(thisRef), outgoing) - } - - operator fun setValue(thisRef: S, property: KProperty<*>, value: List) { - if (edge is KMutableProperty1) { - val callable = edge.setter - callable.isAccessible = true - edge.setter.call(thisRef, PropertyEdge.wrap(value, thisRef as Node, outgoing)) - } - } -} - -/** Similar to a [PropertyEdgeDelegate], but with a [Set] instead of [List]. */ -@Transient -class PropertyEdgeSetDelegate( - val edge: KProperty1>>, - val outgoing: Boolean = true -) { - operator fun getValue(thisRef: S, property: KProperty<*>): MutableSet { - return PropertyEdge.unwrap(edge.get(thisRef).toList(), outgoing).toMutableSet() - } - - operator fun setValue(thisRef: S, property: KProperty<*>, value: MutableSet) { - if (edge is KMutableProperty1) { - val callable = edge.setter - callable.isAccessible = true - edge.setter.call(thisRef, PropertyEdge.wrap(value.toList(), thisRef as Node, outgoing)) - } - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverter.kt deleted file mode 100644 index cd66858896..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverter.kt +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.graph.edge - -import java.util.* -import java.util.function.Function -import org.neo4j.ogm.typeconversion.CompositeAttributeConverter - -class PropertyEdgeConverter : CompositeAttributeConverter> { - /** - * For every PropertyValue that is not a supported type, a serializer and a deserializer must be - * provided Supported Types: - * - * PRIMITIVES = - * char,byte,short,int,long,float,double,boolean,char[],byte[],short[],int[],long[],float[],double[],boolean[] - * AUTOBOXERS = java.lang.Object, java.lang.Character, java.lang.Byte, java.lang.Short, - * java.lang.Integer, java.lang.Long, java.lang.Float, java.lang.Double, java.lang.Boolean, - * java.lang.String, java.lang.Object[], java.lang.Character[], java.lang.Byte[], - * java.lang.Short[], java.lang.Integer[], java.lang.Long[], java.lang.Float[], - * java.lang.Double[], java.lang.Boolean[], java.lang.String[] - */ - // Maps a class to a function that serialized the object from the given class - private val serializer: Map> - - // Maps a string (key of the property) to a function that deserializes the property - private val deserializer: Map> - - init { - serializer = PropertyEdgeConverterManager.instance.serializer - deserializer = PropertyEdgeConverterManager.instance.deserializer - } - - override fun toGraphProperties(value: Map): Map { - val result: MutableMap = HashMap() - for ((key, propertyValue) in value) { - if (propertyValue != null && serializer.containsKey(propertyValue.javaClass.name)) { - result[key.name] = serializer[propertyValue.javaClass.name]?.apply(propertyValue) - } else { - result[key.name] = propertyValue - } - } - return result - } - - override fun toEntityAttribute(value: Map): Map { - val result: MutableMap = EnumMap(Properties::class.java) - for (prop in Properties.entries) { - if (deserializer.containsKey(prop.name)) { - val deserializedProperty = - value[prop.name]?.let { deserializer[prop.name]?.apply(it) } - result[prop] = deserializedProperty - } else { - result[prop] = value[prop.name] - } - } - return result - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt deleted file mode 100644 index 6a044bc58a..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/PropertyEdgeConverterManager.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.graph.edge - -import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import java.util.function.Function - -/** - * Since Neo4J uses the PropertyEdgeConverter (as it implements the CompositeAttributeConverter - * interface), we cannot define it as a singleton, as it requires to have a constructor. We want to - * be able to dynamically define converters for PropertyEdges that have more complex structures such - * as enums or custom classes, we need a singleton to be able to add the converters. Refer to the - * documentation of [PropertyEdgeConverter] to see which primitives are supported by default, and - * which require a custom converter. - */ -class PropertyEdgeConverterManager private constructor() { - // Maps a class to a function that serialized the object from the given class - val serializer: MutableMap> = HashMap() - - // Maps a string (key of the property) to a function that deserializes the property - val deserializer: MutableMap> = HashMap() - - init { - // Add here converters for PropertyEdges - addSerializer(TemplateInitialization::class.java.name) { obj: Any -> obj.toString() } - addDeserializer("INSTANTIATION") { s: Any? -> - if (s != null) TemplateInitialization.valueOf(s.toString()) else null - } - addSerializer(CallExpression::class.java.name) { it.toString() } - addDeserializer("CALLING_CONTEXT_IN") { null } // TODO: Not supported yet - addDeserializer("CALLING_CONTEXT_OUT") { null } // TODO: Not supported yet - } - - fun addSerializer(clazz: String, func: Function) { - serializer[clazz] = func - } - - fun addDeserializer(name: String, func: Function) { - deserializer[name] = func - } - - companion object { - val instance = PropertyEdgeConverterManager() - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt new file mode 100644 index 0000000000..1bb365dd63 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges + +import com.fasterxml.jackson.annotation.JsonBackReference +import com.fasterxml.jackson.annotation.JsonIgnore +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.passes.ProgramDependenceGraphPass +import java.util.* +import kotlin.reflect.KProperty +import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.* + +/** + * This class represents an edge between two [Node] objects in a Neo4J graph. It can be used to + * store additional information that relate to the relationship between the two nodes that belong to + * neither of the two nodes directly. + * + * An example would be the name (in this case `a`) of an argument between a [CallExpression] (`foo`) + * and its argument (a [Literal] of `2`) in languages that support keyword arguments, such as + * Python: + * ```python + * foo("bar", a = 2) + * ``` + */ +@RelationshipEntity +abstract class Edge : Persistable, Cloneable { + /** Required field for object graph mapping. It contains the node id. */ + @field:Id @field:GeneratedValue private val id: Long? = null + + // Node where the edge is outgoing + @JsonIgnore @field:StartNode var start: Node + + // Node where the edge is ingoing + @JsonBackReference @field:EndNode var end: NodeType + + constructor(start: Node, end: NodeType) { + this.start = start + this.end = end + } + + constructor(edge: Edge) { + start = edge.start + end = edge.end + } + + @Transient open val label: String = "EDGE" + + /** + * The index of this node, if it is stored in an + * [de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList]. + */ + var index: Int? = null + + /** An optional name. */ + var name: String? = null + + /** + * The type of dependence (e.g. control or data or none). This field is intentionally nullable, + * because not all [Edge] edges are selected in the PDG. This selection is performed in the + * [ProgramDependenceGraphPass]. + */ + var dependence: DependenceType? = null + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Edge<*>) return false + + return start == other.start && + end == other.end && + index == other.index && + name == other.name && + dependence == other.dependence + } + + fun propertyEquals(obj: Any?): Boolean { + if (this === obj) return true + if (obj !is Edge<*>) return false + return true + } + + override fun hashCode(): Int { + return Objects.hash(start, end, index, name, dependence) + } + + override fun toString(): String { + return ToStringBuilder(this, TO_STRING_STYLE) + .appendSuper(super.toString()) + .append("end", end) + .toString() + } + + public override fun clone(): Edge { + // needs to be implemented by sub-classes + return super.clone() as Edge + } + + companion object { + @JvmStatic + fun propertyEqualsList(edges: List>?, edges2: List>?): Boolean { + // Check, if the first edge is null + if (edges == null) { + // They can only be equal now, if the second one is also null + return edges2 == null + } + + // Otherwise, try to compare the contents of the lists with the propertyEquals (the + // second one still might be null) + if (edges.size == edges2?.size) { + for (i in edges.indices) { + if (!edges[i].propertyEquals(edges2[i])) { + return false + } + } + return true + } + return false + } + } + + fun delegate(): Delegate { + return Delegate() + } + + @Transient + inner class Delegate< + ThisType : Node, + >() { + operator fun getValue(thisRef: ThisType, property: KProperty<*>): NodeType { + var edge = this@Edge + // We only support outgoing edges this way + return edge.end + } + + operator fun setValue(thisRef: ThisType, property: KProperty<*>, value: NodeType) { + this@Edge.end = value + // TODO: trigger some update hook + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt new file mode 100644 index 0000000000..8aa453c0ee --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflow +import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.helpers.identitySetOf + +/** + * Returns all [Edge]s of [EdgeType] directly attached to this [Node]. Optionally, a [predicate] can + * be used to filter the edges even further. + */ +inline fun > Node.edges( + noinline predicate: ((EdgeType) -> Boolean) = { true } +): Collection { + val edges = mutableSetOf() + val fields = SubgraphWalker.getAllEdgeFields(this::class.java) + for (field in fields) { + var obj = + synchronized(field) { + // Disable access mechanisms + field.isAccessible = true + val obj = field[this] + + // Restore old state + field.isAccessible = false + obj + } ?: continue + + // Gather all edges + if (obj is EdgeCollection<*, *>) { + for (edge in obj.toList()) { + if (edge is EdgeType && predicate.invoke(edge)) { + edges += edge + } + } + } + } + + return edges +} + +/** + * This function returns a subgraph containing all [Edge]s starting from this [Node] that are of the + * specific [EdgeType]. Optionally, a [predicate] can be used to filter the edges even further. + */ +inline fun > Node.allEdges( + noinline predicate: ((EdgeType) -> Boolean) = { true } +): Collection { + val alreadySeen = identitySetOf() + val worklist = mutableListOf() + val edges = mutableSetOf() + + worklist += this + alreadySeen += this + + while (worklist.isNotEmpty()) { + val node = worklist.removeFirst() + val toAdd = node.edges(predicate) + + val newStart = toAdd.map { it.start }.filter { it !in alreadySeen } + worklist += newStart + alreadySeen += newStart + + val newEnd = toAdd.map { it.end }.filter { it !in alreadySeen } + worklist += newEnd + alreadySeen += newEnd + + edges += toAdd + } + + return edges +} + +/** A shortcut to return all [AstEdge] edges starting from this node. */ +val Node.astEdges: Collection> + get() { + return allEdges() + } + +/** A shortcut to return all [Dataflow] edges starting from this node. */ +val Node.dataflows: Collection + get() { + return allEdges() + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt new file mode 100644 index 0000000000..e880d20c8c --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Extensions.kt @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSet +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSingletonList +import de.fraunhofer.aisec.cpg.graph.edges.collections.UnwrappedEdgeList +import de.fraunhofer.aisec.cpg.graph.edges.collections.UnwrappedEdgeSet +import kotlin.reflect.KProperty1 +import kotlin.reflect.jvm.isAccessible + +fun > MutableList.add( + target: Node, + builder: EdgeType.() -> Unit +): Boolean { + if (this is UnwrappedEdgeList<*, *>) { + @Suppress("UNCHECKED_CAST") + return (this as UnwrappedEdgeList).add(target, builder) + } + + throw UnsupportedOperationException() +} + +/** See [UnwrappedEdgeList.Delegate]. */ +fun > NodeType.unwrapping( + edgeProperty: KProperty1>, +): UnwrappedEdgeList { + // Create an unwrapped container out of the edge property... + edgeProperty.isAccessible = true + val edge = edgeProperty.call(this) + return edge.unwrap() +} + +/** See [UnwrappedEdgeSet.Delegate]. */ +fun > NodeType.unwrapping( + edgeProperty: KProperty1>, +): UnwrappedEdgeSet { + // Create an unwrapped container out of the edge property... + edgeProperty.isAccessible = true + val edge = edgeProperty.call(this) + return edge.unwrap() +} + +/** See [EdgeSingletonList.UnwrapDelegate]. */ +fun < + PropertyType : Node, + NullablePropertyType : PropertyType?, + NodeType : Node, + EdgeType : Edge +> NodeType.unwrapping( + edgeProperty: + KProperty1>, +): EdgeSingletonList.UnwrapDelegate { + edgeProperty.isAccessible = true + val edge = edgeProperty.call(this) + return edge.delegate() +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt new file mode 100644 index 0000000000..adec108640 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.ast + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSingletonList +import org.neo4j.ogm.annotation.* + +/** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */ +@RelationshipEntity +open class AstEdge(start: Node, end: T) : Edge(start, end) { + init { + end.astParent = start + } +} + +/** Creates an [AstEdges] container starting from this node. */ +fun Node.astEdgesOf( + onAdd: ((AstEdge) -> Unit)? = null, + onRemove: ((AstEdge) -> Unit)? = null, +): AstEdges> { + return AstEdges(thisRef = this, onAdd = onAdd, onRemove = onRemove) +} + +/** + * Creates an single optional [AstEdge] starting from this node (wrapped in a [EdgeSingletonList] + * container). + */ +fun Node.astOptionalEdgeOf( + onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, +): EdgeSingletonList> { + return EdgeSingletonList( + thisRef = this, + init = ::AstEdge, + outgoing = true, + onChanged = onChanged, + of = null + ) +} + +/** + * Creates an single [AstEdge] starting from this node (wrapped in a [EdgeSingletonList] container). + */ +fun Node.astEdgeOf( + of: NodeType, + onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, +): EdgeSingletonList> { + return EdgeSingletonList( + thisRef = this, + init = ::AstEdge, + outgoing = true, + onChanged = onChanged, + of = of + ) +} + +/** This property edge list describes elements that are AST children of a node. */ +open class AstEdges>( + thisRef: Node, + onAdd: ((PropertyEdgeType) -> Unit)? = null, + onRemove: ((PropertyEdgeType) -> Unit)? = null, + @Suppress("UNCHECKED_CAST") + init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end -> + AstEdge(start, end) as PropertyEdgeType + } +) : + EdgeList( + thisRef = thisRef, + init = init, + onAdd = onAdd, + onRemove = onRemove, + ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt similarity index 55% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index 68005bbbc9..b67f7711ea 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,13 +23,19 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.graph.edges.ast -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -/** A simple interface to denote that the implementing class has some kind of [operatorCode]. */ -interface HasOperatorCode { +/** This edge represents a template argument that is attached to a [CallExpression]. */ +class TemplateArgument( + start: Node, + end: NodeType, + var instantiation: TemplateInitialization? = TemplateInitialization.EXPLICIT, +) : AstEdge(start, end) - /** The operator code, identifying an operation executed on one or more [Expression]s */ - val operatorCode: String? -} +/** A container for [TemplateArgument] edges. */ +class TemplateArguments(thisRef: Node) : + AstEdges>(thisRef, init = ::TemplateArgument) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeCollection.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeCollection.kt new file mode 100644 index 0000000000..bf85360f05 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeCollection.kt @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.types.HasType.TypeObserver + +/** + * This interfaces is an extension of [MutableCollection] that holds specific functions for the + * collection of [Edge] edges. + */ +interface EdgeCollection< + NodeType : Node, + EdgeType : Edge, +> : MutableCollection { + var thisRef: Node + var init: (start: Node, end: NodeType) -> EdgeType + var outgoing: Boolean + var onAdd: ((EdgeType) -> Unit)? + var onRemove: ((EdgeType) -> Unit)? + + /** + * Removes all edges with the target node. The target is considered to be either the [Edge.end] + * or [Edge.start] depending on [outgoing]. + */ + fun remove(target: NodeType): Boolean { + val toRemove = + this.filter { + if (outgoing) { + it.end == target + } else { + it.start == target + } + } + return this.removeAll(toRemove.toSet()) + } + + fun addAll(targets: Collection, builder: (EdgeType.() -> Unit)? = null): Boolean { + val edges = + targets.map { + val edge = + if (outgoing) { + init(thisRef, it) + } else { + @Suppress("UNCHECKED_CAST") init(it, thisRef as NodeType) + } + // Apply builder + if (builder != null) { + builder(edge) + } + edge + } + + return addAll(edges) + } + + /** + * Creates a new edge with the target node and an optional [builder] to include edge properties. + * If [outgoing] is true, the edge is created from [thisRef] -> [target], otherwise from + * [target] to [thisRef]. + */ + fun add( + target: NodeType, + init: ((Node, NodeType) -> EdgeType) = this.init, + builder: (EdgeType.() -> Unit)? = null + ): Boolean { + val edge = createEdge(target, init, this.outgoing, builder) + + // Add it + return this.add(edge) + } + + fun > createEdge( + target: NodeType, + init: ((Node, NodeType) -> PropertyEdgeType), + outgoing: Boolean = true, + builder: (PropertyEdgeType.() -> Unit)? = null, + ): PropertyEdgeType { + val edge = + if (outgoing) { + init(thisRef, target) + } else { + @Suppress("UNCHECKED_CAST") init(target, thisRef as NodeType) + } + + // Apply builder + if (builder != null) { + builder(edge) + } + + return edge + } + + fun contains(target: NodeType): Boolean { + return any { if (outgoing) it.end == target else it.start == target } + } + + operator fun plusAssign(end: NodeType) { + add(end) + } + + /** Clears the collection and adds the [nodes]. */ + fun resetTo(nodes: Collection) { + clear() + for (n in nodes) { + this += n + } + } + + /** + * Converts this collection of edges into a collection of nodes for easier access to the + * "target" nodes. + * + * Note, that is an immutable list and only a snapshot. If you want a magic container that is in + * sync with this [EdgeCollection], please use [unwrap]. + */ + fun toNodeCollection(predicate: ((EdgeType) -> Boolean)? = null): Collection + + /** + * Returns an [UnwrappedEdgeCollection] magic container which holds a structure that provides + * easy access to the "target" nodes without edge information, but is mutable and in-sync with + * this collection. + */ + fun unwrap(): UnwrappedEdgeCollection + + /** + * This function will be executed after the edge was added to the container. This can be used to + * propagate the edge to other properties or register additional handlers, e.g. a + * [TypeObserver]. + */ + fun handleOnAdd(edge: EdgeType) { + onAdd?.invoke(edge) + } + + /** + * This function will be executed after an edge was removed from the container. This can be used + * to unregister additional handlers, e.g. a [TypeObserver]. + */ + fun handleOnRemove(edge: EdgeType) { + onRemove?.invoke(edge) + } +} + +/** A helper function for [EdgeCollection.toNodeCollection]. */ +internal fun < + NodeType : Node, + EdgeType : Edge, + CollectionType : MutableCollection +> internalToNodeCollection( + edges: EdgeCollection, + outgoing: Boolean = true, + predicate: ((EdgeType) -> Boolean)? = null, + createCollection: () -> CollectionType +): CollectionType { + val unwrapped = createCollection() + for (edge in edges) { + if (predicate != null && !predicate(edge)) { + continue + } + + @Suppress("UNCHECKED_CAST") + unwrapped += if (outgoing) edge.end else edge.start as NodeType + } + + return unwrapped +} + +/** + * This is a special hashcode implementation that takes into account whether this edge container is + * containing incoming or outgoing edges. It builds the hash-code based on the direction of the + * edges. The reason for this is to avoid loops in the hash-code implementation. + */ +internal fun internalHashcode( + edges: EdgeCollection>, + outgoing: Boolean = true, +): Int { + var hashCode = 0 + + for (edge in edges) { + hashCode = + 31 * hashCode + + if (outgoing) { + edge.end.hashCode() + } else { + edge.start.hashCode() + } + } + + return hashCode +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt new file mode 100644 index 0000000000..f4384112ca --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import java.util.function.Predicate + +/** This class extends a list of edges. This allows us to use lists of edges more conveniently. */ +abstract class EdgeList>( + override var thisRef: Node, + override var init: (start: Node, end: NodeType) -> EdgeType, + override var outgoing: Boolean = true, + override var onAdd: ((EdgeType) -> Unit)? = null, + override var onRemove: ((EdgeType) -> Unit)? = null +) : ArrayList(), EdgeCollection { + + override fun add(element: EdgeType): Boolean { + // Make sure, the index is always set + if (element.index == null) { + element.index = this.size + } + + val ok = super.add(element) + if (ok) { + handleOnAdd(element) + } + return ok + } + + override fun remove(element: EdgeType): Boolean { + val ok = super.remove(element) + if (ok) { + handleOnRemove(element) + } + return ok + } + + override fun removeAll(elements: Collection): Boolean { + val ok = super.removeAll(elements.toSet()) + if (ok) { + elements.forEach { handleOnRemove(it) } + } + return ok + } + + override fun removeAt(index: Int): EdgeType { + val edge = super.removeAt(index) + handleOnRemove(edge) + return edge + } + + override fun removeIf(predicate: Predicate): Boolean { + val edges = filter { predicate.test(it) } + val ok = super.removeIf(predicate) + if (ok) { + edges.forEach { handleOnRemove(it) } + } + return ok + } + + /** Replaces the first occurrence of an edge with [old] with a new edge to [new]. */ + fun replace(old: NodeType, new: NodeType): Boolean { + val idx = this.indexOfFirst { it.end == old } + if (idx != -1) { + this[idx] = init(thisRef, new) + return true + } + + return false + } + + override fun clear() { + // Make a copy of our edges so we can pass a copy to our on-remove handler + val edges = this.toList() + super.clear() + edges.forEach { handleOnRemove(it) } + } + + /** + * This function creates a new edge (of [EdgeType]) to/from the specified node [target] + * (depending on [outgoing]) and adds it to the specified index in the list. + */ + fun add(index: Int, target: NodeType) { + val edge = createEdge(target, init, this.outgoing) + + return add(index, edge) + } + + override fun add(index: Int, element: EdgeType) { + // Make sure, the index is always set + element.index = this.size + + super.add(index, element) + + handleOnAdd(element) + + // We need to re-compute all edges with an index > inserted index + for (i in index until this.size) { + this[i].index = i + } + } + + override fun toNodeCollection(predicate: ((EdgeType) -> Boolean)?): List { + return internalToNodeCollection(this, outgoing, predicate, ::ArrayList) + } + + /** + * Returns an [UnwrappedEdgeList] magic container which holds a structure that provides easy + * access to the "target" nodes without edge information. + */ + override fun unwrap(): UnwrappedEdgeList { + return UnwrappedEdgeList(this) + } + + override fun equals(other: Any?): Boolean { + if (other !is EdgeList<*, *>) return false + + // Otherwise, try to compare the contents of the lists with the propertyEquals method + if (this.size == other.size) { + for (i in this.indices) { + if (this[i] != other[i]) { + return false + } + } + return true + } + + return false + } + + override fun hashCode(): Int { + return internalHashcode(this, outgoing) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSet.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSet.kt new file mode 100644 index 0000000000..d1ca7cb372 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSet.kt @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import java.util.function.Predicate + +/** + * This class extends a list of property edges. This allows us to use list of property edges more + * conveniently. + */ +abstract class EdgeSet>( + override var thisRef: Node, + override var init: (start: Node, end: NodeType) -> EdgeType, + override var outgoing: Boolean = true, + override var onAdd: ((EdgeType) -> Unit)? = null, + override var onRemove: ((EdgeType) -> Unit)? = null +) : HashSet(), EdgeCollection { + override fun add(element: EdgeType): Boolean { + val ok = super.add(element) + if (ok) { + handleOnAdd(element) + } + return ok + } + + override fun removeIf(predicate: Predicate): Boolean { + val edges = filter { predicate.test(it) } + val ok = super.removeIf(predicate) + if (ok) { + edges.forEach { handleOnRemove(it) } + } + return ok + } + + override fun removeAll(elements: Collection): Boolean { + val ok = super.removeAll(elements.toSet()) + if (ok) { + elements.forEach { handleOnRemove(it) } + } + return ok + } + + override fun remove(element: EdgeType): Boolean { + val ok = super.remove(element) + if (ok) { + handleOnRemove(element) + } + return ok + } + + override fun clear() { + // Make a copy of our edges so we can pass a copy to our on-remove handler + val edges = this.toSet() + super.clear() + edges.forEach { handleOnRemove(it) } + } + + override fun toNodeCollection(predicate: ((EdgeType) -> Boolean)?): MutableSet { + return internalToNodeCollection(this, outgoing, predicate, ::HashSet) + } + + /** + * Returns an [UnwrappedEdgeSet] magic container which holds a structure that provides easy + * access to the "target" nodes without edge information. + */ + override fun unwrap(): UnwrappedEdgeSet { + return UnwrappedEdgeSet(this) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is EdgeSet<*, *>) return false + + // Otherwise, try to compare the contents of the lists with the propertyEquals method + return this.containsAll(other) + } + + override fun hashCode(): Int { + return internalHashcode(this, outgoing) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt new file mode 100644 index 0000000000..2b8bab56b2 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.Transient + +/** + * This is a MAJOR workaround since Neo4J OGM does not allow to use our (generic) [Edge] class for + * our AST edges. See https://github.com/neo4j/neo4j-ogm/issues/1132. + * + * Therefore, we need to wrap the edge in a list with a single element. + */ +class EdgeSingletonList>( + override var thisRef: Node, + override var init: (Node, NodeType) -> EdgeType, + var onChanged: ((old: EdgeType?, new: EdgeType?) -> Unit)? = null, + override var outgoing: Boolean, + of: NullableNodeType, +) : EdgeCollection { + + var element: EdgeType? = + if (of == null) { + null + } else { + if (outgoing) { + init(thisRef, of) + } else { + @Suppress("UNCHECKED_CAST") init(of, thisRef as NodeType) + } + } + + override val size: Int + get() = if (element == null) 0 else 1 + + override fun contains(element: EdgeType): Boolean { + return this.element == element + } + + override fun containsAll(elements: Collection): Boolean { + return elements.size == 1 && this.element == elements.firstOrNull() + } + + override fun isEmpty(): Boolean { + return this.element == null + } + + override fun add(element: EdgeType): Boolean { + throw UnsupportedOperationException() + } + + override fun addAll(elements: Collection): Boolean { + throw UnsupportedOperationException() + } + + override fun clear() { + throw UnsupportedOperationException() + } + + override fun iterator(): MutableIterator { + return Iterator(element) + } + + override fun remove(element: EdgeType): Boolean { + throw UnsupportedOperationException() + } + + override fun removeAll(elements: Collection): Boolean { + throw UnsupportedOperationException() + } + + override fun retainAll(elements: Collection): Boolean { + throw UnsupportedOperationException() + } + + override var onAdd: ((EdgeType) -> Unit)? + get() = null + set(_) {} + + override var onRemove: ((EdgeType) -> Unit)? + get() = null + set(_) {} + + override fun toNodeCollection(predicate: ((EdgeType) -> Boolean)?): Collection { + val elements = predicate?.let { toList().filter(it) } ?: toList() + return elements.map { + if (outgoing) { + it.end + } else { + @Suppress("UNCHECKED_CAST") + it.start as NodeType + } + } + } + + override fun unwrap(): UnwrappedEdgeCollection { + TODO("Not yet implemented") + } + + inner class Iterator(val element: EdgeType?) : MutableIterator { + var hasNext = isNotEmpty() + + override fun remove() { + throw UnsupportedOperationException() + } + + override fun hasNext(): Boolean { + return hasNext + } + + override fun next(): EdgeType { + if (hasNext && element != null) { + hasNext = false + return element + } + throw NoSuchElementException() + } + } + + fun resetTo(node: NodeType) { + val old = this.element + this.element = + if (outgoing) { + init(thisRef, node) + } else { + @Suppress("UNCHECKED_CAST") init(node, thisRef as NodeType) + } + onChanged?.invoke(old, this.element) + } + + fun delegate(): UnwrapDelegate { + return UnwrapDelegate() + } + + @Transient + inner class UnwrapDelegate< + ThisType : Node, + >() { + @Suppress("UNCHECKED_CAST") + operator fun getValue(thisRef: ThisType, property: KProperty<*>): NullableNodeType { + return (if (outgoing) { + this@EdgeSingletonList.element?.end + } else { + this@EdgeSingletonList.element?.start as NodeType + }) + as NullableNodeType + } + + operator fun setValue(thisRef: ThisType, property: KProperty<*>, value: NullableNodeType) { + if (value != null) { + this@EdgeSingletonList.resetTo(value) + } + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/MirroredEdgeCollection.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/MirroredEdgeCollection.kt new file mode 100644 index 0000000000..e817aa7bab --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/MirroredEdgeCollection.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import kotlin.reflect.KProperty + +/** + * This interface can be used for edge collections, which "mirror" its content to another property. + * This can be used to automatically populate next/prev flow edges. + */ +interface MirroredEdgeCollection> : + EdgeCollection { + var mirrorProperty: KProperty> + + override fun handleOnRemove(edge: PropertyEdgeType) { + // Handle our mirror property. + if (outgoing) { + var prevOfNext = mirrorProperty.call(edge.end) + if (edge in prevOfNext) { + prevOfNext.remove(edge) + } + } else { + var nextOfPrev = mirrorProperty.call(edge.start) + if (edge in nextOfPrev) { + nextOfPrev.remove(edge) + } + } + + // Execute any remaining pre actions + super.handleOnRemove(edge) + } + + /** + * Adds this particular edge to its [mirrorProperty]. We need the information if this is an + * [outgoing] or incoming edge collection. + */ + override fun handleOnAdd(edge: PropertyEdgeType) { + // Handle our mirror property. We add some extra "in" checks here, otherwise things will + // loop. + if (outgoing) { + var prevOfNext = mirrorProperty.call(edge.end) + if (edge !in prevOfNext) { + prevOfNext.add(edge) + } + } else { + var nextOfPrev = mirrorProperty.call(edge.start) + if (edge !in nextOfPrev) { + nextOfPrev.add(edge) + } + } + + // Execute any remaining post actions + super.handleOnAdd(edge) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeCollection.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeCollection.kt new file mode 100644 index 0000000000..851aa369d9 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeCollection.kt @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge + +/** + * An intelligent [MutableCollection] wrapper around an [EdgeCollection] which supports iterating, + * adding and removing [Node] elements. Basis for [UnwrappedEdgeList] and [UnwrappedEdgeSet]. + */ +abstract class UnwrappedEdgeCollection>( + var collection: EdgeCollection +) : MutableCollection { + + fun add(element: NodeType, builder: (EdgeType.() -> Unit)? = null): Boolean { + return collection.add(element, builder = builder) + } + + override fun add(element: NodeType): Boolean { + return collection.add(element) + } + + override fun addAll(elements: Collection): Boolean { + return collection.addAll(elements) + } + + override fun clear() { + return collection.clear() + } + + override fun iterator(): MutableIterator { + return Iterator(collection.iterator()) + } + + override fun remove(element: NodeType): Boolean { + return collection.remove(element) + } + + override fun removeAll(elements: Collection): Boolean { + TODO("Not yet implemented") + } + + override fun retainAll(elements: Collection): Boolean { + TODO("Not yet implemented") + } + + override val size: Int + get() = collection.size + + override fun contains(element: NodeType): Boolean { + return collection.contains(element) + } + + override fun containsAll(elements: Collection): Boolean { + TODO("Not yet implemented") + } + + override fun isEmpty(): Boolean { + return collection.isEmpty() + } + + fun resetTo(c: Collection) { + return collection.resetTo(c) + } + + inner class Iterator( + var edgeIterator: MutableIterator>, + ) : MutableIterator { + override fun remove() { + return edgeIterator.remove() + } + + override fun hasNext(): Boolean { + return edgeIterator.hasNext() + } + + @Suppress("UNCHECKED_CAST") + override fun next(): NodeType { + var next = edgeIterator.next() + return if (collection.outgoing) { + next.end + } else { + next.start as NodeType + } + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt new file mode 100644 index 0000000000..e46f14c9b7 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.Transient + +/** + * An intelligent [MutableList] wrapper around an [EdgeList] which supports iterating, adding and + * removing [Node] elements. + */ +class UnwrappedEdgeList>( + var list: EdgeList +) : UnwrappedEdgeCollection(list), MutableList { + + override fun add(index: Int, element: NodeType) { + return list.add(index, element) + } + + override fun addAll(index: Int, elements: Collection): Boolean { + TODO("Not yet implemented") + } + + override fun listIterator(): MutableListIterator { + return ListIterator(list.listIterator()) + } + + override fun listIterator(index: Int): MutableListIterator { + return ListIterator(list.listIterator(index)) + } + + override fun removeAt(index: Int): NodeType { + var edge = list.removeAt(index) + return if (list.outgoing) { + edge.end + } else { + @Suppress("UNCHECKED_CAST") + edge.start as NodeType + } + } + + override fun set(index: Int, element: NodeType): NodeType { + TODO("Not yet implemented") + } + + override fun subList(fromIndex: Int, toIndex: Int): MutableList { + TODO("Not yet implemented") + } + + override fun get(index: Int): NodeType { + var edge = list[index] + return if (list.outgoing) { + edge.end + } else { + @Suppress("UNCHECKED_CAST") + edge.start as NodeType + } + } + + override fun indexOf(element: NodeType): Int { + for (i in 0 until this.size) { + if (element == this[i]) { + return i + } + } + + return -1 + } + + override fun lastIndexOf(element: NodeType): Int { + for (i in this.size - 1 downTo 0) { + if (element == this[i]) { + return i + } + } + + return -1 + } + + inner class ListIterator( + var edgeIterator: MutableListIterator, + ) : MutableListIterator { + override fun add(element: NodeType) { + edgeIterator.add(list.createEdge(element, list.init, list.outgoing)) + } + + override fun hasNext(): Boolean { + return edgeIterator.hasNext() + } + + override fun hasPrevious(): Boolean { + return edgeIterator.hasPrevious() + } + + override fun next(): NodeType { + var next = edgeIterator.next() + return if (list.outgoing) { + next.end + } else { + @Suppress("UNCHECKED_CAST") + next.start as NodeType + } + } + + override fun remove() { + return edgeIterator.remove() + } + + override fun set(element: NodeType) { + TODO("Not yet implemented") + } + + override fun nextIndex(): Int { + return edgeIterator.nextIndex() + } + + override fun previous(): NodeType { + var next = edgeIterator.previous() + return if (list.outgoing) { + next.end + } else { + @Suppress("UNCHECKED_CAST") + next.start as NodeType + } + } + + override fun previousIndex(): Int { + return edgeIterator.previousIndex() + } + } + + /** + * Creates a new [Delegate] for this unwrapped list to be used in + * [delegated properties](https://kotlinlang.org/docs/delegated-properties.html). + */ + internal fun delegate(): + UnwrappedEdgeList.Delegate { + return Delegate() + } + + /** + * This class can be used to implement + * [delegated properties](https://kotlinlang.org/docs/delegated-properties.html) in [Node] + * classes. The most common use case is to have a property that is a list of [Edge] objects (for + * persistence) and a second (delegated) property that allows easy access just to the connected + * nodes of the individual edges for in-memory access. It should not be used directly, but + * rather by using [unwrapping]. + * + * For example: + * ```kotlin + * class MyNode { + * @Relationship(value = "EXPRESSIONS", direction = "OUTGOING") + * var expressionsEdges = astChildrenOf() + * var expressions by unwrapping(MyNode::expressionsEdges) + * } + * ``` + * + * This class is intentionally marked with [Transient], so that the delegated properties are not + * transferred to the Neo4J OGM. Only the property that contains the property edges should be + * persisted in the graph database. + */ + @Transient + inner class Delegate< + ThisType : Node, + >() { + operator fun getValue(thisRef: ThisType, property: KProperty<*>): MutableList { + return this@UnwrappedEdgeList + } + + operator fun setValue(thisRef: ThisType, property: KProperty<*>, value: List) { + this@UnwrappedEdgeList.resetTo(value) + } + } + + operator fun provideDelegate( + thisRef: ThisType, + prop: KProperty<*> + ): Delegate { + return Delegate() + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt new file mode 100644 index 0000000000..5924a21204 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeSet.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.Transient + +/** + * An intelligent [MutableSet] wrapper around an [EdgeSet] which supports iterating, adding and + * removing [Node] elements. + */ +class UnwrappedEdgeSet>( + var set: EdgeSet +) : UnwrappedEdgeCollection(set), MutableSet { + + /** + * Creates a new [Delegate] for this unwrapped list to be used in + * [delegated properties](https://kotlinlang.org/docs/delegated-properties.html). + */ + internal fun delegate(): + UnwrappedEdgeSet.Delegate { + return Delegate() + } + + /** See [UnwrappedEdgeList.Delegate], but as a [MutableSet] instead of [MutableList]. */ + @Transient + inner class Delegate< + ThisType : Node, + >() { + operator fun getValue(thisRef: ThisType, property: KProperty<*>): MutableSet { + return this@UnwrappedEdgeSet + } + + operator fun setValue(thisRef: ThisType, property: KProperty<*>, value: Set) { + this@UnwrappedEdgeSet.resetTo(value) + } + } + + operator fun provideDelegate( + thisRef: ThisType, + prop: KProperty<*> + ): Delegate { + return Delegate() + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt new file mode 100644 index 0000000000..e374adfcef --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.RelationshipEntity + +/** + * An edge in a Control Dependence Graph (CDG). Denotes that the [start] node exercises control + * dependence on the [end] node. See [ControlDependenceGraphPass]. + */ +@RelationshipEntity +class ControlDependence( + start: Node, + end: Node, + /** A set of [EvaluationOrder.branch] values. */ + var branches: Set = setOf(), +) : Edge(start, end) { + /** All control dependence edges exercise control dependence. */ + init { + dependence = DependenceType.CONTROL + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is ControlDependence) return false + return this.branches == other.branches && super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + branches.hashCode() + return result + } +} + +/** A container of [ControlDependence] edges. [NodeType] is necessary because of the Neo4J OGM. */ +class ControlDependences : + EdgeList, MirroredEdgeCollection { + + override var mirrorProperty: KProperty> + + constructor( + thisRef: Node, + mirrorProperty: KProperty>, + outgoing: Boolean + ) : super(thisRef = thisRef, init = ::ControlDependence, outgoing = outgoing) { + this.mirrorProperty = mirrorProperty + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt similarity index 53% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt index c8418b0db8..206e554d31 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edge/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt @@ -23,16 +23,18 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph.edge +package de.fraunhofer.aisec.cpg.graph.edges.flows +import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.TupleDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import org.neo4j.ogm.annotation.RelationshipEntity +import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSet +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.HasType +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.* /** * The granularity of the data-flow, e.g., whether the flow contains the whole object, or just a @@ -85,9 +87,21 @@ open class Dataflow( start: Node, end: Node, /** The granularity of this dataflow. */ - val granularity: Granularity = default() -) : PropertyEdge(start, end) { + @Transient @JsonIgnore var granularity: Granularity = default() +) : Edge(start, end) { override val label: String = "DFG" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Dataflow) return false + return this.granularity == other.granularity && super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + granularity.hashCode() + return result + } } sealed interface CallingContext @@ -111,10 +125,67 @@ class CallingContextOut( class ContextSensitiveDataflow( start: Node, end: Node, - /** The calling context affecting this dataflow. */ - val callingContext: CallingContext, /** The granularity of this dataflow. */ - granularity: Granularity, + granularity: Granularity = default(), + val callingContext: CallingContext ) : Dataflow(start, end, granularity) { + override val label: String = "DFG" + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is ContextSensitiveDataflow) return false + return this.callingContext == other.callingContext && super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + callingContext.hashCode() + return result + } +} + +/** This class represents a container of [Dataflow] property edges in a [thisRef]. */ +class Dataflows( + thisRef: Node, + override var mirrorProperty: KProperty>, + outgoing: Boolean, +) : + EdgeSet(thisRef = thisRef, init = ::Dataflow, outgoing = outgoing), + MirroredEdgeCollection { + + /** + * Adds a [ContextSensitiveDataflow] edge from/to (depending on [outgoing]) the node which + * contains this edge container to/from [node], with the given [Granularity]. + */ + fun addContextSensitive( + node: T, + granularity: Granularity = default(), + callingContext: CallingContext + ) { + val edge = + if (outgoing) { + ContextSensitiveDataflow(thisRef, node, granularity, callingContext) + } else { + ContextSensitiveDataflow(node, thisRef, granularity, callingContext) + } + + this.add(edge) + } + + /** + * This connects our dataflow to our "mirror" property. Meaning that if we add a node to + * nextDFG, we add our thisRef to the "prev" of "next" and vice-versa. + */ + override fun handleOnAdd(edge: Dataflow) { + super.handleOnAdd(edge) + val start = edge.start + val thisRef = this.thisRef + + // For references, we want to propagate assigned types all through the previous DFG nodes. + // Therefore, we add a type observer to the previous node (if it is not ourselves) + if (thisRef is Reference && !outgoing && start != thisRef && start is HasType) { + start.registerTypeObserver(thisRef) + } + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt new file mode 100644 index 0000000000..af99e4dcaf --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass +import kotlin.reflect.KProperty +import org.neo4j.ogm.annotation.RelationshipEntity + +/** + * An edge in our Evaluation Order Graph (EOG). It considers the order in which our AST statements + * would be "evaluated" (e.g. by a compiler or interpreter). See [EvaluationOrderGraphPass] for more + * details. + */ +@RelationshipEntity +class EvaluationOrder( + start: Node, + end: Node, + /** + * True, if the edge flows into unreachable code e.g. a branch condition which is always false. + */ + var unreachable: Boolean = false, + + /** + * If we have multiple EOG edges the branch property indicates which EOG edge leads to true + * branch (expression evaluated to true) or the false branch (e.g. with an if/else condition). + * Otherwise, this property is null. + */ + var branch: Boolean? = null +) : Edge(start, end) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is EvaluationOrder) return false + return this.unreachable == other.unreachable && + this.branch == other.branch && + super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + unreachable.hashCode() + result = 31 * result + branch.hashCode() + return result + } +} + +/** + * Holds a container of [EvaluationOrder] edges. The canonical version of this lives in + * [Node.prevEOGEdges] / [Node.nextEOGEdges] and is populated by the [EvaluationOrderGraphPass]. + * + * Note: We would not actually need the type parameter [NodeType] here, since all target nodes are + * of type [Node], but if we skip this parameter, the Neo4J exporter does not recognize this as a + * "list". + */ +class EvaluationOrders( + thisRef: Node, + override var mirrorProperty: KProperty>, + outgoing: Boolean = true, +) : + EdgeList( + thisRef = thisRef, + init = ::EvaluationOrder, + outgoing = outgoing + ), + MirroredEdgeCollection diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt new file mode 100644 index 0000000000..261b4957e4 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import org.neo4j.ogm.annotation.RelationshipEntity + +/** This edge class denotes the invocation of a [FunctionDeclaration] by a [CallExpression]. */ +@RelationshipEntity +class Invoke( + start: Node, + end: FunctionDeclaration, + /** + * True, if this is a "dynamic" invoke, meaning that the call will be resolved during runtime, + * not during compile-time. + */ + var dynamicInvoke: Boolean = false, +) : Edge(start, end) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Invoke) return false + return this.dynamicInvoke == other.dynamicInvoke && super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + dynamicInvoke.hashCode() + return result + } +} + +/** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ +class Invokes(thisRef: CallExpression) : + EdgeList(thisRef = thisRef, init = ::Invoke) { + override fun handleOnAdd(edge: Invoke) { + // TODO: Make thisRef generic :( + edge.end.registerTypeObserver(thisRef as CallExpression) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependence.kt new file mode 100644 index 0000000000..30a4d09db6 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependence.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSet +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import de.fraunhofer.aisec.cpg.passes.ProgramDependenceGraphPass +import kotlin.reflect.KProperty + +/** The types of dependences that might be represented in the CPG */ +enum class DependenceType { + CONTROL, + DATA +} + +/** + * A container of [Edge] edges that act as a program dependence graph (PDG). The canonical version + * of this lives in [Node.prevPDGEdges] / [Node.nextPDGEdges] and is populated by the + * [ProgramDependenceGraphPass]. + * + * After population, this collection will contain a direct combination of two other edge collections + * ([Dataflows] and [ControlDependences]). If we would only handle an in-memory graph, we could just + * store the edges in their original collection (e.g. DFG) as well as in the PDG. But the Neo4J OGM + * does not support this, so unfortunately, we need to clone the edges before inserting them into + * the collection. If we ever got rid of the Neo4J OGM we could potentially also remove the cloning. + */ +class ProgramDependences : + EdgeSet>, MirroredEdgeCollection> { + override var mirrorProperty: KProperty>> + + constructor( + thisRef: Node, + mirrorProperty: KProperty>>, + outgoing: Boolean + ) : super( + thisRef, + init = { start, end -> + throw UnsupportedOperationException( + "This container only allows adding existing edges, but not creating new ones." + ) + }, + outgoing + ) { + this.mirrorProperty = mirrorProperty + } + + override fun add(e: Edge): Boolean { + // Clone the edge before inserting. See comment above for a detailed explanation. + val clonedEdge = e.clone() + return super.add(clonedEdge) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt new file mode 100644 index 0000000000..f5c7a863d7 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.graph.AccessValues +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import org.neo4j.ogm.annotation.RelationshipEntity + +/** This edge class denotes the usage of a [ValueDeclaration] in a [Reference]. */ +@RelationshipEntity +class Usage( + start: Node, + end: Reference, + /** The type of access (read/write/readwrite) of this usage. */ + var access: AccessValues? = null, +) : Edge(start, end) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Usage) return false + return this.access == other.access && super.equals(other) + } + + override fun hashCode(): Int { + var result = super.hashCode() + result = 31 * result + access.hashCode() + return result + } +} + +/** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ +class Usages(thisRef: ValueDeclaration) : + EdgeList(thisRef = thisRef, init = ::Usage) { + override fun handleOnAdd(edge: Usage) { + edge.access = edge.end.access + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index 9507bbabb6..e128ecd112 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -32,6 +32,8 @@ import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement +import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.GeneratedValue @@ -90,6 +92,16 @@ abstract class Scope( */ @Transient var wildcardImports: MutableSet = mutableSetOf() + /** + * In some languages, the lookup scope of a symbol that is being resolved (e.g. of a + * [Reference]) can be adjusted through keywords (such as `global` in Python or PHP). + * + * We store this information in the form of a [LookupScopeStatement] in the AST, but we need to + * also store this information in the scope to avoid unnecessary AST traversals when resolving + * symbols using [lookupSymbol]. + */ + @Transient var predefinedLookupScopes: MutableMap = mutableMapOf() + /** Adds a [declaration] with the defined [symbol]. */ fun addSymbol(symbol: Symbol, declaration: Declaration) { if (declaration is ImportDeclaration && declaration.wildcardImport) { @@ -105,14 +117,34 @@ abstract class Scope( /** * Looks up a list of [Declaration] nodes for the specified [symbol]. Optionally, [predicate] * can be used for additional filtering. + * + * By default, the lookup algorithm will go to the [Scope.parent] if no match was found in the + * current scope. This behaviour can be turned off with [thisScopeOnly]. This is useful for + * qualified lookups, where we want to stay in our lookup-scope. + * + * @param symbol the symbol to lookup + * @param thisScopeOnly whether we should stay in the current scope for lookup or traverse to + * its parents if no match was found. + * @param replaceImports whether any symbols pointing to [ImportDeclaration.importedSymbols] or + * wildcards should be replaced with their actual nodes + * @param predicate An optional predicate which should be used in the lookup. */ fun lookupSymbol( symbol: Symbol, + thisScopeOnly: Boolean = false, replaceImports: Boolean = true, predicate: ((Declaration) -> Boolean)? = null ): List { - // First, try to look for the symbol in the current scope - var scope: Scope? = this + // First, try to look for the symbol in the current scope (unless we have a predefined + // search scope). In the latter case we also need to restrict the lookup to the search scope + var modifiedScoped = this.predefinedLookupScopes[symbol]?.targetScope + var scope: Scope? = + if (modifiedScoped != null) { + modifiedScoped + } else { + this + } + var list: MutableList? = null while (scope != null) { @@ -141,8 +173,13 @@ abstract class Scope( break } - // If we do not have a hit, we can go up one scope - scope = scope.parent + // If we do not have a hit, we can go up one scope, unless thisScopeOnly is set to true + // (or we had a modified scope) + if (thisScopeOnly || modifiedScoped != null) { + break + } else { + scope = scope.parent + } } return list ?: listOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt index a3d2433bcd..bc9910b429 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt @@ -25,17 +25,21 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.Objects +import org.neo4j.ogm.annotation.Relationship /** Represents an assert statement */ class AssertStatement : Statement() { + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition to be evaluated. */ - @AST var condition: Expression? = null + var condition by unwrapping(AssertStatement::conditionEdge) - /** The _optional_ message that is shown, if the assert is evaluated as true */ - @AST var message: Statement? = null + @Relationship(value = "MESSAGE") var messageEdge = astOptionalEdgeOf() + /** The *optional* message that is shown, if the assert is evaluated as true */ + var message by unwrapping(AssertStatement::messageEdge) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt index 9be9b79c0e..e0dfe6f36f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt @@ -25,9 +25,11 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.Objects +import org.neo4j.ogm.annotation.Relationship /** * Case statement of the form `case expression :` that serves as entry point for switch statements, @@ -36,11 +38,14 @@ import java.util.Objects * compound statement. */ class CaseStatement : Statement() { + @Relationship(value = "CASE_EXPRESSION") + var caseExpressionEdge = astOptionalEdgeOf() + /** * Primitive side effect free statement that has to match with the evaluated selector in * SwitchStatement */ - @AST var caseExpression: Expression? = null + var caseExpression by unwrapping(CaseStatement::caseExpressionEdge) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index 8566fce554..5f317eff4c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -25,18 +25,23 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.BranchingNode import de.fraunhofer.aisec.cpg.graph.EOGStarterHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import java.util.Objects +import org.neo4j.ogm.annotation.Relationship class CatchClause : Statement(), BranchingNode, EOGStarterHolder { - @AST var parameter: VariableDeclaration? = null + @Relationship(value = "PARAMETER") var parameterEdge = astOptionalEdgeOf() - @AST var body: Block? = null + var parameter by unwrapping(CatchClause::parameterEdge) + + @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf() + var body by unwrapping(CatchClause::bodyEdge) override val branchedBy: Node? get() = parameter diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt index 5dc54d9509..31285ccc59 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt @@ -25,12 +25,10 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -47,19 +45,14 @@ open class DeclarationStatement : Statement() { * it only contains a single [Declaration]. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - @AST - var declarationEdges: MutableList> = ArrayList() - - override var declarations by PropertyEdgeDelegate(DeclarationStatement::declarationEdges) + var declarationEdges = astEdgesOf() + override var declarations by unwrapping(DeclarationStatement::declarationEdges) var singleDeclaration: Declaration? get() = if (isSingleDeclaration()) declarationEdges[0].end else null set(value) { if (value == null) return - declarationEdges.clear() - val propertyEdge = PropertyEdge(this, value) - propertyEdge.addProperty(Properties.INDEX, 0) - declarationEdges.add(propertyEdge) + declarationEdges.resetTo(listOf(value)) } fun isSingleDeclaration(): Boolean { @@ -70,12 +63,6 @@ open class DeclarationStatement : Statement() { return clazz.cast(singleDeclaration) } - fun addToPropertyEdgeDeclaration(declaration: Declaration) { - val propertyEdge = PropertyEdge(this, declaration) - propertyEdge.addProperty(Properties.INDEX, declarationEdges.size) - declarationEdges.add(propertyEdge) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DistinctLanguageBlock.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DistinctLanguageBlock.kt index 7d32a57a4a..54e44645ae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DistinctLanguageBlock.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DistinctLanguageBlock.kt @@ -31,4 +31,4 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block * A Block of code containing code in a different language that is not parsable with the same * frontend as the enclosing language. */ -class DistinctLanguageBlock : Block() +class DistinctLanguageBlock() : Block() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt index 4c526d50d6..7fcafec4d4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt @@ -25,24 +25,30 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** Represents a conditional loop statement of the form: `do{...}while(...)`. */ class DoStatement : Statement(), ArgumentHolder { + @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() /** * The loop condition that is evaluated after the loop statement and may trigger reevaluation. */ - @AST var condition: Expression? = null + var condition by unwrapping(DoStatement::conditionEdge) + + @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf() /** * The statement that is going to be executed and re-executed, until the condition evaluates to * false for the first time. Usually a [Block]. */ - @AST var statement: Statement? = null + var statement by unwrapping(DoStatement::statementEdge) override fun toString() = ToStringBuilder(this, TO_STRING_STYLE) @@ -63,6 +69,10 @@ class DoStatement : Statement(), ArgumentHolder { return false } + override fun hasArgument(expression: Expression): Boolean { + return condition == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is DoStatement) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt index eeb9e6ac23..be4808ac26 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt @@ -26,61 +26,58 @@ package de.fraunhofer.aisec.cpg.graph.statements import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import java.util.Objects +import org.neo4j.ogm.annotation.Relationship class ForEachStatement : Statement(), BranchingNode, StatementHolder { + + @Relationship("VARIABLE") + var variableEdge = + astOptionalEdgeOf( + onChanged = { _, new -> + val end = new?.end + if (end is Reference) { + end.access = AccessValues.WRITE + } + } + ) + /** * This field contains the iteration variable of the loop. It can be either a new variable * declaration or a reference to an existing variable. */ - @AST - var variable: Statement? = null - set(value) { - if (value is Reference) { - value.access = AccessValues.WRITE - } - field = value - } + var variable by unwrapping(ForEachStatement::variableEdge) + @Relationship("ITERABLE") var iterableEdge = astOptionalEdgeOf() /** This field contains the iteration subject of the loop. */ - @AST var iterable: Statement? = null + var iterable by unwrapping(ForEachStatement::iterableEdge) + @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf() /** This field contains the body of the loop. */ - @AST var statement: Statement? = null + var statement by unwrapping(ForEachStatement::statementEdge) override val branchedBy: Node? get() = iterable - override var statementEdges: MutableList> + override var statementEdges: AstEdges> get() { - val statements = mutableListOf>() - variable?.let { statements.add(PropertyEdge(this, it)) } - iterable?.let { statements.add(PropertyEdge(this, it)) } - statement?.let { statements.add(PropertyEdge(this, it)) } + val statements = astEdgesOf() + variable?.let { statements.add(AstEdge(this, it)) } + iterable?.let { statements.add(AstEdge(this, it)) } + statement?.let { statements.add(AstEdge(this, it)) } return statements } - set(value) { + set(_) { // Nothing to do here } - override fun addStatement(s: Statement) { - when { - variable == null -> variable = s - iterable == null -> iterable = s - statement == null -> statement = s - statement !is Block -> { - val block = Block() - block.language = this.language - statement?.let { block.addStatement(it) } - block.addStatement(s) - statement = block - } - else -> (statement as? Block)?.addStatement(s) - } - } + override var statements by unwrapping(ForEachStatement::statementEdges) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt index 7bc4dff12e..3e36632d7b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt @@ -27,19 +27,29 @@ package de.fraunhofer.aisec.cpg.graph.statements import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.* +import org.neo4j.ogm.annotation.Relationship class ForStatement : Statement(), BranchingNode { - @AST var statement: Statement? = null + @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf() + var statement by unwrapping(ForStatement::statementEdge) - @AST var initializerStatement: Statement? = null + @Relationship("INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() + var initializerStatement by unwrapping(ForStatement::initializerStatementEdge) - @AST var conditionDeclaration: Declaration? = null + @Relationship("CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() + var conditionDeclaration by unwrapping(ForStatement::conditionDeclarationEdge) - @AST var condition: Expression? = null + @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() + var condition by unwrapping(ForStatement::conditionEdge) - @AST var iterationStatement: Statement? = null + @Relationship("ITERATION_STATEMENT") var iterationStatementEdge = astOptionalEdgeOf() + var iterationStatement by unwrapping(ForStatement::iterationStatementEdge) override val branchedBy: Node? get() = condition ?: conditionDeclaration diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt index 88905bc585..346d78b533 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt @@ -25,25 +25,33 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.BranchingNode import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** Represents a condition control flow statement, usually indicating by `If`. */ class IfStatement : Statement(), BranchingNode, ArgumentHolder { + @Relationship(value = "INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() /** C++ initializer statement. */ - @AST var initializerStatement: Statement? = null + var initializerStatement by unwrapping(IfStatement::initializerStatementEdge) + @Relationship(value = "CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() /** C++ alternative to the condition. */ - @AST var conditionDeclaration: Declaration? = null + var conditionDeclaration by unwrapping(IfStatement::conditionDeclarationEdge) + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition to be evaluated. */ - @AST var condition: Expression? = null + var condition by unwrapping(IfStatement::conditionEdge) override val branchedBy: Node? get() = condition ?: conditionDeclaration @@ -51,13 +59,15 @@ class IfStatement : Statement(), BranchingNode, ArgumentHolder { /** C++ constexpr construct. */ var isConstExpression = false + @Relationship(value = "THEN_STATEMENT") var thenStatementEdge = astOptionalEdgeOf() /** The statement that is executed, if the condition is evaluated as true. Usually a [Block]. */ - @AST var thenStatement: Statement? = null + var thenStatement by unwrapping(IfStatement::thenStatementEdge) + @Relationship(value = "ELSE_STATEMENT") var elseStatementEdge = astOptionalEdgeOf() /** * The statement that is executed, if the condition is evaluated as false. Usually a [Block]. */ - @AST var elseStatement: Statement? = null + var elseStatement by unwrapping(IfStatement::elseStatementEdge) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) @@ -77,6 +87,10 @@ class IfStatement : Statement(), BranchingNode, ArgumentHolder { return true } + override fun hasArgument(expression: Expression): Boolean { + return this.condition == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is IfStatement) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt index 9a36428482..88eddde61c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt @@ -25,19 +25,25 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.StatementHolder -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** * A label attached to a statement that is used to change control flow by labeled continue and * breaks (Java) or goto(C++). */ class LabelStatement : Statement(), StatementHolder { + @Relationship(value = "SUB_STATEMENT") var subStatementEdge = astOptionalEdgeOf() + /** Statement that the label is attached to. Can be a simple or compound statement. */ - @AST var subStatement: Statement? = null + var subStatement by unwrapping(LabelStatement::subStatementEdge) /** Label in the form of a String */ var label: String? = null @@ -50,12 +56,18 @@ class LabelStatement : Statement(), StatementHolder { .toString() } - override var statementEdges: MutableList> - get() = subStatement?.let { PropertyEdge.wrap(listOf(it), this) } ?: mutableListOf() + override var statementEdges: AstEdges> + get() { + var list = astEdgesOf() + subStatement?.let { list.resetTo(listOf(it)) } + return list + } set(value) { - subStatement = PropertyEdge.unwrap(value).firstOrNull() + subStatement = value.toNodeCollection().firstOrNull() } + override var statements by unwrapping(LabelStatement::statementEdges) + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is LabelStatement) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LookupScopeStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LookupScopeStatement.kt new file mode 100644 index 0000000000..b0dd317af5 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LookupScopeStatement.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.statements + +import de.fraunhofer.aisec.cpg.graph.newLookupScopeStatement +import de.fraunhofer.aisec.cpg.graph.scopes.Scope +import de.fraunhofer.aisec.cpg.graph.scopes.Symbol +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import java.util.Objects + +/** + * This statement modifies the lookup scope of one or more [Reference] nodes (or more precise it's + * symbols) within the current [Scope]. The most prominent example of this are the Python `global` + * and `nonlocal` keywords. + * + * This node itself does not implement the actual functionality. It is necessary to add this node + * (or the information therein) to [Scope.predefinedLookupScopes]. The reason for this is that we + * want to avoid AST traversals in the scope/identifier lookup. + * + * The [newLookupScopeStatement] node builder will add this automatically, so it is STRONGLY + * encouraged that the node builder is used instead of creating the node itself. + */ +class LookupScopeStatement : Statement() { + + /** The symbols this statement affects. */ + var symbols: List = listOf() + + /** The target scope to which the references are referring to. */ + var targetScope: Scope? = null + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is LookupScopeStatement) return false + return super.equals(other) && symbols == other.symbols && targetScope == other.targetScope + } + + override fun hashCode() = Objects.hash(super.hashCode(), symbols, targetScope) +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt index f3788b225a..420db4def0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt @@ -25,16 +25,20 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** Represents a statement that returns out of the current function. */ class ReturnStatement : Statement(), ArgumentHolder { + @Relationship(value = "RETURN_VALUES") var returnValueEdges = astEdgesOf() + /** The expression whose value will be returned. */ - @AST var returnValues: MutableList = mutableListOf() + var returnValues by unwrapping(ReturnStatement::returnValueEdges) /** * A utility property to handle single-valued return statements. In case [returnValues] contains @@ -70,6 +74,10 @@ class ReturnStatement : Statement(), ArgumentHolder { return true } + override fun hasArgument(expression: Expression): Boolean { + return expression in this.returnValues + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is ReturnStatement) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt index e0b1d4bce4..8c052546d6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt @@ -25,14 +25,13 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.* import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -50,11 +49,10 @@ abstract class Statement : Node(), DeclarationHolder { * TODO: This is actually an AST node just for a subset of nodes, i.e. initializers in for-loops */ @Relationship(value = "LOCALS", direction = Relationship.Direction.OUTGOING) - @AST - var localEdges = mutableListOf>() + var localEdges = astEdgesOf() /** Virtual property to access [localEdges] without property edges. */ - var locals by PropertyEdgeDelegate(Statement::localEdges) + var locals by unwrapping(Statement::localEdges) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt index f9e39ca7e4..e677cc8911 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt @@ -25,12 +25,14 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.BranchingNode import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.Objects +import org.neo4j.ogm.annotation.Relationship /** * Represents a Java or C++ switch statement of the `switch (selector) {...}` that can include case @@ -38,20 +40,26 @@ import java.util.Objects * handled properly. */ class SwitchStatement : Statement(), BranchingNode { + @Relationship(value = "SELECTOR") var selectorEdge = astOptionalEdgeOf() /** Selector that determines the case/default statement of the subsequent execution */ - @AST var selector: Expression? = null + var selector by unwrapping(SwitchStatement::selectorEdge) + @Relationship(value = "INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() /** C++ can have an initializer statement in a switch */ - @AST var initializerStatement: Statement? = null + var initializerStatement by unwrapping(SwitchStatement::initializerStatementEdge) + @Relationship(value = "SELECTOR_DECLARATION") + var selectorDeclarationEdge = astOptionalEdgeOf() /** C++ allows to use a declaration instead of a expression as selector */ - @AST var selectorDeclaration: Declaration? = null + var selectorDeclaration by unwrapping(SwitchStatement::selectorDeclarationEdge) + @Relationship(value = "STATEMENT") var statementEdge = astOptionalEdgeOf() /** * The compound statement that contains break/default statements with regular statements on the * same hierarchy */ - @AST var statement: Statement? = null + var statement by unwrapping(SwitchStatement::statementEdge) override val branchedBy: Node? get() = selector diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt index c6ed76a482..ac955b24ce 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt @@ -25,15 +25,19 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.Objects +import org.neo4j.ogm.annotation.Relationship class SynchronizedStatement : Statement() { - @AST var expression: Expression? = null + @Relationship(value = "EXPRESSION") var expressionEdge = astOptionalEdgeOf() + var expression by unwrapping(SynchronizedStatement::expressionEdge) - @AST var block: Block? = null + @Relationship(value = "BLOCK") var blockEdge = astOptionalEdgeOf() + var block by unwrapping(SynchronizedStatement::blockEdge) override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt index d92011554d..1f887fa207 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt @@ -25,31 +25,57 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import java.util.* import org.neo4j.ogm.annotation.Relationship /** A [Statement] which represents a try/catch block, primarily used for exception handling. */ class TryStatement : Statement() { + /** + * This represents some kind of resource which is typically opened (or similar) while entering + * the [tryBlock]. If this operation fails, we may continue with the [finallyBlock]. However, + * there is no exception raised but if an exception occurs while opening the resource, we won't + * enter the [tryBlock]. + */ @Relationship(value = "RESOURCES", direction = Relationship.Direction.OUTGOING) - @AST - var resourceEdges = mutableListOf>() + var resourceEdges = astEdgesOf() + var resources by unwrapping(TryStatement::resourceEdges) - var resources by PropertyEdgeDelegate(TryStatement::resourceEdges) + /** + * This represents a block whose statements can throw exceptions which are handled by the + * [catchClauses]. + */ + @Relationship(value = "TRY_BLOCK") var tryBlockEdge = astOptionalEdgeOf() + var tryBlock by unwrapping(TryStatement::tryBlockEdge) - @AST var tryBlock: Block? = null + /** + * This represents a block whose statements are only executed if the [tryBlock] finished without + * exceptions. Note that any exception thrown in this block is no longer caught by the + * [catchClauses]. + */ + @Relationship(value = "ELSE_BLOCK") var elseBlockEdge = astOptionalEdgeOf() + var elseBlock by unwrapping(TryStatement::elseBlockEdge) - @AST var finallyBlock: Block? = null + /** + * This represents a block of statements which is always executed after finishing the [tryBlock] + * or one of the [catchClauses]. Note that any exception thrown in this block is no longer + * caught by the [catchClauses]. + */ + @Relationship(value = "FINALLY_BLOCK") var finallyBlockEdge = astOptionalEdgeOf() + var finallyBlock by unwrapping(TryStatement::finallyBlockEdge) + /** + * This represents a set of blocks whose statements handle the exceptions which are thrown in + * the [tryBlock]. There can be multiple catch clauses, but it is also possible that none + * exists. + */ @Relationship(value = "CATCH_CLAUSES", direction = Relationship.Direction.OUTGOING) - @AST - var catchClauseEdges = mutableListOf>() - - var catchClauses by PropertyEdgeDelegate(TryStatement::catchClauseEdges) + var catchClauseEdges = astEdgesOf() + var catchClauses by unwrapping(TryStatement::catchClauseEdges) override fun equals(other: Any?): Boolean { if (this === other) return true @@ -60,9 +86,10 @@ class TryStatement : Statement() { tryBlock == other.tryBlock && finallyBlock == other.finallyBlock && catchClauses == other.catchClauses && + elseBlock == other.elseBlock && propertyEqualsList(catchClauseEdges, other.catchClauseEdges)) } override fun hashCode() = - Objects.hash(super.hashCode(), resources, tryBlock, finallyBlock, catchClauses) + Objects.hash(super.hashCode(), resources, tryBlock, finallyBlock, catchClauses, elseBlock) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt index b121990bcb..11d3f1a9cf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt @@ -25,28 +25,35 @@ */ package de.fraunhofer.aisec.cpg.graph.statements -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.BranchingNode import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** Represents a conditional loop statement of the form: `while(...){...}`. */ class WhileStatement : Statement(), BranchingNode, ArgumentHolder { + @Relationship(value = "CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() /** C++ allows defining a declaration instead of a pure logical expression as condition */ - @AST var conditionDeclaration: Declaration? = null + var conditionDeclaration by unwrapping(WhileStatement::conditionDeclarationEdge) + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition that decides if the block is executed. */ - @AST var condition: Expression? = null + var condition by unwrapping(WhileStatement::conditionEdge) + @Relationship(value = "STATEMENT") var statementEdge = astOptionalEdgeOf() /** * The statement that is going to be executed, until the condition evaluates to false for the * first time. Usually a [Block]. */ - @AST var statement: Statement? = null + var statement by unwrapping(WhileStatement::statementEdge) override val branchedBy: Node? get() = condition ?: conditionDeclaration @@ -68,6 +75,10 @@ class WhileStatement : Statement(), BranchingNode, ArgumentHolder { return true } + override fun hasArgument(expression: Expression): Boolean { + return this.condition == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is WhileStatement) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index c680f7d001..966a913545 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -28,9 +28,12 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.TupleType import de.fraunhofer.aisec.cpg.graph.types.Type +import org.neo4j.ogm.annotation.Relationship import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -56,31 +59,36 @@ class AssignExpression : override var operatorCode: String = "=" - @AST - var lhs: List = listOf() - set(value) { - field = value - field.forEach { - var base = (it as? MemberExpression)?.base as? MemberExpression + /** The expressions on the left-hand side. */ + @Relationship("LHS") + var lhsEdges = + astEdgesOf( + onAdd = { + var end = it.end + var base = (end as? MemberExpression)?.base as? MemberExpression while (base != null) { base.access = AccessValues.READWRITE - base = (base as? MemberExpression)?.base as? MemberExpression + base = base.base as? MemberExpression + } + + if (isSimpleAssignment) { + (end as? Reference)?.access = AccessValues.WRITE + } else { + (end as? Reference)?.access = AccessValues.READWRITE } } - if (isSimpleAssignment) { - field.forEach { (it as? Reference)?.access = AccessValues.WRITE } - } else { - field.forEach { (it as? Reference)?.access = AccessValues.READWRITE } - } - } + ) + var lhs by unwrapping(AssignExpression::lhsEdges) - @AST - var rhs: List = listOf() - set(value) { - field.forEach { it.unregisterTypeObserver(this) } - field = value - value.forEach { it.registerTypeObserver(this) } - } + @Relationship("RHS") + + /** The expressions on the right-hand side. */ + var rhsEdges = + astEdgesOf( + onAdd = { it.end.registerTypeObserver(this) }, + onRemove = { it.end.unregisterTypeObserver(this) }, + ) + var rhs by unwrapping(AssignExpression::rhsEdges) /** * This property specifies, that this is actually used as an expression. Not many languages @@ -117,6 +125,7 @@ class AssignExpression : return operatorCode in (language?.simpleAssignmentOperators ?: setOf()) } + @Relationship("DECLARATIONS") var declarationEdges = astEdgesOf() /** * Some languages, such as Go explicitly allow the definition / declaration of variables in the * assignment (known as a "short assignment"). Some languages, such as Python even implicitly @@ -125,7 +134,7 @@ class AssignExpression : * we need to later resolve this in an additional pass. The declarations are then stored in * [declarations]. */ - @AST override var declarations = mutableListOf() + override var declarations by unwrapping(AssignExpression::declarationEdges) /** Finds the value (of [rhs]) that is assigned to the particular [lhs] expression. */ fun findValue(lhsExpression: HasType): Expression? { @@ -223,21 +232,25 @@ class AssignExpression : override fun addArgument(expression: Expression) { if (lhs.isEmpty()) { - lhs = listOf(expression) + lhs = mutableListOf(expression) } else { - rhs = listOf(expression) + rhs = mutableListOf(expression) } } override fun replaceArgument(old: Expression, new: Expression): Boolean { - return if (lhs == listOf(old)) { - lhs = listOf(new) + return if (lhs.singleOrNull() == old) { + lhs = mutableListOf(new) true - } else if (rhs == listOf(old)) { - rhs = listOf(new) + } else if (rhs.singleOrNull() == old) { + rhs = mutableListOf(new) true } else { false } } + + override fun hasArgument(expression: Expression): Boolean { + return expression in lhs || expression in rhs + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt index 4b1fba69c3..c090750d26 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt @@ -27,10 +27,13 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** * A binary operation expression, such as "a + b". It consists of a left hand expression (lhs), a @@ -39,24 +42,25 @@ import org.apache.commons.lang3.builder.ToStringBuilder * Note: For assignments, i.e., using an `=` or `+=`, etc. the [AssignExpression] MUST be used. */ open class BinaryOperator : - Expression(), HasBase, HasOperatorCode, ArgumentHolder, HasType.TypeObserver { + Expression(), HasOverloadedOperation, ArgumentHolder, HasType.TypeObserver { + /** The left-hand expression. */ - @AST - var lhs: Expression = ProblemExpression("could not parse lhs") - set(value) { - disconnectOldLhs() - field = value - connectNewLhs(value) - } + @Relationship("LHS") + var lhsEdge = + astEdgeOf( + of = ProblemExpression("could not parse lhs"), + onChanged = ::exchangeTypeObserver + ) + var lhs by unwrapping(BinaryOperator::lhsEdge) /** The right-hand expression. */ - @AST - var rhs: Expression = ProblemExpression("could not parse rhs") - set(value) { - disconnectOldRhs() - field = value - connectNewRhs(value) - } + @Relationship("RHS") + var rhsEdge = + astEdgeOf( + of = ProblemExpression("could not parse rhs"), + onChanged = ::exchangeTypeObserver + ) + var rhs by unwrapping(BinaryOperator::rhsEdge) /** The operator code. */ override var operatorCode: String? = null @@ -72,31 +76,6 @@ open class BinaryOperator : } } - private fun connectNewLhs(lhs: Expression) { - lhs.registerTypeObserver(this) - if (lhs is Reference && "=" == operatorCode) { - // declared reference expr is the left-hand side of an assignment -> writing to the var - lhs.access = AccessValues.WRITE - } else if ( - lhs is Reference && operatorCode in (language?.compoundAssignmentOperators ?: setOf()) - ) { - // declared reference expr is the left-hand side of an assignment -> writing to the var - lhs.access = AccessValues.READWRITE - } - } - - private fun disconnectOldLhs() { - lhs.unregisterTypeObserver(this) - } - - private fun connectNewRhs(rhs: Expression) { - rhs.registerTypeObserver(this) - } - - private fun disconnectOldRhs() { - rhs.unregisterTypeObserver(this) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .append("lhs", lhs.name) @@ -129,6 +108,14 @@ open class BinaryOperator : // TODO: replicate something similar like propagateTypeOfBinaryOperation for assigned types } + /** The binary operator operators on the [lhs]. [rhs] is part of the [operatorArguments]. */ + override val operatorArguments: List + get() = listOf(rhs) + + /** The binary operator operators on the [lhs]. [rhs] is part of the [operatorArguments]. */ + override val operatorBase: Expression + get() = lhs + override fun equals(other: Any?): Boolean { if (this === other) { return true @@ -164,7 +151,11 @@ open class BinaryOperator : } } - override val base: Expression? + override fun hasArgument(expression: Expression): Boolean { + return lhs == expression || rhs == expression + } + + val base: Expression? get() { return if (operatorCode == ".*" || operatorCode == "->*") { lhs diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt index 6361e8441c..a8188e8fb4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt @@ -25,10 +25,11 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.StatementHolder import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder @@ -41,9 +42,8 @@ import org.neo4j.ogm.annotation.Relationship open class Block : Expression(), StatementHolder { /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - @AST - override var statementEdges = mutableListOf>() - + override var statementEdges = astEdgesOf() + override var statements by unwrapping(Block::statementEdges) /** * This variable helps to differentiate between static and non static initializer blocks. Static * initializer blocks are executed when the enclosing declaration is first referred to, e.g. @@ -62,7 +62,7 @@ open class Block : Expression(), StatementHolder { if (other !is Block) return false return super.equals(other) && this.statements == other.statements && - PropertyEdge.propertyEqualsList(statementEdges, other.statementEdges) + Edge.propertyEqualsList(statementEdges, other.statementEdges) } override fun hashCode() = Objects.hash(super.hashCode(), statements) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index 1661e45053..cf9056e16a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -27,14 +27,15 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization -import de.fraunhofer.aisec.cpg.graph.edge.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.wrap +import de.fraunhofer.aisec.cpg.graph.edges.* +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.TemplateArguments +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.flows.Invokes import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.* @@ -45,54 +46,48 @@ import org.neo4j.ogm.annotation.Relationship * An expression, which calls another function. It has a list of arguments (list of [Expression]s) * and is connected via the INVOKES edge to its [FunctionDeclaration]. */ -open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { +open class CallExpression : + Expression(), HasOverloadedOperation, HasType.TypeObserver, ArgumentHolder { /** * Connection to its [FunctionDeclaration]. This will be populated by the [SymbolResolver]. This * will have an effect on the [type] */ @PopulatedByPass(SymbolResolver::class) @Relationship(value = "INVOKES", direction = Relationship.Direction.OUTGOING) - var invokeEdges = mutableListOf>() + var invokeEdges = Invokes(this) protected set /** * A virtual property to quickly access the list of declarations that this call invokes without * property edges. */ - @PopulatedByPass(SymbolResolver::class) - var invokes: List - get(): List { - val targets: MutableList = ArrayList() - for (propertyEdge in invokeEdges) { - targets.add(propertyEdge.end) - } - return Collections.unmodifiableList(targets) - } - set(value) { - unwrap(invokeEdges).forEach { it.unregisterTypeObserver(this) } - invokeEdges = wrap(value, this) - value.forEach { it.registerTypeObserver(this) } - } + @PopulatedByPass(SymbolResolver::class) var invokes by unwrapping(CallExpression::invokeEdges) - /** - * The list of arguments of this call expression, backed by a list of [PropertyEdge] objects. - */ + /** The list of arguments of this call expression, backed by a list of [Edge] objects. */ @Relationship(value = "ARGUMENTS", direction = Relationship.Direction.OUTGOING) - @AST - var argumentEdges = mutableListOf>() + var argumentEdges = astEdgesOf() /** * The list of arguments as a simple list. This is a delegated property delegated to * [argumentEdges]. */ - var arguments by PropertyEdgeDelegate(CallExpression::argumentEdges) + var arguments by unwrapping(CallExpression::argumentEdges) + + /** The list of argument types (aka the signature). */ + val signature: List + get() { + return argumentEdges.map { it.end.type } + } /** * The expression that is being "called". This is currently not yet used in the [SymbolResolver] * but will be in the future. In most cases, this is a [Reference] and its [Reference.refersTo] * is intentionally left empty. It is not filled by the [SymbolResolver]. */ - @AST var callee: Expression? = null + @Relationship(value = "CALLEE", direction = Relationship.Direction.OUTGOING) + private var calleeEdge = astEdgeOf(ProblemExpression("could not parse callee")) + + var callee by unwrapping(CallExpression::calleeEdge) /** * The [Name] of this call expression, based on its [callee]. @@ -108,7 +103,7 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { } else if (value is BinaryOperator && value.rhs.type is FunctionPointerType) { value.lhs.type.name.fqn("*" + value.rhs.name.localName) } else { - value?.name ?: Name(EMPTY_NAME) + value.name } } set(_) { @@ -125,12 +120,8 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { /** Adds the specified [expression] with an optional [name] to this call. */ fun addArgument(expression: Expression, name: String? = null) { - val edge = PropertyEdge(this, expression) - edge.addProperty(Properties.INDEX, argumentEdges.size) - - if (name != null) { - edge.addProperty(Properties.NAME, name) - } + val edge = AstEdge(this, expression) + edge.name = name argumentEdges.add(edge) } @@ -146,30 +137,29 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { return true } + override fun hasArgument(expression: Expression): Boolean { + return expression in this.arguments + } + override fun removeArgument(expression: Expression): Boolean { arguments -= expression return true } - /** Returns the function signature as list of types of the call arguments. */ - val signature: List - get() = argumentEdges.map { it.end.type } - /** Specifies, whether this call has any template arguments. */ var template = false - /** If the CallExpression instantiates a template, the call can provide template parameters. */ - @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - @AST - var templateParameterEdges: MutableList>? = null + /** If the CallExpression instantiates a template, the call can provide template arguments. */ + @Relationship(value = "TEMPLATE_ARGUMENTS", direction = Relationship.Direction.OUTGOING) + var templateArgumentEdges: TemplateArguments? = null set(value) { field = value template = value != null } - val templateParameters: List + val templateArguments: List get(): List { - return unwrap(templateParameterEdges ?: listOf()) + return templateArgumentEdges?.toNodeCollection() ?: listOf() } /** @@ -194,14 +184,11 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { templateInitialization: TemplateInitialization? = TemplateInitialization.EXPLICIT ) { if (templateParam is Expression || templateParam is Type) { - if (templateParameterEdges == null) { - templateParameterEdges = mutableListOf() + if (templateArgumentEdges == null) { + templateArgumentEdges = TemplateArguments(this) } - val propertyEdge = PropertyEdge(this, templateParam) - propertyEdge.addProperty(Properties.INDEX, templateParameters.size) - propertyEdge.addProperty(Properties.INSTANTIATION, templateInitialization) - templateParameterEdges?.add(propertyEdge) + templateArgumentEdges?.add(templateParam) { instantiation = templateInitialization } template = true } } @@ -210,37 +197,33 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { initializationType: Map, orderedInitializationSignature: List ) { - if (templateParameterEdges == null) { - templateParameterEdges = mutableListOf() + if (templateArgumentEdges == null) { + templateArgumentEdges = TemplateArguments(this) } - for (edge in templateParameterEdges ?: listOf()) { + for (edge in templateArgumentEdges ?: listOf()) { if ( - edge.getProperty(Properties.INSTANTIATION) != null && - (edge.getProperty(Properties.INSTANTIATION) == - TemplateInitialization.UNKNOWN) && + edge.instantiation != null && + (edge.instantiation == TemplateInitialization.UNKNOWN) && initializationType.containsKey(edge.end) ) { - edge.addProperty(Properties.INSTANTIATION, initializationType[edge.end]) + edge.instantiation = initializationType[edge.end] } } - for (i in (templateParameterEdges?.size ?: 0) until orderedInitializationSignature.size) { - val propertyEdge = PropertyEdge(this, orderedInitializationSignature[i]) - propertyEdge.addProperty(Properties.INDEX, templateParameterEdges?.size) - propertyEdge.addProperty( - Properties.INSTANTIATION, - initializationType.getOrDefault( - orderedInitializationSignature[i], - TemplateInitialization.UNKNOWN - ) - ) - templateParameterEdges?.add(propertyEdge) + for (i in (templateArgumentEdges?.size ?: 0) until orderedInitializationSignature.size) { + templateArgumentEdges?.add(orderedInitializationSignature[i]) { + instantiation = + initializationType.getOrDefault( + orderedInitializationSignature[i], + TemplateInitialization.UNKNOWN + ) + } } } fun instantiatesTemplate(): Boolean { - return templateInstantiation != null || templateParameterEdges != null || template + return templateInstantiation != null || templateArgumentEdges != null || template } override fun typeChanged(newType: Type, src: HasType) { @@ -269,14 +252,28 @@ open class CallExpression : Expression(), HasType.TypeObserver, ArgumentHolder { return ToStringBuilder(this, TO_STRING_STYLE).appendSuper(super.toString()).toString() } + override val operatorCode: String? + get() = "()" + + override val operatorArguments: List + get() = arguments + + /** + * Some languages allow to even overload "()", meaning that basically a normal call to [callee] + * is overloaded. In this case we want the [operatorBase] to point to [callee], so we can take + * its type to lookup the necessary [OperatorDeclaration]. + */ + override val operatorBase: Expression + get() = callee + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is CallExpression) return false return super.equals(other) && arguments == other.arguments && propertyEqualsList(argumentEdges, other.argumentEdges) && - templateParameters == other.templateParameters && - propertyEqualsList(templateParameterEdges, other.templateParameterEdges) && + templateArguments == other.templateArguments && + propertyEqualsList(templateArgumentEdges, other.templateArgumentEdges) && templateInstantiation == other.templateInstantiation && template == other.template } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt index 5af9e6dd71..541cf38a25 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt @@ -26,19 +26,31 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* +import org.neo4j.ogm.annotation.Relationship import org.slf4j.LoggerFactory class CastExpression : Expression(), ArgumentHolder, HasType.TypeObserver { - @AST - var expression: Expression = ProblemExpression("could not parse inner expression") - set(value) { - field.unregisterTypeObserver(this) - field = value - value.registerTypeObserver(this) - } + /** + * The [Expression] that is cast to [castType]. + * + * Note: While the [type] will always stay the same (i.e. the [castType]), we still want to + * register ourselves as a type observer to the expression. The reason for that is that we want + * to propagate the [assignedTypes] of our [expression] to us and then possibly to other nodes. + * This way we can still access the original type of expression (e.g., created by a + * [NewExpression]), even when it is cast. + */ + @Relationship(type = "EXPRESSION") + var expressionEdge = + astEdgeOf( + of = ProblemExpression("could not parse inner expression"), + onChanged = ::exchangeTypeObserver + ) + var expression by unwrapping(CastExpression::expressionEdge) var castType: Type = unknownType() set(value) { @@ -75,6 +87,10 @@ class CastExpression : Expression(), ArgumentHolder, HasType.TypeObserver { return false } + override fun hasArgument(expression: Expression): Boolean { + return this.expression == expression + } + override fun typeChanged(newType: Type, src: HasType) { // Nothing to do, the cast type always stays the same } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt index 02dfd07864..977c85d8ed 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt @@ -27,33 +27,44 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.commonType import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** * Represents an expression containing a ternary operator: `var x = condition ? valueIfTrue : * valueIfFalse`; */ class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasType.TypeObserver { - @AST var condition: Expression = ProblemExpression("could not parse condition expression") + @Relationship("CONDITION") + var conditionEdge = + astEdgeOf(ProblemExpression("could not parse condition expression")) + var condition by unwrapping(ConditionalExpression::conditionEdge) - @AST - var thenExpression: Expression? = null - set(value) { - field?.unregisterTypeObserver(this) - field = value - value?.registerTypeObserver(this) - } + @Relationship("THEN_EXPRESSION") + var thenExpressionEdge = + astOptionalEdgeOf( + onChanged = { old, new -> + old?.end?.unregisterTypeObserver(this) + new?.end?.registerTypeObserver(this) + } + ) + var thenExpression by unwrapping(ConditionalExpression::thenExpressionEdge) - @AST - var elseExpression: Expression? = null - set(value) { - field?.unregisterTypeObserver(this) - field = value - value?.registerTypeObserver(this) - } + @Relationship("ELSE_EXPRESSION") + var elseExpressionEdge = + astOptionalEdgeOf( + onChanged = { old, new -> + old?.end?.unregisterTypeObserver(this) + new?.end?.registerTypeObserver(this) + } + ) + var elseExpression by unwrapping(ConditionalExpression::elseExpressionEdge) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) @@ -68,7 +79,13 @@ class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasTy get() = condition override fun addArgument(expression: Expression) { - // Do nothing + if (condition is ProblemExpression) { + condition = expression + } else if (thenExpression == null) { + thenExpression = expression + } else { + elseExpression = expression + } } override fun replaceArgument(old: Expression, new: Expression): Boolean { @@ -87,6 +104,10 @@ class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasTy } } + override fun hasArgument(expression: Expression): Boolean { + return this.thenExpression == expression || elseExpression == expression + } + override fun typeChanged(newType: Type, src: HasType) { val types = mutableSetOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt index c156cc55a4..c6c81d51ae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt @@ -28,10 +28,13 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.UnknownType import de.fraunhofer.aisec.cpg.passes.SymbolResolver import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** * Represents a call to a constructor, usually as an initializer. @@ -58,11 +61,13 @@ class ConstructExpression : CallExpression() { // Forward to CallExpression. This will also take care of DFG edges. if (value != null) { - invokes = listOf(value as FunctionDeclaration) + invokes = mutableListOf(value as FunctionDeclaration) } } - @AST var anonymousClass: RecordDeclaration? = null + @Relationship("ANONYMOUS_CLASS") var anonymousClassEdge = astOptionalEdgeOf() + + var anonymousClass by unwrapping(ConstructExpression::anonymousClassEdge) /** The [Declaration] of the type this expression instantiates. */ @PopulatedByPass(SymbolResolver::class) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt index 590629b156..dfa39b08f2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt @@ -25,17 +25,20 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.Objects +import org.neo4j.ogm.annotation.Relationship class DeleteExpression : Expression() { - @AST var operand: Expression? = null + @Relationship("OPERANDS") var operandEdges = astEdgesOf() + var operands by unwrapping(DeleteExpression::operandEdges) override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is DeleteExpression) return false - return super.equals(other) && operand == other.operand + return super.equals(other) && operands == other.operands } - override fun hashCode() = Objects.hash(super.hashCode(), operand) + override fun hashCode() = Objects.hash(super.hashCode(), operands) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt index 28de83ce55..f349cc7a33 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt @@ -25,28 +25,17 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement import java.util.* -import kotlin.collections.ArrayList import org.neo4j.ogm.annotation.Relationship class ExpressionList : Expression() { @Relationship(value = "SUBEXPR", direction = Relationship.Direction.OUTGOING) - @AST - var expressionEdges: MutableList> = ArrayList() - - var expressions: List by PropertyEdgeDelegate(ExpressionList::expressionEdges, true) - - fun addExpression(expression: Statement) { - val propertyEdge = PropertyEdge(this, expression) - propertyEdge.addProperty(Properties.INDEX, expressionEdges.size) - expressionEdges.add(propertyEdge) - } + var expressionEdges = astEdgesOf() + var expressions by unwrapping(ExpressionList::expressionEdges) override fun equals(other: Any?): Boolean { if (this === other) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 0cde0292bf..62fa90ca8e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -26,9 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type @@ -46,16 +46,14 @@ import org.neo4j.ogm.annotation.Relationship class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObserver { /** The list of initializers. */ @Relationship(value = "INITIALIZERS", direction = Relationship.Direction.OUTGOING) - @AST - var initializerEdges = mutableListOf>() - set(value) { - field.forEach { it.end.unregisterTypeObserver(this) } - field = value - value.forEach { it.end.registerTypeObserver(this) } - } + var initializerEdges = + astEdgesOf( + onAdd = { it.end.registerTypeObserver(this) }, + onRemove = { it.end.unregisterTypeObserver(this) }, + ) /** Virtual property to access [initializerEdges] without property edges. */ - var initializers by PropertyEdgeDelegate(InitializerListExpression::initializerEdges) + var initializers by unwrapping(InitializerListExpression::initializerEdges) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) @@ -80,6 +78,10 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse return false } + override fun hasArgument(expression: Expression): Boolean { + return expression in this.initializers + } + override fun typeChanged(newType: Type, src: HasType) { // Normally, we would check, if the source comes from our initializers, but we want to limit // the iteration of the initializer list (which can potentially contain tens of thousands of diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt index c5bda8735e..d6b2eaff81 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt @@ -25,9 +25,11 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.* +import org.neo4j.ogm.annotation.Relationship /** * Represents a key / value pair, often found in languages that allow associative arrays or objects, @@ -38,19 +40,22 @@ import java.util.* */ class KeyValueExpression : Expression(), ArgumentHolder { + @Relationship("KEY") var keyEdge = astEdgeOf(ProblemExpression("missing key")) /** * The key of this pair. It is usually a literal, but some languages even allow references to * variables as a key. */ - @AST var key: Expression? = null + var key by unwrapping(KeyValueExpression::keyEdge) + + @Relationship("VALUE") var valueEdge = astEdgeOf(ProblemExpression("missing value")) /** The value of this pair. It can be any expression */ - @AST var value: Expression? = null + var value by unwrapping(KeyValueExpression::valueEdge) override fun addArgument(expression: Expression) { - if (key == null) { + if (key is ProblemExpression) { key = expression - } else if (value == null) { + } else if (value is ProblemExpression) { value = expression } } @@ -67,6 +72,10 @@ class KeyValueExpression : Expression(), ArgumentHolder { return false } + override fun hasArgument(expression: Expression): Boolean { + return key == expression || value == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is KeyValueExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt index eaff56e9d6..2b8823be97 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt @@ -28,9 +28,12 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type +import org.neo4j.ogm.annotation.Relationship /** * This expression denotes the usage of an anonymous / lambda function. It connects the inner @@ -47,13 +50,9 @@ class LambdaExpression : Expression(), HasType.TypeObserver { /** Determines if we can modify variables declared outside the lambda from inside the lambda */ var areVariablesMutable: Boolean = true - @AST - var function: FunctionDeclaration? = null - set(value) { - value?.unregisterTypeObserver(this) - field = value - value?.registerTypeObserver(this) - } + @Relationship("FUNCTION") + var functionEdge = astOptionalEdgeOf(onChanged = ::exchangeTypeObserver) + var function by unwrapping(LambdaExpression::functionEdge) override fun typeChanged(newType: Type, src: HasType) { // Make sure our src is the function diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index 409a5269b2..21c0e2cb85 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -25,33 +25,44 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.HasBase +import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.fqn import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** * Represents access to a member of a [RecordDeclaration], such as `obj.property`. Another common * use-case is access of a member function (method) as part of the [MemberCallExpression.callee] * property of a [MemberCallExpression]. */ -class MemberExpression : Reference(), ArgumentHolder, HasBase { - @AST - override var base: Expression = ProblemExpression("could not parse base expression") - set(value) { - field.unregisterTypeObserver(this) - field = value - updateName() - value.registerTypeObserver(this) - } +class MemberExpression : Reference(), HasOverloadedOperation, ArgumentHolder, HasBase { + @Relationship("BASE") + var baseEdge = + astEdgeOf( + ProblemExpression("could not parse base expression"), + onChanged = { old, new -> + exchangeTypeObserver(old, new) + updateName() + } + ) + override var base by unwrapping(MemberExpression::baseEdge) override var operatorCode: String? = null + override val operatorArguments: List + get() = listOf() + + override val operatorBase: Expression + get() = base + override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) @@ -72,6 +83,10 @@ class MemberExpression : Reference(), ArgumentHolder, HasBase { return false } + override fun hasArgument(expression: Expression): Boolean { + return base == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is MemberExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt index e2b466ebd8..d0bfa683f2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt @@ -25,12 +25,12 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate -import java.util.* +import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import java.util.Objects import org.neo4j.ogm.annotation.Relationship /** @@ -39,15 +39,13 @@ import org.neo4j.ogm.annotation.Relationship */ // TODO Merge and/or refactor with new Expression class NewArrayExpression : Expression() { + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + /** * The initializer of the expression, if present. Many languages, such as Java, either specify * [dimensions] or an initializer. */ - @AST - var initializer: Expression? = null - set(value) { - field = value - } + var initializer by unwrapping(NewArrayExpression::initializerEdge) /** * Specifies the dimensions of the array that is to be created. Many languages, such as Java, @@ -55,11 +53,10 @@ class NewArrayExpression : Expression() { * dimensions. In the graph, this will NOT be done. */ @Relationship(value = "DIMENSIONS", direction = Relationship.Direction.OUTGOING) - @AST - var dimensionEdges = mutableListOf>() + var dimensionEdges = astEdgesOf() /** Virtual property to access [dimensionEdges] without property edges. */ - var dimensions by PropertyEdgeDelegate(NewArrayExpression::dimensionEdges) + var dimensions by unwrapping(NewArrayExpression::dimensionEdges) /** Adds an [Expression] to the existing [dimensions]. */ fun addDimension(expression: Expression) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt index 8d6cd0c4de..807dbcdf29 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt @@ -25,25 +25,29 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasInitializer import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship /** Represents the creation of a new object through the `new` keyword. */ class NewExpression : Expression(), HasInitializer { + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + /** The initializer expression. */ - @AST override var initializer: Expression? = null + override var initializer by unwrapping(NewExpression::initializerEdge) /** * We need a way to store the templateParameters that a NewExpression might have before the * ConstructExpression is created */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - @AST - var templateParameters: List? = null + var templateParameterEdges = astEdgesOf() + var templateParameters by unwrapping(NewExpression::templateParameterEdges) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/OperatorCallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/OperatorCallExpression.kt new file mode 100644 index 0000000000..1a2242c328 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/OperatorCallExpression.kt @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.statements.expressions + +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.OperatorDeclaration + +/** + * This special call expression is used when an operator (such as a [BinaryOperator]) is overloaded. + * In this case, we replace the original [BinaryOperator] with an [OperatorCallExpression], which + * points to its respective [OperatorDeclaration]. + */ +class OperatorCallExpression : CallExpression(), HasOperatorCode, HasBase { + + override var operatorCode: String? = null + + override var name: Name + get() = Name(operatorCode ?: "") + set(_) { + // read-only + } + + /** + * The base object. This is basically a shortcut to accessing the base of the [callee], if it + * has one (i.e., if it implements [HasBase]). This is the case for example, if it is a + * [MemberExpression]. + */ + override val base: Expression? + get() { + return (callee as? HasBase)?.base + } +} + +/** + * Creates a new [OperatorCallExpression] to a [OperatorDeclaration] and also sets the appropriate + * fields such as [CallExpression.invokes] and [Reference.refersTo]. + */ +fun operatorCallFromDeclaration( + decl: OperatorDeclaration, + op: HasOverloadedOperation +): OperatorCallExpression { + return with(decl) { + val ref = + newMemberExpression(decl.name, op.operatorBase, operatorCode = ".") + .implicit(decl.name.localName, location = op.location) + ref.refersTo = decl + val call = + newOperatorCallExpression(operatorCode = op.operatorCode ?: "", ref) + .codeAndLocationFrom(ref) + call.invokes = mutableListOf(decl) + call + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt index 5a56ec7123..0140852024 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt @@ -25,8 +25,10 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import java.util.* +import org.neo4j.ogm.annotation.Relationship /** * Represents the specification of a range (e.g., of an array). Usually used in combination with an @@ -53,19 +55,21 @@ import java.util.* * Individual meaning of the range indices might differ per language. */ class RangeExpression : Expression() { - + @Relationship("FLOOR") var floorEdge = astOptionalEdgeOf() /** The lower bound ("floor") of the range. This index is usually *inclusive*. */ - @AST var floor: Expression? = null + var floor by unwrapping(RangeExpression::floorEdge) + @Relationship("CEILING") var ceilingEdge = astOptionalEdgeOf() /** The upper bound ("ceiling") of the range. This index is usually *exclusive*. */ - @AST var ceiling: Expression? = null + var ceiling by unwrapping(RangeExpression::ceilingEdge) + @Relationship("THIRD") var thirdEdge = astOptionalEdgeOf() /** * Some languages offer a third value. The meaning depends completely on the language. For * example, Python allows specifying a step, while Go allows to control the underlying array's * capacity (not length). */ - @AST var third: Expression? = null + var third by unwrapping(RangeExpression::thirdEdge) /** The operator code that separates the range elements. Common cases are `:` or `...` */ var operatorCode = ":" diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index 568fdf20e9..6f19346014 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -32,9 +32,6 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.CallingContext -import de.fraunhofer.aisec.cpg.graph.edge.Granularity -import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.passes.SymbolResolver @@ -66,7 +63,7 @@ open class Reference : Expression(), HasType.TypeObserver, HasAliases { // set it field = value if (value is ValueDeclaration) { - value.addUsage(this) + value.usageEdges += this } // Register ourselves to get type updates from the declaration @@ -166,17 +163,6 @@ open class Reference : Expression(), HasType.TypeObserver, HasAliases { return super.hashCode() } - override fun addPrevDFG(prev: Node, granularity: Granularity, callingContext: CallingContext?) { - super.addPrevDFG(prev, granularity, callingContext) - - // We want to propagate assigned types all through the previous DFG nodes. Therefore, we - // override the DFG adding function here and add a type observer to the previous node (if it - // is not ourselves) - if (prev != this && prev is HasType) { - prev.registerTypeObserver(this) - } - } - /** * This function builds a tag for the particular reference, based on its [name], * [resolutionHelper] and [scope]. Its purpose is to cache symbol resolutions, similar to LLVMs diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt index bb3c7ddc0c..8254241a74 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt @@ -26,9 +26,12 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* +import org.neo4j.ogm.annotation.Relationship /** * Represents the subscription or access of an array of the form `array[index]`, where both `array` @@ -36,22 +39,24 @@ import java.util.* * overload operators thus changing semantics of array access. */ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, ArgumentHolder { + @Relationship("ARRAY_EXPRESSION") + var arrayExpressionEdge = + astEdgeOf( + of = ProblemExpression("could not parse array expression"), + onChanged = ::exchangeTypeObserver + ) /** The array on which the access is happening. This is most likely a [Reference]. */ - @AST - var arrayExpression: Expression = ProblemExpression("could not parse array expression") - set(value) { - field.unregisterTypeObserver(this) - field = value - type = getSubscriptType(value.type) - value.registerTypeObserver(this) - } + var arrayExpression by unwrapping(SubscriptExpression::arrayExpressionEdge) + @Relationship("SUBSCRIPT_EXPRESSION") + var subscriptExpressionEdge = + astEdgeOf(ProblemExpression("could not parse index expression")) /** * The expression which represents the "subscription" or index on which the array is accessed. * This can for example be a reference to another variable ([Reference]), a [Literal] or a * [RangeExpression]. */ - @AST var subscriptExpression: Expression = ProblemExpression("could not parse index expression") + var subscriptExpression by unwrapping(SubscriptExpression::subscriptExpressionEdge) override val base: Expression get() = arrayExpression @@ -111,6 +116,10 @@ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, Argumen } } + override fun hasArgument(expression: Expression): Boolean { + return arrayExpression == expression || subscriptExpression == expression + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is SubscriptExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index d90d296e0e..3bb1931a57 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -25,28 +25,43 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.AccessValues import de.fraunhofer.aisec.cpg.graph.ArgumentHolder +import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation +import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import org.apache.commons.lang3.builder.ToStringBuilder +import org.neo4j.ogm.annotation.Relationship /** A unary operator expression, involving one expression and an operator, such as `a++`. */ -class UnaryOperator : Expression(), ArgumentHolder, HasType.TypeObserver { +class UnaryOperator : Expression(), HasOverloadedOperation, ArgumentHolder, HasType.TypeObserver { + @Relationship("INPUT") + var inputEdge = + astEdgeOf( + of = ProblemExpression("could not parse input"), + onChanged = { old, new -> + exchangeTypeObserver(old, new) + changeExpressionAccess() + } + ) /** The expression on which the operation is applied. */ - @AST - var input: Expression = ProblemExpression("could not parse input") - set(value) { - field.unregisterTypeObserver(this) - field = value - input.registerTypeObserver(this) - changeExpressionAccess() - } + var input by unwrapping(UnaryOperator::inputEdge) + + /** + * The unary operator does not have any arguments, since [input] is already the [operatorBase]. + */ + override val operatorArguments: List + get() = listOf() + + /** The unary operator operates on [input]. */ + override val operatorBase + get() = input /** The operator code. */ - var operatorCode: String? = null + override var operatorCode: String? = null set(value) { field = value changeExpressionAccess() @@ -126,6 +141,10 @@ class UnaryOperator : Expression(), ArgumentHolder, HasType.TypeObserver { return false } + override fun hasArgument(expression: Expression): Boolean { + return this.input == expression + } + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt index 5574cea7d2..ad2660733f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt @@ -26,10 +26,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.wrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder @@ -46,19 +42,16 @@ import org.neo4j.ogm.annotation.Relationship */ class FunctionPointerType : Type { @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - var parametersPropertyEdge: MutableList> = mutableListOf() - private set + var parameters: List var returnType: Type - var parameters by PropertyEdgeDelegate(FunctionPointerType::parametersPropertyEdge) - constructor( parameters: List = listOf(), language: Language<*>? = null, returnType: Type = UnknownType.getUnknownType(language) ) : super(EMPTY_NAME, language) { - parametersPropertyEdge = wrap(parameters, this) + this.parameters = parameters this.returnType = returnType } @@ -68,7 +61,7 @@ class FunctionPointerType : Type { language: Language<*>? = null, returnType: Type = UnknownType.getUnknownType(language) ) : super(type) { - parametersPropertyEdge = wrap(parameters, this) + this.parameters = parameters this.returnType = returnType this.language = language } @@ -86,11 +79,10 @@ class FunctionPointerType : Type { if (other !is FunctionPointerType) return false return super.equals(other) && parameters == other.parameters && - propertyEqualsList(parametersPropertyEdge, other.parametersPropertyEdge) && returnType == other.returnType } - override fun hashCode() = Objects.hash(super.hashCode(), parametersPropertyEdge, returnType) + override fun hashCode() = Objects.hash(super.hashCode(), parameters, returnType) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt index d50769de83..50c54c1276 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt @@ -29,6 +29,8 @@ import de.fraunhofer.aisec.cpg.graph.ContextProvider import de.fraunhofer.aisec.cpg.graph.LanguageProvider import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSingletonList import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -127,6 +129,18 @@ interface HasType : ContextProvider, LanguageProvider { * [HasType.assignedTypes]. */ fun assignedTypeChanged(assignedTypes: Set, src: HasType) + + /** + * A helper function that can be used for [EdgeSingletonList.onChanged]. It unregisters this + * [TypeObserver] with the [old] node and registers it with the [new] one. + */ + fun exchangeTypeObserver( + old: AstEdge?, + new: AstEdge? + ) { + (old?.end as? HasType)?.unregisterTypeObserver(this) + (new?.end as? HasType)?.registerTypeObserver(this) + } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt index ed8f88a32f..3bb8fa44dd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt @@ -28,10 +28,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.wrap -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.graph.unknownType import de.fraunhofer.aisec.cpg.passes.TypeResolver @@ -57,10 +53,7 @@ open class ObjectType : Type { } @Relationship(value = "GENERICS", direction = Relationship.Direction.OUTGOING) - var genericsPropertyEdges: MutableList> = mutableListOf() - private set - - var generics by PropertyEdgeDelegate(ObjectType::genericsPropertyEdges) + var generics: List private set constructor( @@ -69,7 +62,7 @@ open class ObjectType : Type { primitive: Boolean, language: Language<*>? ) : super(typeName, language) { - this.genericsPropertyEdges = wrap(generics, this) + this.generics = generics isPrimitive = primitive this.language = language } @@ -81,14 +74,14 @@ open class ObjectType : Type { language: Language<*>? ) : super(type) { this.language = language - this.genericsPropertyEdges = wrap(generics, this) + this.generics = generics isPrimitive = primitive } /** Empty default constructor for use in Neo4J persistence. */ constructor() : super() { - genericsPropertyEdges = ArrayList() isPrimitive = false + this.generics = mutableListOf() } /** @return PointerType to a ObjectType, e.g. int* */ @@ -112,9 +105,7 @@ open class ObjectType : Type { if (this === other) return true if (other !is ObjectType) return false if (!super.equals(other)) return false - return generics == other.generics && - propertyEqualsList(genericsPropertyEdges, other.genericsPropertyEdges) && - isPrimitive == other.isPrimitive + return generics == other.generics && isPrimitive == other.isPrimitive } override fun hashCode() = Objects.hash(super.hashCode(), generics, isPrimitive) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt index bba1442f62..ad167ae643 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.types +import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name @@ -142,6 +143,7 @@ abstract class Type : Node { open fun refreshNames() {} + @get:JsonIgnore var root: Type /** * Obtain the root Type Element for a Type Chain (follows Pointer and ReferenceTypes until a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt index df7bfcae12..89f7ce6b66 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/CommentMatcher.kt @@ -49,7 +49,7 @@ class CommentMatcher { ): Node { // If there's an ArtifactLocation specified, it should at least be in the same file. val children = - SubgraphWalker.getAstChildren(node) + node.astChildren .filter { artifactLocation == null || artifactLocation == it.location?.artifactLocation } @@ -58,7 +58,7 @@ class CommentMatcher { // instead. children.addAll( children.filterIsInstance().flatMap { namespace -> - SubgraphWalker.getAstChildren(namespace).filter { it !in children } + namespace.astChildren.filter { it !in children } } ) val enclosing = @@ -95,7 +95,7 @@ class CommentMatcher { } val children = - SubgraphWalker.getAstChildren(smallestEnclosingNode) + smallestEnclosingNode.astChildren .filter { artifactLocation == null || artifactLocation == it.location?.artifactLocation } @@ -105,7 +105,7 @@ class CommentMatcher { // children with a location children.addAll( children.filterIsInstance().flatMap { namespace -> - SubgraphWalker.getAstChildren(namespace).filter { it !in children } + namespace.astChildren.filter { it !in children } } ) @@ -117,9 +117,7 @@ class CommentMatcher { children.addAll( children .filter { node -> node.location == null || node.location?.region == Region() } - .flatMap { locationLess -> - SubgraphWalker.getAstChildren(locationLess).filter { it !in children } - } + .flatMap { locationLess -> locationLess.astChildren.filter { it !in children } } ) // Searching for the closest successor to our comment amongst the children of the smallest diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/IdentitySet.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/IdentitySet.kt index 6f33531cff..d6228a0421 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/IdentitySet.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/IdentitySet.kt @@ -43,7 +43,7 @@ import java.util.concurrent.atomic.AtomicInteger * be very resource-intensive if nodes are very similar but not the *same*, in a work-list however * we only want just to avoid to place the exact node twice. */ -class IdentitySet : MutableSet { +open class IdentitySet : MutableSet { /** * The backing hashmap for our set. The [IdentityHashMap] offers reference-equality for keys and * values. In this case we use it to determine, if a node is already in our set or not. The diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt index 1dcbec6ac2..e5821b00d7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt @@ -27,19 +27,21 @@ package de.fraunhofer.aisec.cpg.helpers import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend -import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder +import de.fraunhofer.aisec.cpg.graph.ContextProvider import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.StatementHolder import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.checkForPropertyEdge -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.processing.strategy.Strategy import java.lang.annotation.AnnotationFormatError import java.lang.reflect.Field import java.util.* import java.util.function.BiConsumer import java.util.function.Consumer -import org.neo4j.ogm.annotation.Relationship import org.slf4j.LoggerFactory /** A type for a node visitor callback for the [SubgraphWalker]. */ @@ -57,7 +59,7 @@ object SubgraphWalker { * @param classType the class type * @return its fields, including the ones from its superclass */ - private fun getAllFields(classType: Class<*>): Collection { + fun getAllEdgeFields(classType: Class<*>): Collection { if (classType.superclass != null) { val cacheKey = classType.name @@ -67,8 +69,8 @@ object SubgraphWalker { return fieldCache[cacheKey] ?: ArrayList() } val fields = ArrayList() - fields.addAll(getAllFields(classType.superclass)) - fields.addAll(listOf(*classType.declaredFields)) + fields.addAll(getAllEdgeFields(classType.superclass)) + fields.addAll(listOf(*classType.declaredFields).filter { it.name.contains("Edge") }) // update the cache fieldCache[cacheKey] = fields @@ -78,8 +80,8 @@ object SubgraphWalker { } /** - * Retrieves a list of AST children of the specified node by iterating all fields that are - * annotated with the [AST] annotation. + * Retrieves a list of AST children of the specified node by iterating all edge fields that are + * of type [AstEdge]. * * Please note, that you SHOULD NOT call this directly in a recursive function, since the AST * might have loops and you will probably run into a [StackOverflowError]. Therefore, use of @@ -94,54 +96,9 @@ object SubgraphWalker { if (node == null) return children val classType: Class<*> = node.javaClass - /*for (member in node::class.members) { - val subGraph = member.findAnnotation() - if (subGraph != null && listOf(*subGraph.value).contains("AST")) { - val old = member.isAccessible - - member.isAccessible = true - - val obj = member.call(node) - - // skip, if null - if (obj == null) { - continue - } - - member.isAccessible = old - - var outgoing = true // default - var relationship = member.findAnnotation() - if (relationship != null) { - outgoing = - relationship.direction == - Relationship.Direction.OUTGOING) - } - if (checkForPropertyEdge(field, obj)) { - obj = unwrap(obj as List>, outgoing) - } - when (obj) { - is Node -> { - children.add(obj) - } - is Collection<*> -> { - children.addAll(obj as Collection) - } - else -> { - throw AnnotationFormatError( - "Found @field:SubGraph(\"AST\") on field of type " + - obj.javaClass + - " but can only used with node graph classes or collections of graph nodes" - ) - } - } - } - }*/ - // We currently need to stick to pure Java reflection, since Kotlin reflection // is EXTREMELY slow. See https://youtrack.jetbrains.com/issue/KT-32198 - for (field in getAllFields(classType)) { - field.getAnnotation(AST::class.java) ?: continue + for (field in getAllEdgeFields(classType)) { try { // We need to synchronize access to the field, because otherwise different // threads might restore the isAccessible property while this thread is still @@ -157,28 +114,15 @@ object SubgraphWalker { obj } ?: continue - // skip, if null - var outgoing = true // default - if (field.getAnnotation(Relationship::class.java) != null) { - outgoing = - (field.getAnnotation(Relationship::class.java).direction == - Relationship.Direction.OUTGOING) - } - if (checkForPropertyEdge(field, obj) && obj is Collection<*>) { - obj = unwrap(obj.filterIsInstance>(), outgoing) - } when (obj) { - is Node -> { - children.add(obj) - } - is Collection<*> -> { - children.addAll(obj.filterIsInstance()) + is EdgeCollection<*, *> -> { + children.addAll(obj.toNodeCollection({ it is AstEdge<*> })) } else -> { throw AnnotationFormatError( - "Found @field:SubGraph(\"AST\") on field of type " + + "Found on field of type " + obj.javaClass + - " but can only used with node graph classes or collections of graph nodes" + " but can only used with edge classes or edge collections" ) } } @@ -401,3 +345,55 @@ object SubgraphWalker { } } } + +/** + * Tries to replace the [old] expression with a [new] one, given the [parent]. + * + * There are two things to consider: + * - First, this only works if [parent] is either an [ArgumentHolder] or [StatementHolder]. + * Otherwise, we cannot instruct the parent to exchange the node + * - Second, since exchanging the node has influence on their edges (such as EOG, DFG, etc.), we + * only support a replacement very early in the pass system. To be specific, we only allow + * replacement before any DFG edges are set. We are re-wiring EOG edges, but nothing else. If one + * tries to replace a node with existing [Node.nextDFG] or [Node.prevDFG], we fail. + */ +context(ContextProvider) +fun SubgraphWalker.ScopedWalker.replace(parent: Node?, old: Expression, new: Expression): Boolean { + // We do not allow to replace nodes where the DFG (or other dependent nodes, such as PDG have + // been set). The reason for that is that these edges contain a lot of information on the edges + // themselves and replacing this edge would be very complicated. + if (old.prevDFG.isNotEmpty() || old.nextDFG.isNotEmpty()) { + return false + } + + val success = + when (parent) { + is ArgumentHolder -> parent.replace(old, new) + is StatementHolder -> parent.replace(old, new) + else -> { + Pass.log.error( + "Parent AST node is not an argument or statement holder. Cannot replace node. Further analysis might not be entirely accurate." + ) + return false + } + } + if (!success) { + Pass.log.error( + "Replacing expression $old was not successful. Further analysis might not be entirely accurate." + ) + } else { + // Store any eventual EOG/DFG nodes and disconnect old node + val oldPrevEOG = old.prevEOG.toMutableList() + val oldNextEOG = old.nextEOG.toMutableList() + old.disconnectFromGraph() + + // Put the stored EOG nodes to the new node + new.prevEOG = oldPrevEOG + new.nextEOG = oldNextEOG + + // Make sure to inform the walker about our change + this.registerReplacement(old, new) + } + + return success +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt index 9fed14d65c..a8ddc2e5d4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/Util.kt @@ -29,8 +29,8 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn -import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextIn +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* @@ -74,21 +74,24 @@ object Util { * - NODE if refs nodes itself are the nodes to connect or SUBTREE if the EOG borders are of * interest * - * @param props - * - All edges must have these properties set to the provided value + * @param branch + * - All edges must have the specified branch property * * @param refs * - Multiple reference nodes that can be passed as varargs * * @return true if all/any of the connections from node connect to n. */ + // TODO: this function needs a major overhaul because it was + // running on the false assumption of the old containsProperty + // return values fun eogConnect( q: Quantifier = Quantifier.ALL, cn: Connect = Connect.SUBTREE, en: Edge, n: Node?, cr: Connect = Connect.SUBTREE, - props: Map = mutableMapOf(), + predicate: ((EvaluationOrder) -> Boolean)? = null, refs: List ): Boolean { if (n == null) { @@ -103,17 +106,17 @@ object Util { val border = SubgraphWalker.getEOGPathEdges(n) if (en == Edge.ENTRIES) { val pe = border.entries.flatMap { it.prevEOGEdges } - if (Quantifier.ALL == q && pe.any { !it.containsProperties(props) }) + if (Quantifier.ALL == q && pe.any { predicate?.invoke(it) == false }) return false - pe.filter { it.containsProperties(props) }.map { it.start } + pe.filter { predicate?.invoke(it) != false }.map { it.start } } else border.exits } else { nodeSide.flatMap { if (en == Edge.ENTRIES) { val pe = it.prevEOGEdges - if (Quantifier.ALL == q && pe.any { !it.containsProperties(props) }) + if (Quantifier.ALL == q && pe.any { predicate?.invoke(it) == false }) return false - pe.filter { it.containsProperties(props) }.map { it.start } + pe.filter { predicate?.invoke(it) != false }.map { it.start } } else listOf(it) } } @@ -124,18 +127,18 @@ object Util { borders.flatMap { border -> if (Edge.ENTRIES == er) { val pe = border.entries.flatMap { it.prevEOGEdges } - if (Quantifier.ALL == q && pe.any { !it.containsProperties(props) }) + if (Quantifier.ALL == q && pe.any { predicate?.invoke(it) == false }) return false - pe.filter { it.containsProperties(props) }.map { it.start } + pe.filter { predicate?.invoke(it) != false }.map { it.start } } else border.exits } } else { refSide.flatMap { node -> if (er == Edge.ENTRIES) { val pe = node?.prevEOGEdges ?: listOf() - if (Quantifier.ALL == q && pe.any { !it.containsProperties(props) }) + if (Quantifier.ALL == q && pe.any { predicate?.invoke(it) == false }) return false - pe.filter { it.containsProperties(props) }.map { it.start } + pe.filter { predicate?.invoke(it) != false }.map { it.start } } else listOf(node) } } @@ -374,7 +377,9 @@ object Util { // Add an incoming DFG edge from a member call's base to the method's receiver if (target is MethodDeclaration && call is MemberCallExpression && !call.isStatic) { target.receiver?.let { receiver -> - call.base?.addNextDFG(receiver, callingContext = CallingContextIn(call)) + call.base + ?.nextDFGEdges + ?.addContextSensitive(receiver, callingContext = CallingContextIn(call)) } } @@ -390,12 +395,18 @@ object Util { if (param.isVariadic) { while (j < arguments.size) { // map all the following arguments to this variadic param - param.addPrevDFG(arguments[j], callingContext = CallingContextIn(call)) + param.prevDFGEdges.addContextSensitive( + arguments[j], + callingContext = CallingContextIn(call) + ) j++ } break } else { - param.addPrevDFG(arguments[j], callingContext = CallingContextIn(call)) + param.prevDFGEdges.addContextSensitive( + arguments[j], + callingContext = CallingContextIn(call) + ) } } j++ @@ -408,10 +419,12 @@ object Util { * @param target * @param arguments */ - fun detachCallParameters(target: FunctionDeclaration, arguments: List) { + fun detachCallParameters(target: FunctionDeclaration, arguments: List) { for (param in target.parameters) { // A param could be variadic, so multiple arguments could be set as incoming DFG - param.prevDFG.filter { o: Node? -> o in arguments }.forEach { param.removeNextDFG(it) } + param.prevDFGEdges + .filter { it.start in arguments } + .forEach { param.nextDFGEdges.remove(it) } } } @@ -423,7 +436,7 @@ object Util { * @return */ fun getAdjacentDFGNodes(n: Node?, incoming: Boolean): MutableList { - val subnodes = SubgraphWalker.getAstChildren(n) + val subnodes = n?.astChildren ?: listOf() val adjacentNodes = if (incoming) { subnodes.filter { it.nextDFG.contains(n) }.toMutableList() @@ -454,7 +467,7 @@ object Util { } else if (branchingDeclaration != null) { conditionNodes = getAdjacentDFGNodes(branchingDeclaration, true) } - conditionNodes.forEach { n.addPrevDFG(it) } + conditionNodes.forEach { n.prevDFGEdges += it } } enum class Connect { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt index 155fd316ac..03d55b1a09 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt @@ -121,7 +121,7 @@ fun applyTemplateInstantiation( // Template. for ((declaration) in initializationSignature) { if (declaration is ParameterDeclaration) { - initializationSignature[declaration]?.let { declaration.addPrevDFG(it) } + initializationSignature[declaration]?.let { declaration.prevDFGEdges += it } } } @@ -281,8 +281,8 @@ fun constructTemplateInitializationSignatureFromTemplateParameters( ): MutableMap? { val instantiationSignature: MutableMap = HashMap() for (i in functionTemplateDeclaration.parameters.indices) { - if (i < templateCall.templateParameters.size) { - val callParameter = templateCall.templateParameters[i] + if (i < templateCall.templateArguments.size) { + val callParameter = templateCall.templateArguments[i] val templateParameter = functionTemplateDeclaration.parameters[i] if (isInstantiated(callParameter, templateParameter)) { instantiationSignature[templateParameter] = callParameter diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlDependenceGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlDependenceGraphPass.kt index d64e7a134a..04c565681f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlDependenceGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlDependenceGraphPass.kt @@ -31,11 +31,10 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.allChildren import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.cyclomaticComplexity -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement -import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConditionalExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ShortCircuitOperator import de.fraunhofer.aisec.cpg.helpers.* @@ -60,7 +59,7 @@ open class ControlDependenceGraphPass(ctx: TranslationContext) : EOGStarterPass( } /** - * Computes the CDG for the given [functionDeclaration]. It performs the following steps: + * Computes the CDG for the given [startNode]. It performs the following steps: * 1) Compute the "parent branching node" for each node and through which path the node is * reached * 2) Find out which branch of a [BranchingNode] is actually conditional. The other ones aren't. @@ -165,8 +164,8 @@ open class ControlDependenceGraphPass(ctx: TranslationContext) : EOGStarterPass( } else -> { // Not sure what to do, there seems to be a cycle but this entry is - // not - // in finalDominators for some reason. Add to finalDominators now. + // not in finalDominators for some reason. Add to finalDominators + // now. finalDominators.add(Pair(newK, newV.toMutableSet())) } } @@ -183,30 +182,29 @@ open class ControlDependenceGraphPass(ctx: TranslationContext) : EOGStarterPass( finalDominators .filter { (k, _) -> k != node } .forEach { (k, v) -> - val properties = EnumMap(Properties::class.java) val branchesSet = k.nextEOGEdges .filter { edge -> edge.end in v } - .mapNotNull { it.getProperty(Properties.BRANCH) } + .mapNotNull { it.branch } .toSet() - when { - branchesSet.size == 1 -> { - properties[Properties.BRANCH] = branchesSet.single() - } - branchesSet.isNotEmpty() -> { - properties[Properties.BRANCH] = branchesSet - } - k is IfStatement && - (branchingNodeConditionals[k]?.size ?: 0) > - 1 -> { // Note: branchesSet must be empty here - // The if statement has only a then branch but there's a way to "jump - // out" of this branch. In this case, we want to set the false property - // here. - properties[Properties.BRANCH] = setOf(false) - } + node.prevCDGEdges.add(k) { + branches = + when { + branchesSet.isNotEmpty() -> { + branchesSet + } + k is IfStatement && + (branchingNodeConditionals[k]?.size ?: 0) > + 1 -> { // Note: branchesSet must be empty here The if + // statement has only a then branch but there's a way + // to "jump out" of this branch. In this case, we + // want to set the false property here. + setOf(false) + } + else -> setOf() + } } - node.addPrevCDG(k, properties) } } } @@ -226,27 +224,23 @@ open class ControlDependenceGraphPass(ctx: TranslationContext) : EOGStarterPass( Pair(functionDeclaration, setOf(functionDeclaration)), *functionDeclaration .allChildren() + .filterIsInstance() .map { branchingNode -> val mergingPoints = - if ( - (branchingNode as? Node)?.nextEOGEdges?.any { - !it.isConditionalBranch() - } == true - ) { + if (branchingNode.nextEOGEdges.any { !it.isConditionalBranch() }) { // There's an unconditional path (case 1), so when reaching this branch, // we're done. Collect all (=1) unconditional branches. - (branchingNode as? Node) - ?.nextEOGEdges - ?.filter { !it.isConditionalBranch() } - ?.map { it.end } - ?.toSet() + branchingNode.nextEOGEdges + .filter { !it.isConditionalBranch() } + .map { it.end } + .toSet() } else { // All branches are executed based on some condition (case 2), so we // collect all these branches. - (branchingNode as Node).nextEOGEdges.map { it.end }.toSet() + branchingNode.nextEOGEdges.map { it.end }.toSet() } // Map this branching node to its merging points - Pair(branchingNode as Node, mergingPoints) + Pair(branchingNode, mergingPoints) } .toTypedArray() ) @@ -265,7 +259,7 @@ open class ControlDependenceGraphPass(ctx: TranslationContext) : EOGStarterPass( * Returns the updated state and true because we always expect an update of the state. */ fun handleEdge( - currentEdge: PropertyEdge, + currentEdge: Edge, currentState: State>> ): State>> { // Check if we start in a branching node and if this edge leads to the conditional @@ -317,13 +311,13 @@ fun handleEdge( * change this if we do not want this behavior (just remove the condition on the start node of the * "false" branch). */ -private fun PropertyEdge.isConditionalBranch(): Boolean { - return if (this.getProperty(Properties.BRANCH) == true) { +private fun EvaluationOrder.isConditionalBranch(): Boolean { + return if (branch == true) { true } else (this.start is IfStatement || this.start is ConditionalExpression || - this.start is ShortCircuitOperator) && this.getProperty(Properties.BRANCH) == false || + this.start is ShortCircuitOperator) && branch == false || (this.start is IfStatement && !(this.start as IfStatement).allBranchesFromMyThenBranchGoThrough( (this.start as IfStatement).nextUnconditionalNode @@ -331,7 +325,7 @@ private fun PropertyEdge.isConditionalBranch(): Boolean { } private val IfStatement.nextUnconditionalNode: Node? - get() = this.nextEOGEdges.firstOrNull { it.getProperty(Properties.BRANCH) == null }?.end + get() = this.nextEOGEdges.firstOrNull { it.branch == null }?.end private fun IfStatement.allBranchesFromMyThenBranchGoThrough(node: Node?): Boolean { if (this.thenStatement.allChildren().isNotEmpty()) return false @@ -339,11 +333,7 @@ private fun IfStatement.allBranchesFromMyThenBranchGoThrough(node: Node?): Boole if (node == null) return true val alreadySeen = mutableSetOf() - val nextNodes = - this.nextEOGEdges - .filter { it.getProperty(Properties.BRANCH) == true } - .map { it.end } - .toMutableList() + val nextNodes = this.nextEOGEdges.filter { it.branch == true }.map { it.end }.toMutableList() while (nextNodes.isNotEmpty()) { val nextNode = nextNodes.removeFirst() @@ -397,7 +387,7 @@ class PrevEOGLattice(override val elements: IdentityHashMap>>() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index fc3f79082c..6883252120 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -28,11 +28,11 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.* -import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement -import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement -import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement -import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContext +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextOut +import de.fraunhofer.aisec.cpg.graph.edges.flows.partial +import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.* import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn @@ -112,12 +112,13 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // elements. We have the indices here, so it's amazingly easy to find the partial // target. key.elements.forEach { element -> - element.addAllPrevDFG( + element.prevDFGEdges.addAll( value.elements.filterNot { (it is VariableDeclaration || it is ParameterDeclaration) && key == it - }, + } + ) { granularity = partial(element) - ) + } } } else { value.elements.forEach { @@ -127,12 +128,12 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass Pair(it, key) in edgePropertiesMap && edgePropertiesMap[Pair(it, key)] is CallingContext ) { - key.addPrevDFG( + key.prevDFGEdges.addContextSensitive( it, - callingContext = (edgePropertiesMap[Pair(it, key)] as? CallingContext) + callingContext = (edgePropertiesMap[Pair(it, key)] as CallingContext) ) } else { - key.addPrevDFG(it) + key.prevDFGEdges += it } } } @@ -155,18 +156,18 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass */ protected fun clearFlowsOfVariableDeclarations(node: Node) { for (varDecl in node.variables /*.filter { it !is FieldDeclaration }*/) { - varDecl.clearPrevDFG() - varDecl.clearNextDFG() + varDecl.prevDFGEdges.clear() + varDecl.nextDFGEdges.clear() } val allChildrenOfFunction = node.allChildren() for (varDecl in node.parameters) { // Clear only prev and next inside this function! - varDecl.nextDFG - .filter { it in allChildrenOfFunction } - .forEach { varDecl.removeNextDFG(it) } - varDecl.prevDFG - .filter { it in allChildrenOfFunction } - .forEach { varDecl.removePrevDFG(it) } + varDecl.nextDFGEdges + .filter { it.end in allChildrenOfFunction } + .forEach { varDecl.nextDFGEdges.remove(it) } + varDecl.prevDFGEdges + .filter { it.start in allChildrenOfFunction } + .forEach { varDecl.prevDFGEdges.remove(it) } } } @@ -180,9 +181,9 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass * even if every path reaching this point already contains a return statement. */ protected open fun transfer( - currentEdge: PropertyEdge, + currentEdge: Edge, state: State>, - worklist: Worklist, Node, Set> + worklist: Worklist, Node, Set> ): State> { // We will set this if we write to a variable val writtenDeclaration: Declaration? @@ -399,7 +400,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass // with the old previousWrites map. val nodesOutsideTheLoop = currentNode.nextEOGEdges.filter { - it.getProperty(Properties.UNREACHABLE) != true && + it.unreachable != true && it.end != currentNode.statement && it.end !in currentNode.statement.allChildren() } @@ -518,7 +519,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass lastStatement.isImplicit && lastStatement !in reachableReturnStatements ) - lastStatement.removeNextDFG(node) + lastStatement.nextDFGEdges.remove(node) } /** @@ -530,7 +531,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass * A mapping of a [Node] to its [LatticeElement]. The keys of this state will later get the * DFG edges from the value! */ - var generalState: State = State(), + var generalState: State = State(), /** * It's main purpose is to store the most recent mapping of a [Declaration] to its * [LatticeElement]. However, it is also used to figure out if we have to continue with the @@ -541,19 +542,17 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : EOGStarterPass var declarationsState: State = State(), /** The [returnStatements] which are reachable. */ - var returnStatements: State = State() + var returnStatements: State = State() ) : State() { override fun duplicate(): DFGPassState { return DFGPassState(generalState.duplicate(), declarationsState.duplicate()) } - override fun get(key: de.fraunhofer.aisec.cpg.graph.Node): LatticeElement? { + override fun get(key: Node): LatticeElement? { return generalState[key] ?: declarationsState[key] } - override fun lub( - other: State - ): Pair, Boolean> { + override fun lub(other: State): Pair, Boolean> { return if (other is DFGPassState) { val (_, generalUpdate) = generalState.lub(other.generalState) val (_, declUpdate) = declarationsState.lub(other.declarationsState) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index c08d6263d8..6d731705ac 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -28,8 +28,8 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut -import de.fraunhofer.aisec.cpg.graph.edge.partial +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextOut +import de.fraunhofer.aisec.cpg.graph.edges.flows.partial import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.IterativeGraphWalker @@ -75,13 +75,17 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { if (param == (invoked as? MethodDeclaration)?.receiver) { (call as? MemberCallExpression) ?.base - ?.addPrevDFG(param, callingContext = CallingContextOut(call)) + ?.prevDFGEdges + ?.addContextSensitive(param, callingContext = CallingContextOut(call)) } else if (param is ParameterDeclaration) { val arg = call.arguments[param.argumentIndex] - arg.addPrevDFG(param, callingContext = CallingContextOut(call)) + arg.prevDFGEdges.addContextSensitive( + param, + callingContext = CallingContextOut(call) + ) (arg as? Reference)?.let { it.access = AccessValues.READWRITE - it.refersTo?.let { it1 -> it.addNextDFG(it1) } + it.refersTo?.let { it1 -> it.nextDFGEdges += it1 } } } } @@ -138,22 +142,22 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { // If this is a compound assign, we also need to model a dataflow to the node itself if (node.isCompoundAssignment) { node.lhs.firstOrNull()?.let { - node.addPrevDFG(it) - node.addNextDFG(it) + node.prevDFGEdges += it + node.nextDFGEdges += it } - node.rhs.firstOrNull()?.let { node.addPrevDFG(it) } + node.rhs.firstOrNull()?.let { node.prevDFGEdges += it } } else { // Find all targets of rhs and connect them node.rhs.forEach { val targets = node.findTargets(it) - targets.forEach { target -> it.addNextDFG(target) } + targets.forEach { target -> it.nextDFGEdges += target } } } // If the assignment is used as an expression, we also model a data flow from the (first) // rhs to the node itself if (node.usedAsExpression) { - node.expressionValue?.addNextDFG(node) + node.expressionValue?.nextDFGEdges += node } } @@ -164,15 +168,15 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { protected fun handleMemberExpression(node: MemberExpression) { when (node.access) { AccessValues.WRITE -> { - node.addNextDFG(node.base, granularity = partial(node.refersTo)) + node.nextDFGEdges.add(node.base) { granularity = partial(node.refersTo) } } AccessValues.READWRITE -> { - node.addNextDFG(node.base, granularity = partial(node.refersTo)) + node.nextDFGEdges.add(node.base) { granularity = partial(node.refersTo) } // We do not make an edge in the other direction on purpose as a workaround for // nested field accesses on the lhs of an assignment. } else -> { - node.addPrevDFG(node.base, granularity = partial(node.refersTo)) + node.prevDFGEdges.add(node.base) { granularity = partial(node.refersTo) } } } } @@ -184,10 +188,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { protected fun handleTupleDeclaration(node: TupleDeclaration) { node.initializer?.let { initializer -> node.elements.withIndex().forEach { - it.value.addPrevDFG( - initializer, - granularity = partial(it.value), - ) + it.value.prevDFGEdges.add(initializer) { granularity = partial(it.value) } } } } @@ -197,7 +198,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * variable. */ protected fun handleVariableDeclaration(node: VariableDeclaration) { - node.initializer?.let { node.addPrevDFG(it) } + node.initializer?.let { node.prevDFGEdges += it } } /** @@ -213,17 +214,16 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { if (!summaryExists) { // If the function is inferred, we connect all parameters to the function - // declaration. - // The condition should make sure that we don't add edges multiple times, i.e., we - // only handle the declaration exactly once. - node.addAllPrevDFG(node.parameters) + // declaration. The condition should make sure that we don't add edges multiple + // times, i.e., we only handle the declaration exactly once. + node.prevDFGEdges.addAll(node.parameters) // If it's a method with a receiver, we connect that one too. if (node is MethodDeclaration) { - node.receiver?.let { node.addPrevDFG(it) } + node.receiver?.let { node.prevDFGEdges += it } } } } else { - node.allChildren().forEach { node.addPrevDFG(it) } + node.allChildren().forEach { node.prevDFGEdges += it } } } @@ -231,7 +231,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * Adds the DFG edge for a [FieldDeclaration]. The data flows from the initializer to the field. */ protected fun handleFieldDeclaration(node: FieldDeclaration) { - node.initializer?.let { node.addPrevDFG(it) } + node.initializer?.let { node.prevDFGEdges += it } } /** @@ -239,7 +239,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * statement. */ protected fun handleReturnStatement(node: ReturnStatement) { - node.returnValues.forEach { node.addPrevDFG(it) } + node.returnValues.forEach { node.prevDFGEdges += it } } /** @@ -254,13 +254,13 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { node.iterable?.let { iterable -> if (node.variable is DeclarationStatement) { (node.variable as DeclarationStatement).declarations.forEach { - it.addPrevDFG(iterable) + it.prevDFGEdges += iterable } } else { - node.variable.variables.lastOrNull()?.addPrevDFG(iterable) + node.variable.variables.lastOrNull()?.prevDFGEdges += iterable } } - node.variable?.let { node.addPrevDFG(it) } + node.variable?.let { node.prevDFGEdges += it } } /** @@ -268,7 +268,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * dependence between data and the branching node. */ protected fun handleDoStatement(node: DoStatement) { - node.condition?.let { node.addPrevDFG(it) } + node.condition?.let { node.prevDFGEdges += it } } /** @@ -329,9 +329,9 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { */ protected fun handleUnaryOperator(node: UnaryOperator) { node.input.let { - node.addPrevDFG(it) + node.prevDFGEdges += it if (node.operatorCode == "++" || node.operatorCode == "--") { - node.addNextDFG(it) + node.nextDFGEdges += it } } } @@ -341,7 +341,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * lambda to the expression. */ protected fun handleLambdaExpression(node: LambdaExpression) { - node.function?.let { node.addPrevDFG(it) } + node.function?.let { node.prevDFGEdges += it } } /** @@ -349,7 +349,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * Check with python and JS implementation */ protected fun handleKeyValueExpression(node: KeyValueExpression) { - node.value?.let { node.addPrevDFG(it) } + node.value?.let { node.prevDFGEdges += it } } /** @@ -357,7 +357,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * this expression. */ protected fun handleInitializerListExpression(node: InitializerListExpression) { - node.initializers.forEach { node.addPrevDFG(it) } + node.initializers.forEach { node.prevDFGEdges += it } } /** @@ -365,7 +365,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * list. */ protected fun handleExpressionList(node: ExpressionList) { - node.expressions.lastOrNull()?.let { node.addPrevDFG(it) } + node.expressions.lastOrNull()?.let { node.prevDFGEdges += it } } /** @@ -373,7 +373,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * expression. */ protected fun handleNewExpression(node: NewExpression) { - node.initializer?.let { node.addPrevDFG(it) } + node.initializer?.let { node.prevDFGEdges += it } } /** @@ -385,11 +385,11 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { protected fun handleReference(node: Reference) { node.refersTo?.let { when (node.access) { - AccessValues.WRITE -> node.addNextDFG(it) - AccessValues.READ -> node.addPrevDFG(it) + AccessValues.WRITE -> node.nextDFGEdges += it + AccessValues.READ -> node.prevDFGEdges += it else -> { - node.addNextDFG(it) - node.addPrevDFG(it) + node.nextDFGEdges += it + node.prevDFGEdges += it } } } @@ -400,8 +400,8 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * expression to the whole expression. */ protected fun handleConditionalExpression(node: ConditionalExpression) { - node.thenExpression?.let { node.addPrevDFG(it) } - node.elseExpression?.let { node.addPrevDFG(it) } + node.thenExpression?.let { node.prevDFGEdges += it } + node.elseExpression?.let { node.prevDFGEdges += it } } /** @@ -409,12 +409,12 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * `x[i]`. */ protected fun handleSubscriptExpression(node: SubscriptExpression) { - node.addPrevDFG(node.arrayExpression) + node.prevDFGEdges += node.arrayExpression } /** Adds the DFG edge to an [NewArrayExpression]. The initializer flows to the expression. */ protected fun handleNewArrayExpression(node: NewArrayExpression) { - node.initializer?.let { node.addPrevDFG(it) } + node.initializer?.let { node.prevDFGEdges += it } } /** @@ -424,26 +424,26 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { protected fun handleBinaryOp(node: BinaryOperator, parent: Node?) { when (node.operatorCode) { "=" -> { - node.rhs.let { node.lhs.addPrevDFG(it) } + node.rhs.let { node.lhs.prevDFGEdges += it } // There are cases where we explicitly want to connect the rhs to the =. // E.g., this is the case in C++ where subexpressions can make the assignment. // Examples: a + (b = 1) or a = a == b ? b = 2: b = 3 // When the parent is a compound statement (or similar block of code), we can safely // assume that we're not in such a sub-expression if (parent == null || parent !is Block) { - node.rhs.addNextDFG(node) + node.rhs.nextDFGEdges += node } } in node.language?.compoundAssignmentOperators ?: setOf() -> { node.lhs.let { - node.addPrevDFG(it) - node.addNextDFG(it) + node.prevDFGEdges += it + node.nextDFGEdges += it } - node.rhs.let { node.addPrevDFG(it) } + node.rhs.let { node.prevDFGEdges += it } } else -> { - node.lhs.let { node.addPrevDFG(it) } - node.rhs.let { node.addPrevDFG(it) } + node.lhs.let { node.prevDFGEdges += it } + node.rhs.let { node.prevDFGEdges += it } } } } @@ -452,15 +452,14 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { * Adds the DFG edge to a [CastExpression]. The inner expression flows to the cast expression. */ protected fun handleCastExpression(castExpression: CastExpression) { - castExpression.expression.let { castExpression.addPrevDFG(it) } + castExpression.expression.let { castExpression.prevDFGEdges += it } } /** Adds the DFG edges to a [CallExpression]. */ fun handleCallExpression(call: CallExpression, inferDfgForUnresolvedSymbols: Boolean) { // Remove existing DFG edges since they are no longer valid (e.g. after updating the // CallExpression with the invokes edges to the called functions) - call.prevDFG.forEach { it.nextDFG.remove(call) } - call.prevDFG.clear() + call.prevDFGEdges.clear() if (call.invokes.isEmpty() && inferDfgForUnresolvedSymbols) { // Unresolved call expression @@ -468,7 +467,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { } else if (call.invokes.isNotEmpty()) { call.invokes.forEach { Util.attachCallParameters(it, call) - call.addPrevDFG(it, callingContext = CallingContextOut(call)) + call.prevDFGEdges.addContextSensitive(it, callingContext = CallingContextOut(call)) if (it.isInferred) { callsInferredFunctions.add(call) } @@ -483,9 +482,9 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { */ protected fun handleUnresolvedCalls(call: CallExpression, dfgTarget: Node) { if (call is MemberCallExpression && !call.isStatic) { - call.base?.let { dfgTarget.addPrevDFG(it) } + call.base?.let { dfgTarget.prevDFGEdges += it } } - call.arguments.forEach { dfgTarget.addPrevDFG(it) } + call.arguments.forEach { dfgTarget.prevDFGEdges += it } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DynamicInvokeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DynamicInvokeResolver.kt index aee325570f..d18696224d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DynamicInvokeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DynamicInvokeResolver.kt @@ -34,8 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.FullDataflowGranularity -import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType @@ -206,7 +205,7 @@ class DynamicInvokeResolver(ctx: TranslationContext) : ComponentPass(ctx) { } call.invokes = invocationCandidates - call.invokeEdges.forEach { it.addProperty(Properties.DYNAMIC_INVOKE, true) } + call.invokeEdges.forEach { it.dynamicInvoke = true } // We have to update the dfg edges because this call could now be resolved (which was not // the case before). diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt deleted file mode 100644 index 3d97fca01d..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EdgeCachePass.kt +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.passes - -import de.fraunhofer.aisec.cpg.TranslationContext -import de.fraunhofer.aisec.cpg.graph.Component -import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn -import de.fraunhofer.aisec.cpg.processing.IVisitor -import de.fraunhofer.aisec.cpg.processing.strategy.Strategy - -enum class EdgeType { - AST, - DFG, - EOG -} - -class Edge(val source: Node, val target: Node, val type: EdgeType) - -/** The edges cache. */ -object Edges { - private val fromMap: MutableMap> = HashMap() - private val toMap: MutableMap> = HashMap() - - fun add(edge: Edge) { - if (fromMap[edge.source] == null) { - fromMap[edge.source] = ArrayList() - } - - if (toMap[edge.target] == null) { - toMap[edge.target] = ArrayList() - } - - fromMap[edge.source]?.add(edge) - toMap[edge.target]?.add(edge) - } - - fun to(node: Node, type: EdgeType): List { - return toMap.computeIfAbsent(node) { mutableListOf() }.filter { it.type == type } - } - - fun from(node: Node, type: EdgeType): List { - return fromMap.computeIfAbsent(node) { mutableListOf() }.filter { it.type == type } - } - - fun size(): Int { - return fromMap.size - } - - fun clear() { - toMap.clear() - fromMap.clear() - } -} - -/** - * This pass creates a simple cache of commonly used edges, such as DFG or AST to quickly traverse - * them in different directions. - * - * The cache itself is stored in the [Edges] object. - */ -@DependsOn(EvaluationOrderGraphPass::class) -@DependsOn(SymbolResolver::class) -@DependsOn(DFGPass::class) -@DependsOn(DynamicInvokeResolver::class) -@DependsOn(ControlFlowSensitiveDFGPass::class) -class EdgeCachePass(ctx: TranslationContext) : ComponentPass(ctx) { - override fun accept(component: Component) { - Edges.clear() - - for (tu in component.translationUnits) { - tu.accept( - Strategy::AST_FORWARD, - object : IVisitor() { - override fun visit(t: Node) { - visitAST(t) - visitDFG(t) - visitEOG(t) - - super.visit(t) - } - } - ) - } - } - - protected fun visitAST(n: Node) { - for (node in SubgraphWalker.getAstChildren(n)) { - val edge = Edge(n, node, EdgeType.AST) - Edges.add(edge) - } - } - - protected fun visitDFG(n: Node) { - for (dfg in n.prevDFG) { - val edge = Edge(dfg, n, EdgeType.DFG) - Edges.add(edge) - } - - for (dfg in n.nextDFG) { - val edge = Edge(n, dfg, EdgeType.DFG) - Edges.add(edge) - } - } - - protected fun visitEOG(n: Node) { - for (eog in n.prevEOG) { - val edge = Edge(eog, n, EdgeType.EOG) - Edges.add(edge) - } - - for (eog in n.nextEOG) { - val edge = Edge(n, eog, EdgeType.EOG) - Edges.add(edge) - } - } - - override fun cleanup() { - // nothing to do - } -} - -val Node.astParent: Node? - get() { - return Edges.to(this, EdgeType.AST).firstOrNull()?.source - } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index 8541fd17f3..04bb795221 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -33,8 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.EOGStarterHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.StatementHolder import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder import de.fraunhofer.aisec.cpg.graph.scopes.* import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* @@ -42,7 +41,6 @@ import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.IdentitySet import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.Util -import de.fraunhofer.aisec.cpg.passes.configuration.ReplacePass import de.fraunhofer.aisec.cpg.tryCast import java.util.* import org.slf4j.LoggerFactory @@ -50,7 +48,7 @@ import org.slf4j.LoggerFactory /** * Creates an Evaluation Order Graph (EOG) based on AST. * - * An EOG is an intraprocedural directed graph whose vertices are executable AST nodes and edges + * An EOG is an intra-procedural directed graph whose vertices are executable AST nodes and edges * connect them in the order they would be executed when running the program. * * An EOG always starts at the header of a method/function and ends in one (virtual) or multiple @@ -73,9 +71,10 @@ import org.slf4j.LoggerFactory */ @Suppress("MemberVisibilityCanBePrivate") open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { + protected val map = mutableMapOf, (Node) -> Unit>() protected var currentPredecessors = mutableListOf() - protected val nextEdgeProperties = EnumMap(Properties::class.java) + protected var nextEdgeBranch: Boolean? = null /** * Allows to register EOG creation logic when a currently visited node can depend on future @@ -158,6 +157,9 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa map[TypeIdExpression::class.java] = { handleDefault(it) } map[Reference::class.java] = { handleDefault(it) } map[LambdaExpression::class.java] = { handleLambdaExpression(it as LambdaExpression) } + map[LookupScopeStatement::class.java] = { + handleLookupScopeStatement(it as LookupScopeStatement) + } } protected fun doNothing() { @@ -183,7 +185,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa val eogNodes = IdentitySet() eogNodes.addAll( SubgraphWalker.flattenAST(tu).filter { - it.prevEOG.isNotEmpty() || it.nextEOG.isNotEmpty() + it.prevEOGEdges.isNotEmpty() || it.nextEOGEdges.isNotEmpty() } ) // only eog entry points @@ -193,17 +195,18 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa val alreadySeen = IdentitySet() while (validStarts.isNotEmpty()) { eogNodes.removeAll(validStarts) - validStarts = validStarts.flatMap { it.nextEOG }.filter { it !in alreadySeen }.toSet() + validStarts = + validStarts + .flatMap { it.nextEOGEdges } + .filter { it.end !in alreadySeen } + .map { it.end } + .toSet() alreadySeen.addAll(validStarts) } // The remaining nodes are unreachable from the entry points. We delete their outgoing EOG // edges. for (unvisitedNode in eogNodes) { - unvisitedNode.nextEOGEdges.forEach { next -> - next.end.removePrevEOGEntry(unvisitedNode) - } - - unvisitedNode.clearNextEOG() + unvisitedNode.nextEOGEdges.clear() } } @@ -282,20 +285,20 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa protected fun handleLambdaExpression(node: LambdaExpression) { val tmpCurrentEOG = currentPredecessors.toMutableList() - val tmpCurrentProperties = nextEdgeProperties.toMutableMap() + val tmpCurrentProperties = nextEdgeBranch val tmpIntermediateNodes = intermediateNodes.toMutableList() - nextEdgeProperties.clear() + nextEdgeBranch = null currentPredecessors.clear() intermediateNodes.clear() createEOG(node.function) - nextEdgeProperties.clear() + nextEdgeBranch = null currentPredecessors.clear() intermediateNodes.clear() - nextEdgeProperties.putAll(tmpCurrentProperties) + nextEdgeBranch = tmpCurrentProperties currentPredecessors.addAll(tmpCurrentEOG) intermediateNodes.addAll(tmpIntermediateNodes) @@ -450,7 +453,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // existing function and the EOG handler for handling function declarations will // reset the // stack - val oldEOG = ArrayList(currentPredecessors) + val oldEOG = currentPredecessors.toMutableList() // analyze the defaults createEOG(declaration) @@ -493,14 +496,12 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // present. // If it is not a conjunctive operator, the check above implies it is a disjunctive // operator. - nextEdgeProperties[Properties.BRANCH] = - lang.conjunctiveOperators.contains(node.operatorCode) + nextEdgeBranch = lang.conjunctiveOperators.contains(node.operatorCode) createEOG(node.rhs) pushToEOG(node) setCurrentEOGs(shortCircuitNodes) // Inverted property to assigne false when true was assigned above. - nextEdgeProperties[Properties.BRANCH] = - !lang.conjunctiveOperators.contains(node.operatorCode) + nextEdgeBranch = !lang.conjunctiveOperators.contains(node.operatorCode) } else { createEOG(node.rhs) } @@ -561,9 +562,9 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa val throwType = input.type pushToEOG(node) if (catchingScope is TryScope) { - catchingScope.catchesOrRelays[throwType] = ArrayList(currentPredecessors) + catchingScope.catchesOrRelays[throwType] = currentPredecessors.toMutableList() } else if (catchingScope is FunctionScope) { - catchingScope.catchesOrRelays[throwType] = ArrayList(currentPredecessors) + catchingScope.catchesOrRelays[throwType] = currentPredecessors.toMutableList() } currentPredecessors.clear() } @@ -583,7 +584,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa protected fun handleAssertStatement(node: AssertStatement) { createEOG(node.condition) - val openConditionEOGs = ArrayList(currentPredecessors) + val openConditionEOGs = currentPredecessors.toMutableList() createEOG(node.message) setCurrentEOGs(openConditionEOGs) pushToEOG(node) @@ -600,7 +601,8 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa node.resources.forEach { createEOG(it) } createEOG(node.tryBlock) - val tmpEOGNodes = ArrayList(currentPredecessors) + val tmpEOGNodes = currentPredecessors.toMutableList() + val catchEnds = mutableListOf() val catchesOrRelays = tryScope?.catchesOrRelays for (catchClause in node.catchClauses) { currentPredecessors.clear() @@ -619,8 +621,21 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa toRemove.forEach { catchesOrRelays?.remove(it) } pushToEOG(catchClause) createEOG(catchClause.body) + catchEnds.addAll(currentPredecessors) + } + + // We need to handle the else block after the catch clauses, as the else could contain a + // throw itself + // that should not be caught be the catch clauses. + if (node.elseBlock != null) { + currentPredecessors.clear() + currentPredecessors.addAll(tmpEOGNodes) + createEOG(node.elseBlock) + // All valid try ends got through the else block. + tmpEOGNodes.clear() tmpEOGNodes.addAll(currentPredecessors) } + tmpEOGNodes.addAll(catchEnds) val canTerminateExceptionfree = tmpEOGNodes.any { reachableFromValidEOGRoot(it) } currentPredecessors.clear() @@ -652,7 +667,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa if (outerScope is TryScope) outerScope.catchesOrRelays else (outerScope as FunctionScope).catchesOrRelays for ((key, value) in catchesOrRelays ?: mapOf()) { - val catches = outerCatchesOrRelays[key] ?: ArrayList() + val catches = outerCatchesOrRelays[key] ?: mutableListOf() catches.addAll(value) outerCatchesOrRelays[key] = catches } @@ -672,7 +687,9 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } protected fun handleDeleteExpression(node: DeleteExpression) { - createEOG(node.operand) + for (operand in node.operands) { + createEOG(operand) + } pushToEOG(node) } @@ -690,9 +707,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa protected fun handleGotoStatement(node: GotoStatement) { pushToEOG(node) node.targetLabel?.let { - processedListener.registerObjectListener(it) { _: Any?, to: Any? -> - addEOGEdge(node, to as Node) - } + processedListener.registerObjectListener(it) { _, to -> addEOGEdge(node, to) } } currentPredecessors.clear() } @@ -744,20 +759,20 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // Generate the EOG inside the anonymous class. It's not linked to the EOG of the outer // part. val tmpCurrentEOG = currentPredecessors.toMutableList() - val tmpCurrentProperties = nextEdgeProperties.toMutableMap() + val tmpCurrentProperties = nextEdgeBranch val tmpIntermediateNodes = intermediateNodes.toMutableList() - nextEdgeProperties.clear() + nextEdgeBranch = null currentPredecessors.clear() intermediateNodes.clear() createEOG(node.anonymousClass) - nextEdgeProperties.clear() + nextEdgeBranch = null currentPredecessors.clear() intermediateNodes.clear() - nextEdgeProperties.putAll(tmpCurrentProperties) + nextEdgeBranch = tmpCurrentProperties currentPredecessors.addAll(tmpCurrentEOG) intermediateNodes.addAll(tmpIntermediateNodes) } @@ -776,13 +791,13 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa addMultipleIncomingEOGEdges(currentPredecessors, node) intermediateNodes.clear() currentPredecessors.clear() - nextEdgeProperties.clear() + nextEdgeBranch = null currentPredecessors.add(node) } fun setCurrentEOGs(nodes: List) { LOGGER.trace("Setting {} to EOGs", nodes) - currentPredecessors = ArrayList(nodes) + currentPredecessors = nodes.toMutableList() } /** @@ -795,7 +810,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // Breaks are connected to the NEXT EOG node and therefore temporarily stored after the loop // context is destroyed currentPredecessors.addAll(loopScope.breakStatements) - val continues = ArrayList(loopScope.continueStatements) + val continues = loopScope.continueStatements.toMutableList() if (continues.isNotEmpty()) { val conditions = loopScope.conditions.map { SubgraphWalker.getEOGPathEdges(it).entries }.flatten() @@ -826,12 +841,10 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa * @param next the next node */ protected fun addEOGEdge(prev: Node, next: Node) { - val propertyEdge = PropertyEdge(prev, next) - propertyEdge.addProperties(nextEdgeProperties) - propertyEdge.addProperty(Properties.INDEX, prev.nextEOG.size) - propertyEdge.addProperty(Properties.UNREACHABLE, false) - prev.addNextEOG(propertyEdge) - next.addPrevEOG(propertyEdge) + val propertyEdge = EvaluationOrder(prev, next, unreachable = false) + propertyEdge.branch = nextEdgeBranch + + prev.nextEOGEdges += propertyEdge } protected fun addMultipleIncomingEOGEdges(prevs: List, next: Node) { @@ -849,12 +862,12 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.condition) // To have semantic information after the condition evaluation pushToEOG(node) - val openConditionEOGs = ArrayList(currentPredecessors) - nextEdgeProperties[Properties.BRANCH] = true + val openConditionEOGs = currentPredecessors.toMutableList() + nextEdgeBranch = true createEOG(node.thenExpression) openBranchNodes.addAll(currentPredecessors) setCurrentEOGs(openConditionEOGs) - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false createEOG(node.elseExpression) openBranchNodes.addAll(currentPredecessors) setCurrentEOGs(openBranchNodes) @@ -864,11 +877,12 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa scopeManager.enterScope(node) createEOG(node.statement) createEOG(node.condition) - node.condition?.let { node.addPrevDFG(it) } + // TODO(oxisto): Do we really want to set DFG edges here? + node.condition?.let { node.prevDFGEdges += it } pushToEOG(node) // To have semantic information after the condition evaluation - nextEdgeProperties[Properties.BRANCH] = true + nextEdgeBranch = true connectCurrentToLoopStart() - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false val currentLoopScope = scopeManager.leaveScope(node) as LoopScope? if (currentLoopScope != null) { exitLoop(currentLoopScope) @@ -881,10 +895,11 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa scopeManager.enterScope(node) createEOG(node.iterable) createEOG(node.variable) - node.variable?.let { node.addPrevDFG(it) } + // TODO(oxisto): Do we really want to set DFG edges here? + node.variable?.let { node.prevDFGEdges += it } pushToEOG(node) // To have semantic information after the variable declaration - nextEdgeProperties[Properties.BRANCH] = true - val tmpEOGNodes = ArrayList(currentPredecessors) + nextEdgeBranch = true + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) connectCurrentToLoopStart() currentPredecessors.clear() @@ -895,7 +910,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa LOGGER.error("Trying to exit foreach loop, but not in loop scope: $node") } currentPredecessors.addAll(tmpEOGNodes) - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false } protected fun handleForStatement(node: ForStatement) { @@ -905,8 +920,8 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.condition) pushToEOG(node) // To have semantic information after the condition evaluation - nextEdgeProperties[Properties.BRANCH] = true - val tmpEOGNodes = ArrayList(currentPredecessors) + nextEdgeBranch = true + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) createEOG(node.iterationStatement) @@ -921,7 +936,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa LOGGER.error("Trying to exit for loop, but no loop scope: $node") } currentPredecessors.addAll(tmpEOGNodes) - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false } protected fun handleIfStatement(node: IfStatement) { @@ -931,13 +946,13 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.conditionDeclaration) createEOG(node.condition) pushToEOG(node) // To have semantic information after the condition evaluation - val openConditionEOGs = ArrayList(currentPredecessors) - nextEdgeProperties[Properties.BRANCH] = true + val openConditionEOGs = currentPredecessors.toMutableList() + nextEdgeBranch = true createEOG(node.thenStatement) openBranchNodes.addAll(currentPredecessors) if (node.elseStatement != null) { setCurrentEOGs(openConditionEOGs) - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false createEOG(node.elseStatement) openBranchNodes.addAll(currentPredecessors) } else { @@ -953,7 +968,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.selectorDeclaration) createEOG(node.selector) pushToEOG(node) // To have semantic information after the condition evaluation - val tmp = ArrayList(currentPredecessors) + val tmp = currentPredecessors.toMutableList() val compound = if (node.statement is DoStatement) { createEOG(node.statement) @@ -961,7 +976,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } else { node.statement as Block } - currentPredecessors = ArrayList() + currentPredecessors = mutableListOf() for (subStatement in compound.statements) { if (subStatement is CaseStatement || subStatement is DefaultStatement) { currentPredecessors.addAll(tmp) @@ -990,8 +1005,8 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.conditionDeclaration) createEOG(node.condition) pushToEOG(node) // To have semantic information after the condition evaluation - nextEdgeProperties[Properties.BRANCH] = true - val tmpEOGNodes = ArrayList(currentPredecessors) + nextEdgeBranch = true + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) connectCurrentToLoopStart() @@ -1004,7 +1019,13 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa LOGGER.error("Trying to exit while loop, but no loop scope: $node") } currentPredecessors.addAll(tmpEOGNodes) - nextEdgeProperties[Properties.BRANCH] = false + nextEdgeBranch = false + } + + private fun handleLookupScopeStatement(stmt: LookupScopeStatement) { + // Include the node as part of the EOG itself, but we do not need to go into any children or + // properties here + pushToEOG(stmt) } companion object { @@ -1022,7 +1043,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa */ protected fun reachableFromValidEOGRoot(node: Node): Boolean { val passedBy = mutableSetOf() - val workList = ArrayList(node.prevEOG) + val workList = node.prevEOG.toMutableList() while (workList.isNotEmpty()) { val toProcess = workList[0] workList.remove(toProcess) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt index eaa0b92981..5227afa584 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt @@ -58,7 +58,7 @@ class ImportResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Let's do some importing. We need to import either a wildcard if (node.wildcardImport) { - val list = scopeManager.findSymbols(node.import, node.location, scope) + val list = scopeManager.lookupSymbolByName(node.import, node.location, scope) val symbol = list.singleOrNull() if (symbol != null) { // In this case, the symbol must point to a name scope @@ -69,7 +69,8 @@ class ImportResolver(ctx: TranslationContext) : ComponentPass(ctx) { } } else { // or a symbol directly - val list = scopeManager.findSymbols(node.import, node.location, scope).toMutableList() + val list = + scopeManager.lookupSymbolByName(node.import, node.location, scope).toMutableList() node.importedSymbols = mutableMapOf(node.symbol to list) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt index 2536b4c64b..ee91444425 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt @@ -35,6 +35,11 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker +import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteFirst +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLast +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait import java.util.concurrent.CompletableFuture @@ -42,8 +47,10 @@ import java.util.function.Consumer import kotlin.reflect.KClass import kotlin.reflect.full.findAnnotation import kotlin.reflect.full.findAnnotations +import kotlin.reflect.full.hasAnnotation import kotlin.reflect.full.isSubclassOf import kotlin.reflect.full.primaryConstructor +import org.apache.commons.lang3.builder.ToStringBuilder import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -153,6 +160,47 @@ sealed class Pass(final override val ctx: TranslationContext) : fun passConfig(): T? { return this.config.passConfigurations[this::class] as? T } + + override fun toString(): String { + val builder = + ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", this::class.simpleName) + + if (this::class.softDependencies.isNotEmpty()) { + builder.append("soft dependencies:", this::class.softDependencies.map { it.simpleName }) + } + + if (this::class.hardDependencies.isNotEmpty()) { + builder.append("hard dependencies:", this::class.hardDependencies.map { it.simpleName }) + } + + if (this::class.softExecuteBefore.isNotEmpty()) { + builder.append( + "execute before (soft): ", + this::class.softExecuteBefore.map { it.simpleName } + ) + } + + if (this::class.hardExecuteBefore.isNotEmpty()) { + builder.append( + "execute before (hard): ", + this::class.hardExecuteBefore.map { it.simpleName } + ) + } + + if (this::class.isFirstPass) { + builder.append("firstPass") + } + + if (this::class.isLastPass) { + builder.append("lastPass") + } + + if (this::class.isLatePass) { + builder.append("latePass") + } + + return builder.toString() + } } fun executePassesInParallel( @@ -318,3 +366,50 @@ fun checkForReplacement( @Suppress("UNCHECKED_CAST") return config.replacedPasses[Pair(cls, language::class)] as? KClass> ?: cls } + +val KClass>.isFirstPass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.isLastPass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.isLatePass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.softDependencies: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == true } + .map { it.value } + .toSet() + } + +val KClass>.hardDependencies: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == false } + .map { it.value } + .toSet() + } + +val KClass>.softExecuteBefore: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == true } + .map { it.other } + .toSet() + } + +val KClass>.hardExecuteBefore: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == false } + .map { it.other } + .toSet() + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt index f61a26239a..7405973bd6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt @@ -31,12 +31,12 @@ import de.fraunhofer.aisec.cpg.graph.allChildren import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate import kotlin.reflect.full.memberProperties import kotlin.reflect.jvm.javaField /** Pass with some graph transformations useful when doing serialization. */ -@ExecuteBefore(FilenameMapper::class) +@ExecuteLate class PrepareSerialization(ctx: TranslationContext) : TranslationUnitPass(ctx) { private val nodeNameField = Node::class diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPass.kt index c2ca1cafdb..a848bd51b5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPass.kt @@ -29,8 +29,8 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.DependenceType -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn @@ -44,6 +44,7 @@ import de.fraunhofer.aisec.cpg.processing.strategy.Strategy @DependsOn(ControlDependenceGraphPass::class) @DependsOn(DFGPass::class) @DependsOn(ControlFlowSensitiveDFGPass::class, softDependency = true) +@DependsOn(DynamicInvokeResolver::class) class ProgramDependenceGraphPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { private val visitor = object : IVisitor() { @@ -55,7 +56,7 @@ class ProgramDependenceGraphPass(ctx: TranslationContext) : TranslationUnitPass( if (t is Reference) { // We filter all prevDFGEdges if the condition affects the variable of t and // if there's a flow from the prevDFGEdge's node through the condition to t. - val prevDFGToConsider = mutableListOf>() + val prevDFGToConsider = mutableListOf() t.prevDFGEdges.forEach { prevDfgEdge -> val prevDfgNode = prevDfgEdge.start // The prevDfgNode also flows into the condition. This is suspicious because @@ -86,11 +87,18 @@ class ProgramDependenceGraphPass(ctx: TranslationContext) : TranslationUnitPass( prevDFGToConsider.add(prevDfgEdge) } } - t.addAllPrevPDGEdges(prevDFGToConsider, DependenceType.DATA) - t.addAllPrevPDGEdges(t.prevCDGEdges, DependenceType.CONTROL) + + prevDFGToConsider.forEach { + it.dependence = DependenceType.DATA + t.prevPDGEdges.add(it) + } + t.prevPDGEdges += t.prevCDGEdges } else { - t.addAllPrevPDGEdges(t.prevDFGEdges, DependenceType.DATA) - t.addAllPrevPDGEdges(t.prevCDGEdges, DependenceType.CONTROL) + t.prevDFGEdges.forEach { + it.dependence = DependenceType.DATA + t.prevPDGEdges.add(it) + } + t.prevPDGEdges += t.prevCDGEdges } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ReplaceCallCastPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt similarity index 53% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ReplaceCallCastPass.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt index 2ee116fbec..42f5ec7c8b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ReplaceCallCastPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt @@ -27,30 +27,35 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.Handler -import de.fraunhofer.aisec.cpg.frontends.HasFunctionalCasts +import de.fraunhofer.aisec.cpg.frontends.HasCallExpressionAmbiguity +import de.fraunhofer.aisec.cpg.frontends.HasFunctionStyleCasts +import de.fraunhofer.aisec.cpg.frontends.HasFunctionStyleConstruction import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.helpers.replace import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait /** - * If a [Language] has the trait [HasFunctionalCasts], we cannot distinguish between a - * [CallExpression] and a [CastExpression] during the initial translation. This stems from the fact - * that we might not know all the types yet. We therefore need to handle them as regular call - * expression in a [LanguageFrontend] or [Handler] and then later replace them with a - * [CastExpression], if the [CallExpression.callee] refers to name of a [Type] rather than a - * function. + * If a [Language] has the trait [HasCallExpressionAmbiguity], we cannot distinguish between + * [CallExpression], [CastExpression] or [ConstructExpression] during the initial translation. This + * stems from the fact that we might not know all the types yet. We therefore need to handle them as + * regular call expression in a [LanguageFrontend] or [Handler] and then later replace them with a + * [CastExpression] or [ConstructExpression], if the [CallExpression.callee] refers to name of a + * [Type] / [RecordDeclaration] rather than a function. */ @ExecuteBefore(EvaluationOrderGraphPass::class) @DependsOn(TypeResolver::class) -@RequiresLanguageTrait(HasFunctionalCasts::class) -class ReplaceCallCastPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { +@RequiresLanguageTrait(HasCallExpressionAmbiguity::class) +class ResolveCallExpressionAmbiguityPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { private lateinit var walker: SubgraphWalker.ScopedWalker override fun accept(tu: TranslationUnitDeclaration) { @@ -71,13 +76,44 @@ class ReplaceCallCastPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { return } + // We really need a parent, otherwise we cannot replace the node + if (parent == null) { + return + } + + // Some local copies for easier smart casting + var callee = call.callee + val language = callee.language + + // Check, if this is cast is really a construct expression (if the language supports + // functional-constructs) + if (language is HasFunctionStyleConstruction) { + // Make sure, we do not accidentally "construct" primitive types + if (language.builtInTypes.contains(callee.name.toString()) == true) { + return + } + + val fqn = + if (callee.name.parent == null) { + scopeManager.currentNamespace.fqn( + callee.name.localName, + delimiter = callee.name.delimiter + ) + } else { + callee.name + } + + // Check for our type. We are only interested in object types + val type = typeManager.lookupResolvedType(fqn) + if (type is ObjectType) { + walker.replaceCallWithConstruct(type, parent, call) + } + } + // We need to check, whether the "callee" refers to a type and if yes, convert it into a // cast expression. And this is only really necessary, if the function call has a single // argument. - var callee = call.callee - if (parent != null && callee != null && call.arguments.size == 1) { - val language = parent.language - + if (language is HasFunctionStyleCasts && call.arguments.size == 1) { var pointer = false // If the argument is a UnaryOperator, unwrap them if (callee is UnaryOperator && callee.operatorCode == "*") { @@ -86,20 +122,25 @@ class ReplaceCallCastPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { } // First, check if this is a built-in type - if (language?.builtInTypes?.contains(callee.name.toString()) == true) { - walker.replaceCallWithCast(callee.name.toString(), parent, call, false) + var builtInType = language.getSimpleTypeOf(callee.name) + if (builtInType != null) { + walker.replaceCallWithCast(builtInType, parent, call, false) } else { // If not, then this could still refer to an existing type. We need to make sure // that we take the current namespace into account val fqn = if (callee.name.parent == null) { - scopeManager.currentNamespace.fqn(callee.name.localName) + scopeManager.currentNamespace.fqn( + callee.name.localName, + delimiter = callee.name.delimiter + ) } else { callee.name } - if (typeManager.typeExists(fqn.toString())) { - walker.replaceCallWithCast(fqn, parent, call, pointer) + val type = typeManager.lookupResolvedType(fqn) + if (type != null) { + walker.replaceCallWithCast(type, parent, call, pointer) } } } @@ -112,7 +153,7 @@ class ReplaceCallCastPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { context(ContextProvider) fun SubgraphWalker.ScopedWalker.replaceCallWithCast( - typeName: CharSequence, + type: Type, parent: Node, call: CallExpression, pointer: Boolean, @@ -123,34 +164,30 @@ fun SubgraphWalker.ScopedWalker.replaceCallWithCast( cast.location = call.location cast.castType = if (pointer) { - call.objectType(typeName).pointer() + type.pointer() } else { - call.objectType(typeName) + type } cast.expression = call.arguments.single() cast.name = cast.castType.name - replaceArgument(parent, call, cast) + replace(parent, call, cast) } context(ContextProvider) -fun SubgraphWalker.ScopedWalker.replaceArgument(parent: Node?, old: Expression, new: Expression) { - if (parent !is ArgumentHolder) { - Pass.log.error( - "Parent AST node of call expression is not an argument holder. Cannot convert to cast expression. Further analysis might not be entirely accurate." - ) - return - } - - val success = parent.replaceArgument(old, new) - if (!success) { - Pass.log.error( - "Replacing expression $old was not successful. Further analysis might not be entirely accurate." - ) - } else { - old.disconnectFromGraph() +fun SubgraphWalker.ScopedWalker.replaceCallWithConstruct( + type: ObjectType, + parent: Node, + call: CallExpression +) { + val construct = newConstructExpression() + construct.code = call.code + construct.language = call.language + construct.location = call.location + construct.callee = call.callee + (construct.callee as? Reference)?.resolutionHelper = construct + construct.arguments = call.arguments + construct.type = type - // Make sure to inform the walker about our change - this.registerReplacement(old, new) - } + replace(parent, call, construct) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index 1edce6cb03..c1572a84dd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -32,11 +32,12 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.NameScope import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope -import de.fraunhofer.aisec.cpg.graph.scopes.StructureDeclarationScope +import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.helpers.replace import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.inference.Inference.TypeInferenceObserver import de.fraunhofer.aisec.cpg.passes.inference.inferFunction @@ -104,7 +105,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { currentTU = tu // Gather all resolution EOG starters; and make sure they really do not have a // predecessor, otherwise we might analyze a node multiple times - val nodes = tu.allEOGStarters.filter { it.prevEOG.isEmpty() } + val nodes = tu.allEOGStarters.filter { it.prevEOGEdges.isEmpty() } for (node in nodes) { walker.iterate(node) @@ -142,7 +143,6 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { if (target == null) { // Determine the scope where we want to start our inference var (scope, _) = scopeManager.extractScope(reference) - if (scope !is NameScope) { scope = null } @@ -178,7 +178,12 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Find a list of candidate symbols. Currently, this is only used the in the "next-gen" call // resolution, but in future this will also be used in resolving regular references. - current.candidates = scopeManager.findSymbols(current.name, current.location).toSet() + current.candidates = scopeManager.lookupSymbolByName(current.name, current.location).toSet() + + // Preparation for a future without legacy call resolving. Taking the first candidate is not + // ideal since we are running into an issue with function pointers here (see workaround + // below). + var wouldResolveTo = current.candidates.singleOrNull() // For now, we need to ignore reference expressions that are directly embedded into call // expressions, because they are the "callee" property. In the future, we will use this @@ -189,21 +194,38 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // of this call expression back to its original variable declaration. In the future, we want // to extend this particular code to resolve all callee references to their declarations, // i.e., their function definitions and get rid of the separate CallResolver. - var wouldResolveTo: Declaration? = null if (current.resolutionHelper is CallExpression) { // Peek into the declaration, and if it is only one declaration and a variable, we can // proceed normally, as we are running into the special case explained above. Otherwise, // we abort here (for now). - wouldResolveTo = current.candidates.singleOrNull() if (wouldResolveTo !is VariableDeclaration && wouldResolveTo !is ParameterDeclaration) { return } } + // Some stupid C++ workaround to use the legacy call resolver when we try to resolve targets + // for function pointers. At least we are only invoking the legacy resolver for a very small + // percentage of references now. + if (wouldResolveTo is FunctionDeclaration) { + // We need to invoke the legacy resolver, just to be sure + var legacy = scopeManager.resolveReference(current) + + // This is just for us to catch these differences in symbol resolving in the future. The + // difference is pretty much only that the legacy system takes parameters of the + // function-pointer-type into account and the new system does not (yet), because it just + // takes the first match. This will be needed to solve in the future. + if (legacy != wouldResolveTo) { + log.warn( + "The legacy symbol resolution and the new system produced different results here. This needs to be investigated in the future. For now, we take the legacy result." + ) + wouldResolveTo = legacy + } + } + // Only consider resolving, if the language frontend did not specify a resolution. If we // already have populated the wouldResolveTo variable, we can re-use this instead of // resolving again - var refersTo = current.refersTo ?: wouldResolveTo ?: scopeManager.resolveReference(current) + var refersTo = current.refersTo ?: wouldResolveTo var recordDeclType: Type? = null if (currentClass != null) { @@ -292,9 +314,25 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { } } - protected fun handleMemberExpression(curClass: RecordDeclaration?, current: Node?) { - if (current !is MemberExpression) { - return + protected fun handleMemberExpression(curClass: RecordDeclaration?, current: MemberExpression) { + // Some locals for easier smart casting + var base = current.base + var language = current.language + + // We need to adjust certain types of the base in case of a "super" expression, and we + // delegate this to the language. If that is successful, we can continue with regular + // resolving. + if ( + language is HasSuperClasses && + curClass != null && + base is Reference && + base.name.endsWith(language.superClassKeyword) + ) { + language.handleSuperExpression( + current, + curClass, + scopeManager, + ) } // For legacy reasons, method and field resolving is split between the VariableUsageResolver @@ -307,57 +345,17 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { return } - var baseTarget: Declaration? = null - if (current.base is Reference) { - val base = current.base as Reference - if ( - current.language is HasSuperClasses && - base.name.toString() == (current.language as HasSuperClasses).superClassKeyword - ) { - if (curClass != null && curClass.superClasses.isNotEmpty()) { - val superType = curClass.superClasses[0] - val superRecord = superType.recordDeclaration - if (superRecord == null) { - log.error( - "Could not find referring super type ${superType.typeName} for ${curClass.name} in the record map. Will set the super type to java.lang.Object" - ) - // TODO: Should be more generic! - base.type = current.objectType(Any::class.java.name) - } else { - // We need to connect this super reference to the receiver of this - // method - val func = scopeManager.currentFunction - if (func is MethodDeclaration) { - baseTarget = func.receiver - } - if (baseTarget != null) { - base.refersTo = baseTarget - // Explicitly set the type of the call's base to the super type - base.type = superType - (base.refersTo as? HasType)?.type = superType - // And set the assigned subtypes, to ensure, that really only our - // super type is in there - base.assignedTypes = mutableSetOf(superType) - (base.refersTo as? ValueDeclaration)?.assignedTypes = - mutableSetOf(superType) - } - } - } else { - // no explicit super type -> java.lang.Object - // TODO: Should be more generic - val objectType = current.objectType(Any::class.java.name) - base.type = objectType - } - } else { - // The base should have been resolved by now. Maybe we have some other clue about - // this base from the type system, so we can set the declaration accordingly. - if (base.refersTo == null) { - base.refersTo = base.type.recordDeclaration - } + if (base is Reference) { + // The base has been resolved by now. Maybe we have some other clue about + // this base from the type system, so we can set the declaration accordingly. + // TODO(oxisto): It is actually not really a good approach, but it is currently + // needed to make the java frontend happy, but this needs to be removed at some point + if (base.refersTo == null) { + base.refersTo = base.type.recordDeclaration } } - val baseType = current.base.type.root + val baseType = base.type.root current.refersTo = resolveMember(baseType, current) } @@ -368,7 +366,29 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { return null } var member: ValueDeclaration? = null - val record = containingClass.recordDeclaration + var type = containingClass + + // Check for a possible overloaded operator-> + if ( + reference.language is HasOperatorOverloading && + reference is MemberExpression && + reference.operatorCode == "->" && + reference.base.type !is PointerType + ) { + val result = resolveOperator(reference) + val op = result?.bestViable?.singleOrNull() + if (result?.success == SUCCESSFUL && op is OperatorDeclaration) { + type = op.returnTypes.singleOrNull()?.root ?: unknownType() + + // We need to insert a new operator call expression in between + val call = operatorCallFromDeclaration(op, reference) + + // Make the call our new base + reference.base = call + } + } + + val record = type.recordDeclaration if (record != null) { member = record.fields @@ -378,8 +398,8 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { } if (member == null) { member = - (containingClass.recordDeclaration?.superTypeDeclarations?.flatMap { it.fields } - ?: listOf()) + type.superTypes + .flatMap { it.recordDeclaration?.fields ?: listOf() } .filter { it.name.localName == reference.name.localName } .map { it.definition } .firstOrNull() @@ -472,286 +492,211 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { is MemberExpression -> handleMemberExpression(currClass, node) is Reference -> handleReference(currClass, node) is ConstructExpression -> handleConstructExpression(node) - is CallExpression -> handleCallExpression(scopeManager.currentRecord, node) + is CallExpression -> handleCallExpression(node) + is HasOverloadedOperation -> handleOverloadedOperator(node) } } - protected fun handleCallExpression(curClass: RecordDeclaration?, call: CallExpression) { + protected fun handleCallExpression(call: CallExpression) { + // Some local variables for easier smart casting + val callee = call.callee + val language = call.language + // Dynamic function invokes (such as function pointers) are handled by extra pass, so we are - // not resolving them here. In this case, our callee refers to a variable rather than a - // function. + // not resolving them here. + // + // We have a dynamic invoke in two cases: + // a) our calleee is not a reference + // b) our reference refers to a variable rather than a function if ( - (call.callee as? Reference)?.refersTo is VariableDeclaration || - (call.callee as? Reference)?.refersTo is ParameterDeclaration + callee !is Reference || + callee.refersTo is VariableDeclaration || + callee.refersTo is ParameterDeclaration ) { return } + // If this is a template call and our language supports templates, we need to directly + // handle this with the template system. This will also take care of inference and + // everything. This will stay in this way until we completely redesign the template system. + if (call.instantiatesTemplate() && language is HasTemplates) { + val (ok, candidates) = + language.handleTemplateFunctionCalls( + scopeManager.currentRecord, + call, + true, + ctx, + currentTU, + false + ) + if (ok) { + call.invokes = candidates.toMutableList() + return + } + } + // We are moving towards a new approach to call resolution. However, we cannot use this for // all nodes yet, so we need to use legacy resolution for some + val isSpecialCXXCase = + call.language.isCPP && scopeManager.currentRecord != null && !callee.name.isQualified() val useLegacyResolution = when { - call.language.isCPP && curClass != null -> true + isSpecialCXXCase -> true call is MemberCallExpression -> true - call is ConstructExpression -> true - call.instantiatesTemplate() && call.language is HasTemplates -> true - call.callee !is Reference -> true else -> { false } } - if (!useLegacyResolution) { - handleCallExpression(call, curClass) - } else { - handleCallExpressionLegacy(call, curClass) - } - } + // Retrieve a list of candidates; either from the "legacy" system or directly from our + // callee + var candidates = + if (useLegacyResolution) { + val (possibleTypes, _) = getPossibleContainingTypes(call) + resolveMemberByName(callee.name.localName, possibleTypes).toSet() + } else { + callee.candidates + } - /** - * This function tries to resolve the call expression according to the [CallExpression.callee]. - */ - private fun handleCallExpression(call: CallExpression, curClass: RecordDeclaration?) { - val result = scopeManager.resolveCall(call) + // There seems to be one more special case and that is a regular function within a record. + // This could either be a member call with an omitted "this" or a regular call. The problem + // is that the legacy system can now only resolve member calls but not regular calls + // (anymore). So if we have this special case and the legacy system does not return any + // candidates, we need to switch to the new system. + if (isSpecialCXXCase && candidates.isEmpty()) { + candidates = callee.candidates + } + val result = resolveWithArguments(candidates, call.arguments, call) when (result.success) { PROBLEMATIC -> { log.error( "Resolution of ${call.name} returned an problematic result and we cannot decide correctly, the invokes edge will contain all possible viable functions" ) - call.invokes = result.bestViable.toList() + call.invokes = result.bestViable.toMutableList() } AMBIGUOUS -> { log.warn( "Resolution of ${call.name} returned an ambiguous result and we cannot decide correctly, the invokes edge will contain the the ambiguous functions" ) - call.invokes = result.bestViable.toList() + call.invokes = result.bestViable.toMutableList() } SUCCESSFUL -> { - call.invokes = result.bestViable.toList() + call.invokes = result.bestViable.toMutableList() } UNRESOLVED -> { - // Resolution has provided no result, we can forward this to the inference system, - // if we want. While this is definitely a function, it could still be a function - // inside a namespace. We therefore have two possible start points, a namespace - // declaration or a translation unit. Nothing else is allowed (fow now). We can - // re-use the information in the ResolutionResult, since this already contains the - // actual start scope (e.g. in case the callee has an FQN). - var scope = result.actualStartScope - if (scope !is NameScope) { - scope = scopeManager.globalScope - } - val func = - when (val start = scope?.astNode) { - is TranslationUnitDeclaration -> start.inferFunction(call, ctx = ctx) - is NamespaceDeclaration -> start.inferFunction(call, ctx = ctx) - else -> null - } - call.invokes = listOfNotNull(func) + call.invokes = tryFunctionInference(call, result).toMutableList() } } // We also set the callee's refersTo - (call.callee as? Reference)?.refersTo = call.invokes.firstOrNull() + callee.refersTo = call.invokes.firstOrNull() } - private fun handleCallExpressionLegacy( - call: CallExpression, - curClass: RecordDeclaration?, - ) { - // At this point, we decide what to do based on the callee property - val callee = call.callee - - // With one exception. If the language supports templates and if this is a template call, we - // delegate it to the language. In the future, we definitely want to do this in a smarter - // way - var candidates = - if (call.instantiatesTemplate() && call.language is HasTemplates) { - val (_, candidates) = - (call.language as HasTemplates).handleTemplateFunctionCalls( - curClass, - call, - true, - ctx, - currentTU, - false - ) - - candidates - } else { - resolveCalleeLegacy(callee, curClass, call) ?: return - } + /** + * This function tries to resolve a set of [candidates] (e.g. coming from a + * [CallExpression.callee]) into the best matching [FunctionDeclaration] (or multiple functions, + * if applicable) based on the supplied [arguments]. The result is returned in the form of a + * [CallResolutionResult] which holds detail information about intermediate results as well as + * the kind of success the resolution had. + * + * The [source] expression specifies the node in the graph that triggered this resolution. This + * is most likely a [CallExpression], but could be other node as well. It is also the source of + * the scope and language used in the resolution. + */ + private fun resolveWithArguments( + candidates: Set, + arguments: List, + source: Expression, + ): CallResolutionResult { + val result = + CallResolutionResult( + source, + arguments, + candidates.filterIsInstance().toSet(), + setOf(), + mapOf(), + setOf(), + CallResolutionResult.SuccessKind.UNRESOLVED, + source.scope, + ) + val language = source.language - // If we do not have any candidates at this point, we will infer one. - if (candidates.isEmpty()) { - // We need to see, whether we have any suitable base (e.g. a class) or not; but only if - // the call itself is not already scoped (e.g. to a namespace) - val (suitableBases, bestGuess) = - if (callee is MemberExpression || callee?.name?.isQualified() == false) { - getPossibleContainingTypes(call) - } else { - Pair(setOf(), null) - } + if (language == null) { + result.success = CallResolutionResult.SuccessKind.PROBLEMATIC + return result + } - candidates = - if (suitableBases.isEmpty()) { - // This is not really the most ideal place, but for now this will do. While this - // is definitely a function, it could still be a function inside a namespace. In - // this case, we want to start inference in that particular namespace and not in - // the TU. It is also a little bit redundant, since ScopeManager.resolveFunction - // (which gets called before) already extracts the scope, but this information - // gets lost. - val (scope, _) = scopeManager.extractScope(call, scopeManager.globalScope) - - // We have two possible start points, a namespace declaration or a translation - // unit. Nothing else is allowed (for now) - val func = - when (val start = scope?.astNode) { - is TranslationUnitDeclaration -> start.inferFunction(call, ctx = ctx) - is NamespaceDeclaration -> start.inferFunction(call, ctx = ctx) - else -> null - } + // Set the start scope. This can either be the call's scope or a scope specified in an FQN + val (scope, _) = ctx.scopeManager.extractScope(source, source.scope) + result.actualStartScope = scope ?: source.scope - listOfNotNull(func) - } else { - createMethodDummies(suitableBases, bestGuess, call) - } + // If the function does not allow function overloading, and we have multiple candidate + // symbols, the result is "problematic" + if (source.language !is HasFunctionOverloading && result.candidateFunctions.size > 1) { + result.success = PROBLEMATIC } - // Set the INVOKES edge to our candidates - call.invokes = candidates + // Filter functions that match the signature of our call, either directly or with casts; + // those functions are "viable". Take default arguments into account if the language has + // them. + result.signatureResults = + result.candidateFunctions + .map { + Pair( + it, + it.matchesSignature( + arguments.map(Expression::type), + arguments, + source.language is HasDefaultArguments, + ) + ) + } + .filter { it.second is SignatureMatches } + .associate { it } + result.viableFunctions = result.signatureResults.keys - // Additionally, also set the REFERS_TO of the callee. In the future, we might make more - // resolution decisions based on the callee itself. Unfortunately we can only set one here, - // so we will take the first one - if (callee is Reference) { - callee.refersTo = candidates.firstOrNull() + // If we have a "problematic" result, we can stop here. In this case we cannot really + // determine anything more. + if (result.success == CallResolutionResult.SuccessKind.PROBLEMATIC) { + result.bestViable = result.viableFunctions + return result } - } - /** - * Resolves [call] to a list of [FunctionDeclaration] nodes, based on the - * [CallExpression.callee] property. - * - * In case a resolution is not possible, `null` can be returned. - */ - protected fun resolveCalleeLegacy( - callee: Expression?, - curClass: RecordDeclaration?, - call: CallExpression - ): List? { - return when (callee) { - is MemberExpression -> resolveMemberCallee(callee, curClass, call) - is Reference -> resolveReferenceCallee(callee, curClass, call) - null -> { - Util.warnWithFileLocation( - call, - log, - "Call expression without callee, maybe because of a parsing error" - ) - null - } - else -> { - Util.errorWithFileLocation( - call, - log, - "Could not resolve callee of unsupported type ${callee.javaClass}" - ) - null - } - } - } + // Otherwise, give the language a chance to narrow down the result (ideally to one) and set + // the success kind. + val pair = language.bestViableResolution(result) + result.bestViable = pair.first + result.success = pair.second - /** - * Resolves a [CallExpression.callee] of type [Reference] to a possible list of - * [FunctionDeclaration] nodes. - */ - protected fun resolveReferenceCallee( - callee: Reference, - curClass: RecordDeclaration?, - call: CallExpression - ): List { - return if (curClass == null || callee.name.isQualified()) { - // We can already forward this to the nextGen resolver. Not quite sure why we ended up - // here in the first place - val result = ctx.scopeManager.resolveCall(call) - result.bestViable.toList() - } else { - resolveCalleeByName(callee.name.localName, curClass, call) - } + return result } - /** - * Resolves a [CallExpression.callee] of type [MemberExpression] to a possible list of - * [FunctionDeclaration] nodes. - */ - fun resolveMemberCallee( - callee: MemberExpression, - curClass: RecordDeclaration?, - call: CallExpression - ): List { - // We need to adjust certain types of the base in case of a super call and we delegate this. - // If that is successful, we can continue with regular resolving. - if ( - curClass != null && - callee.base is Reference && - isSuperclassReference(callee.base as Reference) - ) { - (callee.language as? HasSuperClasses)?.handleSuperCall( - callee, - curClass, - scopeManager, - ) + protected fun resolveMemberByName( + symbol: String, + possibleContainingTypes: Set + ): Set { + var candidates = mutableSetOf() + val records = possibleContainingTypes.mapNotNull { it.root.recordDeclaration }.toSet() + for (record in records) { + candidates.addAll(ctx.scopeManager.lookupSymbolByName(record.name.fqn(symbol))) } - return resolveCalleeByName(callee.name.localName, curClass, call) - } - - protected fun resolveCalleeByName( - localName: String, - curClass: RecordDeclaration?, - call: CallExpression - ): List { - - val (possibleContainingTypes, _) = getPossibleContainingTypes(call) - - // Find function targets. If our languages has a complex call resolution, we need to take - // this into account - var invocationCandidates = - if (call.language is HasComplexCallResolution) { - (call.language as HasComplexCallResolution) - .refineMethodCallResolution( - curClass, - possibleContainingTypes, - call, - ctx, - currentTU, - this - ) - .toMutableList() - } else { - scopeManager.resolveFunctionLegacy(call).toMutableList() - } // Find invokes by supertypes - if ( - invocationCandidates.isEmpty() && - localName.isNotEmpty() && - (!call.language.isCPP || shouldSearchForInvokesInParent(call)) - ) { + if (candidates.isEmpty() && symbol.isNotEmpty()) { val records = possibleContainingTypes.mapNotNull { it.root.recordDeclaration }.toSet() - invocationCandidates = - getInvocationCandidatesFromParents(localName, call, records).toMutableList() + candidates = getInvocationCandidatesFromParents(symbol, records).toMutableSet() } // Add overridden invokes - invocationCandidates.addAll( - invocationCandidates + candidates.addAll( + candidates + .filterIsInstance() .map { getOverridingCandidates(possibleContainingTypes, it) } .flatten() - .toMutableList() ) - return invocationCandidates + return candidates } /** @@ -784,17 +729,6 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { return records.mapNotNull { record -> record.inferMethod(call, ctx = ctx) } } - /** - * In C++ search we don't search in the parent if there is a potential candidate with matching - * name - * - * @param call - * @return true if we should stop searching parent, false otherwise - */ - protected fun shouldSearchForInvokesInParent(call: CallExpression): Boolean { - return scopeManager.resolveFunctionStopScopeTraversalOnDefinition(call).isEmpty() - } - protected fun handleConstructExpression(constructExpression: ConstructExpression) { if (constructExpression.instantiates != null && constructExpression.constructor != null) return @@ -805,10 +739,10 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { template is RecordTemplateDeclaration && recordDeclaration != null && recordDeclaration in template.realizations && - (constructExpression.templateParameters.size <= template.parameters.size) + (constructExpression.templateArguments.size <= template.parameters.size) ) { val defaultDifference = - template.parameters.size - constructExpression.templateParameters.size + template.parameters.size - constructExpression.templateArguments.size if (defaultDifference <= template.parameterDefaults.size) { // Check if predefined template value is used as default in next value addRecursiveDefaultTemplateArgs(constructExpression, template, scopeManager) @@ -816,7 +750,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { // Add missing defaults val missingNewParams: List = template.parameterDefaults.subList( - constructExpression.templateParameters.size, + constructExpression.templateArguments.size, template.parameterDefaults.size ) for (missingParam in missingNewParams) { @@ -838,6 +772,44 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { } } + private fun handleOverloadedOperator(op: HasOverloadedOperation) { + val result = resolveOperator(op) + val decl = result?.bestViable?.singleOrNull() ?: return + + // If the result was successful, we can replace the node + if (result.success == SUCCESSFUL && decl is OperatorDeclaration && op is Expression) { + val call = operatorCallFromDeclaration(decl, op) + walker.replace(op.astParent, op, call) + } + } + + private fun resolveOperator(op: HasOverloadedOperation): CallResolutionResult? { + val language = op.language + val base = op.operatorBase + if (language !is HasOperatorOverloading || language.isPrimitive(base.type)) { + return null + } + + val symbol = language.overloadedOperatorNames[Pair(op::class, op.operatorCode)] + if (symbol == null) { + log.warn( + "Could not resolve operator overloading for unknown operatorCode ${op.operatorCode}" + ) + return null + } + + val possibleTypes = mutableSetOf() + possibleTypes.add(op.operatorBase.type) + possibleTypes.addAll(op.operatorBase.assignedTypes) + + val candidates = + resolveMemberByName(symbol, possibleTypes) + .filterIsInstance() + .toSet() + + return resolveWithArguments(candidates, op.operatorArguments, op as Expression) + } + /** * Returns a set of types in which the callee of our [call] could reside in. More concretely, it * returns a [Pair], where the first element is the set of types and the second is our best @@ -864,78 +836,23 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { return Pair(possibleTypes, bestGuess) } - fun getInvocationCandidatesFromRecord( - recordDeclaration: RecordDeclaration?, - name: String, - call: CallExpression - ): List { - if (recordDeclaration == null) { - return listOf() - } - - // We should not directly access the "methods" property of the record declaration, - // because depending on the programming language, this only may hold methods that are - // declared directly within the original type declaration, but not ones that are - // declared "outside" (e.g, like it is possible in Go and C++). Instead, we should - // retrieve the scope of the record and look for appropriate declarations. - val scope = scopeManager.lookupScope(recordDeclaration) as? StructureDeclarationScope - - val candidateFunctions = - scope - ?.lookupSymbol(name) - ?.filterIsInstance() - ?.toSet() ?: setOf() - - // The following code is unfortunately largely a copy/paste from the new resolveCall - // function; but resolveCall is not yet completely ready to resolve methods, therefore, we - // need to have this duplicate code here, to at least use the new features here. - val signatureResults = - candidateFunctions - .map { - Pair( - it, - it.matchesSignature( - call.signature, - call.language is HasDefaultArguments, - call - ) - ) - } - .filter { it.second is SignatureMatches } - .associate { it } - val viableFunctions = signatureResults.keys - val result = - CallResolutionResult( - call, - candidateFunctions, - viableFunctions, - signatureResults, - setOf(), - UNRESOLVED, - call.scope - ) - val pair = call.language?.bestViableResolution(result) - - return pair?.first?.toList() ?: listOf() - } - protected fun getInvocationCandidatesFromParents( - name: String, - call: CallExpression, - possibleTypes: Set - ): List { + name: Symbol, + possibleTypes: Set, + ): List { val workingPossibleTypes = mutableSetOf(*possibleTypes.toTypedArray()) return if (possibleTypes.isEmpty()) { listOf() } else { val firstLevelCandidates = - possibleTypes.map { getInvocationCandidatesFromRecord(it, name, call) }.flatten() + possibleTypes.map { scopeManager.lookupSymbolByName(it.name.fqn(name)) }.flatten() // C++ does not allow overloading at different hierarchy levels. If we find a // FunctionDeclaration with the same name as the function in the CallExpression we have // to stop the search in the parent even if the FunctionDeclaration does not match with // the signature of the CallExpression - if (call.language.isCPP) { // TODO: Needs a special trait? + // TODO: move this to refineMethodResolution of CXXLanguage + if (possibleTypes.firstOrNull()?.language.isCPP) { // TODO: Needs a special trait? workingPossibleTypes.removeIf { recordDeclaration -> !shouldContinueSearchInParent(recordDeclaration, name) } @@ -943,7 +860,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { firstLevelCandidates.ifEmpty { workingPossibleTypes .map { it.superTypeDeclarations } - .map { getInvocationCandidatesFromParents(name, call, it) } + .map { getInvocationCandidatesFromParents(name, it) } .flatten() } } @@ -981,8 +898,8 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { recordDeclaration.constructors.firstOrNull { it.matchesSignature( signature, + constructExpression.arguments, constructExpression.language is HasDefaultArguments, - constructExpression ) != IncompatibleSignature } @@ -992,6 +909,48 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { ?.createInferredConstructor(constructExpression.signature) } + fun tryFunctionInference( + call: CallExpression, + result: CallResolutionResult, + ): List { + // We need to see, whether we have any suitable base (e.g. a class) or not; There are two + // main cases + // a) we have a member expression -> easy + // b) we have a call expression -> not so easy. This could be a member call with an implicit + // this (in which case we want to explore the base type). But that is only possible if + // the callee is not qualified, because otherwise we are in a static call like + // MyClass::doSomething() or in a namespace call (in case we do not want to explore the + // base type here yet). This will change in a future PR. + val (suitableBases, bestGuess) = + if (call.callee is MemberExpression || !call.callee.name.isQualified()) { + getPossibleContainingTypes(call) + } else { + Pair(setOf(), null) + } + + return if (suitableBases.isEmpty()) { + // Resolution has provided no result, we can forward this to the inference system, + // if we want. While this is definitely a function, it could still be a function + // inside a namespace. We therefore have two possible start points, a namespace + // declaration or a translation unit. Nothing else is allowed (fow now). We can + // re-use the information in the ResolutionResult, since this already contains the + // actual start scope (e.g. in case the callee has an FQN). + var scope = result.actualStartScope + if (scope !is NameScope) { + scope = scopeManager.globalScope + } + val func = + when (val start = scope?.astNode) { + is TranslationUnitDeclaration -> start.inferFunction(call, ctx = ctx) + is NamespaceDeclaration -> start.inferFunction(call, ctx = ctx) + else -> null + } + listOfNotNull(func) + } else { + createMethodDummies(suitableBases, bestGuess, call) + } + } + companion object { val LOGGER: Logger = LoggerFactory.getLogger(SymbolResolver::class.java) @@ -1057,7 +1016,7 @@ fun TranslationContext.tryRecordInference( // At this point, we need to check whether we have any type reference to our parent // name. If we have (e.g. it is used in a function parameter, variable, etc.), then we // have a high chance that this is actually a parent record and not a namespace - var parentType = typeManager.typeExists(parentName) + var parentType = typeManager.lookupResolvedType(parentName) holder = if (parentType != null) { tryRecordInference(parentType, locationHint = locationHint) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt index 61955a1421..91d67e5b72 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TemplateCallResolverHelper.kt @@ -56,7 +56,7 @@ fun addRecursiveDefaultTemplateArgs( var templateParameters: Int do { // Handle Explicit Template Arguments - templateParameters = constructExpression.templateParameters.size + templateParameters = constructExpression.templateArguments.size val templateParametersExplicitInitialization = mutableMapOf() handleExplicitTemplateParameters( constructExpression, @@ -76,7 +76,7 @@ fun addRecursiveDefaultTemplateArgs( templateParameterRealDefaultInitialization, scopeManager ) - } while (templateParameters != constructExpression.templateParameters.size) + } while (templateParameters != constructExpression.templateArguments.size) } /** @@ -92,8 +92,8 @@ fun handleExplicitTemplateParameters( template: RecordTemplateDeclaration, templateParametersExplicitInitialization: MutableMap ) { - for (i in constructExpression.templateParameters.indices) { - val explicit = constructExpression.templateParameters[i] + for (i in constructExpression.templateArguments.indices) { + val explicit = constructExpression.templateArguments[i] if (template.parameters[i] is TypeParameterDeclaration) { templateParametersExplicitInitialization[ (template.parameters[i] as TypeParameterDeclaration).type] = explicit @@ -123,7 +123,7 @@ fun applyMissingParams( with(constructExpression) { val missingParams: List = template.parameterDefaults.subList( - constructExpression.templateParameters.size, + constructExpression.templateArguments.size, template.parameterDefaults.size ) for (m in missingParams) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt index df57786fd1..b27c5dd7d9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt @@ -67,7 +67,7 @@ open class TypeResolver(ctx: TranslationContext) : ComponentPass(ctx) { // constructor declarations and such with the same name. It seems this is ok since most // languages will prefer structs/classes over functions when resolving types. var symbols = - ctx?.scopeManager?.findSymbols(type.name, startScope = type.scope) { + ctx?.scopeManager?.lookupSymbolByName(type.name, startScope = type.scope) { it is DeclaresType } ?: listOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt index ab6a5fe07b..1a0c59249f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt @@ -30,9 +30,10 @@ import kotlin.reflect.KClass /** * Register a dependency for the annotated pass. This ensures that the annotated pass is executed - * before [other] pass. + * before [other] pass. The [softDependency] flag decides whether to treat this as a hard dependency + * (resulting in the pass being registered if not present) or not. */ @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.CLASS) @Repeatable -annotation class ExecuteBefore(val other: KClass>) +annotation class ExecuteBefore(val other: KClass>, val softDependency: Boolean = true) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/EdgeProperty.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt similarity index 75% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/EdgeProperty.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt index 2a6ae6b140..ebe46c8c40 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/EdgeProperty.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,12 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.passes.configuration +/** + * Indicates whether this pass should be executed as late as possible (without breaking any other + * constraints like [ExecuteLast] or [DependsOn], ...) + */ @Retention(AnnotationRetention.RUNTIME) -@Target(AnnotationTarget.ANNOTATION_CLASS) -annotation class EdgeProperty(val key: String) +@Target(AnnotationTarget.CLASS) +annotation class ExecuteLate diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt new file mode 100644 index 0000000000..62ffa62a84 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2022, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes.configuration + +import de.fraunhofer.aisec.cpg.ConfigurationException +import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.passes.Pass +import java.util.* +import kotlin.reflect.KClass +import kotlin.reflect.full.findAnnotations +import org.slf4j.LoggerFactory + +/** + * The goal of this class is to provide ordered passes when invoking the [order] function. + * * soft dependencies ([DependsOn] with `softDependency == true`): all passes registered as soft + * dependency will be executed before the current pass, if they are registered independently + * * hard dependencies ([DependsOn] with `softDependency == false (default)`): all passes registered + * as hard dependency will be executed before the current pass (hard dependencies will be + * registered even if the user did not specifically register them) + * * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the beginning + * * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end + * * late pass [ExecuteLate]: a pass that is executed as late as possible without violating any of + * the other constraints + * * [ExecuteBefore] (with soft and hard dependencies): the pass is to be executed before the other + * pass (soft: if the other pass is also configured) + * + * This class works by + * 1. Setup + * 1. Iterating through the configured passes and registering them (plus all `hard == true` + * dependencies) as [PassWithDependencies] containers: + * 1. [ExecuteFirst] and [ExecuteLast] passes are stored in separate lists to keep the + * ordering logic simple. + * 2. Normal passes are stored in the [workingList] + * 2. [ExecuteBefore] passes: "`A` execute before `B`" implies that "`B` depends on `A`" -> the + * dependency is stored in the [PassWithDependencies.dependenciesRemaining] of "`B`". This + * logic is implemented in [populateExecuteBeforeDependencies]. + * 3. [DependsOn] passes: [PassWithDependencies.dependenciesRemaining] keeps track of these + * dependencies. [populateNormalDependencies] implements this logic. + * 4. A [sanityCheck] is performed. + * 2. Ordering + * 1. [firstPassesList] passes are moved to the result. + * 2. While the [workingList] is not empty: + * 1. All passes ready to be scheduled are moved to the result excluding late passes. If + * this found at least one pass, then the loop starts again. + * 2. Late passes are considered for scheduling as well. If this found at least one pass, + * then the loop starts again. Otherwise, the dependencies cannot be satisfied. + * 3. [lastPassesList] passes are moved to the result. + * + * Note: whenever a pass is moved to the result: + * - it is removed from the [workingList] (and [firstPassesList] / [lastPassesList]) + * - the other pass's [PassWithDependencies.dependenciesRemaining] are updated. + */ +class PassOrderingHelper { + /** This list stores all non-{first,last} passes which have not yet been ordered. */ + private val workingList: MutableList = ArrayList() + + /** This list stores all first passes. Stored separately to keep the sorting logic simpler. */ + private val firstPassesList: MutableList = ArrayList() + + /** This list stores all last passes. Stored separately to keep the sorting logic simpler. */ + private val lastPassesList: MutableList = ArrayList() + + companion object { + private val log = LoggerFactory.getLogger(PassOrderingHelper::class.java) + } + + /** + * Collects the requested passes provided as [passes] and populates the internal [workingList] + * consisting of pairs of passes and their dependencies. Also, this function adds all + * `hardDependencies` + */ + constructor(passes: List>>) { + for (pass in passes) { + addToWorkingList(pass) + } + + // translate "A `executeBefore` B" to "B `dependsOn` A" + populateExecuteBeforeDependencies() + + // clean up soft dependencies which are not registered in the workingList + populateNormalDependencies() + + // finally, run a sanity check + sanityCheck() + } + + /** Register all (soft and hard) dependencies. */ + private fun populateNormalDependencies() { + for (pass in workingList) { + pass.passClass.hardDependencies.forEach { pass.dependenciesRemaining += it } + pass.passClass.softDependencies + .filter { workingList.map { it.passClass }.contains(it) } + .forEach { pass.dependenciesRemaining += it } + } + } + + /** + * Add a pass to the internal [workingList], iff it does not exist. + * + * Also, add + * * hard dependencies + * * [ExecuteBefore] dependencies + */ + private fun addToWorkingList(newElement: KClass>) { + if ( + (workingList + firstPassesList + lastPassesList) + .filter { it.passClass == newElement } + .isNotEmpty() + ) { + // we already know about this pass + return + } + + var firstOrLastPass = false + if (newElement.findAnnotations().isNotEmpty()) { + firstOrLastPass = true + firstPassesList.add(PassWithDependencies(newElement, mutableSetOf())) + } + if (newElement.findAnnotations().isNotEmpty()) { + firstOrLastPass = true + lastPassesList.add(PassWithDependencies(newElement, mutableSetOf())) + } + if (!firstOrLastPass) { + workingList.add(PassWithDependencies(newElement, mutableSetOf())) + } + + // take care of hard dependencies + for (dep in newElement.findAnnotations()) { + if (!dep.softDependency) { // only hard dependencies + addToWorkingList(dep.value) + } + } + + // take care of [ExecuteBefore] dependencies + for (dep in newElement.findAnnotations()) { + if (!dep.softDependency) { + addToWorkingList(dep.other) + } + } + } + + /** + * Order the passes. This function honors + * - [DependsOn] with soft and hard dependencies + * - [ExecuteFirst] + * - [ExecuteLast] + * - [ExecuteLate] + * - [ExecuteBefore] with soft and hard dependencies + * + * @return a sorted list of passes, with passes that can be run in parallel together in a nested + * list. + * @throws [ConfigurationException] if the passes cannot be ordered. + */ + fun order(): List>>> { + val result = mutableListOf>>>() + + // [ExecuteFirst] + getAndRemoveFirstPasses()?.let { + result.add(listOf(it)) + } // there can only be one because of [sanityCheck] + + // "normal / middle" passes + while (workingList.isNotEmpty()) { + val noLatePassesAllowed = getAndRemoveNextPasses(allowLatePasses = false) + if (noLatePassesAllowed.isNotEmpty()) { + result.add(noLatePassesAllowed) + } else { + val latePassesAllowed = getAndRemoveNextPasses(allowLatePasses = true) + if (latePassesAllowed.isNotEmpty()) { + result.add(latePassesAllowed) + } else { + throw ConfigurationException("Failed to satisfy ordering requirements.") + } + } + } + + // [ExecuteLast] + lastPassesList.firstOrNull()?.let { + result.add(listOf(selectPass(it))) + } // there can only be one because of [sanityCheck] + + log.info( + "Passes after enforcing order: {}", + result.map { list -> list.map { it.simpleName } } + ) + return result + } + + /** + * A pass annotated with [ExecuteBefore] implies that the other pass depends on it. We populate + * the [de.fraunhofer.aisec.cpg.passes.configuration.PassWithDependencies.dependenciesRemaining] + * field in the other pass to make the analysis simpler. + */ + private fun populateExecuteBeforeDependencies() { + for (pass in + (workingList + firstPassesList + lastPassesList)) { // iterate over entire workingList + for (executeBeforePass in + (pass.passClass.softExecuteBefore + + pass.passClass.hardExecuteBefore)) { // iterate over all executeBefore passes + (workingList + firstPassesList + lastPassesList) + .map { it } + .filter { it.passClass == executeBeforePass } // find the executeBeforePass + .forEach { + it.dependenciesRemaining += pass.passClass + } // add the original pass to the dependency list + } + } + } + + /** + * Iterate through all elements and remove the provided dependency [cls] from all passes in the + * working lists. + */ + private fun removeDependencyByClass(cls: KClass>) { + for (pass in workingList) { + pass.dependenciesRemaining.remove(cls) + } + for (pass in firstPassesList) { + pass.dependenciesRemaining.remove(cls) + } + for (pass in lastPassesList) { + pass.dependenciesRemaining.remove(cls) + } + } + + /** + * Finds the first passes which have all their dependencies satisfied. These passes are then + * returned. + * + * @return The first passes which have no active dependencies on success. An empty list + * otherwise. + */ + private fun getAndRemoveNextPasses(allowLatePasses: Boolean): List>> { + return workingList + .filter { + it.dependenciesRemaining.isEmpty() && it.passClass.isLatePass == allowLatePasses + } + .map { selectPass(it) } + } + + /** + * Checks for passes marked as first pass by [ExecuteFirst] + * + * If found, this pass is returned and removed from the working list. + * + * @return The first pass if present. Otherwise, null. + */ + private fun getAndRemoveFirstPasses(): KClass>? { + return when (firstPassesList.isEmpty()) { + true -> null + false -> selectPass(firstPassesList.first()) + } + } + + /** + * Removes a pass from the other passes dependencies and the workingList. + * + * @return the (unpacked) pass + */ + private fun selectPass(pass: PassWithDependencies): KClass> { + // remove it from the other passes dependencies + removeDependencyByClass(pass.passClass) + + // remove it from the workingList + workingList.remove(pass) + firstPassesList.remove(pass) + lastPassesList.remove(pass) + + // return the pass (not the [PassWithDependencies] container) + return pass.passClass + } + + /** + * Perform a sanity check on the configured [workingList]. Currently, this only checks that + * * there is at most one [ExecuteFirst] and + * * at most one [ExecuteLast] pass configured and + * * the first pass does not have a hard dependency and + * * the last pass is not to be executed before other passes. + * + * This does not check, whether the requested ordering can be satisfied. + */ + private fun sanityCheck() { + if (firstPassesList.size > 1) { + throw ConfigurationException( + "More than one pass registered as first pass: \"${firstPassesList.map { it.passClass }}\"." + ) + } + if (lastPassesList.size > 1) { + throw ConfigurationException( + "More than one pass registered as last pass: \"${lastPassesList.map { it.passClass }}\"." + ) + } + + firstPassesList.map { firstPass -> + if (firstPass.passClass.hardDependencies.isNotEmpty()) { + throw ConfigurationException( + "The first pass \"${firstPass.passClass}\" has a hard dependency: \"${firstPass.passClass.hardDependencies}\"." + ) + } + } + + lastPassesList.map { lastPass -> + if (lastPass.passClass.softExecuteBefore.isNotEmpty()) { + throw ConfigurationException( + "The last pass \"${lastPass.passClass}\" is supposed to be executed before another pass: \"${lastPass.passClass.softExecuteBefore}\"." + ) + } + if (lastPass.passClass.hardExecuteBefore.isNotEmpty()) { + throw ConfigurationException( + "The last pass \"${lastPass.passClass}\" is supposed to be executed before another pass: \"${lastPass.passClass.hardExecuteBefore}\"." + ) + } + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt index 50ff80b028..18044c6104 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt @@ -25,71 +25,17 @@ */ package de.fraunhofer.aisec.cpg.passes.configuration -import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.Pass import kotlin.reflect.KClass -import kotlin.reflect.full.hasAnnotation -import org.apache.commons.lang3.builder.ToStringBuilder -/** A simple helper class to match a pass with dependencies. */ +/** + * A simple helper class to match a pass with its dependencies. [dependenciesRemaining] shows the + * currently remaining / unsatisfied dependencies. These values are updated during the ordering + * procedure. + */ data class PassWithDependencies( - val pass: KClass>, - val softDependencies: MutableSet>>, - val hardDependencies: MutableSet>> -) { - val dependencies: Set>> - get() { - return softDependencies + hardDependencies - } - - val isFirstPass: Boolean - get() { - return pass.hasAnnotation() - } - - val isLastPass: Boolean - get() { - return pass.hasAnnotation() - } - - override fun toString(): String { - val builder = ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", pass.simpleName) - - if (softDependencies.isNotEmpty()) { - builder.append("softDependencies", softDependencies.map { it.simpleName }) - } - - if (hardDependencies.isNotEmpty()) { - builder.append("hardDependencies", hardDependencies.map { it.simpleName }) - } - return builder.toString() - } - - /** - * Checks whether the [dependencies] of this pass are met. The list of [softDependencies] and - * [hardDependencies] is removed step-by-step in - * [PassWithDepsContainer.getAndRemoveFirstPassWithoutDependencies]. - */ - fun dependenciesMet(workingList: MutableList): Boolean { - // In the simplest case all our dependencies are empty since they were already removed by - // the selecting algorithm. - if (this.dependencies.isEmpty() && !this.isLastPass) { - return true - } - - // We also need to check, whether we still "soft" depend on passes that are just not - // there (after all hard dependencies are met), in this case we can select the pass - // as well - val remainingClasses = workingList.map { it.pass } - if ( - this.hardDependencies.isEmpty() && - this.softDependencies.all { !remainingClasses.contains(it) } && - !this.isLastPass - ) { - return true - } - - // Otherwise, we still depend on an unselected pass - return false - } -} + /** the pass itself */ + val passClass: KClass>, + /** currently unsatisfied dependencies (soft / hard / [ExecuteBefore] from other passes) */ + val dependenciesRemaining: MutableSet>> +) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt deleted file mode 100644 index c3219e55c2..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2022, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.passes.configuration - -import de.fraunhofer.aisec.cpg.ConfigurationException -import de.fraunhofer.aisec.cpg.passes.Pass -import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log -import java.util.* -import kotlin.reflect.KClass -import kotlin.reflect.full.findAnnotations - -/** - * A simple helper class for keeping track of passes and their (currently not satisfied) - * dependencies during ordering. - */ -class PassWithDepsContainer { - private val workingList: MutableList - - init { - workingList = ArrayList() - } - - fun getWorkingList(): List { - return workingList - } - - fun addToWorkingList(newElement: PassWithDependencies) { - workingList.add(newElement) - } - - val isEmpty: Boolean - get() = workingList.isEmpty() - - fun size(): Int { - return workingList.size - } - - /** - * Iterate through all elements and remove the provided dependency [cls] from all passes in the - * working list. - */ - private fun removeDependencyByClass(cls: KClass>) { - for (pass in workingList) { - pass.softDependencies.remove(cls) - pass.hardDependencies.remove(cls) - } - } - - override fun toString(): String { - return workingList.toString() - } - - fun getFirstPasses(): List { - return workingList.filter { it.isFirstPass } - } - - fun getLastPasses(): List { - return workingList.filter { it.isLastPass } - } - - private fun dependencyPresent(dep: KClass>): Boolean { - var result = false - for (currentElement in workingList) { - if (dep == currentElement.pass) { - result = true - break - } - } - - return result - } - - private fun createNewPassWithDependency(cls: KClass>): PassWithDependencies { - val softDependencies = mutableSetOf>>() - val hardDependencies = mutableSetOf>>() - - val dependencies = cls.findAnnotations() - for (d in dependencies) { - if (d.softDependency) { - softDependencies += d.value - } else { - hardDependencies += d.value - } - } - - return PassWithDependencies(cls, softDependencies, hardDependencies) - } - - /** - * Recursively iterates the workingList and adds all hard dependencies [DependsOn] and their - * dependencies to the workingList. - */ - fun addMissingDependencies() { - val it = workingList.listIterator() - while (it.hasNext()) { - val current = it.next() - for (dependency in current.hardDependencies) { - if (!dependencyPresent(dependency)) { - log.info( - "Registering a required hard dependency which was not registered explicitly: {}", - dependency - ) - it.add(createNewPassWithDependency(dependency)) - } - } - } - - // add required dependencies to the working list - val missingPasses: MutableList>> = ArrayList() - - // initially populate the missing dependencies list given the current passes - for (currentElement in workingList) { - for (dependency in currentElement.hardDependencies) { - if (!dependencyPresent(dependency)) { - missingPasses.add(dependency) - } - } - } - } - - /** - * Finds the first pass that has all its dependencies satisfied. This pass is then removed from - * the other passes dependencies and returned. - * - * @return The first pass that has no active dependencies on success. null otherwise. - */ - fun getAndRemoveFirstPassWithoutDependencies(): List>> { - val results = mutableListOf>>() - val it = workingList.listIterator() - - while (it.hasNext()) { - val currentElement = it.next() - if (results.isEmpty() && currentElement.isLastPass && workingList.size == 1) { - it.remove() - return listOf(currentElement.pass) - } - - // Keep going until our dependencies are met, this will collect passes that can run in - // parallel in results - if (currentElement.dependenciesMet(workingList)) { - // no unsatisfied dependencies - val result = currentElement.pass - results.add(result) - - // remove pass from the work-list - it.remove() - } else { - continue - } - } - - // remove the selected passes from the other pass's dependencies - results.forEach { removeDependencyByClass(it) } - - return results - } - - /** - * Checks for passes marked as first pass by [ExecuteFirst] - * - * If found, this pass is returned and removed from the working list. - * - * @return The first pass if present. Otherwise, null. - */ - fun getAndRemoveFirstPass(): KClass>? { - val firstPasses = getFirstPasses() - if (firstPasses.size > 1) { - throw ConfigurationException( - "More than one pass requires to be run as first pass: {}".format(firstPasses) - ) - } - return if (firstPasses.isNotEmpty()) { - val firstPass = firstPasses.first() - if (firstPass.hardDependencies.isNotEmpty()) { - throw ConfigurationException("The first pass has a hard dependency.") - } else { - removeDependencyByClass(firstPass.pass) - workingList.remove(firstPass) - firstPass.pass - } - } else { - null - } - } -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 47a78280c0..61e58c87d8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -299,7 +299,7 @@ class DFGFunctionSummaries { } // TODO: It would make sense to model properties here. Could be the index of a return // value, full vs. partial flow or whatever comes to our minds in the future - to?.let { from?.addNextDFG(it) } + to?.let { from?.nextDFGEdges += it } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt index 9b047ffff1..ce3e2e3f21 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt @@ -317,11 +317,11 @@ class Inference internal constructor(val start: Node, override val ctx: Translat tu?.inferFunction(call, ctx = ctx) } - inferredRealization?.let { inferred.addRealization(it) } + inferredRealization?.let { inferred.realization += it } var typeCounter = 0 var nonTypeCounter = 0 - for (node in call.templateParameters) { + for (node in call.templateArguments) { if (node is TypeExpression) { // Template Parameter val inferredTypeIdentifier = "T$typeCounter" @@ -329,7 +329,7 @@ class Inference internal constructor(val start: Node, override val ctx: Translat inferred.startInference(ctx)?.inferTemplateParameter(inferredTypeIdentifier) typeCounter++ if (typeParamDeclaration != null) { - inferred.addParameter(typeParamDeclaration) + inferred.parameters += typeParamDeclaration } } else if (node is Expression) { val inferredNonTypeIdentifier = "N$nonTypeCounter" @@ -338,11 +338,11 @@ class Inference internal constructor(val start: Node, override val ctx: Translat .startInference(ctx) ?.inferNonTypeTemplateParameter(inferredNonTypeIdentifier) if (paramVariableDeclaration != null) { - node.addNextDFG(paramVariableDeclaration) + node.nextDFGEdges += paramVariableDeclaration } nonTypeCounter++ if (paramVariableDeclaration != null) { - inferred.addParameter(paramVariableDeclaration) + inferred.parameters += paramVariableDeclaration } } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/processing/strategy/Strategy.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/processing/strategy/Strategy.kt index e8f73db852..e3b43c97bf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/processing/strategy/Strategy.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/processing/strategy/Strategy.kt @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.processing.strategy import de.fraunhofer.aisec.cpg.graph.Node -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.getAstChildren import java.util.* /** Strategies (iterators) for traversing graphs to be used by visitors. */ @@ -88,6 +87,6 @@ object Strategy { * @return */ fun AST_FORWARD(x: Node): Iterator { - return getAstChildren(x).iterator() + return x.astChildren.iterator() } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt index 0adb743f22..605ec1fa82 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.newInitializerListExpression import de.fraunhofer.aisec.cpg.graph.newVariableDeclaration import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin.POINTER -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.net.URI @@ -56,7 +55,7 @@ class GraphExamples { declare { variable("i", t("int")) { val initList = newInitializerListExpression() - initList.initializers = listOf(call("foo")) + initList.initializers = mutableListOf(call("foo")) initializer = initList } } @@ -1062,7 +1061,6 @@ class GraphExamples { config: TranslationConfiguration = TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) = @@ -1134,7 +1132,6 @@ class GraphExamples { config: TranslationConfiguration = TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) = diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index 3dccf39421..da08592c2f 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -32,9 +32,9 @@ import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* -import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn -import de.fraunhofer.aisec.cpg.graph.edge.CallingContextOut -import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextIn +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextOut +import de.fraunhofer.aisec.cpg.graph.edges.flows.ContextSensitiveDataflow import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt new file mode 100644 index 0000000000..ffc2c65bb6 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ArgumentHolderTest.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class ArgumentHolderTest { + @Test + fun testHasArgument() { + with(TestLanguageFrontend()) { + var ref = newReference("test") + var list = + listOf( + newBinaryOperator("?"), + newUnaryOperator("?", false, false), + newCallExpression(), + newCastExpression(), + newIfStatement(), + newReturnStatement(), + newConditionalExpression(newLiteral(true)), + newDoStatement(), + newInitializerListExpression(), + newKeyValueExpression(key = ProblemExpression(), value = ProblemExpression()), + newSubscriptExpression(), + newWhileStatement(), + newAssignExpression(), + newVariableDeclaration("test"), + ) + + for (node in list) { + assertFalse(node.hasArgument(ref), "hasArgument failed for ${node::class}") + } + + for (node in list) { + node += ref + assertTrue(node.hasArgument(ref), "hasArgument failed for ${node::class}") + } + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt index f5ce80f271..2756e92e7a 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilderTest.kt @@ -26,9 +26,9 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.CallingContextIn -import de.fraunhofer.aisec.cpg.graph.edge.ContextSensitiveDataflow -import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.edges.flows.CallingContextIn +import de.fraunhofer.aisec.cpg.graph.edges.flows.ContextSensitiveDataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.PartialDataflowGranularity import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -43,7 +43,7 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addPrevDFG(node2, granularity, callingContextIn) + node1.prevDFGEdges.addContextSensitive(node2, granularity, callingContextIn) val clone = node1.duplicate(false) val clonedPrevDFG = clone.prevDFGEdges.single() @@ -60,7 +60,7 @@ class ExpressionBuilderTest { val node2 = Reference() val granularity = PartialDataflowGranularity(FieldDeclaration()) val callingContextIn = CallingContextIn(CallExpression()) - node1.addNextDFG(node2, granularity, callingContextIn) + node1.nextDFGEdges.addContextSensitive(node2, granularity, callingContextIn) val clone = node1.duplicate(false) val clonedPrevDFG = clone.nextDFGEdges.single() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasAliases.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/InterfacesTest.kt similarity index 68% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasAliases.kt rename to cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/InterfacesTest.kt index afb92e0a50..ececbb06d3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasAliases.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/InterfacesTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,11 +25,18 @@ */ package de.fraunhofer.aisec.cpg.graph -/** - * Some nodes have aliases, i.e., it potentially references another variable. This means that - * writing to this node, also writes to its [aliases] and vice-versa. - */ -interface HasAliases { - /** The aliases which this node has. */ - var aliases: MutableSet +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import kotlin.test.Test +import kotlin.test.assertTrue + +class InterfacesTest { + + @Test + fun testRemoveArgument() { + val v: HasInitializer = VariableDeclaration() + val lit = Literal() + v.initializer = lit + assertTrue(v.removeArgument(lit)) + } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt index b000e343f3..c1e929a2e1 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import de.fraunhofer.aisec.cpg.test.* import kotlin.test.* import org.junit.jupiter.api.BeforeAll @@ -232,7 +231,6 @@ class ShortcutsTest { GraphExamples.getShortcutClass( TranslationConfiguration.builder() .defaultPasses() - .registerPass() .registerLanguage(TestLanguage(".")) .build() ) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilderTest.kt new file mode 100644 index 0000000000..340362daf8 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/StatementBuilderTest.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.ScopeManager +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.TypeManager +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.builder.translationResult +import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope +import de.fraunhofer.aisec.cpg.test.assertRefersTo +import kotlin.test.Test +import kotlin.test.assertIs +import kotlin.test.assertNotNull + +class StatementBuilderTest { + @Test + fun testNewLookupScopeStatement() { + val frontend = + TestLanguageFrontend( + ctx = + TranslationContext( + TranslationConfiguration.builder().defaultPasses().build(), + ScopeManager(), + TypeManager() + ) + ) + val result = + frontend.build { + translationResult { + var tu = + with(frontend) { + var tu = newTranslationUnitDeclaration("main.file") + scopeManager.resetToGlobal(tu) + + var globalA = newVariableDeclaration("a") + scopeManager.addDeclaration(globalA) + + var func = newFunctionDeclaration("main") + scopeManager.enterScope(func) + + var body = newBlock() + scopeManager.enterScope(body) + + var localA = newVariableDeclaration("a") + var stmt = newDeclarationStatement() + stmt.declarations += localA + scopeManager.addDeclaration(localA) + body += stmt + + body += newLookupScopeStatement(listOf("a"), scopeManager.globalScope) + body += newReference("a") + + scopeManager.leaveScope(body) + func.body = body + scopeManager.leaveScope(func) + + scopeManager.addDeclaration(func) + scopeManager.leaveScope(tu) + tu + } + + components.firstOrNull()?.translationUnits?.add(tu) + } + } + + val globalA = result.variables["a"] + assertNotNull(globalA) + assertIs(globalA.scope) + + val a = result.refs["a"] + assertRefersTo(a, globalA) + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/WalkerTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/WalkerTest.kt index 9dfd348400..75c3bde692 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/WalkerTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/WalkerTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.edges.astEdges import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal @@ -69,9 +70,8 @@ class WalkerTest : BaseTest() { lit.value = k decl.initializer = lit - stmt.addToPropertyEdgeDeclaration(decl) - - comp.addStatement(stmt) + stmt.declarationEdges += decl + comp.statementEdges += stmt } method.body = comp @@ -108,6 +108,17 @@ class WalkerTest : BaseTest() { log.info("Flat AST has {} nodes", flat.size) } + + // Alternative approach using new edge extensions + val bench = Benchmark(WalkerTest::class.java, "Speed of Walker") + val flat = tu.astEdges.map { it.end } + bench.stop() + + assertNotNull(flat) + + assertEquals(82618, flat.size) + + log.info("Flat AST has {} nodes", flat.size) } // 741ms with branch @@ -119,7 +130,7 @@ class WalkerTest : BaseTest() { val decl = VariableDeclaration() decl.name = Name("var${k}") - stmt.addToPropertyEdgeDeclaration(decl) + stmt.declarationEdges.add(decl) } val bench = Benchmark(WalkerTest::class.java, "Speed of Walker") diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclarationTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclarationTest.kt index 99f889d777..a2bda404c4 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclarationTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclarationTest.kt @@ -142,7 +142,7 @@ class TupleDeclarationTest { ), newCallExpression(newReference("func")) ) - this.addToPropertyEdgeDeclaration(tuple) + this.declarationEdges += tuple scopeManager.addDeclaration(tuple) tuple.elements.forEach { scopeManager.addDeclaration(it) } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalkerTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalkerTest.kt new file mode 100644 index 0000000000..33e90c0cc3 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalkerTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.allEdges +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.astEdges +import de.fraunhofer.aisec.cpg.graph.edges.edges +import de.fraunhofer.aisec.cpg.graph.newBlock +import de.fraunhofer.aisec.cpg.graph.newCallExpression +import de.fraunhofer.aisec.cpg.graph.newFunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.newReference +import kotlin.test.Test +import kotlin.test.assertEquals + +class EdgeWalkerTest { + @Test + fun testExtract() { + with(TestLanguageFrontend()) { + var node = newFunctionDeclaration("do") + node.body = newBlock() + + node.prevDFG += newCallExpression(newReference("do")) + + var edges = node.edges>() + assertEquals(2, edges.size) + + edges = node.edges { it is AstEdge } + assertEquals(1, edges.size) + + var all = node.allEdges>() + assertEquals(3, all.size) + + var allAst = node.astEdges + assertEquals(1, allAst.size) + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/AST.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgesTest.kt similarity index 57% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/AST.kt rename to cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgesTest.kt index 238a1535f3..d5724048ca 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/AST.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgesTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,13 +23,24 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.graph.edges -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.newMethodDeclaration +import de.fraunhofer.aisec.cpg.graph.newRecordDeclaration +import kotlin.test.Test +import kotlin.test.assertEquals -/** - * Annotates single member variables of supertype [Node] or a collection of nodes to be part of the - * AST of the current [Node]. This is used to iterate over all AST sub-nodes with - * [SubgraphWalker.getAstChildren]. - */ -@Target(AnnotationTarget.FIELD) @Retention(AnnotationRetention.RUNTIME) annotation class AST +class EdgesTest { + @Test + fun testUnwrap() { + with(TestLanguageFrontend()) { + var record = newRecordDeclaration("myRecord", kind = "class") + var method = newMethodDeclaration("myFunc") + record.methods += method + + assertEquals(1, record.methods.size) + assertEquals(method, record.methods.firstOrNull()) + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasDefault.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ExtensionsTest.kt similarity index 59% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasDefault.kt rename to cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ExtensionsTest.kt index 88139e7388..acade637ab 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasDefault.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ExtensionsTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,13 +23,23 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.graph.edges -/** - * Interface that allows us to mark nodes that contain a default value - * - * @param type of the default node - */ -interface HasDefault { - var default: T +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrder +import de.fraunhofer.aisec.cpg.graph.newLiteral +import kotlin.test.Test + +class ExtensionsTest { + @Test + fun testBuilder() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + + node1.nextEOGEdges.add(node2) { unreachable = true } + node1.nextEOG.add(node3) { unreachable = true } + } + } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt new file mode 100644 index 0000000000..500dc8ea53 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.newLiteral +import kotlin.test.Test +import kotlin.test.assertEquals + +class EdgeListTest { + @Test + fun testAddIndex() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + var node4 = newLiteral(4) + + var list = AstEdges>(thisRef = node1) + list += node2 + list += node3 + + assertEquals(2, list.size) + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // insert something at position 1, this should shift the existing entries (after the + // position) + 1 + list.add(1, AstEdge(node1, node4)) + assertEquals(3, list.size) + + // indices should still be in sync afterward + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // the order should be node2, node4, node3 + var unwrapped = list.unwrap() + assertEquals>(listOf(node2, node4, node3), unwrapped) + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonListTest.kt similarity index 53% rename from cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt rename to cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonListTest.kt index d200b24825..465640e25d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonListTest.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,19 +23,32 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.graph.edges.collections +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.graph.newLiteral import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import kotlin.test.Test +import kotlin.test.assertNull +import kotlin.test.assertSame -/** Specifies that a certain node has a base on which it executes an operation. */ -interface HasBase : HasOperatorCode { +class EdgeSingletonListTest { + @Test + fun testNullable() { + with(TestLanguageFrontend()) { + class MyNode : Node() { + var edge = astOptionalEdgeOf() + var unwrapped by unwrapping(MyNode::edge) + } - /** The base. */ - val base: Expression? + var node = MyNode() + assertNull(node.unwrapped) - /** - * The operator that is used to access the base. Usually either `.` or `->`, but some languages - * offer additional operator codes. - */ - override val operatorCode: String? + node.unwrapped = newLiteral(1) + assertSame(node.unwrapped, node.edge.element?.end) + } + } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt new file mode 100644 index 0000000000..2d318ae5a6 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.collections + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges +import de.fraunhofer.aisec.cpg.graph.newLiteral +import kotlin.test.Test +import kotlin.test.assertEquals + +class UnwrappedEdgeListTest { + @Test + fun testAdd() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + + node1.nextEOGEdges += node2 + + // this should trigger add of the edge underneath (node1.nextEOGEdges += node3) + node1.nextEOG += node3 + + // should contain 2 nodes now + assertEquals(2, node1.nextEOGEdges.size) + assertEquals(2, node1.nextEOG.size) + // mirroring should also work + assertEquals(1, node2.prevEOGEdges.size) + assertEquals(1, node3.prevEOGEdges.size) + assertEquals(1, node3.prevEOG.size) + } + } + + @Test + fun testAddIndex() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + var node4 = newLiteral(4) + + var list = AstEdges>(thisRef = node1) + list += node2 + list += node3 + + assertEquals(2, list.size) + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // insert something at position 1 (using the unwrapped list), this should shift the + // existing entries (after the position) + 1 + var unwrapped = list.unwrap() + unwrapped.add(1, node4) + assertEquals(3, list.size) + + // indices should still be in sync afterward + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // the order should be node2, node4, node3 + assertEquals>(listOf(node2, node4, node3), unwrapped) + } + } + + @Test + fun testIterator() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + + node1.nextEOGEdges += node2 + + node1.nextEOG += node3 + + var list = node1.nextEOG.toList() + assertEquals(2, list.size) + + // test our mutable iterator + var iter = node1.nextEOG.iterator() + iter.next() + iter.remove() + + assertEquals(1, node1.nextEOGEdges.size) + + // test our list iterator + var listIter = node1.nextEOG.listIterator() + listIter.add(node2) + + assertEquals(2, node1.nextEOGEdges.size) + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependenceTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependenceTest.kt new file mode 100644 index 0000000000..55b77a35e0 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependenceTest.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.newLiteral +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertSame + +class ControlDependenceTest { + @Test + fun testOnAdd() { + with(TestLanguageFrontend()) { + // -- CDG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextCDGEdges.add(node2) { branches = setOf(false) } + + // should contain 1 prevCDG edge now + assertEquals(1, node2.prevCDGEdges.size) + // and it should be the same as the nextDFG of node1 + assertSame(node1.nextCDGEdges.firstOrNull(), node2.prevCDGEdges.firstOrNull()) + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/DataflowTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/DataflowTest.kt new file mode 100644 index 0000000000..d01d9b9b60 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/DataflowTest.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.newLiteral +import de.fraunhofer.aisec.cpg.graph.newReference +import kotlin.collections.firstOrNull +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertEquals +import kotlin.test.assertSame + +class DataflowTest { + @Test + fun testDataflows() { + with(TestLanguageFrontend()) { + // -- DFG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextDFGEdges += node2 + // should contain 1 prevDFG edge now + assertEquals(1, node2.prevDFGEdges.size) + // and it should be the same as the nextDFG of node1 + assertSame(node1.nextDFGEdges.firstOrNull(), node2.prevDFGEdges.firstOrNull()) + } + } + + @Test + fun testReferenceTypeListener() { + with(TestLanguageFrontend()) { + // -- DFG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newReference("a") + + node1.nextDFGEdges += node2 + // should contain 1 prevDFG edge now + assertEquals(1, node2.prevDFGEdges.size) + + // node2 should now be a type observer on node1 + assertContains(node1.typeObservers, node2) + } + } + + @Test + fun testRemove() { + with(TestLanguageFrontend()) { + // -- DFG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextDFGEdges += node2 + assertEquals(1, node2.prevDFGEdges.size) + + node1.nextDFGEdges.removeIf { it.end == node2 } + // should contain 0 prevDFG edge now + assertEquals(0, node2.prevDFGEdges.size) + } + } + + @Test + fun testClear() { + with(TestLanguageFrontend()) { + // -- DFG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextDFGEdges += node2 + assertEquals(1, node2.prevDFGEdges.size) + + node1.nextDFGEdges.clear() + // should contain 0 prevDFG edge now + assertEquals(0, node2.prevDFGEdges.size) + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrderTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrderTest.kt new file mode 100644 index 0000000000..e333d854a8 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrderTest.kt @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.* +import kotlin.test.* + +class EvaluationOrderTest { + @Test + fun testEvaluationOrders() { + with(TestLanguageFrontend()) { + // -- EOG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextEOGEdges.add(node2) + // should contain 1 prevDFG edge now + assertEquals(1, node2.prevEOGEdges.size) + // and it should be the same as the nextDFG of node1 + assertSame(node1.nextEOGEdges.firstOrNull(), node2.prevEOGEdges.firstOrNull()) + + node1.nextEOGEdges.removeAt(0) + // should contain 0 prevDFG edge now + assertEquals(0, node2.prevEOGEdges.size) + } + } + + @Test + fun testClear() { + with(TestLanguageFrontend()) { + // -- EOG --> + // this should be 1 nextDFG for node1 and 1 prevDFG for node2 + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextEOGEdges.add(node2) + assertEquals(1, node2.prevEOGEdges.size) + + node1.nextEOGEdges.clear() + // should contain 0 prevEOG edge now + assertEquals(0, node2.prevEOGEdges.size) + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependenceTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependenceTest.kt new file mode 100644 index 0000000000..7ad5f1ce61 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ProgramDependenceTest.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.flows + +import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.newLiteral +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull + +class ProgramDependenceTest { + @Test + fun testCombinedAdd() { + with(TestLanguageFrontend()) { + // -- DFG --> + // -- CDG --> + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextDFGEdges += node2 + node1.nextCDGEdges.add(node2) { branches = setOf(false) } + + // Add the combined PDG edges. We always to this in an incoming way. This simulates what + // the PDG pass does. + // This should result in a combined PDG of 2 edges + var combined = mutableListOf>() + combined += node2.prevDFGEdges + combined += node2.prevCDGEdges + node2.prevPDGEdges += combined + + // Should contain 2 PDG edges now + assertEquals(2, node2.prevPDGEdges.size) + + // The content should be "equal". We can only do this with a union because we are + // comparing sets and lists here + assertEquals(node2.prevPDGEdges.toSet(), node2.prevPDGEdges.union(combined)) + + // Assert the mirror property + assertEquals(node1.nextPDGEdges, node2.prevPDGEdges) + } + } + + @Test + fun testEquals() { + with(TestLanguageFrontend()) { + // -- DFG --> + // -- CDG --> + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + node1.nextDFGEdges += node2 + node1.nextCDGEdges.add(node2) { branches = setOf(false) } + + var dfgEdge = node1.nextDFGEdges.firstOrNull() + assertNotNull(dfgEdge) + + var cdgEdge = node1.nextCDGEdges.firstOrNull() + assertNotNull(cdgEdge) + + assertNotEquals>(dfgEdge, cdgEdge) + } + } + + @Test + fun testUnsupported() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(value = 1) + var node2 = newLiteral(value = 1) + + assertFailsWith { + // We do not allow to "create" new edges, but we can only put existing edges (as in + // DFG, CDG) in the PDG container + node1.nextPDGEdges.add(node2) + } + } + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ScopeTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ScopeTest.kt new file mode 100644 index 0000000000..77f6db5b6a --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/ScopeTest.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.scopes + +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import kotlin.test.Test +import kotlin.test.assertEquals + +class ScopeTest { + @Test + fun testLookup() { + // some mock variable declarations, global and local + var globalA = VariableDeclaration() + globalA.name = Name("a") + var localA = VariableDeclaration() + localA.name = Name("a") + + // two scopes, global and local + val globalScope = GlobalScope() + globalScope.addSymbol("a", globalA) + val scope = BlockScope(Block()) + scope.parent = globalScope + scope.addSymbol("a", localA) + + // if we try to resolve "a" now, this should point to the local A since we start there and + // move upwards + var result = scope.lookupSymbol("a") + assertEquals(listOf(localA), result) + + // now, we pretend to have a lookup scope modifier for a symbol, e.g. through "global" in + // Python + var stmt = LookupScopeStatement() + stmt.targetScope = globalScope + stmt.symbols = listOf("a") + scope.predefinedLookupScopes["a"] = stmt + + // let's try the lookup again, this time it should point to the global A + result = scope.lookupSymbol("a") + assertEquals(listOf(globalA), result) + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt index 730a2c1eb8..bffb6cc963 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt @@ -100,7 +100,7 @@ class AssignExpressionTest { assertNotNull(refErr) // This should now set the correct type of the call expression - call.invokes = listOf(func) + call.invokes = mutableListOf(func) assertIs(call.type) // We should at least know the "assigned" type of the references. Their declared diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt new file mode 100644 index 0000000000..0eab8a9046 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.helpers + +import de.fraunhofer.aisec.cpg.GraphExamples.Companion.testFrontend +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.frontends.TestLanguage +import de.fraunhofer.aisec.cpg.graph.builder.body +import de.fraunhofer.aisec.cpg.graph.builder.declare +import de.fraunhofer.aisec.cpg.graph.builder.function +import de.fraunhofer.aisec.cpg.graph.builder.problemDecl +import de.fraunhofer.aisec.cpg.graph.builder.translationResult +import de.fraunhofer.aisec.cpg.graph.builder.translationUnit +import de.fraunhofer.aisec.cpg.graph.problems +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.test.BaseTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +internal class ExtensionsTest : BaseTest() { + val problemDeclText = "This is a problem declaration." + val problemExprText = "This is a problem expression." + + fun getTranslationResultWithProblems( + config: TranslationConfiguration = + TranslationConfiguration.builder() + .defaultPasses() + .registerLanguage(TestLanguage(".")) + .build() + ) = + testFrontend(config).build { + translationResult { + translationUnit("foo.bar") { + function("foo") { body { declare { problemDecl(problemDeclText) } } } + .additionalProblems += ProblemExpression(problemExprText) + } + } + } + + @Test + fun testProblemsExtension() { + val test = getTranslationResultWithProblems() + assertNotNull(test) + assertEquals(2, test.problems.size, "Expected two problems.") + assertNotNull( + test.problems.filter { it.problem == problemDeclText }, + "Failed to find the problem declaration." + ) + assertNotNull( + test.problems.filter { it.problem == problemExprText }, + "Failed to find the problem expression." + ) + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPassTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPassTest.kt index 82a517eb3a..52e4ea72f7 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPassTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPassTest.kt @@ -31,9 +31,9 @@ import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.edge.Dataflow -import de.fraunhofer.aisec.cpg.graph.edge.FullDataflowGranularity -import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.edges.flows.PartialDataflowGranularity import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPassTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPassTest.kt index 946fbc5719..704ee16532 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPassTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ProgramDependenceGraphPassTest.kt @@ -27,17 +27,13 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.TestLanguage -import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.frontends.testFrontend import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.builder.* -import de.fraunhofer.aisec.cpg.graph.edge.DependenceType -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.get import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy -import java.util.* import java.util.stream.Stream import kotlin.test.assertNotNull import kotlin.test.assertTrue @@ -59,19 +55,15 @@ class ProgramDependenceGraphPassTest { object : IVisitor() { override fun visit(t: Node) { val expectedPrevEdges = - t.prevCDGEdges.map { - it.apply { addProperty(Properties.DEPENDENCE, DependenceType.CONTROL) } - } + - t.prevDFG.mapNotNull { + t.prevCDGEdges + + t.prevDFGEdges.filter { if ( - "remove next" in (it.comment ?: "") && + "remove next" in (it.start.comment ?: "") && "remove prev" in (t.comment ?: "") ) { - null + false } else { - PropertyEdge(it, t).apply { - addProperty(Properties.DEPENDENCE, DependenceType.DATA) - } + true } } assertTrue( @@ -79,23 +71,19 @@ class ProgramDependenceGraphPassTest { "expectedPrevEdges: ${expectedPrevEdges.sortedBy { it.hashCode() }}\n" + "actualPrevEdges: ${t.prevPDGEdges.sortedBy { it.hashCode() }}" ) { - compareCollectionWithoutOrder(expectedPrevEdges, t.prevPDGEdges) + t.prevPDGEdges.union(expectedPrevEdges) == t.prevPDGEdges } val expectedNextEdges = - t.nextCDGEdges.map { - it.apply { addProperty(Properties.DEPENDENCE, DependenceType.CONTROL) } - } + - t.nextDFG.mapNotNull { + t.nextCDGEdges + + t.nextDFGEdges.filter { if ( "remove next" in (t.comment ?: "") && - "remove prev" in (it.comment ?: "") + "remove prev" in (it.end.comment ?: "") ) { - null + false } else { - PropertyEdge(t, it).apply { - addProperty(Properties.DEPENDENCE, DependenceType.DATA) - } + true } } assertTrue( @@ -103,31 +91,14 @@ class ProgramDependenceGraphPassTest { "\nexpectedNextEdges: ${expectedNextEdges.sortedBy { it.hashCode() }}" + "\nactualNextEdges: ${t.nextPDGEdges.sortedBy { it.hashCode() }}" ) { - compareCollectionWithoutOrder(expectedNextEdges, t.nextPDGEdges) + t.prevPDGEdges.union(expectedPrevEdges) == t.prevPDGEdges } } } ) } - private fun compareCollectionWithoutOrder( - expected: Collection, - actual: Collection - ): Boolean { - val expectedWithDuplicatesGrouped = expected.groupingBy { it }.eachCount() - val actualWithDuplicatesGrouped = actual.groupingBy { it }.eachCount() - - return expected.size == actual.size && - expectedWithDuplicatesGrouped == actualWithDuplicatesGrouped - } - companion object { - fun testFrontend(config: TranslationConfiguration): TestLanguageFrontend { - val ctx = TranslationContext(config, ScopeManager(), TypeManager()) - val language = config.languages.filterIsInstance().first() - return TestLanguageFrontend(language.namespaceDelimiter, language, ctx) - } - @JvmStatic fun provideTranslationResultForPDGTest() = Stream.of( @@ -136,14 +107,12 @@ class ProgramDependenceGraphPassTest { ) private fun getIfTest() = - testFrontend( - TranslationConfiguration.builder() - .registerLanguage(TestLanguage("::")) - .defaultPasses() - .registerPass() - .registerPass() - .build() - ) + testFrontend { + it.registerLanguage(TestLanguage(".")) + it.defaultPasses() + it.registerPass() + it.registerPass() + } .build { translationResult { translationUnit("if.cpp") { @@ -174,14 +143,12 @@ class ProgramDependenceGraphPassTest { } private fun getWhileLoopTest() = - testFrontend( - TranslationConfiguration.builder() - .registerLanguage(TestLanguage("::")) - .defaultPasses() - .registerPass() - .registerPass() - .build() - ) + testFrontend { + it.registerLanguage(TestLanguage(".")) + it.defaultPasses() + it.registerPass() + it.registerPass() + } .build { translationResult { translationUnit("loop.cpp") { diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt index 47eef6eb9d..3fb62a3bfe 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/scopes/ScopeManagerTest.kt @@ -100,7 +100,7 @@ internal class ScopeManagerTest : BaseTest() { // resolve symbol val call = frontend.newCallExpression(frontend.newReference("A::func1"), "A::func1", false) - val func = final.resolveFunctionLegacy(call).firstOrNull() + val func = final.lookupSymbolByName(call.callee!!.name).firstOrNull() assertEquals(func1, func) } diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/frontends/TestLanguage.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/frontends/TestLanguage.kt index 839c57538b..859e58adec 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/frontends/TestLanguage.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/frontends/TestLanguage.kt @@ -41,9 +41,9 @@ import kotlin.reflect.KClass * This is a test language that can be used for unit test, where we need a language but do not have * a specific one. */ -open class TestLanguage(namespaceDelimiter: String = "::") : Language() { +open class TestLanguage(final override var namespaceDelimiter: String = "::") : + Language() { override val fileExtensions: List = listOf() - final override val namespaceDelimiter: String override val frontend: KClass = TestLanguageFrontend::class override val compoundAssignmentOperators = setOf("+=", "-=", "*=", "/=", "%=", "<<=", ">>=", "&=", "|=", "^=") @@ -61,10 +61,6 @@ open class TestLanguage(namespaceDelimiter: String = "::") : Language Unit): TestLanguageFrontend { + var config = TranslationConfiguration.builder().also(builder).build() + + var ctx: TranslationContext = TranslationContext(config, ScopeManager(), TypeManager()) + return TestLanguageFrontend(ctx = ctx) +} + open class TestLanguageFrontend( namespaceDelimiter: String = "::", language: Language = TestLanguage(namespaceDelimiter), diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index b510edd302..ad24b11905 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.frontends.* -import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.types.* import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient @@ -130,18 +129,6 @@ open class CLanguage : return ImplicitCast } - // Another special rule is that if we have a const reference (e.g. const T&) in a function - // call, this will match the type T because this means that the parameter is given by - // reference rather than by value. - if ( - targetType is ReferenceType && - targetType.elementType == type && - targetHint is ParameterDeclaration && - CONST in targetHint.modifiers - ) { - return DirectMatch - } - return CastNotPossible } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt index c8e328f590..23d7b942fd 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt @@ -25,16 +25,25 @@ */ package de.fraunhofer.aisec.cpg.frontends.cxx +import de.fraunhofer.aisec.cpg.CallResolutionResult +import de.fraunhofer.aisec.cpg.SignatureMatches import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.* +import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.primitiveType +import de.fraunhofer.aisec.cpg.graph.scopes.Symbol +import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator import de.fraunhofer.aisec.cpg.graph.types.* +import de.fraunhofer.aisec.cpg.matchesSignature import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.inference.startInference +import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient /** The C++ language. */ @@ -42,16 +51,63 @@ open class CPPLanguage : CLanguage(), HasDefaultArguments, HasTemplates, - HasComplexCallResolution, HasStructs, HasClasses, HasUnknownType, - HasFunctionalCasts, - HasFunctionOverloading { + HasFunctionStyleCasts, + HasFunctionOverloading, + HasOperatorOverloading { override val fileExtensions = listOf("cpp", "cc", "cxx", "c++", "hpp", "hh") override val elaboratedTypeSpecifier = listOf("class", "struct", "union", "enum") override val unknownTypeString = listOf("auto") + @Transient + override val overloadedOperatorNames: + Map, String>, Symbol> = + mapOf( + // Arithmetic operators. See + // https://en.cppreference.com/w/cpp/language/operator_arithmetic + UnaryOperator::class of "+" to "operator+", + UnaryOperator::class of "-" to "operator-", + BinaryOperator::class of "+" to "operator+", + BinaryOperator::class of "-" to "operator-", + BinaryOperator::class of "*" to "operator*", + BinaryOperator::class of "/" to "operator/", + BinaryOperator::class of "%" to "operator%", + UnaryOperator::class of "~" to "operator~", + BinaryOperator::class of "&" to "operator&", + BinaryOperator::class of "|" to "operator|", + BinaryOperator::class of "^" to "operator^", + BinaryOperator::class of "<<" to "operator<<", + BinaryOperator::class of ">>" to "operator>>", + + // Increment/decrement operators. See + // https://en.cppreference.com/w/cpp/language/operator_incdec + UnaryOperator::class of "++" to "operator++", + UnaryOperator::class of "--" to "operator--", + + // Comparison operators. See + // https://en.cppreference.com/w/cpp/language/operator_comparison + BinaryOperator::class of "==" to "operator==", + BinaryOperator::class of "!=" to "operator!=", + BinaryOperator::class of "<" to "operator<", + BinaryOperator::class of ">" to "operator>", + BinaryOperator::class of "<=" to "operator<=", + BinaryOperator::class of "=>" to "operator=>", + + // Member access operators. See + // https://en.cppreference.com/w/cpp/language/operator_member_access + MemberExpression::class of "[]" to "operator[]", + UnaryOperator::class of "*" to "operator*", + UnaryOperator::class of "&" to "operator&", + MemberExpression::class of "->" to "operator->", + MemberExpression::class of "->*" to "operator->*", + + // Other operators. See https://en.cppreference.com/w/cpp/language/operator_other + MemberCallExpression::class of "()" to "operator()", + BinaryOperator::class of "," to "operator,", + ) + /** * The list of built-in types. See https://en.cppreference.com/w/cpp/language/types for a * reference. We only list equivalent types here and use the canonical form of integer values. @@ -107,43 +163,6 @@ open class CPPLanguage : "__int128" to IntegerType("__int128", 128, this, NumericType.Modifier.SIGNED), ) - /** - * @param call - * @return FunctionDeclarations that are invocation candidates for the MethodCall call using C++ - * resolution techniques - */ - override fun refineMethodCallResolution( - curClass: RecordDeclaration?, - possibleContainingTypes: Set, - call: CallExpression, - ctx: TranslationContext, - currentTU: TranslationUnitDeclaration, - callResolver: SymbolResolver - ): List { - var invocationCandidates = mutableListOf() - val records = possibleContainingTypes.mapNotNull { it.root.recordDeclaration }.toSet() - for (record in records) { - invocationCandidates.addAll( - callResolver.getInvocationCandidatesFromRecord(record, call.name.localName, call) - ) - } - if (invocationCandidates.isEmpty()) { - // This could be a regular function call that somehow ends up here because of weird - // complexity of the old call resolver - val result = ctx.scopeManager.resolveCall(call) - invocationCandidates.addAll(result.bestViable) - } - - // Make sure, that our invocation candidates for member call expressions are really METHODS, - // otherwise this will lead to false positives. This is a hotfix until we rework the call - // resolver completely. - if (call is MemberCallExpression) { - invocationCandidates = - invocationCandidates.filterIsInstance().toMutableList() - } - return invocationCandidates - } - override fun tryCast( type: Type, targetType: Type, @@ -155,6 +174,17 @@ open class CPPLanguage : return match } + // Another special rule is that if we have a (const) reference (e.g. const T&) in a function + // call, this will match the type T because this means that the parameter is given by + // reference rather than by value. + if ( + targetType is ReferenceType && + targetType.elementType == type && + targetHint is ParameterDeclaration + ) { + return DirectMatch + } + // In C++, it is possible to have conversion constructors. We will not have full support for // them yet, but at least we should have some common cases here, such as const char* to // std::string @@ -169,6 +199,29 @@ open class CPPLanguage : return CastNotPossible } + override fun bestViableResolution( + result: CallResolutionResult + ): Pair, CallResolutionResult.SuccessKind> { + // There is a sort of weird workaround in C++ to select a prefix vs. postfix operator for + // increment and decrement operators. See + // https://en.cppreference.com/w/cpp/language/operator_incdec. If it is a postfix, we need + // to match for a function with a fake "int" parameter + val expr = result.source + if ( + expr is UnaryOperator && + (expr.operatorCode == "++" || expr.operatorCode == "--") && + expr.isPostfix + ) { + result.signatureResults = + result.candidateFunctions + .map { Pair(it, it.matchesSignature(listOf(primitiveType("int")))) } + .filter { it.second is SignatureMatches } + .associate { it } + } + + return super.bestViableResolution(result) + } + override val startCharacter = '<' override val endCharacter = '>' @@ -197,7 +250,7 @@ open class CPPLanguage : val orderedInitializationSignature = mutableMapOf() val explicitInstantiation = mutableListOf() if ( - (templateCall.templateParameters.size <= + (templateCall.templateArguments.size <= functionTemplateDeclaration.parameters.size) && (templateCall.arguments.size <= functionTemplateDeclaration.realization[0].parameters.size) @@ -249,13 +302,10 @@ open class CPPLanguage : val functionTemplateDeclaration = holder?.startInference(ctx)?.inferFunctionTemplate(templateCall) templateCall.templateInstantiation = functionTemplateDeclaration - val edges = templateCall.templateParameterEdges + val edges = templateCall.templateArgumentEdges // Set instantiation propertyEdges - for (instantiationParameter in edges ?: listOf()) { - instantiationParameter.addProperty( - Properties.INSTANTIATION, - TemplateDeclaration.TemplateInitialization.EXPLICIT - ) + for (edge in edges ?: listOf()) { + edge.instantiation = TemplateDeclaration.TemplateInitialization.EXPLICIT } if (functionTemplateDeclaration == null) { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt index eec0f337ad..28b5ee5ed8 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt @@ -378,7 +378,7 @@ open class CXXLanguageFrontend(language: Language, ctx: Tra */ fun processAttributes(node: Node, owner: IASTNode) { if (config.processAnnotations && owner is IASTAttributeOwner) { // set attributes - node.addAnnotations(handleAttributes(owner)) + node.annotations += handleAttributes(owner) } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt index 74451bf7ca..210aa97d68 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt @@ -218,7 +218,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : if (lastStatement !is ReturnStatement) { val returnStatement = newReturnStatement() returnStatement.isImplicit = true - bodyStatement.addStatement(returnStatement) + bodyStatement.statements += returnStatement } declaration.body = bodyStatement } @@ -314,7 +314,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : val defaultType = frontend.typeOf(templateParameter.defaultType) typeParamDecl.default = defaultType } - templateDeclaration.addParameter(typeParamDecl) + templateDeclaration.parameters += typeParamDecl } else if (templateParameter is CPPASTParameterDeclaration) { // Handle Value Parameters val nonTypeTemplateParamDeclaration = @@ -329,11 +329,11 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : ) nonTypeTemplateParamDeclaration.default = defaultExpression defaultExpression?.let { - nonTypeTemplateParamDeclaration.addPrevDFG(it) - it.addNextDFG(nonTypeTemplateParamDeclaration) + nonTypeTemplateParamDeclaration.prevDFGEdges += it + it.nextDFGEdges += nonTypeTemplateParamDeclaration } } - templateDeclaration.addParameter(nonTypeTemplateParamDeclaration) + templateDeclaration.parameters += nonTypeTemplateParamDeclaration frontend.scopeManager.addDeclaration(nonTypeTemplateParamDeclaration) } } @@ -422,7 +422,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : handleDeclarationSpecifier(declSpecifier, ctx, sequence) // Fill template params, if needed - val templateParams: List? = extractTemplateParams(ctx, declSpecifier) + val templateParams = extractTemplateParams(ctx, declSpecifier) // Loop through all declarators, as we can potentially have multiple declarations here for (declarator in ctx.declarators) { @@ -484,7 +484,9 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : // initializer. if (declaration is VariableDeclaration) { // Set template parameters of the variable (if any) - declaration.templateParameters = templateParams + if (templateParams != null) { + declaration.templateParameters = templateParams + } // Parse the initializer, if we have one declarator.initializer?.let { @@ -786,9 +788,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : val problems = problematicIncludes[includeString] val includeDeclaration = newIncludeDeclaration(includeString ?: "") - if (problems != null) { - includeDeclaration.addProblems(problems) - } + problems?.forEach { includeDeclaration.problems += it } includeMap[includeString] = includeDeclaration } } @@ -801,8 +801,11 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : // attach to remaining nodes for ((key, value) in allIncludes) { val includeDeclaration = includeMap[key] + if (includeDeclaration == null) { + continue + } for (s in value) { - includeDeclaration?.addInclude(includeMap[s]) + includeMap[s]?.let { includeDeclaration.includes += it } } } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt index 0b83ba0909..ddf3ab683d 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import de.fraunhofer.aisec.cpg.ResolveInFrontend +import de.fraunhofer.aisec.cpg.frontends.isKnownOperatorName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.NameScope @@ -33,7 +34,6 @@ import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util -import java.util.* import java.util.function.Supplier import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCompositeTypeSpecifier @@ -158,7 +158,7 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : * checks if the scope is a namespace or a record and if the name matches to the record (in case * of a constructor). */ - private fun createFunctionOrMethodOrConstructor( + private fun createAppropriateFunction( name: Name, scope: Scope?, ctx: IASTNode, @@ -168,9 +168,14 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : val func = when { + // Check if it's an operator + name.isKnownOperatorName -> { + // retrieve the operator code + val operatorCode = name.localName.drop("operator".length) + newOperatorDeclaration(name, operatorCode, rawNode = ctx) + } // Check, if it's a constructor. This is the case if the local names of the function - // and the - // record declaration match + // and the record declaration match holder is RecordDeclaration && name.localName == holder.name.localName -> { newConstructorDeclaration(name, holder, rawNode = ctx) } @@ -219,9 +224,11 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : /* * As always, there are some special cases to consider and one of those are C++ operators. - * They are regarded as functions and eclipse CDT for some reason introduces a whitespace in the function name, which will complicate things later on + * They are regarded as functions and eclipse CDT for some reason introduces a whitespace + * in the function name, which will complicate things later on. But we only want to replace + * the whitespace for "standard" operators. */ - if (name.startsWith("operator")) { + if (nameDecl.name is CPPASTOperatorName && name.replace(" ", "").isKnownOperatorName) { name = name.replace(" ", "") } val declaration: FunctionDeclaration @@ -242,15 +249,11 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : frontend.scopeManager.currentNamespace.fqn(parent.toString()).toString() ) - declaration = createFunctionOrMethodOrConstructor(name, parentScope, ctx.parent) + declaration = createAppropriateFunction(name, parentScope, ctx.parent) } else if (frontend.scopeManager.isInRecord) { // If the current scope is already a record, it's a method declaration = - createFunctionOrMethodOrConstructor( - name, - frontend.scopeManager.currentScope, - ctx.parent - ) + createAppropriateFunction(name, frontend.scopeManager.currentScope, ctx.parent) } else { // a plain old function, outside any named scope declaration = newFunctionDeclaration(name, rawNode = ctx.parent) diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt index 88393994a0..1647567d08 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt @@ -120,7 +120,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : val initializer = frontend.initializerHandler.handle(node.initializer) if (initializer is InitializerListExpression) { val construct = newConstructExpression(rawNode = node) - construct.arguments = initializer.initializers.toList() + construct.arguments = initializer.initializers construct } else initializer ?: newProblemExpression("could not parse initializer") } @@ -261,10 +261,11 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // we also need to "forward" our template parameters (if we have any) to the construct // expression since the construct expression will do the actual template instantiation - if (newExpression.templateParameters?.isNotEmpty() == true) { - newExpression.templateParameters?.let { - addImplicitTemplateParametersToCall(it, initializer as ConstructExpression) - } + if (newExpression.templateParameters.isNotEmpty() == true) { + addImplicitTemplateParametersToCall( + newExpression.templateParameters, + initializer as ConstructExpression + ) } // our initializer, such as a construct expression, will have the non-pointer type @@ -281,8 +282,8 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : * @param template * @return List of Nodes containing the all the arguments the template was instantiated with. */ - private fun getTemplateArguments(template: CPPASTTemplateId): List { - val templateArguments: MutableList = ArrayList() + private fun getTemplateArguments(template: CPPASTTemplateId): MutableList { + val templateArguments = mutableListOf() for (argument in template.templateArguments) { when (argument) { is IASTTypeId -> { @@ -320,7 +321,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : for (name in ctx.implicitDestructorNames) { log.debug("Implicit constructor name {}", name) } - deleteExpression.operand = handle(ctx.operand) + handle(ctx.operand)?.let { deleteExpression.operands.add(it) } return deleteExpression } @@ -530,7 +531,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : private fun handleExpressionList(exprList: IASTExpressionList): ExpressionList { val expressionList = newExpressionList(rawNode = exprList) for (expr in exprList.expressions) { - handle(expr)?.let { expressionList.addExpression(it) } + handle(expr)?.let { expressionList.expressions += it } } return expressionList } @@ -868,7 +869,9 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // The only supported initializer is an initializer list (ctx.initializer as? IASTInitializerList)?.let { construct.arguments = - it.clauses.map { handle(it) ?: newProblemExpression("could not parse argument") } + it.clauses + .map { handle(it) ?: newProblemExpression("could not parse argument") } + .toMutableList() } return construct diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt index d2c4cdcb8f..3dbae3df3c 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt @@ -27,8 +27,6 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.newConstructExpression import de.fraunhofer.aisec.cpg.graph.newInitializerListExpression import de.fraunhofer.aisec.cpg.graph.newProblemExpression @@ -83,11 +81,8 @@ class InitializerHandler(lang: CXXLanguageFrontend) : for (clause in ctx.clauses) { frontend.expressionHandler.handle(clause)?.let { - val edge = PropertyEdge(expression, it) - edge.addProperty(Properties.INDEX, expression.initializerEdges.size) - - expression.initializerEdges.add(edge) - expression.addPrevDFG(it) + expression.initializerEdges.add(it) + expression.prevDFGEdges += it } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt index 986f46bcd3..2dfc0f60b9 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/StatementHandler.kt @@ -297,7 +297,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : val declarationStatement = newDeclarationStatement(rawNode = ctx) val declaration = frontend.declarationHandler.handle(ctx.declaration) if (declaration is DeclarationSequence) { - declarationStatement.declarations = declaration.asList() + declarationStatement.declarations = declaration.asMutableList() } else { declarationStatement.singleDeclaration = declaration } @@ -324,7 +324,7 @@ class StatementHandler(lang: CXXLanguageFrontend) : for (statement in ctx.statements) { val handled = handle(statement) if (handled != null) { - block.addStatement(handled) + block.statements += handled } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt index f7b2a84114..b6eeae1365 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXExtraPass.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.scopes.ValueDeclarationScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.recordDeclaration import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.helpers.replace import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore @@ -45,7 +46,7 @@ import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore * type information. */ @ExecuteBefore(EvaluationOrderGraphPass::class) -@ExecuteBefore(ReplaceCallCastPass::class) +@ExecuteBefore(ResolveCallExpressionAmbiguityPass::class) @DependsOn(TypeResolver::class) class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { @@ -76,14 +77,14 @@ class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { * the graph. */ private fun removeBracketOperators(node: UnaryOperator, parent: Node?) { - if (node.operatorCode == "()" && !typeManager.typeExists(node.input.name.toString())) { + if (node.operatorCode == "()" && !typeManager.typeExists(node.input.name)) { // It was really just parenthesis around an identifier, but we can only make this // distinction now. // // In theory, we could just keep this meaningless unary expression, but it in order // to reduce nodes, we unwrap the reference and exchange it in the arguments of the // binary op - walker.replaceArgument(parent, node, node.input) + walker.replace(parent, node, node.input) } } @@ -92,9 +93,10 @@ class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { * operator where some arguments are wrapped in parentheses. This function tries to resolve * this. * - * Note: This is done especially for the C++ frontend. [ReplaceCallCastPass.handleCall] handles - * the more general case (which also applies to C++), in which a cast and a call are - * indistinguishable and need to be resolved once all types are known. + * Note: This is done especially for the C++ frontend. + * [ResolveCallExpressionAmbiguityPass.handleCall] handles the more general case (which also + * applies to C++), in which a cast and a call are indistinguishable and need to be resolved + * once all types are known. */ private fun convertOperators(binOp: BinaryOperator, parent: Node?) { val fakeUnaryOp = binOp.lhs @@ -107,7 +109,7 @@ class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { language != null && fakeUnaryOp is UnaryOperator && fakeUnaryOp.operatorCode == "()" && - typeManager.typeExists(fakeUnaryOp.input.name.toString()) + typeManager.typeExists(fakeUnaryOp.input.name) ) { // If the name (`long` in the example) is a type, then the unary operator (`(long)`) // is really a cast and our binary operator is really a unary operator `&addr`. @@ -116,7 +118,7 @@ class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { // referred to in the op. val cast = newCastExpression().codeAndLocationFrom(fakeUnaryOp) cast.language = language - cast.castType = language.objectType(fakeUnaryOp.input.name) + cast.castType = fakeUnaryOp.objectType(fakeUnaryOp.input.name) // * create a unary operator with the rhs of the binary operator (and the same // operator code). @@ -143,7 +145,7 @@ class CXXExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { // * replace the binary operator with the cast expression in the parent argument // holder - walker.replaceArgument(parent, binOp, cast) + walker.replace(parent, binOp, cast) } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt index f03dd0146f..a1afbe40ff 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt @@ -27,17 +27,15 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage -import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage +import de.fraunhofer.aisec.cpg.frontends.cxx.CXXLanguageFrontend import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge import de.fraunhofer.aisec.cpg.graph.newLiteral +import de.fraunhofer.aisec.cpg.graph.primitiveType import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression -import de.fraunhofer.aisec.cpg.graph.types.IntegerType -import de.fraunhofer.aisec.cpg.graph.types.NumericType import de.fraunhofer.aisec.cpg.helpers.Benchmark -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.test.* import java.time.Duration import java.time.temporal.ChronoUnit @@ -89,16 +87,7 @@ class PerformanceRegressionTest { val list = InitializerListExpression() for (i in 0 until 50000) { - list.initializerEdges.add( - PropertyEdge( - list, - newLiteral( - i, - IntegerType("int", 32, CPPLanguage(), NumericType.Modifier.UNSIGNED), - null - ) - ) - ) + list.initializerEdges.add(AstEdge(list, newLiteral(i, primitiveType("int"), null))) } decl.initializer = list @@ -115,7 +104,7 @@ class PerformanceRegressionTest { } fun doNothing(node: Node) { - for (child in SubgraphWalker.getAstChildren(node)) { + for (child in node.astChildren) { doNothing(child) } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt index 5a1d30aadf..4d4cc1d831 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.graph.SearchModifier.UNIQUE import de.fraunhofer.aisec.cpg.graph.allChildren import de.fraunhofer.aisec.cpg.graph.declarations.ConstructorDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker @@ -111,7 +110,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifSimple.thenStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifSimple) ) ) @@ -130,7 +129,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifSimple, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifSimple.thenStatement) ) ) @@ -162,7 +161,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifBranched, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifBranched.thenStatement) ) ) @@ -171,7 +170,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifBranched, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(ifBranched.elseStatement) ) ) @@ -184,7 +183,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifBranched.thenStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifBranched) ) ) @@ -195,7 +194,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifBranched.elseStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(ifBranched) ) ) @@ -284,7 +283,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.EXITS, n = fs, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[1]) ) ) @@ -345,7 +344,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.EXITS, n = fs, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[2]) ) ) @@ -406,7 +405,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.EXITS, n = fs, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[4]) ) ) @@ -511,7 +510,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = wstat.statement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(wstat) ) ) @@ -523,7 +522,7 @@ internal class EOGTest : BaseTest() { cn = Connect.SUBTREE, en = Util.Edge.EXITS, n = wstat, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[1]) ) ) @@ -556,7 +555,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = dostat, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(dostat.statement) ) ) @@ -577,7 +576,7 @@ internal class EOGTest : BaseTest() { cn = Connect.SUBTREE, en = Util.Edge.EXITS, n = dostat, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[2]) ) ) diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt index 578bb0d6d8..b81cd85e79 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt @@ -32,7 +32,6 @@ import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression import de.fraunhofer.aisec.cpg.test.* @@ -165,7 +164,7 @@ internal class FunctionPointerTest : BaseTest() { } if (functions.size == 0) { variable.usageEdges - .filter { it.getProperty(Properties.ACCESS) == AccessValues.WRITE } + .filter { it.access == AccessValues.WRITE } .forEach { worklist.push(it.end) } while (!worklist.isEmpty()) { val curr = worklist.pop() diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/ClassTemplateTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/ClassTemplateTest.kt index 7256379f70..d7df4a7a05 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/ClassTemplateTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/ClassTemplateTest.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg.enhancements.templates import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.ObjectType @@ -122,7 +121,7 @@ internal class ClassTemplateTest : BaseTest() { assertLocalName("int", instantiatedType.generics[0]) assertLocalName("int", instantiatedType.generics[1]) - val templateParameters = constructExpression.templateParameters + val templateParameters = constructExpression.templateArguments assertNotNull(templateParameters) assertEquals(2, templateParameters.size) assertLocalName("int", (templateParameters[0] as TypeExpression).type) @@ -243,13 +242,13 @@ internal class ClassTemplateTest : BaseTest() { assertEquals(literal3, point1.templateParameters?.get(2)) // Test Invocation - val templateParameters = constructExpr.templateParameters + val templateParameters = constructExpr.templateArguments assertNotNull(templateParameters) assertEquals(3, templateParameters.size) assertEquals(literal3Implicit, templateParameters[2]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpr.templateParameterEdges?.get(2)?.getProperty(Properties.INSTANTIATION) + constructExpr.templateArgumentEdges?.get(2)?.instantiation ) assertEquals(pair, constructExpr.instantiates) assertEquals(template, constructExpr.templateInstantiation) @@ -265,20 +264,16 @@ internal class ClassTemplateTest : BaseTest() { assertEquals(pair, constructExpression.instantiates) assertEquals(template, constructExpression.templateInstantiation) assertEquals(pairConstructorDeclaration, constructExpression.constructor) - assertEquals(2, constructExpression.templateParameters.size) - assertLocalName("int", constructExpression.templateParameters[0]) + assertEquals(2, constructExpression.templateArguments.size) + assertLocalName("int", constructExpression.templateArguments[0]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpression.templateParameterEdges - ?.get(0) - ?.getProperty(Properties.INSTANTIATION) + constructExpression.templateArgumentEdges?.get(0)?.instantiation ) - assertLocalName("int", constructExpression.templateParameters[1]) + assertLocalName("int", constructExpression.templateArguments[1]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpression.templateParameterEdges - ?.get(1) - ?.getProperty(Properties.INSTANTIATION) + constructExpression.templateArgumentEdges?.get(1)?.instantiation ) val pairTypeInstantiated = constructExpression.type as ObjectType @@ -370,26 +365,26 @@ internal class ClassTemplateTest : BaseTest() { findByUniquePredicate(result.literals) { it.value == 2 && it.isImplicit } assertEquals(pair, constructExpr.instantiates) assertEquals(template, constructExpr.templateInstantiation) - assertEquals(4, constructExpr.templateParameters.size) - assertLocalName("int", constructExpr.templateParameters[0]) + assertEquals(4, constructExpr.templateArguments.size) + assertLocalName("int", constructExpr.templateArguments[0]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpr.templateParameterEdges?.get(0)?.getProperty(Properties.INSTANTIATION) + constructExpr.templateArgumentEdges?.get(0)?.instantiation ) - assertLocalName("int", constructExpr.templateParameters[1]) + assertLocalName("int", constructExpr.templateArguments[1]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpr.templateParameterEdges?.get(1)?.getProperty(Properties.INSTANTIATION) + constructExpr.templateArgumentEdges?.get(1)?.instantiation ) - assertEquals(literal2Implicit, constructExpr.templateParameters[2]) + assertEquals(literal2Implicit, constructExpr.templateArguments[2]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpr.templateParameterEdges?.get(2)?.getProperty(Properties.INSTANTIATION) + constructExpr.templateArgumentEdges?.get(2)?.instantiation ) - assertEquals(literal2Implicit, constructExpr.templateParameters[3]) + assertEquals(literal2Implicit, constructExpr.templateArguments[3]) assertEquals( TemplateDeclaration.TemplateInitialization.DEFAULT, - constructExpr.templateParameterEdges?.get(3)?.getProperty(Properties.INSTANTIATION) + constructExpr.templateArgumentEdges?.get(3)?.instantiation ) val type = constructExpr.type as ObjectType @@ -427,51 +422,31 @@ internal class ClassTemplateTest : BaseTest() { assertEquals(paramA, (paramB.default as Reference).refersTo) assertEquals(pair, constructExpression.instantiates) assertEquals(template, constructExpression.templateInstantiation) - assertEquals(4, constructExpression.templateParameters.size) - assertLocalName("int", (constructExpression.templateParameters[0] as TypeExpression).type) + assertEquals(4, constructExpression.templateArguments.size) + assertLocalName("int", (constructExpression.templateArguments[0] as TypeExpression).type) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - constructExpression.templateParameterEdges - ?.get(0) - ?.getProperty(Properties.INSTANTIATION) + constructExpression.templateArgumentEdges?.get(0)?.instantiation ) - assertEquals( - 0, - constructExpression.templateParameterEdges?.get(0)?.getProperty(Properties.INDEX) - ) - assertLocalName("int", (constructExpression.templateParameters[1] as TypeExpression).type) + assertEquals(0, constructExpression.templateArgumentEdges?.get(0)?.index) + assertLocalName("int", (constructExpression.templateArguments[1] as TypeExpression).type) assertEquals( TemplateDeclaration.TemplateInitialization.DEFAULT, - constructExpression.templateParameterEdges - ?.get(1) - ?.getProperty(Properties.INSTANTIATION) - ) - assertEquals( - 1, - constructExpression.templateParameterEdges?.get(1)?.getProperty(Properties.INDEX) + constructExpression.templateArgumentEdges?.get(1)?.instantiation ) - assertEquals(literal1, constructExpression.templateParameters[2]) + assertEquals(1, constructExpression.templateArgumentEdges?.get(1)?.index) + assertEquals(literal1, constructExpression.templateArguments[2]) assertEquals( TemplateDeclaration.TemplateInitialization.DEFAULT, - constructExpression.templateParameterEdges - ?.get(2) - ?.getProperty(Properties.INSTANTIATION) + constructExpression.templateArgumentEdges?.get(2)?.instantiation ) - assertEquals( - 2, - constructExpression.templateParameterEdges?.get(2)?.getProperty(Properties.INDEX) - ) - assertEquals(literal1, constructExpression.templateParameters[3]) + assertEquals(2, constructExpression.templateArgumentEdges?.get(2)?.index) + assertEquals(literal1, constructExpression.templateArguments[3]) assertEquals( TemplateDeclaration.TemplateInitialization.DEFAULT, - constructExpression.templateParameterEdges - ?.get(3) - ?.getProperty(Properties.INSTANTIATION) - ) - assertEquals( - 3, - constructExpression.templateParameterEdges?.get(3)?.getProperty(Properties.INDEX) + constructExpression.templateArgumentEdges?.get(3)?.instantiation ) + assertEquals(3, constructExpression.templateArgumentEdges?.get(3)?.index) // Test Type val type = constructExpression.type as ObjectType @@ -526,8 +501,8 @@ internal class ClassTemplateTest : BaseTest() { } assertEquals(template, constructExpr.templateInstantiation) assertEquals(array, constructExpr.instantiates) - assertLocalName("int", constructExpr.templateParameters[0]) - assertEquals(literal10, constructExpr.templateParameters[1]) + assertLocalName("int", constructExpr.templateArguments[0]) + assertEquals(literal10, constructExpr.templateArguments[1]) assertLocalName("Array", constructExpr.type) val instantiatedType = constructExpr.type as ObjectType @@ -570,10 +545,10 @@ internal class ClassTemplateTest : BaseTest() { val newExpression = findByUniqueName(result.allChildren(), "") assertEquals(array, constructExpression.instantiates) assertEquals(template, constructExpression.templateInstantiation) - assertEquals(2, constructExpression.templateParameters.size) - assertLocalName("int", (constructExpression.templateParameters[0] as TypeExpression).type) - assertTrue(constructExpression.templateParameters[0].isImplicit) - assertEquals(literal5Implicit, constructExpression.templateParameters[1]) + assertEquals(2, constructExpression.templateArguments.size) + assertLocalName("int", (constructExpression.templateArguments[0] as TypeExpression).type) + assertTrue(constructExpression.templateArguments[0].isImplicit) + assertEquals(literal5Implicit, constructExpression.templateArguments[1]) assertEquals(2, arrayVariable.templateParameters?.size) assertLocalName("int", (arrayVariable.templateParameters?.get(0) as TypeExpression).type) assertFalse(arrayVariable.templateParameters!![0].isImplicit) diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt index be7dd17c67..bd91e4a0ee 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg.enhancements.templates import de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.test.* @@ -68,18 +67,18 @@ internal class FunctionTemplateTest : BaseTest() { floatType: ObjectType, int3: Literal<*> ) { - assertEquals(2, callFloat3.templateParameters.size) - assertEquals(floatType, (callFloat3.templateParameters[0] as TypeExpression).type) - assertEquals(0, callFloat3.templateParameterEdges!![0].getProperty(Properties.INDEX)) + assertEquals(2, callFloat3.templateArguments.size) + assertEquals(floatType, (callFloat3.templateArguments[0] as TypeExpression).type) + assertEquals(0, callFloat3.templateArgumentEdges!![0].index) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - callFloat3.templateParameterEdges!![0].getProperty(Properties.INSTANTIATION) + callFloat3.templateArgumentEdges!![0].instantiation ) - assertEquals(int3, callFloat3.templateParameters[1]) - assertEquals(1, callFloat3.templateParameterEdges!![1].getProperty(Properties.INDEX)) + assertEquals(int3, callFloat3.templateArguments[1]) + assertEquals(1, callFloat3.templateArgumentEdges!![1].index) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - callFloat3.templateParameterEdges!![1].getProperty(Properties.INSTANTIATION) + callFloat3.templateArgumentEdges!![1].instantiation ) } @@ -220,9 +219,9 @@ internal class FunctionTemplateTest : BaseTest() { // Check template parameters val doubleType = FloatingPointType("double", 64, CPPLanguage(), NumericType.Modifier.SIGNED) val literal5 = findByUniquePredicate(result.literals) { l: Literal<*> -> l.value == 5 } - assertEquals(2, call.templateParameters.size) - assertEquals(doubleType, (call.templateParameters[0] as TypeExpression).type) - assertEquals(literal5, call.templateParameters[1]) + assertEquals(2, call.templateArguments.size) + assertEquals(doubleType, (call.templateArguments[0] as TypeExpression).type) + assertEquals(literal5, call.templateArguments[1]) // Check return value assertEquals(doubleType, call.type) @@ -262,9 +261,9 @@ internal class FunctionTemplateTest : BaseTest() { // Check template parameters val literal5 = findByUniquePredicate(result.literals) { l: Literal<*> -> l.value == 5 } - assertEquals(2, call.templateParameters.size) - assertLocalName("double", call.templateParameters[0]) - assertEquals(literal5, call.templateParameters[1]) + assertEquals(2, call.templateArguments.size) + assertLocalName("double", call.templateArguments[0]) + assertEquals(literal5, call.templateArguments[1]) // Check return value assertLocalName("double", call.type) @@ -311,9 +310,9 @@ internal class FunctionTemplateTest : BaseTest() { } val literal5 = findByUniquePredicate>(result.literals) { l: Literal<*> -> l.value == 5 } - assertEquals(2, call.templateParameters.size) - assertEquals(intType, (call.templateParameters[0] as TypeExpression).type) - assertEquals(literal5, call.templateParameters[1]) + assertEquals(2, call.templateArguments.size) + assertEquals(intType, (call.templateArguments[0] as TypeExpression).type) + assertEquals(literal5, call.templateArguments[1]) // Check return value assertEquals(intType, call.type) @@ -357,9 +356,9 @@ internal class FunctionTemplateTest : BaseTest() { // Check template parameters val doubleType = FloatingPointType("double", 64, CPPLanguage(), NumericType.Modifier.SIGNED) val literal5 = findByUniquePredicate(result.literals) { l: Literal<*> -> l.value == 5 } - assertEquals(2, call.templateParameters.size) - assertEquals(doubleType, (call.templateParameters[0] as TypeExpression).type) - assertEquals(literal5, call.templateParameters[1]) + assertEquals(2, call.templateArguments.size) + assertEquals(doubleType, (call.templateArguments[0] as TypeExpression).type) + assertEquals(literal5, call.templateArguments[1]) // Check return value assertEquals(doubleType, call.type) @@ -406,9 +405,9 @@ internal class FunctionTemplateTest : BaseTest() { t.name.localName == "int" } val literal5 = findByUniquePredicate(result.literals) { l: Literal<*> -> l.value == 5 } - assertEquals(2, call.templateParameters.size) - assertEquals(intType, (call.templateParameters[0] as TypeExpression).type) - assertEquals(literal5, call.templateParameters[1]) + assertEquals(2, call.templateArguments.size) + assertEquals(intType, (call.templateArguments[0] as TypeExpression).type) + assertEquals(literal5, call.templateArguments[1]) // Check return value assertEquals(intType, call.type) @@ -512,20 +511,20 @@ internal class FunctionTemplateTest : BaseTest() { assertEquals(1, callExpression.invokes.size) assertEquals(methodDeclaration, callExpression.invokes[0]) assertEquals(templateDeclaration, callExpression.templateInstantiation) - assertEquals(2, callExpression.templateParameters.size) - assertLocalName("int", callExpression.templateParameters[0]) + assertEquals(2, callExpression.templateArguments.size) + assertLocalName("int", callExpression.templateArguments[0]) assertEquals( TemplateDeclaration.TemplateInitialization.EXPLICIT, - callExpression.templateParameterEdges?.get(0)?.getProperty(Properties.INSTANTIATION) + callExpression.templateArgumentEdges?.get(0)?.instantiation ) - assertEquals(0, callExpression.templateParameterEdges!![0].getProperty(Properties.INDEX)) + assertEquals(0, callExpression.templateArgumentEdges!![0].index) val int5 = findByUniquePredicate(result.literals, Predicate { l: Literal<*> -> l.value == 5 }) - assertEquals(int5, callExpression.templateParameters[1]) - assertEquals(1, callExpression.templateParameterEdges!![1].getProperty(Properties.INDEX)) + assertEquals(int5, callExpression.templateArguments[1]) + assertEquals(1, callExpression.templateArgumentEdges!![1].index) assertEquals( TemplateDeclaration.TemplateInitialization.DEFAULT, - callExpression.templateParameterEdges!![1].getProperty(Properties.INSTANTIATION) + callExpression.templateArgumentEdges!![1].instantiation ) } @@ -565,7 +564,7 @@ internal class FunctionTemplateTest : BaseTest() { assertEquals(1, callInt2.invokes.size) assertEquals(fixedDivision, callInt2.invokes[0]) assertTrue( - callInt2.templateParameters[1].nextDFG.contains(templateDeclaration.parameters[1]) + callInt2.templateArguments[1].nextDFG.contains(templateDeclaration.parameters[1]) ) // Check inferred for second fixed_division call @@ -591,7 +590,7 @@ internal class FunctionTemplateTest : BaseTest() { assertEquals(1, callDouble3.invokes.size) assertEquals(fixedDivision, callDouble3.invokes[0]) assertTrue( - callDouble3.templateParameters[1].nextDFG.contains(templateDeclaration.parameters[1]) + callDouble3.templateArguments[1].nextDFG.contains(templateDeclaration.parameters[1]) ) // Check return values diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CDataflowTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CDataflowTest.kt index d7f3e58152..1565041436 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CDataflowTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CDataflowTest.kt @@ -27,8 +27,8 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.edge.Dataflow -import de.fraunhofer.aisec.cpg.graph.edge.PartialDataflowGranularity +import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflow +import de.fraunhofer.aisec.cpg.graph.edges.flows.PartialDataflowGranularity import de.fraunhofer.aisec.cpg.test.* import java.io.File import kotlin.test.Test diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXAmbiguitiesTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXAmbiguitiesTest.kt index 76f6187a5e..aadb414edc 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXAmbiguitiesTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXAmbiguitiesTest.kt @@ -57,8 +57,10 @@ class CXXAmbiguitiesTest { } assertNotNull(tu) - // make sure we still have only one declaration in the file (the record) - assertEquals(1, tu.declarations.size) + // we have 3 (record) declarations in our TU now. 1 of the original MyClass and two because + // CDT thinks that "call" is the return type and "crazy" the type of the parameter. We infer + // record declarations for all types, so we end up with 3 declarations here. + assertEquals(3, tu.declarations.size) val myClass = tu.records["MyClass"] assertNotNull(myClass) diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXDeclarationTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXDeclarationTest.kt index bf197aeb8a..b874b2a2ea 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXDeclarationTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXDeclarationTest.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.OperatorCallExpression import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.test.* import java.io.File @@ -232,4 +233,77 @@ class CXXDeclarationTest { assertEquals(assertResolvedType("ABC::A"), a.type) } } + + @Test + fun testArithmeticOperator() { + val file = File("src/test/resources/cxx/operators/arithmetic.cpp") + val result = + analyze(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + } + assertNotNull(result) + + val integer = result.records["Integer"] + assertNotNull(integer) + + val plusplus = integer.operators["operator++"] + assertNotNull(plusplus) + assertEquals("++", plusplus.operatorCode) + + val plus = integer.operators("operator+") + assertEquals(2, plus.size) + assertEquals("+", plus.map { it.operatorCode }.distinct().singleOrNull()) + + val main = result.functions["main"] + assertNotNull(main) + + val unaryOp = main.operatorCalls["++"] + assertNotNull(unaryOp) + assertInvokes(unaryOp, plusplus) + + val binaryOp0 = main.operatorCalls("+").getOrNull(0) + assertNotNull(binaryOp0) + assertInvokes(binaryOp0, plus.getOrNull(0)) + + val binaryOp1 = main.operatorCalls("+").getOrNull(1) + assertNotNull(binaryOp1) + assertInvokes(binaryOp1, plus.getOrNull(1)) + } + + @Test + fun testMemberAccessOperator() { + val file = File("src/test/resources/cxx/operators/member_access.cpp") + val result = + analyze(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + } + assertNotNull(result) + + var proxy = result.records["Proxy"] + assertNotNull(proxy) + + var op = proxy.operators["operator->"] + assertNotNull(op) + + var data = result.records["Data"] + assertNotNull(data) + + var size = data.fields["size"] + assertNotNull(size) + + val p = result.refs["p"] + assertNotNull(p) + assertEquals(proxy.toType(), p.type) + + var sizeRef = result.memberExpressions["size"] + assertNotNull(sizeRef) + assertRefersTo(sizeRef, size) + + // we should now have an implicit call to our operator in-between "p" and "size" + val opCall = sizeRef.base + assertNotNull(opCall) + assertIs(opCall) + assertEquals(p, opCall.base) + assertInvokes(opCall, op) + } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index a61639add9..3e1b661e95 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -1740,4 +1740,22 @@ internal class CXXLanguageFrontendTest : BaseTest() { val test = result.functions["test"] assertNotNull(test) } + + @Test + @Throws(Exception::class) + fun testCastToInferredType() { + val file = File("src/test/resources/c/cast_to_inferred.c") + val tu = + analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) { + it.registerLanguage() + } + assertNotNull(tu) + + val assign = tu.assigns.firstOrNull() + assertNotNull(assign) + + val cast = assign.rhs.singleOrNull() + assertIs(cast) + assertLocalName("mytype", cast.castType) + } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt index 520b59f9e8..abd331f2d9 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolverTest.kt @@ -717,6 +717,7 @@ class CallResolverTest : BaseTest() { true ) { it.registerLanguage() + it.inferenceConfiguration(InferenceConfiguration.builder().enabled(false).build()) } val calls = result.calls @@ -727,8 +728,7 @@ class CallResolverTest : BaseTest() { } false } - assertEquals(1, calcCall.invokes.size) - assertFalse(calcCall.invokes[0].isInferred) + assertEquals(0, calcCall.invokes.size) } @Test diff --git a/cpg-language-cxx/src/test/resources/c/cast_to_inferred.c b/cpg-language-cxx/src/test/resources/c/cast_to_inferred.c new file mode 100644 index 0000000000..8058090828 --- /dev/null +++ b/cpg-language-cxx/src/test/resources/c/cast_to_inferred.c @@ -0,0 +1,6 @@ +int foo(const mytype *p) +{ + mytype *v; + *v = (mytype)*p; + return 0; +} \ No newline at end of file diff --git a/cpg-language-cxx/src/test/resources/calls/cxxprioresolution/methodresolution/overloadnoresolution.cpp b/cpg-language-cxx/src/test/resources/calls/cxxprioresolution/methodresolution/overloadnoresolution.cpp index 5d1da9cea6..9eab39df76 100644 --- a/cpg-language-cxx/src/test/resources/calls/cxxprioresolution/methodresolution/overloadnoresolution.cpp +++ b/cpg-language-cxx/src/test/resources/calls/cxxprioresolution/methodresolution/overloadnoresolution.cpp @@ -19,6 +19,6 @@ class Overloaded : public Base int main() { Overloaded overload; - cout << overload.calc(1) << '\n'; + overload.calc(1); return 0; } \ No newline at end of file diff --git a/cpg-language-cxx/src/test/resources/cxx/operators/arithmetic.cpp b/cpg-language-cxx/src/test/resources/cxx/operators/arithmetic.cpp new file mode 100644 index 0000000000..a888301d6b --- /dev/null +++ b/cpg-language-cxx/src/test/resources/cxx/operators/arithmetic.cpp @@ -0,0 +1,27 @@ +class Integer { +public: + Integer(int i) : i(i) {} + void operator++(int) { + i++; + } + + Integer operator+(int j) { + return Integer(this->i+j); + } + + Integer operator+(Integer &j) { + return Integer(this->i+j.i); + } + + void test() {}; + + int i; +}; + +int main() { + Integer i(5); + i++; + + Integer j = i + 2; + auto k = i + j; +} \ No newline at end of file diff --git a/cpg-language-cxx/src/test/resources/cxx/operators/member_access.cpp b/cpg-language-cxx/src/test/resources/cxx/operators/member_access.cpp new file mode 100644 index 0000000000..5136efc645 --- /dev/null +++ b/cpg-language-cxx/src/test/resources/cxx/operators/member_access.cpp @@ -0,0 +1,24 @@ +class Data { +public: + int size; +}; + +class Proxy { +public: + Proxy() { + this->data = new Data(); + } + + Data* operator->() { + return data; + } + + Data* data; +}; + +int main() { + Proxy p; + int size = p->size; + + // int another_size = p.operator->()->size; +} \ No newline at end of file diff --git a/cpg-language-go/build.gradle.kts b/cpg-language-go/build.gradle.kts index 342d0b874c..9aea87fd3a 100644 --- a/cpg-language-go/build.gradle.kts +++ b/cpg-language-go/build.gradle.kts @@ -40,7 +40,7 @@ publishing { } dependencies { - implementation("net.java.dev.jna:jna:5.14.0") + implementation("net.java.dev.jna:jna:5.15.0") testImplementation(project(":cpg-analysis")) } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt index 9dda501c57..b1f17ca3bd 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt @@ -42,7 +42,7 @@ class GoLanguage : HasStructs, HasFirstClassFunctions, HasAnonymousIdentifier, - HasFunctionalCasts { + HasFunctionStyleCasts { override val fileExtensions = listOf("go") override val namespaceDelimiter = "." @Transient override val frontend = GoLanguageFrontend::class diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt index dcc0154fac..a735e5baec 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt @@ -190,7 +190,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : // Add the variable to the declaration statement as well as to the current scope (aka // our block wrapper) - stmt.addToPropertyEdgeDeclaration(decl) + stmt.declarationEdges += decl frontend.scopeManager.addDeclaration(decl) if (block != null) { @@ -226,7 +226,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : for (declaration in sequence.declarations) { frontend.scopeManager.addDeclaration(declaration) } - stmt.declarations = sequence.asList() + stmt.declarations = sequence.asMutableList() } else { frontend.scopeManager.addDeclaration(sequence) stmt.singleDeclaration = sequence @@ -329,7 +329,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : if (ref is Reference) { val key = newVariableDeclaration(ref.name, rawNode = it) frontend.scopeManager.addDeclaration(key) - stmt.addToPropertyEdgeDeclaration(key) + stmt.declarationEdges += key } } @@ -339,7 +339,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : if (ref is Reference) { val key = newVariableDeclaration(ref.name, rawNode = it) frontend.scopeManager.addDeclaration(key) - stmt.addToPropertyEdgeDeclaration(key) + stmt.declarationEdges += key } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt index 1683c1df8f..5297e13826 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt @@ -109,8 +109,8 @@ import de.fraunhofer.aisec.cpg.passes.inference.startInference */ @ExecuteBefore(SymbolResolver::class) @ExecuteBefore(EvaluationOrderGraphPass::class) -@ExecuteBefore(DFGPass::class) @DependsOn(ImportResolver::class) +@DependsOn(TypeResolver::class) class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { private lateinit var walker: SubgraphWalker.ScopedWalker @@ -183,7 +183,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { return with(builtin) { val len = newFunctionDeclaration("len", localNameOnly = true) - len.parameters = listOf(newParameterDeclaration("v", autoType())) + len.parameters = mutableListOf(newParameterDeclaration("v", autoType())) len.returnTypes = listOf(primitiveType("int")) addBuiltInFunction(len) @@ -194,7 +194,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { */ val append = newFunctionDeclaration("append", localNameOnly = true) append.parameters = - listOf( + mutableListOf( newParameterDeclaration("slice", autoType().array()), newParameterDeclaration("elems", autoType(), variadic = true), ) @@ -207,7 +207,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { * ``` */ val panic = newFunctionDeclaration("panic", localNameOnly = true) - panic.parameters = listOf(newParameterDeclaration("v", primitiveType("any"))) + panic.parameters = mutableListOf(newParameterDeclaration("v", primitiveType("any"))) addBuiltInFunction(panic) /** @@ -403,14 +403,10 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx) { // Try to see if we already know about this namespace somehow val namespace = - scopeManager.findSymbols(import.name, null).filter { + scopeManager.lookupSymbolByName(import.name, null).filter { it is NamespaceDeclaration && it.path == import.importURL } - scopeManager.resolve(scopeManager.globalScope, true) { - it.name == import.name && it.path == import.importURL - } - // If not, we can infer a namespace declaration, so we can bundle all inferred function // declarations in there if (namespace.isEmpty()) { diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index d07a097e5e..d19f75f502 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -25,17 +25,14 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.analysis.MultiValueEvaluator import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType -import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import de.fraunhofer.aisec.cpg.test.* import java.io.File import java.nio.file.Path @@ -1173,7 +1170,7 @@ class GoLanguageFrontendTest : BaseTest() { val funcy = result.calls["funcy"] assertNotNull(funcy) - funcy.invokeEdges.all { it.getProperty(Properties.DYNAMIC_INVOKE) == true } + funcy.invokeEdges.all { it.dynamicInvoke == true } // We should be able to resolve the call from our stored "do" function to funcy assertInvokes(funcy, result.functions["do"]) @@ -1208,7 +1205,6 @@ class GoLanguageFrontendTest : BaseTest() { val tu = analyzeAndGetFirstTU(listOf(topLevel.resolve("eval.go").toFile()), topLevel, true) { it.registerLanguage() - it.registerPass() } assertNotNull(tu) diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt index e8119feac9..27c44a126d 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/DeclarationHandler.kt @@ -41,7 +41,6 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.declarations.EnumConstantDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.EnumDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.NewArrayExpression @@ -82,7 +81,7 @@ open class DeclarationHandler(lang: JavaLanguageFrontend) : parameter.isVarArgs, rawNode = parameter ) - declaration.addParameter(param) + declaration.parameterEdges += param frontend.scopeManager.addDeclaration(param) } @@ -136,7 +135,7 @@ open class DeclarationHandler(lang: JavaLanguageFrontend) : parameter.isVarArgs, rawNode = parameter ) - functionDeclaration.addParameter(param) + functionDeclaration.parameterEdges += param frontend.processAnnotations(param, parameter) frontend.scopeManager.addDeclaration(param) } @@ -314,7 +313,7 @@ open class DeclarationHandler(lang: JavaLanguageFrontend) : val entries = enumDecl.entries.mapNotNull { handle(it) as EnumConstantDeclaration? } entries.forEach { it.type = this.objectType(enumDeclaration.name) } - enumDeclaration.entries = entries + enumDeclaration.entries = entries.toMutableList() frontend.scopeManager.leaveScope(enumDeclaration) @@ -361,7 +360,7 @@ open class DeclarationHandler(lang: JavaLanguageFrontend) : val initializerBlock = frontend.statementHandler.handleBlockStatement(decl.body) initializerBlock.isStaticBlock = decl.isStatic - recordDeclaration.addStatement(initializerBlock) + recordDeclaration.statements += initializerBlock } else -> { log.debug( diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt index a78c169526..063d163c2a 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt @@ -60,7 +60,7 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : val resolvedType = frontend.getTypeAsGoodAsPossible(parameter.type) val param = newParameterDeclaration(parameter.nameAsString, resolvedType, parameter.isVarArgs) - anonymousFunction.addParameter(param) + anonymousFunction.parameterEdges += param frontend.processAnnotations(param, parameter) frontend.scopeManager.addDeclaration(param) } @@ -149,6 +149,7 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : .java .cast(it) } + .toMutableList() initList.initializers = initializers return initList } @@ -232,7 +233,7 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : val declarationStatement = newDeclarationStatement(rawNode = expr) for (variable in variableDeclarationExpr.variables) { val declaration = frontend.declarationHandler.handleVariableDeclarator(variable) - declarationStatement.addToPropertyEdgeDeclaration(declaration) + declarationStatement.declarationEdges += declaration frontend.processAnnotations(declaration, variableDeclarationExpr) frontend.scopeManager.addDeclaration(declaration) } @@ -837,9 +838,8 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : .implicit(anonymousRecord.name.localName) ctor.arguments.forEachIndexed { i, arg -> - constructorDeclaration.addParameter( + constructorDeclaration.parameterEdges += newParameterDeclaration("arg${i}", arg.type) - ) } anonymousRecord.addConstructor(constructorDeclaration) ctor.anonymousClass = anonymousRecord diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt index db622510d6..4a3ed16ee2 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguage.kt @@ -39,13 +39,13 @@ import org.neo4j.ogm.annotation.Transient /** The Java language. */ open class JavaLanguage : Language(), - // HasComplexCallResolution, HasClasses, HasSuperClasses, HasGenerics, HasQualifier, HasUnknownType, - HasShortCircuitOperators { + HasShortCircuitOperators, + HasFunctionOverloading { override val fileExtensions = listOf("java") override val namespaceDelimiter = "." @Transient override val frontend: KClass = JavaLanguageFrontend::class @@ -108,11 +108,11 @@ open class JavaLanguage : } else super.propagateTypeOfBinaryOperation(operation) } - override fun handleSuperCall( - callee: MemberExpression, + override fun handleSuperExpression( + memberExpression: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager, - ) = JavaCallResolverHelper.handleSuperCall(callee, curClass, scopeManager) + ) = JavaCallResolverHelper.handleSuperExpression(memberExpression, curClass, scopeManager) override val startCharacter = '<' override val endCharacter = '>' diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt index ad108ff396..78609f9b7e 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt @@ -440,7 +440,7 @@ open class JavaLanguageFrontend(language: Language, ctx: T owner: NodeWithAnnotations<*> ) { if (config.processAnnotations) { - node.addAnnotations(handleAnnotations(owner)) + node.annotations += handleAnnotations(owner) } } diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt index 4e75aef65a..df320075fe 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt @@ -180,7 +180,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : val initExprList = this.newExpressionList() for (initExpr in forStmt.initialization) { val s = frontend.expressionHandler.handle(initExpr) - s?.let { initExprList.addExpression(it) } + s?.let { initExprList.expressions += it } // can not update location if (s?.location == null) { @@ -213,7 +213,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : val s = frontend.expressionHandler.handle(updateExpr) s?.let { // make sure location is set - iterationExprList.addExpression(it) + iterationExprList.expressions += it } // can not update location @@ -291,7 +291,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : frontend.scopeManager.enterScope(compoundStatement) for (child in blockStmt.statements) { val statement = handle(child) - statement?.let { compoundStatement.addStatement(it) } + statement?.let { compoundStatement.statements += it } } frontend.scopeManager.leaveScope(compoundStatement) return compoundStatement @@ -431,15 +431,14 @@ class StatementHandler(lang: JavaLanguageFrontend?) : compoundStatement.location = getLocationsFromTokens(switchStatement.location, start, end) for (sentry in switchStmt.entries) { if (sentry.labels.isEmpty()) { - compoundStatement.addStatement(handleCaseDefaultStatement(null, sentry)) + compoundStatement.statements += handleCaseDefaultStatement(null, sentry) } for (caseExp in sentry.labels) { - compoundStatement.addStatement(handleCaseDefaultStatement(caseExp, sentry)) + compoundStatement.statements += handleCaseDefaultStatement(caseExp, sentry) } for (subStmt in sentry.statements) { - compoundStatement.addStatement( + compoundStatement.statements += handle(subStmt) ?: ProblemExpression("Could not parse statement") - ) } } switchStatement.statement = compoundStatement @@ -479,6 +478,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : explicitConstructorInvocationStmt.arguments .map(frontend.expressionHandler::handle) .filterIsInstance() + .toMutableList() node.arguments = arguments return node @@ -489,9 +489,11 @@ class StatementHandler(lang: JavaLanguageFrontend?) : val tryStatement = newTryStatement(rawNode = stmt) frontend.scopeManager.enterScope(tryStatement) val resources = - tryStmt.resources.mapNotNull { ctx -> frontend.expressionHandler.handle(ctx) } + tryStmt.resources + .mapNotNull { ctx -> frontend.expressionHandler.handle(ctx) } + .toMutableList() val tryBlock = handleBlockStatement(tryStmt.tryBlock) - val catchClauses = tryStmt.catchClauses.map(::handleCatchClause) + val catchClauses = tryStmt.catchClauses.map(::handleCatchClause).toMutableList() val finallyBlock = tryStmt.finallyBlock.map(::handleBlockStatement).orElse(null) frontend.scopeManager.leaveScope(tryStatement) tryStatement.resources = resources diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt index 25d313b048..a9eef663a8 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt @@ -29,7 +29,6 @@ import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.HasType @@ -42,18 +41,17 @@ class JavaCallResolverHelper { companion object { /** - * Handle calls in the form of `super.call()` or `ClassName.super.call()`, conforming to - * JLS13 §15.12.1. + * Handle expressions in the form of `super.property` or `ClassName.super.call()`, + * conforming to JLS13 §15.12.1. * * This function basically sets the correct type of the [Reference] containing the "super" - * keyword. Afterwards, we can use the regular [SymbolResolver.resolveMemberCallee] to - * resolve the [MemberCallExpression]. + * keyword. * - * @param callee The callee of the call expression that needs to be adjusted + * @param memberExpression The member expression that needs to be adjusted * @param curClass The class containing the call */ - fun handleSuperCall( - callee: MemberExpression, + fun handleSuperExpression( + memberExpression: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager ): Boolean { @@ -61,7 +59,7 @@ class JavaCallResolverHelper { // still need to connect the super reference to the receiver of this method. val func = scopeManager.currentFunction if (func is MethodDeclaration) { - (callee.base as Reference?)?.refersTo = func.receiver + (memberExpression.base as Reference?)?.refersTo = func.receiver } // In the next step we can "cast" the base to the correct type, by setting the base @@ -69,37 +67,36 @@ class JavaCallResolverHelper { // In case the reference is just called "super", this is a direct superclass, either // defined explicitly or java.lang.Object by default - if (callee.base.name.toString() == JavaLanguage().superClassKeyword) { + if (memberExpression.base.name.toString() == JavaLanguage().superClassKeyword) { if (curClass.superClasses.isNotEmpty()) { target = curClass.superClasses[0].root.recordDeclaration } else { Util.warnWithFileLocation( - callee, + memberExpression, LOGGER, "super call without direct superclass! Expected java.lang.Object to be present at least!" ) } } else { // BaseName.super.call(), might either be in order to specify an enclosing class or - // an - // interface that is implemented - target = handleSpecificSupertype(callee, curClass) + // an interface that is implemented + target = handleSpecificSupertype(memberExpression, curClass) } if (target != null) { val superType = target.toType() // Explicitly set the type of the call's base to the super type, basically "casting" // the "this" object to the super class - callee.base.type = superType + memberExpression.base.type = superType - val refersTo = (callee.base as? Reference)?.refersTo + val refersTo = (memberExpression.base as? Reference)?.refersTo if (refersTo is HasType) { refersTo.type = superType refersTo.assignedTypes = mutableSetOf(superType) } // Make sure that really only our super class is in the list of assigned types - callee.base.assignedTypes = mutableSetOf(superType) + memberExpression.base.assignedTypes = mutableSetOf(superType) return true } diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt index e427a59a15..49333511b7 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/EOGTest.kt @@ -30,8 +30,6 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.allChildren import de.fraunhofer.aisec.cpg.graph.declarations.ConstructorDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -112,7 +110,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifSimple.thenStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifSimple) ) ) @@ -131,7 +129,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifSimple, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifSimple.thenStatement) ) ) @@ -163,7 +161,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifBranched, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifBranched.thenStatement) ) ) @@ -172,7 +170,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = ifBranched, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(ifBranched.elseStatement) ) ) @@ -185,7 +183,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifBranched.thenStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(ifBranched) ) ) @@ -196,7 +194,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = ifBranched.elseStatement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(ifBranched) ) ) @@ -238,7 +236,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = bo.rhs, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to (bo.operatorCode == "&&")), + predicate = { it.branch == (bo.operatorCode == "&&") }, refs = listOf(bo.lhs) ) ) @@ -269,7 +267,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = bo, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to (bo.operatorCode != "&&")), + predicate = { it.branch == (bo.operatorCode != "&&") }, refs = listOf(bo.lhs) ) ) @@ -400,7 +398,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.EXITS, n = fs, cr = Connect.SUBTREE, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[2]) ) ) @@ -473,7 +471,7 @@ internal class EOGTest : BaseTest() { en = Util.Edge.ENTRIES, n = wstat.statement, cr = Connect.NODE, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(wstat) ) ) @@ -485,7 +483,7 @@ internal class EOGTest : BaseTest() { cn = Connect.SUBTREE, en = Util.Edge.EXITS, n = wstat, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[1]) ) ) @@ -518,7 +516,7 @@ internal class EOGTest : BaseTest() { cn = Connect.NODE, en = Util.Edge.EXITS, n = dostat, - props = mutableMapOf(Properties.BRANCH to true), + predicate = { it.branch == true }, refs = listOf(dostat.statement) ) ) @@ -539,7 +537,7 @@ internal class EOGTest : BaseTest() { cn = Connect.SUBTREE, en = Util.Edge.EXITS, n = dostat, - props = mutableMapOf(Properties.BRANCH to false), + predicate = { it.branch == false }, refs = listOf(prints[2]) ) ) @@ -759,17 +757,17 @@ internal class EOGTest : BaseTest() { assertNotNull(a) val b = result.refs[{ it.location?.region?.startLine == 7 && it.name.localName == "b" }] assertNotNull(b) - var nextEOG: List> = firstIf.nextEOGEdges + var nextEOG = firstIf.nextEOGEdges assertEquals(2, nextEOG.size) for (edge in nextEOG) { assertEquals(firstIf, edge.start) if (edge.end == b) { - assertEquals(true, edge.getProperty(Properties.BRANCH)) - assertEquals(0, edge.getProperty(Properties.INDEX)) + assertEquals(true, edge.branch) + assertEquals(0, edge.index) } else { assertEquals(a, edge.end) - assertEquals(false, edge.getProperty(Properties.BRANCH)) - assertEquals(1, edge.getProperty(Properties.INDEX)) + assertEquals(false, edge.branch) + assertEquals(1, edge.index) } } val elseIf: IfStatement = @@ -786,12 +784,12 @@ internal class EOGTest : BaseTest() { for (edge in nextEOG) { assertEquals(elseIf, edge.start) if (edge.end == b2) { - assertEquals(true, edge.getProperty(Properties.BRANCH)) - assertEquals(0, edge.getProperty(Properties.INDEX)) + assertEquals(true, edge.branch) + assertEquals(0, edge.index) } else { assertEquals(x, edge.end) - assertEquals(false, edge.getProperty(Properties.BRANCH)) - assertEquals(1, edge.getProperty(Properties.INDEX)) + assertEquals(false, edge.branch) + assertEquals(1, edge.index) } } } diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/SuperCallTest.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/SuperCallTest.kt index fdef17f2cd..ca83ba515f 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/SuperCallTest.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/SuperCallTest.kt @@ -52,7 +52,7 @@ internal class SuperCallTest : BaseTest() { val target = findByUniqueName(methods, "target") val calls = target.calls val superCall = findByUniquePredicate(calls) { "super.target();" == it.code } - assertEquals(listOf(superTarget), superCall.invokes) + assertInvokes(superCall, superTarget) } @Test @@ -74,8 +74,8 @@ internal class SuperCallTest : BaseTest() { findByUniquePredicate(calls) { "Interface1.super.target();" == it.code } val interface2Call = findByUniquePredicate(calls) { "Interface2.super.target();" == it.code } - assertEquals(listOf(interface1Target), interface1Call.invokes) - assertEquals(listOf(interface2Target), interface2Call.invokes) + assertInvokes(interface1Call, interface1Target) + assertInvokes(interface2Call, interface2Target) } @Test diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StaticImportsTest.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StaticImportsTest.kt index f7f0275762..0def824fc9 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StaticImportsTest.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StaticImportsTest.kt @@ -59,7 +59,7 @@ internal class StaticImportsTest : BaseTest() { val main = findByUniqueName(methods, "main") val call = main.calls.firstOrNull() assertNotNull(call) - assertEquals(listOf(test), call.invokes) + assertInvokes(call, test) val testFields = result.fields { it.name.localName == "test" } assertEquals(1, testFields.size) @@ -94,20 +94,20 @@ internal class StaticImportsTest : BaseTest() { for (call in main.calls) { when (call.name.localName) { "a" -> { - assertEquals(listOf(findByUniqueName(methods, "a")), call.invokes) + assertInvokes(call, methods["a"]) assertTrue((call.invokes[0] as MethodDeclaration).isStatic) } "b" -> { val bs = methods { it.name.localName == "b" && it.isStatic } assertEquals( - call.invokes, + call.invokes.toList(), bs { it.matchesSignature(call.signature) != IncompatibleSignature } ) } "nonStatic" -> { val nonStatic = findByUniqueName(b.methods, "nonStatic") assertTrue(nonStatic.isInferred) - assertEquals(listOf(nonStatic), call.invokes) + assertInvokes(call, nonStatic) } } } diff --git a/cpg-language-jvm/build.gradle.kts b/cpg-language-jvm/build.gradle.kts new file mode 100644 index 0000000000..6502dfa23a --- /dev/null +++ b/cpg-language-jvm/build.gradle.kts @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +plugins { + id("cpg.frontend-conventions") +} + +publishing { + publications { + named("cpg-language-jvm") { + pom { + artifactId = "cpg-language-jvm" + name.set("Code Property Graph - JVM bytecode Frontend") + description.set("A JVM bytecode frontend for the CPG") + } + } + } +} + +dependencies { + api(libs.bundles.sootup) + // needed until https://github.com/antlr/antlr4/issues/3895 is fixed + runtimeOnly("org.antlr:antlr4-runtime") { + version { + strictly("4.9.3") + } + } +} diff --git a/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/DeclarationHandler.kt b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/DeclarationHandler.kt new file mode 100644 index 0000000000..97b57b5acd --- /dev/null +++ b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/DeclarationHandler.kt @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.frontends.Handler +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.* +import sootup.core.jimple.basic.Local +import sootup.core.model.SootClass +import sootup.core.model.SootField +import sootup.core.model.SootMethod +import sootup.java.core.JavaSootClass +import sootup.java.core.JavaSootField +import sootup.java.core.JavaSootMethod +import sootup.java.core.jimple.basic.JavaLocal + +class DeclarationHandler(frontend: JVMLanguageFrontend) : + Handler(::ProblemDeclaration, frontend) { + init { + map.put(SootClass::class.java) { handleClass(it as SootClass) } + map.put(JavaSootClass::class.java) { handleClass(it as SootClass) } + map.put(SootMethod::class.java) { handleMethod(it as SootMethod) } + map.put(JavaSootMethod::class.java) { handleMethod(it as SootMethod) } + map.put(SootField::class.java) { handleField(it as SootField) } + map.put(JavaSootField::class.java) { handleField(it as SootField) } + map.put(Local::class.java) { handleLocal(it as Local) } + map.put(JavaLocal::class.java) { handleLocal(it as Local) } + } + + private fun handleClass(sootClass: SootClass): RecordDeclaration { + val record = + newRecordDeclaration( + sootClass.getName(), + if (sootClass.isInterface()) { + "interface" + } else { + "class" + }, + rawNode = sootClass + ) + + // Collect super class + val o = sootClass.superclass + if (o.isPresent) { + record.addSuperClass(frontend.typeOf(o.get())) + } + + // Collect implemented interfaces + for (i in sootClass.interfaces) { + record.implementedInterfaces += frontend.typeOf(i) + } + + // Enter the class scope + frontend.scopeManager.enterScope(record) + + // Loop through all fields + for (sootField in sootClass.fields) { + val field = handle(sootField) + frontend.scopeManager.addDeclaration(field) + } + + // Loop through all methods + for (sootMethod in sootClass.methods) { + val method = handle(sootMethod) + frontend.scopeManager.addDeclaration(method) + } + + // Leave the class scope + frontend.scopeManager.leaveScope(record) + + return record + } + + private fun handleMethod(sootMethod: SootMethod): MethodDeclaration { + val record = frontend.scopeManager.currentRecord + + val method = + if (sootMethod.name == "") { + newConstructorDeclaration(sootMethod.name, record, rawNode = sootMethod) + } else { + newMethodDeclaration( + sootMethod.name, + sootMethod.isStatic, + frontend.scopeManager.currentRecord, + rawNode = sootMethod, + ) + } + + // Enter method scope + frontend.scopeManager.enterScope(method) + + // Add "@this" as the receiver + method.receiver = + newVariableDeclaration("@this", method.recordDeclaration?.toType() ?: unknownType()) + .implicit("@this") + frontend.scopeManager.addDeclaration(method.receiver) + + // Add method parameters + for ((index, type) in sootMethod.parameterTypes.withIndex()) { + val param = newParameterDeclaration("@parameter${index}", frontend.typeOf(type)) + frontend.scopeManager.addDeclaration(param) + } + + if (sootMethod.isConcrete) { + // Handle method body + method.body = frontend.statementHandler.handle(sootMethod.body) + } + + // Leave method scope + frontend.scopeManager.leaveScope(method) + + return method + } + + fun handleField(field: SootField): FieldDeclaration { + return newFieldDeclaration( + field.name, + frontend.typeOf(field.type), + field.modifiers.map { it.name.lowercase() }, + rawNode = field + ) + } + + private fun handleLocal(local: Local): VariableDeclaration { + return newVariableDeclaration(local.name, frontend.typeOf(local.type), rawNode = local) + } +} diff --git a/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/ExpressionHandler.kt b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/ExpressionHandler.kt new file mode 100644 index 0000000000..a124c2bc3f --- /dev/null +++ b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/ExpressionHandler.kt @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.frontends.Handler +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.passes.SymbolResolver +import sootup.core.jimple.basic.Local +import sootup.core.jimple.basic.Value +import sootup.core.jimple.common.constant.* +import sootup.core.jimple.common.expr.* +import sootup.core.jimple.common.ref.* +import sootup.core.signatures.MethodSignature +import sootup.core.signatures.SootClassMemberSignature +import sootup.java.core.jimple.basic.JavaLocal + +class ExpressionHandler(frontend: JVMLanguageFrontend) : + Handler( + ::ProblemExpression, + frontend, + ) { + + init { + map.put(Local::class.java) { handleLocal(it as Local) } + map.put(JavaLocal::class.java) { handleLocal(it as Local) } + map.put(JThisRef::class.java) { handleThisRef(it as JThisRef) } + map.put(JParameterRef::class.java) { handleParameterRef(it as JParameterRef) } + map.put(JInstanceFieldRef::class.java) { handleInstanceFieldRef(it as JInstanceFieldRef) } + map.put(JStaticFieldRef::class.java) { handleStaticFieldRef(it as JStaticFieldRef) } + map.put(JArrayRef::class.java) { handleArrayRef(it as JArrayRef) } + map.put(JInterfaceInvokeExpr::class.java) { + handleInterfaceInvokeExpr(it as JInterfaceInvokeExpr) + } + map.put(JVirtualInvokeExpr::class.java) { + handleVirtualInvokeExpr(it as JVirtualInvokeExpr) + } + map.put(JDynamicInvokeExpr::class.java) { + handleDynamicInvokeExpr(it as JDynamicInvokeExpr) + } + map.put(JSpecialInvokeExpr::class.java) { handleSpecialInvoke(it as JSpecialInvokeExpr) } + map.put(JStaticInvokeExpr::class.java) { handleStaticInvoke(it as JStaticInvokeExpr) } + map.put(JNewExpr::class.java) { handleNewExpr(it as JNewExpr) } + map.put(JNewArrayExpr::class.java) { handleNewArrayExpr(it as JNewArrayExpr) } + map.put(JNewMultiArrayExpr::class.java) { + handleNewMultiArrayExpr(it as JNewMultiArrayExpr) + } + map.put(JCastExpr::class.java) { handleCastExpr(it as JCastExpr) } + + // Binary operators + // - Equality checks + map.put(JEqExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JNeExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JGeExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JGtExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JLeExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JLtExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + + // - Numeric comparisons + map.put(JCmpExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JCmplExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JCmpgExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + + // - Simple arithmetics + map.put(JAddExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JDivExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JMulExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JRemExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JSubExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + + // - Binary arithmetics + map.put(JAndExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JOrExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JShlExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JShrExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JUshrExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + map.put(JXorExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + + // Fallback, just to be sure + map.put(AbstractBinopExpr::class.java) { handleAbstractBinopExpr(it as AbstractBinopExpr) } + + // Unary operator + map.put(JNegExpr::class.java) { handleNegExpr(it as JNegExpr) } + + // Special operators, which we need to model as binary/unary operators + map.put(JInstanceOfExpr::class.java) { handleInstanceOfExpr(it as JInstanceOfExpr) } + map.put(JLengthExpr::class.java) { handleLengthExpr(it as JLengthExpr) } + + // Constants + map.put(BooleanConstant::class.java) { handleBooleanConstant(it as BooleanConstant) } + map.put(FloatConstant::class.java) { handleFloatConstant(it as FloatConstant) } + map.put(DoubleConstant::class.java) { handleDoubleConstant(it as DoubleConstant) } + map.put(IntConstant::class.java) { handleIntConstant(it as IntConstant) } + map.put(LongConstant::class.java) { handleLongConstant(it as LongConstant) } + map.put(StringConstant::class.java) { handleStringConstant(it as StringConstant) } + map.put(NullConstant::class.java) { handleNullConstant(it as NullConstant) } + map.put(ClassConstant::class.java) { handleClassConstant(it as ClassConstant) } + } + + private fun handleLocal(local: Local): Expression { + // Apparently, a local can either be a reference to variable or a literal + return if (local.name.startsWith("\"")) { + val lit = newLiteral(local.name.substring(1, local.name.length - 2), rawNode = local) + lit.type = objectType("java.lang.String") + + lit + } else { + val ref = newReference(local.name, frontend.typeOf(local.type), rawNode = local) + + ref + } + } + + private fun handleThisRef(thisRef: JThisRef): Reference { + val ref = newReference("@this", frontend.typeOf(thisRef.type), rawNode = thisRef) + + return ref + } + + private fun handleParameterRef(parameterRef: JParameterRef): Reference { + val ref = + newReference( + "@parameter${parameterRef.index}", + frontend.typeOf(parameterRef.type), + rawNode = parameterRef + ) + + return ref + } + + private fun handleInstanceFieldRef(instanceFieldRef: JInstanceFieldRef): Reference { + val base = handle(instanceFieldRef.base) ?: newProblemExpression("missing base") + + val ref = + newMemberExpression( + instanceFieldRef.fieldSignature.name, + base, + frontend.typeOf(instanceFieldRef.fieldSignature.type), + rawNode = instanceFieldRef + ) + + return ref + } + + private fun handleStaticFieldRef(staticFieldRef: JStaticFieldRef) = + staticFieldRef.fieldSignature.toStaticRef() + + private fun handleArrayRef(arrayRef: JArrayRef): SubscriptExpression { + val sub = newSubscriptExpression(rawNode = arrayRef) + sub.arrayExpression = handle(arrayRef.base) ?: newProblemExpression("missing base") + sub.subscriptExpression = handle(arrayRef.index) ?: newProblemExpression("missing index") + + return sub + } + + private fun handleAbstractInstanceInvokeExpr( + invokeExpr: AbstractInstanceInvokeExpr + ): MemberCallExpression { + val base = handle(invokeExpr.base) ?: newProblemExpression("could not parse base") + // Not really necessary, but since we already have the type information, we can use it + base.type = frontend.typeOf(invokeExpr.methodSignature.declClassType) + + val callee = newMemberExpression(invokeExpr.methodSignature.name, base) + + val call = newMemberCallExpression(callee, rawNode = invokeExpr) + call.arguments = invokeExpr.args.mapNotNull { handle(it) }.toMutableList() + + return call + } + + private fun handleVirtualInvokeExpr(invokeExpr: JVirtualInvokeExpr): MemberCallExpression { + return handleAbstractInstanceInvokeExpr(invokeExpr) + } + + private fun handleInterfaceInvokeExpr(invokeExpr: JInterfaceInvokeExpr): MemberCallExpression { + return handleAbstractInstanceInvokeExpr(invokeExpr) + } + + /** + * The difference between [JSpecialInvokeExpr] and a regular [JVirtualInvokeExpr] is that the + * invoked function is not part of the declared class, but rather it is a function of its base + * class(es). + * + * We currently can only model this as a regular call and hope that the [SymbolResolver] will + * pick the correct function. Maybe we can supply some kind of hint to the resolver to make this + * better. + */ + private fun handleSpecialInvoke(invokeExpr: JSpecialInvokeExpr): Expression { + // This is probably a constructor call + return if (invokeExpr.methodSignature.name == "") { + val type = frontend.typeOf(invokeExpr.methodSignature.declClassType) + val construct = newConstructExpression(rawNode = invokeExpr) + construct.callee = newReference(Name("", type.name)) + construct.type = type + + construct.arguments = invokeExpr.args.mapNotNull { handle(it) }.toMutableList() + + construct + } else { + // Just a normal call + return handleAbstractInstanceInvokeExpr(invokeExpr) + } + } + + private fun handleDynamicInvokeExpr(dynamicInvokeExpr: AbstractInvokeExpr): CallExpression { + // Model this as a static call to the method. Not sure if this is really that good or if we + // want to somehow "call" the underlying bootstrap method. + // TODO(oxisto): This is actually somewhat related to a LambdaExpression, but not really + // sure ow to model this + val callee = dynamicInvokeExpr.methodSignature.toStaticRef() + val call = newCallExpression(callee, rawNode = dynamicInvokeExpr) + call.arguments = dynamicInvokeExpr.args.mapNotNull { handle(it) }.toMutableList() + call.type = frontend.typeOf(dynamicInvokeExpr.methodSignature.type) + + return call + } + + private fun handleStaticInvoke(staticInvokeExpr: JStaticInvokeExpr): CallExpression { + val ref = staticInvokeExpr.methodSignature.toStaticRef() + + val call = newCallExpression(ref, rawNode = staticInvokeExpr) + call.arguments = staticInvokeExpr.args.mapNotNull { handle(it) }.toMutableList() + call.type = frontend.typeOf(staticInvokeExpr.type) + + return call + } + + /** + * In the jimple IR, the "new" and the constructor calls are split into two expressions. This + * will only handle the "new" expression, a later call to "invokespecial" will handle the + * constructor call. + */ + private fun handleNewExpr(newExpr: JNewExpr) = + newNewExpression(frontend.typeOf(newExpr.type), rawNode = newExpr) + + private fun handleNewArrayExpr(newArrayExpr: JNewArrayExpr): NewArrayExpression { + val new = newNewArrayExpression(rawNode = newArrayExpr) + new.type = frontend.typeOf(newArrayExpr.type) + new.dimensions = listOfNotNull(handle(newArrayExpr.size)).toMutableList() + + return new + } + + private fun handleNewMultiArrayExpr(newMultiArrayExpr: JNewMultiArrayExpr): NewArrayExpression { + val new = newNewArrayExpression(rawNode = newMultiArrayExpr) + new.type = frontend.typeOf(newMultiArrayExpr.type) + new.dimensions = newMultiArrayExpr.sizes.mapNotNull { handle(it) }.toMutableList() + + return new + } + + private fun handleCastExpr(castExpr: JCastExpr): CastExpression { + val cast = newCastExpression(rawNode = castExpr) + cast.expression = handle(castExpr.op) ?: newProblemExpression("missing expression") + cast.castType = frontend.typeOf(castExpr.type) + + return cast + } + + private fun handleAbstractBinopExpr(expr: AbstractBinopExpr): BinaryOperator { + val op = newBinaryOperator(expr.symbol.trim(), rawNode = expr) + op.lhs = handle(expr.op1) ?: newProblemExpression("missing lhs") + op.rhs = handle(expr.op2) ?: newProblemExpression("missing rhs") + op.type = frontend.typeOf(expr.type) + + return op + } + + private fun handleNegExpr(expr: AbstractUnopExpr): UnaryOperator { + val op = newUnaryOperator("-", postfix = false, prefix = true, rawNode = expr) + op.input = handle(expr.op) ?: newProblemExpression("missing input") + op.type = frontend.typeOf(expr.type) + + return op + } + + private fun handleInstanceOfExpr(instanceOfExpr: JInstanceOfExpr): BinaryOperator { + val op = newBinaryOperator("instanceof", rawNode = instanceOfExpr) + op.lhs = handle(instanceOfExpr.op) ?: newProblemExpression("missing lhs") + + val type = frontend.typeOf(instanceOfExpr.checkType) + op.rhs = newTypeExpression("", type, rawNode = type) + op.rhs.name = type.name + op.type = frontend.typeOf(instanceOfExpr.type) + + return op + } + + private fun handleLengthExpr(lengthExpr: JLengthExpr): UnaryOperator { + val op = newUnaryOperator("lengthof", prefix = true, postfix = false, rawNode = lengthExpr) + op.input = handle(lengthExpr.op) ?: newProblemExpression("missing input") + op.type = frontend.typeOf(lengthExpr.type) + + return op + } + + private fun handleBooleanConstant(constant: BooleanConstant) = + newLiteral( + constant.equalEqual(BooleanConstant.getTrue()), + primitiveType("boolean"), + rawNode = constant + ) + + private fun handleFloatConstant(constant: FloatConstant) = + newLiteral(constant.value, primitiveType("float"), rawNode = constant) + + private fun handleDoubleConstant(constant: DoubleConstant) = + newLiteral(constant.value, primitiveType("double"), rawNode = constant) + + private fun handleIntConstant(constant: IntConstant) = + newLiteral(constant.value, primitiveType("int"), rawNode = constant) + + private fun handleLongConstant(constant: LongConstant) = + newLiteral(constant.value, primitiveType("long"), rawNode = constant) + + private fun handleStringConstant(constant: StringConstant) = + newLiteral(constant.value, primitiveType("java.lang.String"), rawNode = constant) + + private fun handleNullConstant(constant: NullConstant) = + newLiteral(null, unknownType(), rawNode = constant) + + /** + * We need to keep the class name as a string, rather than a [Class], because otherwise we would + * try to find the specified class on the classpath, which can lead to unwanted results. + */ + private fun handleClassConstant(constant: ClassConstant) = + newLiteral(constant.value, primitiveType("java.lang.Class"), rawNode = constant) + + private fun MethodSignature.toStaticRef(): Reference { + // First, construct the name using . + val ref = (this as SootClassMemberSignature<*>).toStaticRef() + + // We can also provide a function type, since these are all statically known. This might + // help in inferring some (unknown) functions later + ref.type = + FunctionType( + this.name, + this.parameterTypes.map { frontend.typeOf(it) }, + listOf(frontend.typeOf(this.type)), + frontend.language + ) + + return ref + } + + private fun SootClassMemberSignature<*>.toStaticRef(): Reference { + // First, construct the name using . + val ref = newReference("${this.declClassType.fullyQualifiedName}.${this.name}") + + // Make it static + ref.isStaticAccess = true + + return ref + } +} diff --git a/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguage.kt b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguage.kt new file mode 100644 index 0000000000..f5bae138b7 --- /dev/null +++ b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguage.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.graph.types.* +import kotlin.reflect.KClass + +class JVMLanguage : Language() { + override val fileExtensions: List + get() = listOf("class", "java", "jimple", "jar") + + override val namespaceDelimiter: String + get() = "." + + override val frontend: KClass + get() = JVMLanguageFrontend::class + + override val builtInTypes: Map + get() = + mapOf( + "float" to FloatingPointType("float", 32, this), + "double" to FloatingPointType("double", 64, this), + "char" to IntegerType("char", 8, this, NumericType.Modifier.UNSIGNED), + "boolean" to BooleanType("boolean", 1, this), + "byte" to IntegerType("byte", 8, this), + "short" to IntegerType("short", 16, this), + "int" to IntegerType("int", 32, this), + "long" to IntegerType("long", 64, this), + "java.lang.String" to StringType("java.lang.String", this), + "java.lang.Class" to ObjectType("java.lang.Class", listOf(), true, this) + ) + + override val compoundAssignmentOperators: Set + get() = setOf() +} diff --git a/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontend.kt b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontend.kt new file mode 100644 index 0000000000..9b561079a7 --- /dev/null +++ b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontend.kt @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend +import de.fraunhofer.aisec.cpg.frontends.TranslationException +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation +import java.io.File +import sootup.core.model.Body +import sootup.core.model.SootMethod +import sootup.core.model.SourceType +import sootup.core.types.ArrayType +import sootup.core.types.UnknownType +import sootup.core.util.printer.NormalStmtPrinter +import sootup.java.bytecode.inputlocation.JavaClassPathAnalysisInputLocation +import sootup.java.core.interceptors.* +import sootup.java.core.views.JavaView +import sootup.java.sourcecode.inputlocation.JavaSourcePathAnalysisInputLocation +import sootup.jimple.parser.JimpleAnalysisInputLocation +import sootup.jimple.parser.JimpleView + +typealias SootType = sootup.core.types.Type + +class JVMLanguageFrontend( + language: Language>, + ctx: TranslationContext +) : LanguageFrontend(language, ctx) { + + val declarationHandler = DeclarationHandler(this) + val statementHandler = StatementHandler(this) + val expressionHandler = ExpressionHandler(this) + + lateinit var view: JavaView + + var body: Body? = null + + var printer: NormalStmtPrinter? = null + + /** + * Because of a limitation in SootUp, we can only specify the whole classpath for soot to parse. + * But in the CPG we need to specify one file. In this case, we take the + * [TranslationConfiguration.topLevel] and hand it over to soot, which parses all appropriate + * files within this folder/classpath. This means that the returned [TranslationUnitDeclaration] + * will contain not just the content of one file but the whole directory. + */ + override fun parse(file: File): TranslationUnitDeclaration { + val view = + when (file.extension) { + "class" -> { + JavaView( + JavaClassPathAnalysisInputLocation( + ctx.config.topLevel!!.path, + SourceType.Library, + listOf( + NopEliminator(), + CastAndReturnInliner(), + UnreachableCodeEliminator(), + Aggregator(), + CopyPropagator(), + // ConditionalBranchFolder(), + EmptySwitchEliminator(), + TypeAssigner(), + LocalNameStandardizer() + ) + ) + ) + } + "jar" -> { + JavaView( + JavaClassPathAnalysisInputLocation( + file.path, + SourceType.Library, + listOf( + NopEliminator(), + CastAndReturnInliner(), + UnreachableCodeEliminator(), + Aggregator(), + CopyPropagator(), + // ConditionalBranchFolder(), + EmptySwitchEliminator(), + TypeAssigner(), + LocalNameStandardizer() + ) + ) + ) + } + "java" -> { + JavaView(JavaSourcePathAnalysisInputLocation(ctx.config.topLevel!!.path)) + } + "jimple" -> { + JimpleView(JimpleAnalysisInputLocation(ctx.config.topLevel!!.toPath())) + } + else -> { + throw TranslationException("unsupported file") + } + } + // This contains the whole directory + val tu = newTranslationUnitDeclaration(file.parent) + scopeManager.resetToGlobal(tu) + + val packages = mutableMapOf() + + for (sootClass in view.classes) { + // Create an appropriate namespace, if it does not already exist + val pkg = + packages.computeIfAbsent(sootClass.type.packageName.name) { + val pkg = newNamespaceDeclaration(it) + scopeManager.addDeclaration(pkg) + pkg + } + + // Enter namespace scope + scopeManager.enterScope(pkg) + + val decl = declarationHandler.handle(sootClass) + scopeManager.addDeclaration(decl) + + // Leave namespace scope + scopeManager.leaveScope(pkg) + + // We need to clear the processed because they need to be per-file and we only have one + // frontend for all files + clearProcessed() + } + + return tu + } + + override fun setComment(node: Node, astNode: Any) {} + + override fun locationOf(astNode: Any): PhysicalLocation? { + // We do not really have a location anyway. maybe in jimple? + return null + } + + override fun codeOf(astNode: Any): String? { + if (astNode is SootMethod && astNode.isConcrete) { + return astNode.body.toString() + } + // We do not really have a source anyway. maybe in jimple? + return "" + } + + override fun typeOf(type: SootType): Type { + return when (type) { + is UnknownType -> { + unknownType() + } + is ArrayType -> { + typeOf(type.baseType).array() + } + else -> { + // TODO(oxisto): primitive types + val out = objectType(type.toString()) + + out + } + } + } +} diff --git a/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/StatementHandler.kt b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/StatementHandler.kt new file mode 100644 index 0000000000..c969dd3757 --- /dev/null +++ b/cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/StatementHandler.kt @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.frontends.Handler +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import sootup.core.jimple.common.stmt.* +import sootup.core.model.Body +import sootup.core.util.printer.NormalStmtPrinter + +class StatementHandler(frontend: JVMLanguageFrontend) : + Handler(::ProblemExpression, frontend) { + init { + map.put(Body::class.java) { handleBody(it as Body) } + map.put(JAssignStmt::class.java) { handleAbstractDefinitionStmt(it as JAssignStmt) } + map.put(JIdentityStmt::class.java) { handleAbstractDefinitionStmt(it as JIdentityStmt) } + map.put(JIfStmt::class.java) { handleIfStmt(it as JIfStmt) } + map.put(JGotoStmt::class.java) { handleGotoStmt(it as JGotoStmt) } + map.put(JInvokeStmt::class.java) { handleInvokeStmt(it as JInvokeStmt) } + map.put(JReturnStmt::class.java) { handleReturnStmt(it as JReturnStmt) } + map.put(JReturnVoidStmt::class.java) { handleReturnVoidStmt(it as JReturnVoidStmt) } + } + + private fun handleBody(body: Body): Block { + // The first block contains all our other blocks and this will be the one we return + val outerBlock = newBlock(rawNode = body) + + val printer = NormalStmtPrinter() + printer.initializeSootMethod(body.stmtGraph) + + frontend.printer = printer + frontend.body = body + + // Parse locals, these are always at the beginning of the function + for (local in body.locals) { + val decl = frontend.declarationHandler.handle(local) + + if (decl != null) { + // We need to wrap them into a declaration statement and put them into the outer + // block + val stmt = newDeclarationStatement(rawNode = local) + stmt.declarationEdges += decl + frontend.scopeManager.addDeclaration(decl) + outerBlock += stmt + } + } + + // Parse statements and segment them into (sub)-blocks. + var block = outerBlock + for (sootStmt in body.stmts) { + val label = printer.labels[sootStmt] + if (label != null) { + // If we have a label, we need to create a new label statement, that starts a new + // block + val stmt = newLabelStatement() + block = newBlock() + stmt.label = label + stmt.subStatement = block + + // We need to inform our processing system, since we do it outside of a handler, so + // the created goto statements will be informed about our new label + frontend.process(Any(), stmt) + + // Always add it to the outer block + outerBlock += stmt + } + + // Parse the statement + val stmt = handle(sootStmt) + if (stmt != null) { + block += stmt + } + } + + // Always return the outer block, since it comprises all the other sub-blocks. + return outerBlock + } + + private fun handleAbstractDefinitionStmt(defStmt: AbstractDefinitionStmt): AssignExpression { + val assign = newAssignExpression("=", rawNode = defStmt) + assign.lhs = + listOfNotNull(frontend.expressionHandler.handle(defStmt.leftOp)).toMutableList() + assign.rhs = + listOfNotNull(frontend.expressionHandler.handle(defStmt.rightOp)).toMutableList() + + return assign + } + + private fun handleIfStmt(ifStmt: JIfStmt): IfStatement { + val stmt = newIfStatement(rawNode = ifStmt) + stmt.condition = + frontend.expressionHandler.handle(ifStmt.condition) + ?: newProblemExpression("missing condition") + stmt.thenStatement = handleBranchingStmt(ifStmt) + + return stmt + } + + private fun handleGotoStmt(gotoStmt: JGotoStmt): GotoStatement { + return handleBranchingStmt(gotoStmt) + } + + private fun handleBranchingStmt(branchingStmt: BranchingStmt): GotoStatement { + val stmt = newGotoStatement(rawNode = branchingStmt) + + frontend.body?.let { + val target = branchingStmt.getTargetStmts(it).firstOrNull() + val label = frontend.printer?.labels?.get(target) + if (label != null) { + stmt.labelName = label + } + + // Register a predicate listener that informs us as soon as new label statement that + // matches our label name is created. + frontend.registerPredicateListener({ _, to -> + (to is LabelStatement && to.label == stmt.labelName) + }) { _, to -> + stmt.targetLabel = to as LabelStatement + } + } + + return stmt + } + + private fun handleInvokeStmt(invokeStmt: JInvokeStmt) = + frontend.expressionHandler.handle(invokeStmt.invokeExpr) + + private fun handleReturnStmt(returnStmt: JReturnStmt): ReturnStatement { + val stmt = newReturnStatement(rawNode = returnStmt) + stmt.returnValue = + frontend.expressionHandler.handle(returnStmt.op) + ?: newProblemExpression("missing return value") + + return stmt + } + + private fun handleReturnVoidStmt(returnStmt: JReturnVoidStmt) = + newReturnStatement(rawNode = returnStmt) +} diff --git a/cpg-language-jvm/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontendTest.kt b/cpg-language-jvm/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontendTest.kt new file mode 100644 index 0000000000..a30901c681 --- /dev/null +++ b/cpg-language-jvm/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontendTest.kt @@ -0,0 +1,412 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.jvm + +import de.fraunhofer.aisec.cpg.* +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.PointerType +import de.fraunhofer.aisec.cpg.test.analyze +import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.test.assertFullName +import de.fraunhofer.aisec.cpg.test.assertInvokes +import de.fraunhofer.aisec.cpg.test.assertLiteralValue +import de.fraunhofer.aisec.cpg.test.assertLocalName +import de.fraunhofer.aisec.cpg.test.assertRefersTo +import java.nio.file.Path +import kotlin.test.* +import org.junit.jupiter.api.Disabled + +class JVMLanguageFrontendTest { + @Test + fun testHelloJimple() { + val topLevel = Path.of("src", "test", "resources", "jimple", "helloworld") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("HelloWorld.jimple").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val helloWorld = tu.records["HelloWorld"] + assertNotNull(helloWorld) + + val constructor = helloWorld.constructors.firstOrNull() + assertNotNull(constructor) + + // All references should be resolved (except Object., which should be a construct + // expression anyway) + val refs = constructor.refs.filter { it.name.toString() != "java.lang.Object." } + refs.forEach { + val refersTo = it.refersTo + assertNotNull(refersTo, "${it.name} could not be resolved") + assertFalse( + refersTo.isInferred, + "${it.name} should not be resolved to an inferred node" + ) + } + + val main = helloWorld.methods["main"] + assertNotNull(main) + assertTrue(main.isStatic) + + val param0 = main.refs["@parameter0"] + assertNotNull(param0) + + val refersTo = param0.refersTo + assertNotNull(refersTo) + assertFalse(refersTo.isInferred) + } + + @Test + fun testMethodsClass() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "methods") + val tu = + analyzeAndGetFirstTU( + // We just need to specify one file to trigger the class byte loader + listOf(topLevel.resolve("mypackage/Adder.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + assertEquals(0, tu.problems.size) + + val pkg = tu.namespaces["mypackage"] + assertNotNull(pkg) + + val adder = pkg.records["Adder"] + assertNotNull(adder) + + val add = adder.methods["add"] + assertNotNull(add) + + val main = pkg.methods["Main.main"] + assertNotNull(main) + + println(main.code) + + // r5 contains our adder + val r5 = main.variables["r5"] + assertNotNull(r5) + assertFullName("mypackage.Adder", r5.type) + + // r3 should be the result of the add call + val r3 = main.variables["r3"] + assertNotNull(r3) + + val r3ref = r3.usages.firstOrNull { it.access == AccessValues.WRITE } + assertNotNull(r3ref) + + // Call to add should be resolved + val call = r3ref.prevDFG.firstOrNull() + assertIs(call) + assertLocalName("add", call) + assertInvokes(call, add) + assertEquals(listOf("Integer", "Integer"), call.arguments.map { it.type.name.localName }) + + // All references (which are not part of a call) and not to the stdlib should be resolved + val refs = tu.refs + refs + .filter { it.astParent !is CallExpression } + .filter { !it.name.startsWith("java.") } + .forEach { + val refersTo = it.refersTo + assertNotNull(refersTo, "${it.name} could not be resolved") + assertFalse( + refersTo.isInferred, + "${it.name} should not be resolved to an inferred node" + ) + } + } + + @Test + fun testLiteralsClass() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "literals") + val result = + analyze( + // We just need to specify one file to trigger the byte code loader + listOf(topLevel.resolve("mypackage/Literals.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + result.methods.forEach { + println(it.name) + println(it.code) + } + + assertEquals(0, result.problems.size) + } + + @Test + fun testLiteralsJar() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "jar", "literals") + val tu = + analyzeAndGetFirstTU( + // In case of a jar, the jar is directly used as a class path + listOf(topLevel.resolve("literals.jar").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + assertEquals(0, tu.problems.size) + tu.methods.forEach { println(it.code) } + } + + @Test + fun testInheritanceClass() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "inheritance") + val tu = + analyzeAndGetFirstTU( + // In case of a jar, the jar is directly used as a class path + listOf(topLevel.resolve("mypackage/Application.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + tu.methods.forEach { println(it.code) } + assertEquals(0, tu.problems.size) + + val myInterface = tu.records["mypackage.MyInterface"] + assertNotNull(myInterface) + assertEquals("interface", myInterface.kind) + + val baseClass = tu.records["mypackage.BaseClass"] + assertNotNull(baseClass) + + val extendedClass = tu.records["mypackage.ExtendedClass"] + assertNotNull(extendedClass) + assertContains(extendedClass.implementedInterfaces, myInterface.toType()) + assertContains(extendedClass.superTypeDeclarations, baseClass) + assertContains(extendedClass.superTypeDeclarations, myInterface) + + val anotherExtendedClass = tu.records["mypackage.AnotherExtendedClass"] + assertNotNull(anotherExtendedClass) + assertContains(anotherExtendedClass.superTypeDeclarations, baseClass) + + assertEquals( + baseClass.toType(), + listOf(extendedClass.toType(), anotherExtendedClass.toType()).commonType + ) + + val appInit = tu.methods["mypackage.Application."] + assertNotNull(appInit) + + val appDoSomething = tu.methods["mypackage.Application.doSomething"] + assertNotNull(appDoSomething) + assertLocalName("MyInterface", appDoSomething.parameters.firstOrNull()?.type) + + // Call doSomething in Application. with an object of ExtendedClass, which should + // fulfill the MyInterface of the needed parameter + val doSomethingCall1 = appInit.calls["doSomething"] + assertNotNull(doSomethingCall1) + assertLocalName("ExtendedClass", doSomethingCall1.arguments.firstOrNull()?.type) + assertInvokes(doSomethingCall1, appDoSomething) + + val extended = appInit.variables["r4"] + assertNotNull(extended) + + val getMyProperty = + appInit.calls[ + { + it.name.localName == "getMyProperty" && + it is MemberCallExpression && + it.base in extended.usages + }] + assertNotNull(getMyProperty) + assertInvokes(getMyProperty, baseClass.methods["getMyProperty"]) + + val setMyProperty = + appInit.calls[ + { + it.name.localName == "setMyProperty" && + it is MemberCallExpression && + it.base in extended.usages + }] + assertNotNull(setMyProperty) + assertInvokes(setMyProperty, extendedClass.methods["setMyProperty"]) + } + + @Test + fun testFieldsClass() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "fields") + val tu = + analyzeAndGetFirstTU( + // We just need to specify one file to trigger the byte code loader + listOf(topLevel.resolve("mypackage/Fields.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + assertEquals(0, tu.problems.size) + tu.methods.forEach { println(it.code) } + + val refs = tu.refs.filterIsInstance() + refs.forEach { + val refersTo = it.refersTo + assertNotNull(refersTo, "${it.name} could not be resolved") + assertFalse( + refersTo.isInferred, + "${it.name} should not be resolved to an inferred node" + ) + } + + val setACall = tu.calls["setA"] + assertNotNull(setACall) + + val lit10 = setACall.arguments.firstOrNull() + assertIs>(lit10) + assertLiteralValue(10, lit10) + } + + @Disabled + @Test + fun testLiteralsSource() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "literals") + val tu = + analyzeAndGetFirstTU( + // We just need to specify one file to trigger the source code loader + listOf(topLevel.resolve("mypackage/Literals.java").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val haveFun = tu.methods["haveFunWithLiterals"] + assertNotNull(haveFun) + + println(haveFun.code) + } + + @Test + fun testArraysClass() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "arrays") + val tu = + analyzeAndGetFirstTU( + // We just need to specify one file to trigger the class byte loader + listOf(topLevel.resolve("mypackage/Arrays.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + tu.methods.forEach { println(it.code) } + assertEquals(0, tu.problems.size) + + val create = tu.methods["create"] + assertNotNull(create) + + val r3 = create.variables["r3"] + assertNotNull(r3) + + var arrayType = r3.type + assertIs(arrayType) + assertTrue(arrayType.isArray) + assertFullName("mypackage.Element", arrayType.elementType) + + val r3write = r3.usages.firstOrNull { it.access == AccessValues.WRITE } + assertNotNull(r3write) + + var expr = r3write.prevDFG.singleOrNull() + assertIs(expr) + assertLiteralValue(2, expr.dimensions.singleOrNull()) + + var r1 = create.variables["r1"] + assertNotNull(r1) + assertEquals(arrayType.elementType, r1.type) + + val r2 = create.variables["r2"] + assertNotNull(r2) + assertEquals(arrayType.elementType, r2.type) + + val r2write = r2.usages.firstOrNull { it.access == AccessValues.WRITE } + assertNotNull(r2write) + + val prevDFG = r2write.prevDFG.singleOrNull() + assertIs(prevDFG) + assertRefersTo(prevDFG.arrayExpression, r3) + + val createMulti = tu.methods["createMulti"] + assertNotNull(createMulti) + + r1 = createMulti.variables["r1"] + assertNotNull(r1) + + arrayType = r1.type + assertIs(arrayType) + assertTrue(arrayType.isArray) + assertFullName("mypackage.Element", arrayType.elementType) + + val r1write = r1.usages.firstOrNull { it.access == AccessValues.WRITE } + assertNotNull(r1write) + + expr = r1write.prevDFG.singleOrNull() + assertIs(expr) + listOf(2, 10).forEachIndexed { index, i -> assertLiteralValue(i, expr.dimensions[index]) } + } + + @Disabled + @Test + fun testExceptional() { + // This will be our classpath + val topLevel = Path.of("src", "test", "resources", "class", "exception") + val tu = + analyzeAndGetFirstTU( + // We just need to specify one file to trigger the class byte loader + listOf(topLevel.resolve("mypackage/Exceptional.class").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + tu.methods.forEach { println(it.code) } + } +} diff --git a/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.class b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.class new file mode 100644 index 0000000000..194c12155e Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.class differ diff --git a/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.java b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.java new file mode 100644 index 0000000000..e91aeb1999 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Arrays.java @@ -0,0 +1,21 @@ +package mypackage; + +public class Arrays { + + public Element[] create() { + var arrays = new Element[2]; + arrays[0] = new Element(); + arrays[1] = arrays[0]; + + int len = arrays.length; + + return arrays; + } + + public Element[][] createMulti() { + var multi = new Element[2][10]; + + return multi; + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.class b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.class new file mode 100644 index 0000000000..80ab8317cc Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.class differ diff --git a/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.java b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.java new file mode 100644 index 0000000000..06a2c9a062 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/arrays/mypackage/Element.java @@ -0,0 +1,5 @@ +package mypackage; + +public class Element { + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.class b/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.class new file mode 100644 index 0000000000..3ef2237bee Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.class differ diff --git a/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.java b/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.java new file mode 100644 index 0000000000..3a7fed1211 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/fields/mypackage/Fields.java @@ -0,0 +1,18 @@ +package mypackage; + +public class Fields { + + private int a = 2; + + Fields() { + resetA(); + } + + public void setA(int a) { + this.a = a; + } + + private void resetA() { + setA(10); + } +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.class b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.class new file mode 100644 index 0000000000..144452d835 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.class differ diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.java b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.java new file mode 100644 index 0000000000..06fea507f4 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/AnotherExtendedClass.java @@ -0,0 +1,4 @@ +package mypackage; + +public class AnotherExtendedClass extends BaseClass { +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.class b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.class new file mode 100644 index 0000000000..0104bfda3e Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.class differ diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.java b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.java new file mode 100644 index 0000000000..8341691082 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/Application.java @@ -0,0 +1,27 @@ +package mypackage; + +public class Application { + public Application() { + var extended = new ExtendedClass(); + int old = extended.getMyProperty(); + extended.setMyProperty(10); + doSomething(extended); + + BaseClass base; + if(Math.random() == 1.0) { + base = (BaseClass) extended; + } else { + base = new AnotherExtendedClass(); + } + base.setMyProperty(10); + + if(base instanceof ExtendedClass) { + System.out.println("Is extended!"); + } + } + + public void doSomething(MyInterface i) { + i.doSomething(); + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.class b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.class new file mode 100644 index 0000000000..ce1f1370c1 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.class differ diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.java b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.java new file mode 100644 index 0000000000..3962a24f17 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/BaseClass.java @@ -0,0 +1,14 @@ +package mypackage; + +public class BaseClass { + + public int getMyProperty() { + return myProperty; + } + + public void setMyProperty(int myProperty) { + this.myProperty = myProperty; + } + + protected int myProperty = 5; +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.class b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.class new file mode 100644 index 0000000000..21739710f4 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.class differ diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.java b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.java new file mode 100644 index 0000000000..6dee7893f8 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/ExtendedClass.java @@ -0,0 +1,18 @@ +package mypackage; + +public class ExtendedClass extends BaseClass implements MyInterface { + + public void setMyProperty(int myProperty) { + informSomebody(this.myProperty, myProperty); + super.setMyProperty(myProperty); + } + private void informSomebody(int oldValue, int newValue) { + System.out.println("We changed the value from " + oldValue + " to " + newValue); + } + + @Override + public void doSomething() { + + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.class b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.class new file mode 100644 index 0000000000..45fc9318b3 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.class differ diff --git a/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.java b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.java new file mode 100644 index 0000000000..0c08afdd7d --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/inheritance/mypackage/MyInterface.java @@ -0,0 +1,7 @@ +package mypackage; + +public interface MyInterface { + + public void doSomething(); + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.class b/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.class new file mode 100644 index 0000000000..03362eb916 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.class differ diff --git a/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.java b/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.java new file mode 100644 index 0000000000..918c3befdb --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/literals/mypackage/Literals.java @@ -0,0 +1,37 @@ +package mypackage; + +import java.util.function.Supplier; + +public class Literals { + + void haveFunWithLiterals() { + float f = 2; + double d = 4d; + String str = "mystring"; + short s = 10; + int i = 2000; + long l = 2000L; + boolean b = false; + Literals obj; + if(Math.random() == 10) { + obj = null; + } else { + obj = this; + } + + Integer i2 = 1000; + Long l2 = 1000L; + + Class clazz = Literals.class; + test(this::mySupplier); + } + + void test(Supplier s) { + s.get(); + } + + Integer mySupplier() { + return 1; + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.class b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.class new file mode 100644 index 0000000000..9493eaf6a0 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.class differ diff --git a/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.java b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.java new file mode 100644 index 0000000000..954e22c6a4 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Adder.java @@ -0,0 +1,11 @@ +package mypackage; + +import java.lang.Integer; + +public class Adder { + + Integer add(Integer a, Integer b) { + return a + b; + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.class b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.class new file mode 100644 index 0000000000..4d448cfe92 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.class differ diff --git a/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.java b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.java new file mode 100644 index 0000000000..fdad05b5f8 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/class/methods/mypackage/Main.java @@ -0,0 +1,12 @@ +package mypackage; + +public class Main { + + public static void main(String[] args) { + var adder = new Adder(); + var sum = adder.add(1, 2); + + System.out.println(sum); + } + +} \ No newline at end of file diff --git a/cpg-language-jvm/src/test/resources/jar/literals/literals.jar b/cpg-language-jvm/src/test/resources/jar/literals/literals.jar new file mode 100644 index 0000000000..4931b86ad1 Binary files /dev/null and b/cpg-language-jvm/src/test/resources/jar/literals/literals.jar differ diff --git a/cpg-language-jvm/src/test/resources/jimple/helloworld/HelloWorld.jimple b/cpg-language-jvm/src/test/resources/jimple/helloworld/HelloWorld.jimple new file mode 100644 index 0000000000..256700b174 --- /dev/null +++ b/cpg-language-jvm/src/test/resources/jimple/helloworld/HelloWorld.jimple @@ -0,0 +1,22 @@ +public class HelloWorld extends java.lang.Object +{ + public void () + { + HelloWorld r0; + r0 := @this: HelloWorld; + specialinvoke r0.()>(); + return; + } + + public static void main(java.lang.String[]) + { + java.lang.String[] r0; + java.io.PrintStream $r1; + + r0 := @parameter0: java.lang.String[]; + $r1 = ; + virtualinvoke $r1.("Hello world!"); + return; + } +} diff --git a/cpg-language-jvm/src/test/resources/log4j2.xml b/cpg-language-jvm/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..5b73082e2c --- /dev/null +++ b/cpg-language-jvm/src/test/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/DeclarationHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/DeclarationHandler.kt index 948340673c..df86cdf4c5 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/DeclarationHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/DeclarationHandler.kt @@ -152,13 +152,13 @@ class DeclarationHandler(lang: LLVMIRLanguageFrontend) : } else if (LLVMGetEntryBasicBlock(func) == bb) { functionDeclaration.body = newBlock() if (stmt != null) { - (functionDeclaration.body as Block).addStatement(stmt) + (functionDeclaration.body as Block).statements += stmt } } else { // add the label statement, containing this basic block as a compound statement to // our body (if we have none, which we should) if (stmt != null) { - (functionDeclaration.body as? Block)?.addStatement(stmt) + (functionDeclaration.body as? Block)?.statements += stmt } } @@ -194,12 +194,7 @@ class DeclarationHandler(lang: LLVMIRLanguageFrontend) : } // try to see, if the struct already exists as a record declaration - var record = - frontend.scopeManager - .resolve(frontend.scopeManager.globalScope, true) { - it.name.toString() == name - } - .firstOrNull() + var record = frontend.scopeManager.getRecordForName(Name(name)) // if yes, return it if (record != null) { diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt index 194c5cda29..6376fc98ab 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt @@ -481,12 +481,7 @@ class ExpressionHandler(lang: LLVMIRLanguageFrontend) : var record = (baseType as? ObjectType)?.recordDeclaration if (record == null) { - record = - frontend.scopeManager - .resolve(frontend.scopeManager.globalScope, true) { - it.name == baseType.name - } - .firstOrNull() + record = frontend.scopeManager.getRecordForName(baseType.name) if (record != null) { (baseType as? ObjectType)?.recordDeclaration = record } diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt index 4ac3b40a64..35e3163ac2 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt @@ -31,7 +31,6 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -246,11 +245,7 @@ class LLVMIRLanguageFrontend(language: Language, ctx: Tr /** Determines if a struct with [name] exists in the scope. */ fun isKnownStructTypeName(name: String): Boolean { - return this.scopeManager - .resolve(this.scopeManager.globalScope, true) { - it.name.toString() == name - } - .isNotEmpty() + return this.scopeManager.getRecordForName(Name(name)) != null } fun getOperandValueAtIndex(instr: LLVMValueRef, idx: Int): Expression { diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt index 3d0c8ea8f4..1c2687fa68 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt @@ -277,7 +277,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : dummyCall.addArgument(parent, "parent") val tokenGeneration = declarationOrNot(dummyCall, instr) as DeclarationStatement - compoundStatement.addStatement(tokenGeneration) + compoundStatement.statements += tokenGeneration val ifStatement = newIfStatement(rawNode = instr) var currentIfStatement: IfStatement? = null @@ -342,7 +342,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : currentIfStatement?.elseStatement = throwOperation } - compoundStatement.addStatement(ifStatement) + compoundStatement.statements += ifStatement return compoundStatement } @@ -705,8 +705,8 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : val compoundStatement = newBlock(rawNode = instr) val assignment = newAssignExpression("=", listOf(base), listOf(valueToSet), rawNode = instr) - compoundStatement.addStatement(copy) - compoundStatement.addStatement(assignment) + compoundStatement.statements += copy + compoundStatement.statements += assignment return compoundStatement } @@ -835,7 +835,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : construct.addArgument(cmpExprConstruct) val decl = declarationOrNot(construct, instr) - compoundStatement.addStatement(decl) + compoundStatement.statements += decl } val ptrDerefAssign = newUnaryOperator("*", false, true, rawNode = instr) @@ -848,7 +848,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : ifStatement.condition = cmpExpr ifStatement.thenStatement = assignment - compoundStatement.addStatement(ifStatement) + compoundStatement.statements += ifStatement return compoundStatement } @@ -871,31 +871,31 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : val ptrDerefExch = newUnaryOperator("*", postfix = false, prefix = true, rawNode = instr) ptrDerefExch.input = frontend.getOperandValueAtIndex(instr, 0) - exchOp.lhs = listOf(ptrDerefExch) + exchOp.lhs = mutableListOf(ptrDerefExch) when (operation) { LLVMAtomicRMWBinOpXchg -> { - exchOp.rhs = listOf(value) + exchOp.rhs = mutableListOf(value) } LLVMAtomicRMWBinOpFAdd, LLVMAtomicRMWBinOpAdd -> { val binaryOperator = newBinaryOperator("+", rawNode = instr) binaryOperator.lhs = ptrDeref binaryOperator.rhs = value - exchOp.rhs = listOf(binaryOperator) + exchOp.rhs = mutableListOf(binaryOperator) } LLVMAtomicRMWBinOpFSub, LLVMAtomicRMWBinOpSub -> { val binaryOperator = newBinaryOperator("-", rawNode = instr) binaryOperator.lhs = ptrDeref binaryOperator.rhs = value - exchOp.rhs = listOf(binaryOperator) + exchOp.rhs = mutableListOf(binaryOperator) } LLVMAtomicRMWBinOpAnd -> { val binaryOperator = newBinaryOperator("&", rawNode = instr) binaryOperator.lhs = ptrDeref binaryOperator.rhs = value - exchOp.rhs = listOf(binaryOperator) + exchOp.rhs = mutableListOf(binaryOperator) } LLVMAtomicRMWBinOpNand -> { val binaryOperator = newBinaryOperator("|", rawNode = instr) @@ -903,19 +903,19 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : binaryOperator.rhs = value val unaryOperator = newUnaryOperator("~", false, true, rawNode = instr) unaryOperator.input = binaryOperator - exchOp.rhs = listOf(unaryOperator) + exchOp.rhs = mutableListOf(unaryOperator) } LLVMAtomicRMWBinOpOr -> { val binaryOperator = newBinaryOperator("|", rawNode = instr) binaryOperator.lhs = ptrDeref binaryOperator.rhs = value - exchOp.rhs = listOf(binaryOperator) + exchOp.rhs = mutableListOf(binaryOperator) } LLVMAtomicRMWBinOpXor -> { val binaryOperator = newBinaryOperator("^", rawNode = instr) binaryOperator.lhs = ptrDeref binaryOperator.rhs = value - exchOp.rhs = listOf(binaryOperator) + exchOp.rhs = mutableListOf(binaryOperator) } LLVMAtomicRMWBinOpMax, LLVMAtomicRMWBinOpMin -> { @@ -938,7 +938,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : value, ty, ) - exchOp.rhs = listOf(conditional) + exchOp.rhs = mutableListOf(conditional) } LLVMAtomicRMWBinOpUMax, LLVMAtomicRMWBinOpUMin -> { @@ -968,7 +968,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : value, ty, ) - exchOp.rhs = listOf(conditional) + exchOp.rhs = mutableListOf(conditional) } else -> { throw TranslationException("LLVMAtomicRMWBinOp $operation not supported") @@ -984,7 +984,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : ptrDerefAssignment.input = frontend.getOperandValueAtIndex(instr, 0) compoundStatement.statements = - listOf(declarationOrNot(ptrDerefAssignment, instr), exchOp) + mutableListOf(declarationOrNot(ptrDerefAssignment, instr), exchOp) compoundStatement } else { // only perform the replacement @@ -1017,11 +1017,11 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : val caseStatement = newCaseStatement(rawNode = instr) caseStatement.caseExpression = newLiteral(caseBBAddress, primitiveType("i64"), rawNode = instr) - caseStatements.addStatement(caseStatement) + caseStatements.statements += caseStatement // Get the label of the goto statement. val gotoStatement = assembleGotoStatement(instr, LLVMGetOperand(instr, idx)) - caseStatements.addStatement(gotoStatement) + caseStatements.statements += gotoStatement idx++ } @@ -1078,18 +1078,18 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : // Get the comparison value and add it to the CaseStatement val caseStatement = newCaseStatement(rawNode = instr) caseStatement.caseExpression = frontend.getOperandValueAtIndex(instr, idx) - caseStatements.addStatement(caseStatement) + caseStatements.statements += caseStatement idx++ // Get the "case" statements and add it to the CaseStatement val gotoStatement = assembleGotoStatement(instr, LLVMGetOperand(instr, idx)) - caseStatements.addStatement(gotoStatement) + caseStatements.statements += gotoStatement idx++ } // Get the label of the "default" branch - caseStatements.addStatement(newDefaultStatement(rawNode = instr)) + caseStatements.statements += newDefaultStatement(rawNode = instr) val defaultGoto = assembleGotoStatement(instr, LLVMGetOperand(instr, 1)) - caseStatements.addStatement(defaultGoto) + caseStatements.statements += defaultGoto switchStatement.statement = caseStatements @@ -1148,8 +1148,8 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : val tryStatement = newTryStatement(rawNode = instr) frontend.scopeManager.enterScope(tryStatement) val tryBlock = newBlock(rawNode = instr) - tryBlock.addStatement(declarationOrNot(callExpr, instr)) - tryBlock.addStatement(tryContinue) + tryBlock.statements += declarationOrNot(callExpr, instr) + tryBlock.statements += tryContinue tryStatement.tryBlock = tryBlock frontend.scopeManager.leaveScope(tryStatement) @@ -1164,7 +1164,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : ) val catchBlockStatement = newBlock(rawNode = instr) - catchBlockStatement.addStatement(gotoCatch) + catchBlockStatement.statements += gotoCatch catchClause.body = catchBlockStatement tryStatement.catchClauses = mutableListOf(catchClause) @@ -1230,7 +1230,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : // TODO: Probably we should make a proper copy of the array val newArrayDecl = declarationOrNot(frontend.getOperandValueAtIndex(instr, 0), instr) - compoundStatement.addStatement(newArrayDecl) + compoundStatement.statements += newArrayDecl val decl = newArrayDecl.declarations[0] as? VariableDeclaration val arrayExpr = newSubscriptExpression(rawNode = instr) @@ -1249,7 +1249,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : listOf(frontend.getOperandValueAtIndex(instr, 1)), rawNode = instr ) - compoundStatement.addStatement(assignExpr) + compoundStatement.statements += assignExpr return compoundStatement } @@ -1489,7 +1489,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : val stmt = frontend.statementHandler.handle(instr) if (stmt != null) { - compound.addStatement(stmt) + compound.statements += stmt } instr = LLVMGetNextInstruction(instr) diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CompressLLVMPass.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CompressLLVMPass.kt index 4c9565c5f5..082de02842 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CompressLLVMPass.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CompressLLVMPass.kt @@ -155,7 +155,7 @@ class CompressLLVMPass(ctx: TranslationContext) : ComponentPass(ctx) { // This is the most generic one val clauseToAdd = caseBody?.statements?.get(0) as CatchClause catchClauses.add(clauseToAdd) - caseBody.statements = caseBody.statements.drop(1) + caseBody.statements = caseBody.statements.drop(1).toMutableList() catchClauses[0].body = caseBody if (node.catchClauses[0].parameter != null) { catchClauses[0].parameter = node.catchClauses[0].parameter @@ -227,7 +227,7 @@ class CompressLLVMPass(ctx: TranslationContext) : ComponentPass(ctx) { alreadyChecked.add(currentNode) // We exclude sub-try statements as they would mess up with the results val toAdd = - SubgraphWalker.getAstChildren(currentNode).filter { n -> + currentNode.astChildren.filter { n -> n !is TryStatement && !alreadyChecked.contains(n) && !worklist.contains(n) } worklist.addAll(toAdd) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 5c5cdf4cb9..951d01cf62 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -28,13 +28,11 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import jep.python.PyObject class ExpressionHandler(frontend: PythonLanguageFrontend) : - PythonHandler(::ProblemExpression, frontend) { + PythonHandler(::ProblemExpression, frontend) { /* Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue): @@ -49,35 +47,35 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : private val formattedValConversionRepr = 114L private val formattedValConversionASCII = 97L - override fun handleNode(node: Python.ASTBASEexpr): Expression { + override fun handleNode(node: Python.AST.BaseExpr): Expression { return when (node) { - is Python.ASTName -> handleName(node) - is Python.ASTCall -> handleCall(node) - is Python.ASTConstant -> handleConstant(node) - is Python.ASTAttribute -> handleAttribute(node) - is Python.ASTBinOp -> handleBinOp(node) - is Python.ASTUnaryOp -> handleUnaryOp(node) - is Python.ASTCompare -> handleCompare(node) - is Python.ASTDict -> handleDict(node) - is Python.ASTIfExp -> handleIfExp(node) - is Python.ASTTuple -> handleTuple(node) - is Python.ASTList -> handleList(node) - is Python.ASTBoolOp -> handleBoolOp(node) - is Python.ASTSubscript -> handleSubscript(node) - is Python.ASTSlice -> handleSlice(node) - is Python.ASTLambda -> handleLambda(node) - is Python.ASTSet -> handleSet(node) - is Python.ASTFormattedValue -> handleFormattedValue(node) - is Python.ASTJoinedStr -> handleJoinedStr(node) - is Python.ASTStarred -> handleStarred(node) - is Python.ASTNamedExpr, - is Python.ASTGeneratorExp, - is Python.ASTListComp, - is Python.ASTSetComp, - is Python.ASTDictComp, - is Python.ASTAwait, - is Python.ASTYield, - is Python.ASTYieldFrom -> + is Python.AST.Name -> handleName(node) + is Python.AST.Call -> handleCall(node) + is Python.AST.Constant -> handleConstant(node) + is Python.AST.Attribute -> handleAttribute(node) + is Python.AST.BinOp -> handleBinOp(node) + is Python.AST.UnaryOp -> handleUnaryOp(node) + is Python.AST.Compare -> handleCompare(node) + is Python.AST.Dict -> handleDict(node) + is Python.AST.IfExp -> handleIfExp(node) + is Python.AST.Tuple -> handleTuple(node) + is Python.AST.List -> handleList(node) + is Python.AST.BoolOp -> handleBoolOp(node) + is Python.AST.Subscript -> handleSubscript(node) + is Python.AST.Slice -> handleSlice(node) + is Python.AST.Lambda -> handleLambda(node) + is Python.AST.Set -> handleSet(node) + is Python.AST.FormattedValue -> handleFormattedValue(node) + is Python.AST.JoinedStr -> handleJoinedStr(node) + is Python.AST.Starred -> handleStarred(node) + is Python.AST.NamedExpr -> handleNamedExpr(node) + is Python.AST.GeneratorExp, + is Python.AST.ListComp, + is Python.AST.SetComp, + is Python.AST.DictComp, + is Python.AST.Await, + is Python.AST.Yield, + is Python.AST.YieldFrom -> newProblemExpression( "The expression of class ${node.javaClass} is not supported yet", rawNode = node @@ -85,7 +83,25 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } - private fun handleFormattedValue(node: Python.ASTFormattedValue): Expression { + /** + * Translates a Python [`NamedExpr`](https://docs.python.org/3/library/ast.html#ast.NamedExpr) + * into an [AssignExpression]. + * + * As opposed to the Assign node, both target and value must be single nodes. + */ + private fun handleNamedExpr(node: Python.AST.NamedExpr): AssignExpression { + val assignExpression = + newAssignExpression( + operatorCode = ":=", + lhs = listOf(handle(node.target)), + rhs = listOf(handle(node.value)), + rawNode = node + ) + assignExpression.usedAsExpression = true + return assignExpression + } + + private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression { if (node.format_spec != null) { return newProblemExpression( "Cannot handle formatted value with format_spec ${node.format_spec} yet", @@ -124,7 +140,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } - private fun handleJoinedStr(node: Python.ASTJoinedStr): Expression { + private fun handleJoinedStr(node: Python.AST.JoinedStr): Expression { val values = node.values.map(::handle) return if (values.isEmpty()) { newLiteral("", primitiveType("str"), rawNode = node) @@ -143,13 +159,13 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } - private fun handleStarred(node: Python.ASTStarred): Expression { + private fun handleStarred(node: Python.AST.Starred): Expression { val unaryOp = newUnaryOperator("*", postfix = false, prefix = false, rawNode = node) unaryOp.input = handle(node.value) return unaryOp } - private fun handleSlice(node: Python.ASTSlice): Expression { + private fun handleSlice(node: Python.AST.Slice): Expression { val slice = newRangeExpression(rawNode = node) slice.floor = node.lower?.let { handle(it) } slice.ceiling = node.upper?.let { handle(it) } @@ -157,32 +173,52 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return slice } - private fun handleSubscript(node: Python.ASTSubscript): Expression { + private fun handleSubscript(node: Python.AST.Subscript): Expression { val subscriptExpression = newSubscriptExpression(rawNode = node) subscriptExpression.arrayExpression = handle(node.value) subscriptExpression.subscriptExpression = handle(node.slice) return subscriptExpression } - private fun handleBoolOp(node: Python.ASTBoolOp): Expression { + /** + * This method handles the python + * [`BoolOp`](https://docs.python.org/3/library/ast.html#ast.BoolOp). + * + * Generates a (potentially nested) [BinaryOperator] from a `BoolOp`. Less than two operands in + * [Python.AST.BoolOp.values] don't make sense and will generate a [ProblemExpression]. If only + * two operands exist, a simple [BinaryOperator] will be generated. More than two operands will + * lead to a nested [BinaryOperator]. E.g., if [Python.AST.BoolOp.values] contains the operators + * `[a, b, c]`, the result will be `a OP (b OP c)`. + */ + private fun handleBoolOp(node: Python.AST.BoolOp): Expression { val op = when (node.op) { - is Python.ASTAnd -> "and" - is Python.ASTOr -> "or" + is Python.AST.And -> "and" + is Python.AST.Or -> "or" } - val ret = newBinaryOperator(operatorCode = op, rawNode = node) - if (node.values.size != 2) { - return newProblemExpression( - "Expected exactly two expressions but got " + node.values.size, + + return if (node.values.size <= 1) { + newProblemExpression( + "Expected exactly two expressions but got ${node.values.size}", rawNode = node ) + } else { + // Start with the last two operands, then keep prepending the previous ones until the + // list is finished. + val lastTwo = newBinaryOperator(op, rawNode = node) + lastTwo.rhs = handle(node.values.last()) + lastTwo.lhs = handle(node.values[node.values.size - 2]) + return node.values.subList(0, node.values.size - 2).foldRight(lastTwo) { newVal, start + -> + val nextValue = newBinaryOperator(op, rawNode = node) + nextValue.rhs = start + nextValue.lhs = handle(newVal) + nextValue + } } - ret.lhs = handle(node.values[0]) - ret.rhs = handle(node.values[1]) - return ret } - private fun handleList(node: Python.ASTList): Expression { + private fun handleList(node: Python.AST.List): Expression { val lst = mutableListOf() for (e in node.elts) { lst += handle(e) @@ -193,7 +229,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ile } - private fun handleSet(node: Python.ASTSet): Expression { + private fun handleSet(node: Python.AST.Set): Expression { val lst = mutableListOf() for (e in node.elts) { lst += handle(e) @@ -204,18 +240,18 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ile } - private fun handleTuple(node: Python.ASTTuple): Expression { + private fun handleTuple(node: Python.AST.Tuple): Expression { val lst = mutableListOf() for (e in node.elts) { lst += handle(e) } val ile = newInitializerListExpression(rawNode = node) ile.type = frontend.objectType("tuple") - ile.initializers = lst.toList() + ile.initializers = lst return ile } - private fun handleIfExp(node: Python.ASTIfExp): Expression { + private fun handleIfExp(node: Python.AST.IfExp): Expression { return newConditionalExpression( condition = handle(node.test), thenExpression = handle(node.body), @@ -224,39 +260,40 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : ) } - private fun handleDict(node: Python.ASTDict): Expression { + private fun handleDict(node: Python.AST.Dict): Expression { val lst = mutableListOf() for (i in node.values.indices) { // TODO: keys longer than values possible? // Here we can not use node as raw node as it spans all keys and values lst += newKeyValueExpression( - key = node.keys[i]?.let { handle(it) }, + key = + node.keys[i]?.let { handle(it) } ?: newProblemExpression("missing key"), value = handle(node.values[i]), ) .codeAndLocationFromChildren(node) } val ile = newInitializerListExpression(rawNode = node) ile.type = frontend.objectType("dict") - ile.initializers = lst.toList() + ile.initializers = lst return ile } - private fun handleCompare(node: Python.ASTCompare): Expression { + private fun handleCompare(node: Python.AST.Compare): Expression { if (node.comparators.size != 1 || node.ops.size != 1) { return newProblemExpression("Multi compare is not (yet) supported.", rawNode = node) } val op = when (node.ops.first()) { - is Python.ASTEq -> "==" - is Python.ASTNotEq -> "!=" - is Python.ASTLt -> "<" - is Python.ASTLtE -> "<=" - is Python.ASTGt -> ">" - is Python.ASTGtE -> ">=" - is Python.ASTIs -> "is" - is Python.ASTIsNot -> "is not" - is Python.ASTIn -> "in" - is Python.ASTNotIn -> "not in" + is Python.AST.Eq -> "==" + is Python.AST.NotEq -> "!=" + is Python.AST.Lt -> "<" + is Python.AST.LtE -> "<=" + is Python.AST.Gt -> ">" + is Python.AST.GtE -> ">=" + is Python.AST.Is -> "is" + is Python.AST.IsNot -> "is not" + is Python.AST.In -> "in" + is Python.AST.NotIn -> "not in" } val ret = newBinaryOperator(op, rawNode = node) ret.lhs = handle(node.left) @@ -264,7 +301,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ret } - private fun handleBinOp(node: Python.ASTBinOp): Expression { + private fun handleBinOp(node: Python.AST.BinOp): Expression { val op = frontend.operatorToString(node.op) val ret = newBinaryOperator(operatorCode = op, rawNode = node) ret.lhs = handle(node.left) @@ -272,7 +309,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ret } - private fun handleUnaryOp(node: Python.ASTUnaryOp): Expression { + private fun handleUnaryOp(node: Python.AST.UnaryOp): Expression { val op = frontend.operatorUnaryToString(node.op) val ret = newUnaryOperator(operatorCode = op, postfix = false, prefix = false, rawNode = node) @@ -280,7 +317,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ret } - private fun handleAttribute(node: Python.ASTAttribute): Expression { + private fun handleAttribute(node: Python.AST.Attribute): Expression { var base = handle(node.value) // We do a quick check, if this refers to an import. This is faster than doing @@ -298,7 +335,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return ref } - private fun handleConstant(node: Python.ASTConstant): Expression { + private fun handleConstant(node: Python.AST.Constant): Expression { // TODO: this is ugly return if ( @@ -316,7 +353,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } - private fun easyConstant(node: Python.ASTConstant): Expression { + private fun easyConstant(node: Python.AST.Constant): Expression { // TODO check and add missing types val tpe = when (node.value) { @@ -342,27 +379,14 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * * TODO: cast, memberexpression, magic */ - private fun handleCall(node: Python.ASTCall): Expression { + private fun handleCall(node: Python.AST.Call): Expression { var callee = frontend.expressionHandler.handle(node.func) val ret = if (callee is MemberExpression) { newMemberCallExpression(callee, rawNode = node) } else { - // try to resolve -> [ConstructExpression] - val currentScope = frontend.scopeManager.currentScope - val record = - currentScope?.let { frontend.scopeManager.getRecordForName(callee.name) } - - if (record != null) { - // construct expression - val constructExpr = - newConstructExpression((node.func as? Python.ASTName)?.id, rawNode = node) - constructExpr.type = record.toType() - constructExpr - } else { - newCallExpression(callee, rawNode = node) - } + newCallExpression(callee, rawNode = node) } for (arg in node.args) { @@ -370,7 +394,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } for (keyword in node.keywords) { - ret.addArgument(handle(keyword.value), keyword.arg) + ret.argumentEdges.add(handle(keyword.value)) { name = keyword.arg } } return ret @@ -384,7 +408,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return decl?.isNotEmpty() ?: false } - private fun handleName(node: Python.ASTName): Expression { + private fun handleName(node: Python.AST.Name): Expression { val r = newReference(name = node.id, rawNode = node) /* @@ -406,7 +430,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : return r } - private fun handleLambda(node: Python.ASTLambda): Expression { + private fun handleLambda(node: Python.AST.Lambda): Expression { val lambda = newLambdaExpression(rawNode = node) val function = newFunctionDeclaration(name = "", rawNode = node) frontend.scopeManager.enterScope(function) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt index f7675d4879..1640a316b0 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt @@ -33,11 +33,15 @@ import jep.JepConfig import jep.MainInterpreter import jep.SharedInterpreter import kotlin.io.path.exists +import org.slf4j.Logger +import org.slf4j.LoggerFactory /** * Takes care of configuring Jep according to some well known paths on popular operating systems. */ object JepSingleton { + val log: Logger = LoggerFactory.getLogger(JepSingleton::class.java) + init { // TODO logging val config = JepConfig() @@ -101,20 +105,23 @@ object JepSingleton { wellKnownPaths.add(Paths.get("/", "usr", "lib", "libjep.so")) wellKnownPaths.add(Paths.get("/", "Library", "Java", "Extensions", "libjep.jnilib")) - wellKnownPaths.forEach { - if (it.exists()) { + for (path in wellKnownPaths) { + if (path.exists()) { // Jep's configuration must be set before the first instance is created. Later // calls to setJepLibraryPath and co result in failures. - MainInterpreter.setJepLibraryPath(it.toString()) + MainInterpreter.setJepLibraryPath(path.toString()) + + log.info("Using Jep native library in {}", path.toString()) // also add include path so that Python can find jep in case of virtual environment // fixes: jep.JepException: : No module named 'jep' if ( - it.parent.fileName.toString() == "jep" && - (Paths.get(it.parent.toString(), "__init__.py").exists()) + path.parent.fileName.toString() == "jep" && + (Paths.get(path.parent.toString(), "__init__.py").exists()) ) { - config.addIncludePaths(it.parent.parent.toString()) + config.addIncludePaths(path.parent.parent.toString()) } + break } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index 0aef6a52f5..774bd91c0b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -28,1264 +28,1355 @@ package de.fraunhofer.aisec.cpg.frontends.python import jep.python.PyObject /** - * This interface makes Python's `ast` nodes accessible to Kotlin. It does not contain any complex - * logic but rather aims at making all Python `ast` properties accessible to Kotlin (under the same - * name as in Python). - * - * Python's AST object are mapped as close as possible to the original. Exceptions: - * - `identifier` fields are mapped as Kotlin `String`s - * - Python's `int` is mapped to `Int` - * - Constants are mapped as `Any` (thus Jep's conversion to Java makes the translation) + * This interface encapsulates Python <-> Kotlin translation objects. It consists mainly, of + * translation objects from Python's `ast` class (see [Python.AST]), but other Python classes (like + * `complex`, ...) can be included, too. */ interface Python { /** - * `ast.stmt` [ASTBASEstmt] and `ast.expr` [ASTBASEexpr] nodes have extra location properties as - * implemented here. + * This is an abstract class that is common to all our python objects. Represents python's + * `object`. */ - interface WithPythonLocation { // TODO make the fields accessible `by lazy` - val pyObject: PyObject - - /** Maps to the `lineno` filed from Python's ast. */ - val lineno: Int - get() { - return (pyObject.getAttr("lineno") as? Long)?.toInt() ?: TODO() - } - - /** Maps to the `col_offset` filed from Python's ast. */ - val col_offset: Int - get() { - return (pyObject.getAttr("col_offset") as? Long)?.toInt() ?: TODO() - } - - /** Maps to the `end_lineno` filed from Python's ast. */ - val end_lineno: Int - get() { - return (pyObject.getAttr("end_lineno") as? Long)?.toInt() ?: TODO() - } + abstract class BaseObject(var pyObject: PyObject) - /** Maps to the `end_col_offset` filed from Python's ast. */ - val end_col_offset: Int - get() { - return (pyObject.getAttr("end_col_offset") as? Long)?.toInt() ?: TODO() - } - } + /** The `ellipsis` class. */ + class Ellipsis(pyObject: PyObject) : BaseObject(pyObject) - /** - * Represents a `ast.AST` node as returned by Python's `ast` parser. - * - * @param pyObject The Python object returned by jep. - */ - abstract class AST(val pyObject: PyObject) + /** The `complex` class. */ + class Complex(pyObject: PyObject) : BaseObject(pyObject) /** - * ``` - * ast.mod = class mod(AST) - * | mod = Module(stmt* body, type_ignore* type_ignores) - * | | Interactive(stmt* body) - * | | Expression(expr body) - * | | FunctionType(expr* argtypes, expr returns) - * ``` + * This interface makes Python's `ast` nodes accessible to Kotlin. It does not contain any + * complex logic but rather aims at making all Python `ast` properties accessible to Kotlin + * (under the same name as in Python). * - * Note: We currently only support `Module`s. - */ - abstract class ASTBASEmod(pyObject: PyObject) : AST(pyObject) - - /** - * ``` - * ast.Module = class Module(mod) - * | Module(stmt* body, type_ignore* type_ignores) - * ``` - */ - class ASTModule(pyObject: PyObject) : AST(pyObject) { - val body: List by lazy { "body" of pyObject } - - val type_ignores: List by lazy { "type_ignores" of pyObject } - } - - /** - * ``` - * ast.stmt = class stmt(AST) - * | stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - * | | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - * | | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) - * | | Return(expr? value) - * | | Delete(expr* targets) - * | | Assign(expr* targets, expr value, string? type_comment) - * | | AugAssign(expr target, operator op, expr value) - * | | AnnAssign(expr target, expr annotation, expr? value, int simple) - * | | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - * | | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - * | | While(expr test, stmt* body, stmt* orelse) - * | | If(expr test, stmt* body, stmt* orelse) - * | | With(withitem* items, stmt* body, string? type_comment) - * | | AsyncWith(withitem* items, stmt* body, string? type_comment) - * | | Match(expr subject, match_case* cases) - * | | Raise(expr? exc, expr? cause) - * | | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - * | | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - * | | Assert(expr test, expr? msg) - * | | Import(alias* names) - * | | ImportFrom(identifier? module, alias* names, int? level) - * | | Global(identifier* names) - * | | Nonlocal(identifier* names) - * | | Expr(expr value) - * | | Pass - * | | Break - * | | Continue - * ``` - */ - sealed class ASTBASEstmt(pyObject: PyObject) : AST(pyObject), WithPythonLocation - - /** - * ``` - * ast.FunctionDef = class FunctionDef(stmt) - * | FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - * ``` - */ - class ASTFunctionDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val name: String by lazy { "name" of pyObject } - - val args: ASTarguments by lazy { "args" of pyObject } - - val body: List by lazy { "body" of pyObject } - - val decorator_list: List by lazy { "decorator_list" of pyObject } - - val returns: ASTBASEexpr? by lazy { "returns" of pyObject } - - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.AsyncFunctionDef = class AsyncFunctionDef(stmt) - * | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) - * ``` - */ - class ASTAsyncFunctionDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val name: String by lazy { "name" of pyObject } - - val args: ASTarguments by lazy { "args" of pyObject } - - val body: List by lazy { "body" of pyObject } - - val decorator_list: List by lazy { "decorator_list" of pyObject } - - val returns: ASTBASEexpr? by lazy { "returns" of pyObject } - - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.ClassDef = class ClassDef(stmt) - * | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) - * ``` - */ - class ASTClassDef(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val name: String by lazy { "name" of pyObject } - - val bases: List by lazy { "bases" of pyObject } - - val keywords: List by lazy { "keywords" of pyObject } - - val body: List by lazy { "body" of pyObject } - - val decorator_list: List by lazy { "decorator_list" of pyObject } - } - - /** - * ``` - * ast.Return = class Return(stmt) - * | Return(expr? value) - * ``` - */ - class ASTReturn(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val value: ASTBASEexpr? by lazy { "value" of pyObject } - } - - /** - * ``` - * ast.Delete = class Delete(stmt) - * | Delete(expr* targets) - * ``` - */ - class ASTDelete(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val targets: List by lazy { "targets" of pyObject } - } - - /** - * ``` - * ast.Assign = class Assign(stmt) - * | Assign(expr* targets, expr value, string? type_comment) - * ``` - */ - class ASTAssign(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val targets: List by lazy { "targets" of pyObject } - - val value: ASTBASEexpr by lazy { "value" of pyObject } - - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.AugAssign = class AugAssign(stmt) - * | AugAssign(expr target, operator op, expr value) - * ``` - */ - class ASTAugAssign(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val op: ASTBASEoperator by lazy { "op" of pyObject } - val value: ASTBASEexpr by lazy { "value" of pyObject } - } - - /** - * ``` - * ast.AnnAssign = class AnnAssign(stmt) - * | AnnAssign(expr target, expr annotation, expr? value, int simple) - * ``` - */ - class ASTAnnAssign(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val annotation: ASTBASEexpr by lazy { "annotation" of pyObject } - val value: ASTBASEexpr? by lazy { "value" of pyObject } - val simple: Long by lazy { "simple" of pyObject } - } - - /** - * ``` - * ast.For = class For(stmt) - * | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - * ``` - */ - class ASTFor(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val iter: ASTBASEexpr by lazy { "iter" of pyObject } - val body: List by lazy { "body" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.AsyncFor = class AsyncFor(stmt) - * | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - * ``` - */ - class ASTAsyncFor(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val iter: ASTBASEexpr by lazy { "iter" of pyObject } - val body: List by lazy { "body" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.While = class While(stmt) - * | While(expr test, stmt* body, stmt* orelse) - * ``` - */ - class ASTWhile(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val test: ASTBASEexpr by lazy { "test" of pyObject } - val body: List by lazy { "body" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - } - - /** - * ``` - * ast.If = class If(stmt) - * | If(expr test, stmt* body, stmt* orelse) - * ``` - */ - class ASTIf(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val test: ASTBASEexpr by lazy { "test" of pyObject } - val body: List by lazy { "body" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - } - - /** - * ``` - * ast.With = class With(stmt) - * | With(withitem* items, stmt* body, string? type_comment) - * ``` - */ - class ASTWith(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val items: ASTwithitem by lazy { "items" of pyObject } - val body: List by lazy { "body" of pyObject } - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.AsyncWith = class AsyncWith(stmt) - * | AsyncWith(withitem* items, stmt* body, string? type_comment) - * ``` - */ - class ASTAsyncWith(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val iter: ASTBASEexpr by lazy { "iter" of pyObject } - val body: List by lazy { "body" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - val type_comment: String? by lazy { "type_comment" of pyObject } - } - - /** - * ``` - * ast.Match = class Match(stmt) - * | Match(expr subject, match_case* cases) - * ``` - */ - class ASTMatch(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val subject: ASTBASEexpr by lazy { "subject" of pyObject } - val cases: List by lazy { "cases" of pyObject } - } - - /** - * ``` - * ast.Raise = class Raise(stmt) - * | Raise(expr? exc, expr? cause) - * ``` - */ - class ASTRaise(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val exc: ASTBASEexpr? by lazy { "exc" of pyObject } - val cause: ASTBASEexpr? by lazy { "cause" of pyObject } - } - - /** - * ``` - * ast.Try = class Try(stmt) - * | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - * ``` - */ - class ASTTry(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val body: List by lazy { "body" of pyObject } - val handlers: List by lazy { "handlers" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - val stmt: List by lazy { "StmtBase" of pyObject } - } - - /** - * ``` - * ast.TryStar = class TryStar(stmt) - * | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) - * ``` - */ - class ASTTryStar(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val body: List by lazy { "body" of pyObject } - val handlers: List by lazy { "handlers" of pyObject } - val orelse: List by lazy { "orelse" of pyObject } - val finalbody: List by lazy { "finalbody" of pyObject } - } - - /** - * ``` - * ast.Assert = class Assert(stmt) - * | Assert(expr test, expr? msg) - * ``` - */ - class ASTAssert(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val test: ASTBASEexpr by lazy { "test" of pyObject } - val msg: ASTBASEexpr? by lazy { "msg" of pyObject } - } - - /** - * ``` - * ast.Import = class Import(stmt) - * | Import(alias* names) - * ``` - */ - class ASTImport(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val names: List by lazy { "names" of pyObject } - } - - /** - * ``` - * ast.ImportFrom = class ImportFrom(stmt) - * | ImportFrom(identifier? module, alias* names, int? level) - * ``` - */ - class ASTImportFrom(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val module: String? by lazy { "module" of pyObject } - val names: List by lazy { "names" of pyObject } - val level: Long? by lazy { "level" of pyObject } - } - - /** - * ``` - * ast.Global = class Global(stmt) - * | Global(identifier* names) - * ``` - */ - class ASTGlobal(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val names: List by lazy { "names" of pyObject } - } + * Python's AST object are mapped as close as possible to the original. Exceptions: + * - `identifier` fields are mapped as Kotlin [String]s + * - Python's `int` is mapped to [Int] + * - Constants are mapped as [Any] (thus Jep's conversion to Java makes the translation) + */ + interface AST { + + /** + * Some nodes, such as `ast.stmt` [AST.BaseStmt] and `ast.expr` [AST.BaseExpr] nodes have + * extra location properties as implemented here. + */ + interface WithLocation { // TODO make the fields accessible `by lazy` + val pyObject: PyObject + + /** Maps to the `lineno` filed from Python's ast. */ + val lineno: Int + get() { + return (pyObject.getAttr("lineno") as? Long)?.toInt() ?: TODO() + } + + /** Maps to the `col_offset` filed from Python's ast. */ + val col_offset: Int + get() { + return (pyObject.getAttr("col_offset") as? Long)?.toInt() ?: TODO() + } + + /** Maps to the `end_lineno` filed from Python's ast. */ + val end_lineno: Int + get() { + return (pyObject.getAttr("end_lineno") as? Long)?.toInt() ?: TODO() + } + + /** Maps to the `end_col_offset` filed from Python's ast. */ + val end_col_offset: Int + get() { + return (pyObject.getAttr("end_col_offset") as? Long)?.toInt() ?: TODO() + } + } - /** - * ``` - * ast.Nonlocal = class Nonlocal(stmt) - * | Nonlocal(identifier* names) - * ``` - */ - class ASTNonlocal(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val names: List by lazy { "names" of pyObject } - } + /** + * Represents a `ast.AST` node as returned by Python's `ast` parser. + * + * @param pyObject The Python object returned by jep. + */ + abstract class AST(pyObject: PyObject) : BaseObject(pyObject) + + /** + * ``` + * ast.mod = class mod(AST) + * | mod = Module(stmt* body, type_ignore* type_ignores) + * | | Interactive(stmt* body) + * | | Expression(expr body) + * | | FunctionType(expr* argtypes, expr returns) + * ``` + * + * Note: We currently only support `Module`s. + */ + abstract class BaseMod(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.Module = class Module(mod) + * | Module(stmt* body, type_ignore* type_ignores) + * ``` + */ + class Module(pyObject: PyObject) : AST(pyObject) { + val body: kotlin.collections.List by lazy { "body" of pyObject } + + val type_ignores: kotlin.collections.List by lazy { + "type_ignores" of pyObject + } + } - /** - * Represents `ast.Expr` expressions. Note: do not confuse with - * - [ASTBASEexpr] -> the expression class - * - [Expression] -> the expression as part of `mod` - * - * ``` - * ast.Expr = class Expr(stmt) - * | Expr(expr value) - * ``` - */ - class ASTExpr(pyObject: PyObject) : ASTBASEstmt(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + /** + * ``` + * ast.stmt = class stmt(AST) + * | stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) + * | | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) + * | | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) + * | | Return(expr? value) + * | | Delete(expr* targets) + * | | Assign(expr* targets, expr value, string? type_comment) + * | | AugAssign(expr target, operator op, expr value) + * | | AnnAssign(expr target, expr annotation, expr? value, int simple) + * | | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + * | | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + * | | While(expr test, stmt* body, stmt* orelse) + * | | If(expr test, stmt* body, stmt* orelse) + * | | With(withitem* items, stmt* body, string? type_comment) + * | | AsyncWith(withitem* items, stmt* body, string? type_comment) + * | | Match(expr subject, match_case* cases) + * | | Raise(expr? exc, expr? cause) + * | | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + * | | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + * | | Assert(expr test, expr? msg) + * | | Import(alias* names) + * | | ImportFrom(identifier? module, alias* names, int? level) + * | | Global(identifier* names) + * | | Nonlocal(identifier* names) + * | | Expr(expr value) + * | | Pass + * | | Break + * | | Continue + * ``` + */ + sealed class BaseStmt(pyObject: PyObject) : AST(pyObject), WithLocation + + /** + * Several classes are duplicated in the python AST for async and non-async variants. This + * interface is a common interface for those AST classes. + */ + interface AsyncOrNot : WithLocation + + /** This interface denotes that this is an "async" node. */ + interface IsAsync : AsyncOrNot + + /** + * ast.FunctionDef and ast.AsyncFunctionDef are not related according to the Python syntax. + * However, they are so similar, that we make use of this interface to avoid a lot of + * duplicate code. + */ + interface NormalOrAsyncFunctionDef : AsyncOrNot { + val name: String + val args: arguments + val body: kotlin.collections.List + val decorator_list: kotlin.collections.List + val returns: BaseExpr? + val type_comment: String? + } - /** - * ``` - * ast.Pass = class Pass(stmt) - * | Pass - * ``` - */ - class ASTPass(pyObject: PyObject) : ASTBASEstmt(pyObject) + /** + * ``` + * ast.FunctionDef = class FunctionDef(stmt) + * | FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) + * ``` + */ + class FunctionDef(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFunctionDef { + override val name: String by lazy { "name" of pyObject } - /** - * ``` - * ast.Break = class Break(stmt) - * | Break - * ``` - */ - class ASTBreak(pyObject: PyObject) : ASTBASEstmt(pyObject) + override val args: arguments by lazy { "args" of pyObject } - /** - * ``` - * ast.Continue = class Continue(stmt) - * | Continue - * ``` - */ - class ASTContinue(pyObject: PyObject) : ASTBASEstmt(pyObject) + override val body: kotlin.collections.List by lazy { "body" of pyObject } - /** - * Represents `ast.expr` expressions. Note: do not confuse with - * - [ASTExpr] -> the expression statement - * - [Expression] -> the expression as part of `mod` - * - * ast.expr = class expr(AST) - */ - sealed class ASTBASEexpr(pyObject: PyObject) : AST(pyObject), WithPythonLocation + override val decorator_list: kotlin.collections.List by lazy { + "decorator_list" of pyObject + } - /** - * ``` - * ast.BoolOp = class BoolOp(expr) - * | BoolOp(boolop op, expr* values) - * ``` - */ - class ASTBoolOp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val op: ASTBASEboolop by lazy { "op" of pyObject } - val values: List by lazy { "values" of pyObject } - } + override val returns: BaseExpr? by lazy { "returns" of pyObject } - /** - * ``` - * ast.NamedExpr = class NamedExpr(expr) - * | NamedExpr(expr target, expr value) - * ``` - */ - class ASTNamedExpr(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.BinOp = class BinOp(expr) - * | BinOp(expr left, operator op, expr right) - * ``` - */ - class ASTBinOp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val left: ASTBASEexpr by lazy { "left" of pyObject } - val op: ASTBASEoperator by lazy { "op" of pyObject } - val right: ASTBASEexpr by lazy { "right" of pyObject } - } + /** + * ``` + * ast.AsyncFunctionDef = class AsyncFunctionDef(stmt) + * | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment) + * ``` + */ + class AsyncFunctionDef(pyObject: PyObject) : + BaseStmt(pyObject), NormalOrAsyncFunctionDef, IsAsync { + override val name: String by lazy { "name" of pyObject } - /** - * ``` - * ast.UnaryOp = class UnaryOp(expr) - * | UnaryOp(unaryop op, expr operand) - * ``` - */ - class ASTUnaryOp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val op: ASTBASEunaryop by lazy { "op" of pyObject } - val operand: ASTBASEexpr by lazy { "operand" of pyObject } - } + override val args: arguments by lazy { "args" of pyObject } - /** - * ``` - * ast.Lambda = class Lambda(expr) - * | Lambda(arguments args, expr body) - * ``` - */ - class ASTLambda(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val args: ASTarguments by lazy { "args" of pyObject } - val body: ASTBASEexpr by lazy { "body" of pyObject } - } + override val body: kotlin.collections.List by lazy { "body" of pyObject } - /** - * ``` - * ast.IfExp = class IfExp(expr) - * | IfExp(expr test, expr body, expr orelse) - * ``` - */ - class ASTIfExp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val test: ASTBASEexpr by lazy { "test" of pyObject } - val body: ASTBASEexpr by lazy { "body" of pyObject } - val orelse: ASTBASEexpr by lazy { "orelse" of pyObject } - } + override val decorator_list: kotlin.collections.List by lazy { + "decorator_list" of pyObject + } - /** - * ``` - * ast.Dict = class Dict(expr) - * | Dict(expr* keys, expr* values) - * ``` - */ - class ASTDict(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val keys: List by lazy { "keys" of pyObject } - val values: List by lazy { "values" of pyObject } - } + override val returns: BaseExpr? by lazy { "returns" of pyObject } - /** - * ``` - * ast.Set = class Set(expr) - * | Set(expr* elts) - * ``` - */ - class ASTSet(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elts: List by lazy { "elts" of pyObject } - } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.ListComp = class ListComp(expr) - * | ListComp(expr elt, comprehension* generators) - * ``` - */ - class ASTListComp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elt: ASTBASEexpr by lazy { "elt" of pyObject } - val generators: List by lazy { "generators" of pyObject } - } + /** + * ``` + * ast.ClassDef = class ClassDef(stmt) + * | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list) + * ``` + */ + class ClassDef(pyObject: PyObject) : BaseStmt(pyObject) { + val name: String by lazy { "name" of pyObject } - /** - * ``` - * ast.SetComp = class SetComp(expr) - * | SetComp(expr elt, comprehension* generators) - * ``` - */ - class ASTSetComp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elt: ASTBASEexpr by lazy { "elt" of pyObject } - val generators: List by lazy { "generators" of pyObject } - } + val bases: kotlin.collections.List by lazy { "bases" of pyObject } - /** - * ``` - * ast.DictComp = class DictComp(expr) - * | DictComp(expr key, expr value, comprehension* generators) - * ``` - */ - class ASTDictComp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val key: ASTBASEexpr by lazy { "key" of pyObject } - val value: ASTBASEexpr by lazy { "value" of pyObject } - val generators: List by lazy { "generators" of pyObject } - } + val keywords: kotlin.collections.List by lazy { "keywords" of pyObject } - /** - * ``` - * ast.GeneratorExp = class GeneratorExp(expr) - * | GeneratorExp(expr elt, comprehension* generators) - * ``` - */ - class ASTGeneratorExp(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elt: ASTBASEexpr by lazy { "elt" of pyObject } - val generators: List by lazy { "generators" of pyObject } - } + val body: kotlin.collections.List by lazy { "body" of pyObject } - /** - * ``` - * ast.Await = class Await(expr) - * | Await(expr value) - * ``` - */ - class ASTAwait(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + val decorator_list: kotlin.collections.List by lazy { + "decorator_list" of pyObject + } + } - /** - * ``` - * ast.Yield = class Yield(expr) - * | Yield(expr? value) - * ``` - */ - class ASTYield(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr? by lazy { "value" of pyObject } - } + /** + * ``` + * ast.Return = class Return(stmt) + * | Return(expr? value) + * ``` + */ + class Return(pyObject: PyObject) : BaseStmt(pyObject) { + val value: BaseExpr? by lazy { "value" of pyObject } + } - /** - * ``` - * ast.YieldFrom = class YieldFrom(expr) - * | YieldFrom(expr value) - * ``` - */ - class ASTYieldFrom(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + /** + * ``` + * ast.Delete = class Delete(stmt) + * | Delete(expr* targets) + * ``` + */ + class Delete(pyObject: PyObject) : BaseStmt(pyObject) { + val targets: kotlin.collections.List by lazy { "targets" of pyObject } + } - /** - * ``` - * ast.Compare = class Compare(expr) - * | Compare(expr left, cmpop* ops, expr* comparators) - * ``` - */ - class ASTCompare(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val left: ASTBASEexpr by lazy { "left" of pyObject } - val ops: List by lazy { "ops" of pyObject } - val comparators: List by lazy { "comparators" of pyObject } - } + /** + * ``` + * ast.Assign = class Assign(stmt) + * | Assign(expr* targets, expr value, string? type_comment) + * ``` + */ + class Assign(pyObject: PyObject) : BaseStmt(pyObject) { + val targets: kotlin.collections.List by lazy { "targets" of pyObject } - /** - * ``` - * ast.Call = class Call(expr) - * | Call(expr func, expr* args, keyword* keywords) - * ``` - */ - class ASTCall(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val func: ASTBASEexpr by lazy { "func" of pyObject } + val value: BaseExpr by lazy { "value" of pyObject } - val args: List by lazy { "args" of pyObject } + val type_comment: String? by lazy { "type_comment" of pyObject } + } - val keywords: List by lazy { "keywords" of pyObject } - } + /** + * ``` + * ast.AugAssign = class AugAssign(stmt) + * | AugAssign(expr target, operator op, expr value) + * ``` + */ + class AugAssign(pyObject: PyObject) : BaseStmt(pyObject) { + val target: BaseExpr by lazy { "target" of pyObject } + val op: BaseOperator by lazy { "op" of pyObject } + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.FormattedValue = class FormattedValue(expr) - * | FormattedValue(expr value, int conversion, expr? format_spec) - * ``` - */ - class ASTFormattedValue(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - val conversion: Long? by lazy { "conversion" of pyObject } - val format_spec: ASTBASEexpr? by lazy { "format_spec" of pyObject } - } + /** + * ``` + * ast.AnnAssign = class AnnAssign(stmt) + * | AnnAssign(expr target, expr annotation, expr? value, int simple) + * ``` + */ + class AnnAssign(pyObject: PyObject) : BaseStmt(pyObject) { + val target: BaseExpr by lazy { "target" of pyObject } + val annotation: BaseExpr by lazy { "annotation" of pyObject } + val value: BaseExpr? by lazy { "value" of pyObject } + val simple: Long by lazy { "simple" of pyObject } + } - /** - * ``` - * ast.JoinedStr = class JoinedStr(expr) - * | JoinedStr(expr* values) - * ``` - */ - class ASTJoinedStr(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val values: List by lazy { "values" of pyObject } - } + /** + * ast.For and ast.AsyncFor are not related according to the Python syntax. However, they + * are so similar, that we make use of this interface to avoid a lot of duplicate code. + */ + interface NormalOrAsyncFor : AsyncOrNot { + val target: BaseExpr + val iter: BaseExpr + val body: kotlin.collections.List + val orelse: kotlin.collections.List + val type_comment: String? + } - /** - * ``` - * ast.Constant = class Constant(expr) - * | Constant(constant value, string? kind) - * ``` - */ - class ASTConstant(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: Any by lazy { "value" of pyObject } - val kind: String? by lazy { "kind" of pyObject } - } + /** + * ``` + * ast.For = class For(stmt) + * | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + * ``` + */ + class For(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFor { + override val target: BaseExpr by lazy { "target" of pyObject } + override val iter: BaseExpr by lazy { "iter" of pyObject } + override val body: kotlin.collections.List by lazy { "body" of pyObject } + override val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.Attribute = class Attribute(expr) - * | Attribute(expr value, identifier attr, expr_context ctx) - * ``` - */ - class ASTAttribute(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - val attr: String by lazy { "attr" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ``` + * ast.AsyncFor = class AsyncFor(stmt) + * | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + * ``` + */ + class AsyncFor(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFor, IsAsync { + override val target: BaseExpr by lazy { "target" of pyObject } + override val iter: BaseExpr by lazy { "iter" of pyObject } + override val body: kotlin.collections.List by lazy { "body" of pyObject } + override val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.Subscript = class Subscript(expr) - * | Subscript(expr value, expr slice, expr_context ctx) - * ``` - */ - class ASTSubscript(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - val slice: ASTBASEexpr by lazy { "slice" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ``` + * ast.While = class While(stmt) + * | While(expr test, stmt* body, stmt* orelse) + * ``` + */ + class While(pyObject: PyObject) : BaseStmt(pyObject) { + val test: BaseExpr by lazy { "test" of pyObject } + val body: kotlin.collections.List by lazy { "body" of pyObject } + val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + } - /** - * ``` - * ast.Starred = class Starred(expr) - * | Starred(expr value, expr_context ctx) - * ``` - */ - class ASTStarred(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ``` + * ast.If = class If(stmt) + * | If(expr test, stmt* body, stmt* orelse) + * ``` + */ + class If(pyObject: PyObject) : BaseStmt(pyObject) { + val test: BaseExpr by lazy { "test" of pyObject } + val body: kotlin.collections.List by lazy { "body" of pyObject } + val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + } - /** - * ``` - * ast.Name = class Name(expr) - * | Name(identifier id, expr_context ctx) - * ``` - */ - class ASTName(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val id: String by lazy { "id" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ast.With and ast.AsyncWith are not related according to the Python syntax. However, they + * are so similar, that we make use of this interface to avoid a lot of duplicate code. + */ + interface NormalOrAsyncWith : AsyncOrNot { + val items: kotlin.collections.List + val body: kotlin.collections.List + val type_comment: String? + } - /** - * ``` - * ast.List = class List(expr) - * | List(expr* elts, expr_context ctx) - * ``` - */ - class ASTList(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elts: List by lazy { "elts" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ``` + * ast.With = class With(stmt) + * | With(withitem* items, stmt* body, string? type_comment) + * ``` + */ + class With(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncWith { + override val items: kotlin.collections.List by lazy { "items" of pyObject } + override val body: kotlin.collections.List by lazy { "body" of pyObject } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.Tuple = class Tuple(expr) - * | Tuple(expr* elts, expr_context ctx) - * ``` - */ - class ASTTuple(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val elts: List by lazy { "elts" of pyObject } - val ctx: ASTBASEexpr_context by lazy { "ctx" of pyObject } - } + /** + * ``` + * ast.AsyncWith = class AsyncWith(stmt) + * | AsyncWith(withitem* items, stmt* body, string? type_comment) + * ``` + */ + class AsyncWith(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncWith, IsAsync { + override val items: kotlin.collections.List by lazy { "items" of pyObject } + override val body: kotlin.collections.List by lazy { "body" of pyObject } + override val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.Slice = class Slice(expr) - * | Slice(expr? lower, expr? upper, expr? step) - * ``` - */ - class ASTSlice(pyObject: PyObject) : ASTBASEexpr(pyObject) { - val lower: ASTBASEexpr? by lazy { "lower" of pyObject } - val upper: ASTBASEexpr? by lazy { "upper" of pyObject } - val step: ASTBASEexpr? by lazy { "step" of pyObject } - } + /** + * ``` + * ast.Match = class Match(stmt) + * | Match(expr subject, match_case* cases) + * ``` + */ + class Match(pyObject: PyObject) : BaseStmt(pyObject) { + val subject: BaseExpr by lazy { "subject" of pyObject } + val cases: kotlin.collections.List by lazy { "cases" of pyObject } + } - /** - * ``` - * ast.boolop = class boolop(AST) - * | boolop = And | Or - * ``` - */ - sealed class ASTBASEboolop(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.Raise = class Raise(stmt) + * | Raise(expr? exc, expr? cause) + * ``` + */ + class Raise(pyObject: PyObject) : BaseStmt(pyObject) { + val exc: BaseExpr? by lazy { "exc" of pyObject } + val cause: BaseExpr? by lazy { "cause" of pyObject } + } - /** - * ``` - * ast.And = class And(boolop) - * | And - * ``` - */ - class ASTAnd(pyObject: PyObject) : ASTBASEboolop(pyObject) + /** + * ``` + * ast.Try = class Try(stmt) + * | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + * ``` + */ + class Try(pyObject: PyObject) : BaseStmt(pyObject) { + val body: kotlin.collections.List by lazy { "body" of pyObject } + val handlers: kotlin.collections.List by lazy { + "handlers" of pyObject + } + val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + val finalbody: kotlin.collections.List by lazy { "finalbody" of pyObject } + } - /** - * ``` - * ast.Or = class Or(boolop) - * | Or - */ - class ASTOr(pyObject: PyObject) : ASTBASEboolop(pyObject) + /** + * ``` + * ast.TryStar = class TryStar(stmt) + * | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + * ``` + */ + class TryStar(pyObject: PyObject) : BaseStmt(pyObject) { + val body: kotlin.collections.List by lazy { "body" of pyObject } + val handlers: kotlin.collections.List by lazy { + "handlers" of pyObject + } + val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } + val finalbody: kotlin.collections.List by lazy { "finalbody" of pyObject } + } - /** - * ``` - * ast.cmpop = class cmpop(AST) - * | cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - * ``` - */ - sealed class ASTBASEcmpop(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.Assert = class Assert(stmt) + * | Assert(expr test, expr? msg) + * ``` + */ + class Assert(pyObject: PyObject) : BaseStmt(pyObject) { + val test: BaseExpr by lazy { "test" of pyObject } + val msg: BaseExpr? by lazy { "msg" of pyObject } + } - /** - * ``` - * ast.Eq = class Eq(cmpop) - * | Eq - * ``` - */ - class ASTEq(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.Import = class Import(stmt) + * | Import(alias* names) + * ``` + */ + class Import(pyObject: PyObject) : BaseStmt(pyObject) { + val names: kotlin.collections.List by lazy { "names" of pyObject } + } - /** - * ``` - * ast.NotEq = class NotEq(cmpop) - * | NotEq - * ``` - */ - class ASTNotEq(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.ImportFrom = class ImportFrom(stmt) + * | ImportFrom(identifier? module, alias* names, int? level) + * ``` + */ + class ImportFrom(pyObject: PyObject) : BaseStmt(pyObject) { + val module: String? by lazy { "module" of pyObject } + val names: kotlin.collections.List by lazy { "names" of pyObject } + val level: Long? by lazy { "level" of pyObject } + } - /** - * ``` - * ast.Lt = class Lt(cmpop) - * | Lt - * ``` - */ - class ASTLt(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.Global = class Global(stmt) + * | Global(identifier* names) + * ``` + */ + class Global(pyObject: PyObject) : BaseStmt(pyObject) { + val names: kotlin.collections.List by lazy { "names" of pyObject } + } - /** - * ``` - * ast.LtE = class LtE(cmpop) - * | LtE - * ``` - */ - class ASTLtE(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.Nonlocal = class Nonlocal(stmt) + * | Nonlocal(identifier* names) + * ``` + */ + class Nonlocal(pyObject: PyObject) : BaseStmt(pyObject) { + val names: kotlin.collections.List by lazy { "names" of pyObject } + } - /** - * ``` - * ast.Gt = class Gt(cmpop) - * | Gt - * ``` - */ - class ASTGt(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * Represents `ast.Expr` expressions. Note: do not confuse with + * - [BaseExpr] -> the expression class + * - [Expression] -> the expression as part of `mod` + * + * ``` + * ast.Expr = class Expr(stmt) + * | Expr(expr value) + * ``` + */ + class Expr(pyObject: PyObject) : BaseStmt(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.GtE = class GtE(cmpop) - * | GtE - * ``` - */ - class ASTGtE(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.Pass = class Pass(stmt) + * | Pass + * ``` + */ + class Pass(pyObject: PyObject) : BaseStmt(pyObject) + + /** + * ``` + * ast.Break = class Break(stmt) + * | Break + * ``` + */ + class Break(pyObject: PyObject) : BaseStmt(pyObject) + + /** + * ``` + * ast.Continue = class Continue(stmt) + * | Continue + * ``` + */ + class Continue(pyObject: PyObject) : BaseStmt(pyObject) + + /** + * Represents `ast.expr` expressions. Note: do not confuse with + * - [Expr] -> the expression statement + * - [Expression] -> the expression as part of `mod` + * + * ast.expr = class expr(AST) + */ + sealed class BaseExpr(pyObject: PyObject) : AST(pyObject), WithLocation + + /** + * ``` + * ast.BoolOp = class BoolOp(expr) + * | BoolOp(boolop op, expr* values) + * ``` + */ + class BoolOp(pyObject: PyObject) : BaseExpr(pyObject) { + val op: BaseBoolOp by lazy { "op" of pyObject } + val values: kotlin.collections.List by lazy { "values" of pyObject } + } - /** - * ``` - * ast.Is = class Is(cmpop) - * | Is - * ``` - */ - class ASTIs(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.NamedExpr = class NamedExpr(expr) + * | NamedExpr(expr target, expr value) + * ``` + */ + class NamedExpr(pyObject: PyObject) : BaseExpr(pyObject) { + val target: BaseExpr by lazy { "target" of pyObject } + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.IsNot = class IsNot(cmpop) - * | IsNot - * ``` - */ - class ASTIsNot(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.BinOp = class BinOp(expr) + * | BinOp(expr left, operator op, expr right) + * ``` + */ + class BinOp(pyObject: PyObject) : BaseExpr(pyObject) { + val left: BaseExpr by lazy { "left" of pyObject } + val op: BaseOperator by lazy { "op" of pyObject } + val right: BaseExpr by lazy { "right" of pyObject } + } - /** - * ``` - * ast.In = class In(cmpop) - * | In - * ``` - */ - class ASTIn(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.UnaryOp = class UnaryOp(expr) + * | UnaryOp(unaryop op, expr operand) + * ``` + */ + class UnaryOp(pyObject: PyObject) : BaseExpr(pyObject) { + val op: BaseUnaryOp by lazy { "op" of pyObject } + val operand: BaseExpr by lazy { "operand" of pyObject } + } - /** - * ``` - * ast.NotIn = class NotIn(cmpop) - * | NotIn - * ``` - */ - class ASTNotIn(pyObject: PyObject) : ASTBASEcmpop(pyObject) + /** + * ``` + * ast.Lambda = class Lambda(expr) + * | Lambda(arguments args, expr body) + * ``` + */ + class Lambda(pyObject: PyObject) : BaseExpr(pyObject) { + val args: arguments by lazy { "args" of pyObject } + val body: BaseExpr by lazy { "body" of pyObject } + } - /** - * ``` - * ast.expr_context = class expr_context(AST) - * | expr_context = Load | Store | Del - * ``` - */ - sealed class ASTBASEexpr_context(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.IfExp = class IfExp(expr) + * | IfExp(expr test, expr body, expr orelse) + * ``` + */ + class IfExp(pyObject: PyObject) : BaseExpr(pyObject) { + val test: BaseExpr by lazy { "test" of pyObject } + val body: BaseExpr by lazy { "body" of pyObject } + val orelse: BaseExpr by lazy { "orelse" of pyObject } + } - /** - * ``` - * ast.Load = class Load(expr_context) - * | Load - * ``` - */ - class ASTLoad(pyObject: PyObject) : ASTBASEexpr_context(pyObject) + /** + * ``` + * ast.Dict = class Dict(expr) + * | Dict(expr* keys, expr* values) + * ``` + */ + class Dict(pyObject: PyObject) : BaseExpr(pyObject) { + val keys: kotlin.collections.List by lazy { "keys" of pyObject } + val values: kotlin.collections.List by lazy { "values" of pyObject } + } - /** - * ``` - * ast.Store = class Store(expr_context) - * | Store - * ``` - */ - class ASTStore(pyObject: PyObject) : ASTBASEexpr_context(pyObject) + /** + * ``` + * ast.Set = class Set(expr) + * | Set(expr* elts) + * ``` + */ + class Set(pyObject: PyObject) : BaseExpr(pyObject) { + val elts: kotlin.collections.List by lazy { "elts" of pyObject } + } - /** - * ``` - * ast.Del = class Del(expr_context) - * | Del - * ``` - */ - class ASTDel(pyObject: PyObject) : ASTBASEexpr_context(pyObject) + /** + * ``` + * ast.ListComp = class ListComp(expr) + * | ListComp(expr elt, comprehension* generators) + * ``` + */ + class ListComp(pyObject: PyObject) : BaseExpr(pyObject) { + val elt: BaseExpr by lazy { "elt" of pyObject } + val generators: kotlin.collections.List by lazy { + "generators" of pyObject + } + } - /** - * ``` - * ast.operator = class operator(AST) - * | operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv - * ``` - */ - sealed class ASTBASEoperator(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.SetComp = class SetComp(expr) + * | SetComp(expr elt, comprehension* generators) + * ``` + */ + class SetComp(pyObject: PyObject) : BaseExpr(pyObject) { + val elt: BaseExpr by lazy { "elt" of pyObject } + val generators: kotlin.collections.List by lazy { + "generators" of pyObject + } + } - /** - * ``` - * ast.Add = class Add(operator) - * | Add - * ``` - */ - class ASTAdd(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.DictComp = class DictComp(expr) + * | DictComp(expr key, expr value, comprehension* generators) + * ``` + */ + class DictComp(pyObject: PyObject) : BaseExpr(pyObject) { + val key: BaseExpr by lazy { "key" of pyObject } + val value: BaseExpr by lazy { "value" of pyObject } + val generators: kotlin.collections.List by lazy { + "generators" of pyObject + } + } - /** - * ``` - * ast.Sub = class Sub(operator) - * | Sub - * ``` - */ - class ASTSub(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.GeneratorExp = class GeneratorExp(expr) + * | GeneratorExp(expr elt, comprehension* generators) + * ``` + */ + class GeneratorExp(pyObject: PyObject) : BaseExpr(pyObject) { + val elt: BaseExpr by lazy { "elt" of pyObject } + val generators: kotlin.collections.List by lazy { + "generators" of pyObject + } + } - /** - * ``` - * ast.Mult = class Mult(operator) - * | Mult - * ``` - */ - class ASTMult(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Await = class Await(expr) + * | Await(expr value) + * ``` + */ + class Await(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.MatMult = class MatMult(operator) - * | MatMult - * ``` - */ - class ASTMatMult(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Yield = class Yield(expr) + * | Yield(expr? value) + * ``` + */ + class Yield(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr? by lazy { "value" of pyObject } + } - /** - * ``` - * ast.Div = class Div(operator) - * | Div - * ``` - */ - class ASTDiv(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.YieldFrom = class YieldFrom(expr) + * | YieldFrom(expr value) + * ``` + */ + class YieldFrom(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.Mod = class Mod(operator) - * | Mod - * ``` - */ - class ASTMod(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Compare = class Compare(expr) + * | Compare(expr left, cmpop* ops, expr* comparators) + * ``` + */ + class Compare(pyObject: PyObject) : BaseExpr(pyObject) { + val left: BaseExpr by lazy { "left" of pyObject } + val ops: kotlin.collections.List by lazy { "ops" of pyObject } + val comparators: kotlin.collections.List by lazy { "comparators" of pyObject } + } - /** - * ``` - * ast.Pow = class Pow(operator) - * | Pow - * ``` - */ - class ASTPow(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Call = class Call(expr) + * | Call(expr func, expr* args, keyword* keywords) + * ``` + */ + class Call(pyObject: PyObject) : BaseExpr(pyObject) { + val func: BaseExpr by lazy { "func" of pyObject } - /** - * ``` - * ast.LShift = class LShift(operator) - * | LShift - * ``` - */ - class ASTLShift(pyObject: PyObject) : ASTBASEoperator(pyObject) + val args: kotlin.collections.List by lazy { "args" of pyObject } - /** - * ``` - * ast.RShift = class RShift(operator) - * | RShift - * ``` - */ - class ASTRShift(pyObject: PyObject) : ASTBASEoperator(pyObject) + val keywords: kotlin.collections.List by lazy { "keywords" of pyObject } + } - /** - * ``` - * ast.BitOr = class BitOr(operator) - * | BitOr - * ``` - */ - class ASTBitOr(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.FormattedValue = class FormattedValue(expr) + * | FormattedValue(expr value, int conversion, expr? format_spec) + * ``` + */ + class FormattedValue(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + val conversion: Long? by lazy { "conversion" of pyObject } + val format_spec: BaseExpr? by lazy { "format_spec" of pyObject } + } - /** - * ``` - * ast.BitXor = class BitXor(operator) - * | BitXor - * ``` - */ - class ASTBitXor(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.JoinedStr = class JoinedStr(expr) + * | JoinedStr(expr* values) + * ``` + */ + class JoinedStr(pyObject: PyObject) : BaseExpr(pyObject) { + val values: kotlin.collections.List by lazy { "values" of pyObject } + } - /** - * ``` - * ast.BitAnd = class BitAnd(operator) - * | BitAnd - * ``` - */ - class ASTBitAnd(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Constant = class Constant(expr) + * | Constant(constant value, string? kind) + * ``` + */ + class Constant(pyObject: PyObject) : BaseExpr(pyObject) { + val value: Any by lazy { "value" of pyObject } + val kind: String? by lazy { "kind" of pyObject } + } - /** - * ``` - * ast.FloorDiv = class FloorDiv(operator) - * | FloorDiv - * ``` - */ - class ASTFloorDiv(pyObject: PyObject) : ASTBASEoperator(pyObject) + /** + * ``` + * ast.Attribute = class Attribute(expr) + * | Attribute(expr value, identifier attr, expr_context ctx) + * ``` + */ + class Attribute(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + val attr: String by lazy { "attr" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.pattern = class pattern(AST) - * | pattern = MatchValue(expr value) - * | | MatchSingleton(constant value) - * | | MatchSequence(pattern* patterns) - * | | MatchMapping(expr* keys, pattern* patterns, identifier? rest) - * | | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) - * | | MatchStar(identifier? name) - * | | MatchAs(pattern? pattern, identifier? name) - * | | MatchOr(pattern* patterns) - * ``` - */ - abstract class ASTBASEpattern(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.Subscript = class Subscript(expr) + * | Subscript(expr value, expr slice, expr_context ctx) + * ``` + */ + class Subscript(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + val slice: BaseExpr by lazy { "slice" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.MatchValue = class MatchValue(pattern) - * | MatchValue(expr value) - * ``` - */ - class ASTMatchValue(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + /** + * ``` + * ast.Starred = class Starred(expr) + * | Starred(expr value, expr_context ctx) + * ``` + */ + class Starred(pyObject: PyObject) : BaseExpr(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.MatchSingleton = class MatchSingleton(pattern) - * | MatchSingleton(constant value) - * ``` - */ - class ASTMatchSingleton(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val value: Any by lazy { "value" of pyObject } - } + /** + * ``` + * ast.Name = class Name(expr) + * | Name(identifier id, expr_context ctx) + * ``` + */ + class Name(pyObject: PyObject) : BaseExpr(pyObject) { + val id: String by lazy { "id" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.MatchSequence = class MatchSequence(pattern) - * | MatchSequence(pattern* patterns) - * ``` - */ - class ASTMatchSequence(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val patterns: List by lazy { "patterns" of pyObject } - } + /** + * ``` + * ast.List = class List(expr) + * | List(expr* elts, expr_context ctx) + * ``` + */ + class List(pyObject: PyObject) : BaseExpr(pyObject) { + val elts: kotlin.collections.List by lazy { "elts" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.MatchMapping = class MatchMapping(pattern) - * | MatchMapping(expr* keys, pattern* patterns, identifier? rest) - * ``` - */ - class ASTMatchMapping(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val key: List by lazy { "keys" of pyObject } - val patterns: List by lazy { "patterns" of pyObject } - val rest: String? by lazy { "rest" of pyObject } - } + /** + * ``` + * ast.Tuple = class Tuple(expr) + * | Tuple(expr* elts, expr_context ctx) + * ``` + */ + class Tuple(pyObject: PyObject) : BaseExpr(pyObject) { + val elts: kotlin.collections.List by lazy { "elts" of pyObject } + val ctx: BaseExprContext by lazy { "ctx" of pyObject } + } - /** - * ``` - * ast.MatchClass = class MatchClass(pattern) - * | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) - * ``` - */ - class ASTMatchClass(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val cls: ASTBASEexpr by lazy { "cls" of pyObject } - val patterns: List by lazy { "patterns" of pyObject } - val kwd_attrs: List by lazy { "kwd_attrs" of pyObject } - val kwd_patterns: List by lazy { "kwd_patterns" of pyObject } - } + /** + * ``` + * ast.Slice = class Slice(expr) + * | Slice(expr? lower, expr? upper, expr? step) + * ``` + */ + class Slice(pyObject: PyObject) : BaseExpr(pyObject) { + val lower: BaseExpr? by lazy { "lower" of pyObject } + val upper: BaseExpr? by lazy { "upper" of pyObject } + val step: BaseExpr? by lazy { "step" of pyObject } + } - /** - * ``` - * ast.MatchStar = class MatchStar(pattern) - * | MatchStar(identifier? name) - * ``` - */ - class ASTMatchStar(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val name: String? by lazy { "name" of pyObject } - } + /** + * ``` + * ast.boolop = class boolop(AST) + * | boolop = And | Or + * ``` + */ + sealed class BaseBoolOp(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.And = class And(boolop) + * | And + * ``` + */ + class And(pyObject: PyObject) : BaseBoolOp(pyObject) + + /** + * ``` + * ast.Or = class Or(boolop) + * | Or + */ + class Or(pyObject: PyObject) : BaseBoolOp(pyObject) + + /** + * ``` + * ast.cmpop = class cmpop(AST) + * | cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn + * ``` + */ + sealed class BaseCmpOp(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.Eq = class Eq(cmpop) + * | Eq + * ``` + */ + class Eq(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.NotEq = class NotEq(cmpop) + * | NotEq + * ``` + */ + class NotEq(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.Lt = class Lt(cmpop) + * | Lt + * ``` + */ + class Lt(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.LtE = class LtE(cmpop) + * | LtE + * ``` + */ + class LtE(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.Gt = class Gt(cmpop) + * | Gt + * ``` + */ + class Gt(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.GtE = class GtE(cmpop) + * | GtE + * ``` + */ + class GtE(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.Is = class Is(cmpop) + * | Is + * ``` + */ + class Is(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.IsNot = class IsNot(cmpop) + * | IsNot + * ``` + */ + class IsNot(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.In = class In(cmpop) + * | In + * ``` + */ + class In(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.NotIn = class NotIn(cmpop) + * | NotIn + * ``` + */ + class NotIn(pyObject: PyObject) : BaseCmpOp(pyObject) + + /** + * ``` + * ast.expr_context = class expr_context(AST) + * | expr_context = Load | Store | Del + * ``` + */ + sealed class BaseExprContext(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.Load = class Load(expr_context) + * | Load + * ``` + */ + class Load(pyObject: PyObject) : BaseExprContext(pyObject) + + /** + * ``` + * ast.Store = class Store(expr_context) + * | Store + * ``` + */ + class Store(pyObject: PyObject) : BaseExprContext(pyObject) + + /** + * ``` + * ast.Del = class Del(expr_context) + * | Del + * ``` + */ + class Del(pyObject: PyObject) : BaseExprContext(pyObject) + + /** + * ``` + * ast.operator = class operator(AST) + * | operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv + * ``` + */ + sealed class BaseOperator(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.Add = class Add(operator) + * | Add + * ``` + */ + class Add(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.Sub = class Sub(operator) + * | Sub + * ``` + */ + class Sub(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.Mult = class Mult(operator) + * | Mult + * ``` + */ + class Mult(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.MatMult = class MatMult(operator) + * | MatMult + * ``` + */ + class MatMult(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.Div = class Div(operator) + * | Div + * ``` + */ + class Div(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.Mod = class Mod(operator) + * | Mod + * ``` + */ + class Mod(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.Pow = class Pow(operator) + * | Pow + * ``` + */ + class Pow(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.LShift = class LShift(operator) + * | LShift + * ``` + */ + class LShift(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.RShift = class RShift(operator) + * | RShift + * ``` + */ + class RShift(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.BitOr = class BitOr(operator) + * | BitOr + * ``` + */ + class BitOr(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.BitXor = class BitXor(operator) + * | BitXor + * ``` + */ + class BitXor(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.BitAnd = class BitAnd(operator) + * | BitAnd + * ``` + */ + class BitAnd(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.FloorDiv = class FloorDiv(operator) + * | FloorDiv + * ``` + */ + class FloorDiv(pyObject: PyObject) : BaseOperator(pyObject) + + /** + * ``` + * ast.pattern = class pattern(AST) + * | pattern = MatchValue(expr value) + * | | MatchSingleton(constant value) + * | | MatchSequence(pattern* patterns) + * | | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + * | | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + * | | MatchStar(identifier? name) + * | | MatchAs(pattern? pattern, identifier? name) + * | | MatchOr(pattern* patterns) + * ``` + */ + abstract class BasePattern(pyObject: PyObject) : AST(pyObject), WithLocation + + /** + * ``` + * ast.MatchValue = class MatchValue(pattern) + * | MatchValue(expr value) + * ``` + */ + class MatchValue(pyObject: PyObject) : BasePattern(pyObject) { + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.MatchAs = class MatchAs(pattern) - * | MatchAs(pattern? pattern, identifier? name) - * ``` - */ - class ASTMatchAs(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val pattern: ASTBASEpattern? by lazy { "pattern" of pyObject } - val name: String? by lazy { "name" of pyObject } - } + /** + * ``` + * ast.MatchSingleton = class MatchSingleton(pattern) + * | MatchSingleton(constant value) + * ``` + */ + class MatchSingleton(pyObject: PyObject) : BasePattern(pyObject) { + val value: Any by lazy { "value" of pyObject } + } - /** - * ``` - * ast.MatchOr = class MatchOr(pattern) - * | MatchOr(pattern* patterns) - * ``` - */ - class ASTMatchOr(pyObject: PyObject) : ASTBASEpattern(pyObject) { - val patterns: List by lazy { "patterns" of pyObject } - } + /** + * ``` + * ast.MatchSequence = class MatchSequence(pattern) + * | MatchSequence(pattern* patterns) + * ``` + */ + class MatchSequence(pyObject: PyObject) : BasePattern(pyObject) { + val patterns: kotlin.collections.List by lazy { "patterns" of pyObject } + } - /** - * ``` - * ast.unaryop = class unaryop(AST) - * | unaryop = Invert | Not | UAdd | USub - * ``` - */ - sealed class ASTBASEunaryop(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.MatchMapping = class MatchMapping(pattern) + * | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + * ``` + */ + class MatchMapping(pyObject: PyObject) : BasePattern(pyObject) { + val key: kotlin.collections.List by lazy { "keys" of pyObject } + val patterns: kotlin.collections.List by lazy { "patterns" of pyObject } + val rest: String? by lazy { "rest" of pyObject } + } - /** - * ``` - * ast.Invert = class Invert(unaryop) - * | Invert - * ``` - */ - class ASTInvert(pyObject: PyObject) : ASTBASEunaryop(pyObject) + /** + * ``` + * ast.MatchClass = class MatchClass(pattern) + * | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + * ``` + */ + class MatchClass(pyObject: PyObject) : BasePattern(pyObject) { + val cls: BaseExpr by lazy { "cls" of pyObject } + val patterns: kotlin.collections.List by lazy { "patterns" of pyObject } + val kwd_attrs: kotlin.collections.List by lazy { "kwd_attrs" of pyObject } + val kwd_patterns: kotlin.collections.List by lazy { + "kwd_patterns" of pyObject + } + } - /** - * ``` - * ast.Not = class Not(unaryop) - * | Not - * ``` - */ - class ASTNot(pyObject: PyObject) : ASTBASEunaryop(pyObject) - /** - * ``` - * ast.UAdd = class UAdd(unaryop) - * | UAdd - * ``` - */ - class ASTUAdd(pyObject: PyObject) : ASTBASEunaryop(pyObject) + /** + * ``` + * ast.MatchStar = class MatchStar(pattern) + * | MatchStar(identifier? name) + * ``` + */ + class MatchStar(pyObject: PyObject) : BasePattern(pyObject) { + val name: String? by lazy { "name" of pyObject } + } - /** - * ``` - * ast.USub = class USub(unaryop) - * | USub - * ``` - */ - class ASTUSub(pyObject: PyObject) : ASTBASEunaryop(pyObject) + /** + * ``` + * ast.MatchAs = class MatchAs(pattern) + * | MatchAs(pattern? pattern, identifier? name) + * ``` + */ + class MatchAs(pyObject: PyObject) : BasePattern(pyObject) { + val pattern: BasePattern? by lazy { "pattern" of pyObject } + val name: String? by lazy { "name" of pyObject } + } - /** - * ``` - * ast.alias = class alias(AST) - * | alias(identifier name, identifier? asname) - * ``` - */ - class ASTalias(pyObject: PyObject) : AST(pyObject) { - val name: String by lazy { "name" of pyObject } - val asname: String? by lazy { "asname" of pyObject } - } + /** + * ``` + * ast.MatchOr = class MatchOr(pattern) + * | MatchOr(pattern* patterns) + * ``` + */ + class MatchOr(pyObject: PyObject) : BasePattern(pyObject) { + val patterns: kotlin.collections.List by lazy { "patterns" of pyObject } + } - /** - * ``` - * ast.arg = class arg(AST) - * | arg(identifier arg, expr? annotation, string? type_comment) - * ``` - */ - class ASTarg(pyObject: PyObject) : AST(pyObject), WithPythonLocation { - val arg: String by lazy { "arg" of pyObject } - val annotation: ASTBASEexpr? by lazy { "annotation" of pyObject } - val type_comment: String? by lazy { "type_comment" of pyObject } - } + /** + * ``` + * ast.unaryop = class unaryop(AST) + * | unaryop = Invert | Not | UAdd | USub + * ``` + */ + sealed class BaseUnaryOp(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.Invert = class Invert(unaryop) + * | Invert + * ``` + */ + class Invert(pyObject: PyObject) : BaseUnaryOp(pyObject) + + /** + * ``` + * ast.Not = class Not(unaryop) + * | Not + * ``` + */ + class Not(pyObject: PyObject) : BaseUnaryOp(pyObject) + + /** + * ``` + * ast.UAdd = class UAdd(unaryop) + * | UAdd + * ``` + */ + class UAdd(pyObject: PyObject) : BaseUnaryOp(pyObject) + + /** + * ``` + * ast.USub = class USub(unaryop) + * | USub + * ``` + */ + class USub(pyObject: PyObject) : BaseUnaryOp(pyObject) + + /** + * ``` + * ast.alias = class alias(AST) + * | alias(identifier name, identifier? asname) + * ``` + */ + class alias(pyObject: PyObject) : AST(pyObject), WithLocation { + val name: String by lazy { "name" of pyObject } + val asname: String? by lazy { "asname" of pyObject } + } - /** - * ``` - * ast.arguments = class arguments(AST) - * | arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults) - * ``` - */ - class ASTarguments(pyObject: PyObject) : AST(pyObject) { - val posonlyargs: List by lazy { "posonlyargs" of pyObject } - val args: List by lazy { "args" of pyObject } - val vararg: ASTarg? by lazy { "vararg" of pyObject } - val kwonlyargs: List by lazy { "kwonlyargs" of pyObject } - val kw_defaults: List by lazy { "kw_defaults" of pyObject } - val kwarg: ASTarg? by lazy { "kwarg" of pyObject } - val defaults: List by lazy { "defaults" of pyObject } - } + /** + * ``` + * ast.arg = class arg(AST) + * | arg(identifier arg, expr? annotation, string? type_comment) + * ``` + */ + class arg(pyObject: PyObject) : AST(pyObject), WithLocation { + val arg: String by lazy { "arg" of pyObject } + val annotation: BaseExpr? by lazy { "annotation" of pyObject } + val type_comment: String? by lazy { "type_comment" of pyObject } + } - /** - * ``` - * ast.comprehension = class comprehension(AST) - * | comprehension(expr target, expr iter, expr* ifs, int is_async) - * ``` - */ - class ASTcomprehension(pyObject: PyObject) : AST(pyObject) { - val target: ASTBASEexpr by lazy { "target" of pyObject } - val iter: ASTBASEexpr by lazy { "iter" of pyObject } - val ifs: List by lazy { "ifs" of pyObject } - val is_async: Long by lazy { "is_async" of pyObject } - } + /** + * ``` + * ast.arguments = class arguments(AST) + * | arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults) + * ``` + */ + class arguments(pyObject: PyObject) : AST(pyObject) { + val posonlyargs: kotlin.collections.List by lazy { "posonlyargs" of pyObject } + val args: kotlin.collections.List by lazy { "args" of pyObject } + val vararg: arg? by lazy { "vararg" of pyObject } + val kwonlyargs: kotlin.collections.List by lazy { "kwonlyargs" of pyObject } + val kw_defaults: kotlin.collections.List by lazy { "kw_defaults" of pyObject } + val kwarg: arg? by lazy { "kwarg" of pyObject } + val defaults: kotlin.collections.List by lazy { "defaults" of pyObject } + } - /** - * ``` - * ast.excepthandler = class excepthandler(AST) - * | excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) - * ``` - * - * TODO: excepthandler <-> ExceptHandler - */ - class ASTexcepthandler(pyObject: PyObject) : AST(pyObject) { - val type: ASTBASEexpr by lazy { "type" of pyObject } - val name: String by lazy { "name" of pyObject } - val body: List by lazy { "body" of pyObject } - } + /** + * ``` + * ast.comprehension = class comprehension(AST) + * | comprehension(expr target, expr iter, expr* ifs, int is_async) + * ``` + */ + class comprehension(pyObject: PyObject) : AST(pyObject) { + val target: BaseExpr by lazy { "target" of pyObject } + val iter: BaseExpr by lazy { "iter" of pyObject } + val ifs: kotlin.collections.List by lazy { "ifs" of pyObject } + val is_async: Long by lazy { "is_async" of pyObject } + } - /** - * ``` - * ast.keyword = class keyword(AST) - * | keyword(identifier? arg, expr value) - * ``` - */ - class ASTkeyword(pyObject: PyObject) : AST(pyObject) { - val arg: String? by lazy { "arg" of pyObject } - val value: ASTBASEexpr by lazy { "value" of pyObject } - } + /** + * ``` + * ast.excepthandler = class excepthandler(AST) + * | excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + * ``` + */ + sealed class BaseExcepthandler(python: PyObject) : AST(python), WithLocation + + /** + * ast.ExceptHandler = class ExceptHandler(excepthandler) | ExceptHandler(expr? type, + * identifier? name, stmt* body) + */ + class ExceptHandler(pyObject: PyObject) : BaseExcepthandler(pyObject) { + val type: BaseExpr? by lazy { "type" of pyObject } + val name: String? by lazy { "name" of pyObject } + val body: kotlin.collections.List by lazy { "body" of pyObject } + } - /** - * ``` - * ast.match_case = class match_case(AST) - * | match_case(pattern pattern, expr? guard, stmt* body) - * ``` - */ - class ASTmatch_case(pyObject: PyObject) : AST(pyObject) { - val pattern: ASTBASEpattern by lazy { "pattern" of pyObject } - val guard: ASTBASEexpr? by lazy { "guard" of pyObject } - val body: List by lazy { "body" of pyObject } - } + /** + * ``` + * ast.keyword = class keyword(AST) + * | keyword(identifier? arg, expr value) + * ``` + */ + class keyword(pyObject: PyObject) : AST(pyObject), WithLocation { + val arg: String? by lazy { "arg" of pyObject } + val value: BaseExpr by lazy { "value" of pyObject } + } - /** - * ``` - * ast.type_ignore = class type_ignore(AST) - * | type_ignore = TypeIgnore(int lineno, string tag) - * ``` - * - * TODO - */ - class ASTtype_ignore(pyObject: PyObject) : AST(pyObject) + /** + * ``` + * ast.match_case = class match_case(AST) + * | match_case(pattern pattern, expr? guard, stmt* body) + * ``` + */ + class match_case(pyObject: PyObject) : AST(pyObject) { + val pattern: BasePattern by lazy { "pattern" of pyObject } + val guard: BaseExpr? by lazy { "guard" of pyObject } + val body: kotlin.collections.List by lazy { "body" of pyObject } + } - /** - * ``` - * ast.withitem = class withitem(AST) - * | withitem(expr context_expr, expr? optional_vars) - * ``` - */ - class ASTwithitem(pyObject: PyObject) : AST(pyObject) { - val context_expr: ASTBASEexpr by lazy { "context_expr" of pyObject } - val optional_vars: ASTBASEexpr? by lazy { "optional_vars" of pyObject } + /** + * ``` + * ast.type_ignore = class type_ignore(AST) + * | type_ignore = TypeIgnore(int lineno, string tag) + * ``` + * + * TODO + */ + class type_ignore(pyObject: PyObject) : AST(pyObject) + + /** + * ``` + * ast.withitem = class withitem(AST) + * | withitem(expr context_expr, expr? optional_vars) + * ``` + */ + class withitem(pyObject: PyObject) : AST(pyObject) { + val context_expr: BaseExpr by lazy { "context_expr" of pyObject } + val optional_vars: BaseExpr? by lazy { "optional_vars" of pyObject } + } } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt index ff15ea7ebe..23a48a3327 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt @@ -29,7 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.graph.Node import java.util.function.Supplier -abstract class PythonHandler( +abstract class PythonHandler( configConstructor: Supplier, lang: PythonLanguageFrontend ) : Handler(configConstructor, lang) { @@ -49,4 +49,12 @@ abstract class PythonHandler( } abstract fun handleNode(node: HandlerNode): ResultNode + + companion object { + /** + * A prefix to add to random names when handling for loops with multiple variables and + * having to add implicit assignments for the unwrapping process. + */ + const val LOOP_VAR_PREFIX = "loopMultiVarHelperVar" + } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 2c6a16daa5..49fd56726f 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -25,17 +25,24 @@ */ package de.fraunhofer.aisec.cpg.frontends.python -import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators -import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.frontends.* +import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.autoType +import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration +import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator import de.fraunhofer.aisec.cpg.graph.types.* import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient /** The Python language. */ -class PythonLanguage : Language(), HasShortCircuitOperators { - override val fileExtensions = listOf("py") +class PythonLanguage : + Language(), + HasShortCircuitOperators, + HasOperatorOverloading, + HasFunctionStyleConstruction { + override val fileExtensions = listOf("py", "pyi") override val namespaceDelimiter = "." @Transient override val frontend: KClass = PythonLanguageFrontend::class @@ -49,6 +56,55 @@ class PythonLanguage : Language(), HasShortCircuitOperat override val compoundAssignmentOperators = setOf("+=", "-=", "*=", "**=", "/=", "//=", "%=", "<<=", ">>=", "&=", "|=", "^=", "@=") + // https://docs.python.org/3/reference/datamodel.html#special-method-names + @Transient + override val overloadedOperatorNames: + Map, String>, Symbol> = + mapOf( + UnaryOperator::class of + "[]" to + "__getitem__", // ... then x[i] is roughly equivalent to type(x).__getitem__(x, i) + BinaryOperator::class of "<" to "__lt__", + BinaryOperator::class of "<=" to "__le__", + BinaryOperator::class of "==" to "__eq__", + BinaryOperator::class of "!=" to "__ne__", + BinaryOperator::class of ">" to "__gt__", + BinaryOperator::class of ">=" to "__ge__", + BinaryOperator::class of "+" to "__add__", + BinaryOperator::class of "-" to "__sub__", + BinaryOperator::class of "*" to "__mul__", + BinaryOperator::class of "@" to "__matmul__", + BinaryOperator::class of "/" to "__truediv__", + BinaryOperator::class of "//" to "__floordiv__", + BinaryOperator::class of "%" to "__mod__", + BinaryOperator::class of "**" to "__pow__", + BinaryOperator::class of "<<" to "__lshift__", + BinaryOperator::class of ">>" to "__rshift__", + BinaryOperator::class of "&" to "__and__", + BinaryOperator::class of "^" to "__xor__", + BinaryOperator::class of "|" to "__or__", + BinaryOperator::class of "+=" to "__iadd__", + BinaryOperator::class of "-=" to "__isub__", + BinaryOperator::class of "*=" to "__imul__", + BinaryOperator::class of "@=" to "__imatmul__", + BinaryOperator::class of "/=" to "__itruediv__", + BinaryOperator::class of "//=" to "__ifloordiv__", + BinaryOperator::class of "%=" to "__imod__", + BinaryOperator::class of "**=" to "__ipow__", + BinaryOperator::class of "<<=" to "__ilshift__", + BinaryOperator::class of ">>=" to "__irshift__", + BinaryOperator::class of "&=" to "__iand__", + BinaryOperator::class of "^=" to "__ixor__", + BinaryOperator::class of "|=" to "__ior__", + UnaryOperator::class of "-" to "__neg__", + UnaryOperator::class of "+" to "__pos__", + UnaryOperator::class of "~" to "__invert__", + UnaryOperator::class of + "()" to + "__call__", // ... x(arg1, arg2, ...) roughly translates to type(x).__call__(x, + // arg1, ...) + ) + /** See [Documentation](https://docs.python.org/3/library/stdtypes.html#). */ @Transient override val builtInTypes = @@ -104,4 +160,20 @@ class PythonLanguage : Language(), HasShortCircuitOperat // The rest behaves like other languages return super.propagateTypeOfBinaryOperation(operation) } + + companion object { + /** + * This is a "modifier" to differentiate parameters in functions that are "positional" only. + * This information will be stored in [ParameterDeclaration.modifiers] so that we can use is + * later in call resolving. + */ + const val MODIFIER_POSITIONAL_ONLY_ARGUMENT = "posonlyarg" + + /** + * This is a "modifier" to differentiate parameters in functions that are "keyword" only. + * This information will be stored in [ParameterDeclaration.modifiers] so that we can use is + * later in call resolving. + */ + const val MODIFIER_KEYWORD_ONLY_ARGUMENT = "kwonlyarg" + } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index 22a9d80fe7..5e46113d1d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -47,7 +47,7 @@ import kotlin.math.min @RegisterExtraPass(PythonAddDeclarationsPass::class) class PythonLanguageFrontend(language: Language, ctx: TranslationContext) : - LanguageFrontend(language, ctx) { + LanguageFrontend(language, ctx) { private val lineSeparator = '\n' // TODO private val tokenTypeIndex = 0 private val jep = JepSingleton // configure Jep @@ -137,40 +137,38 @@ class PythonLanguageFrontend(language: Language, ctx: Tr * present, we parse it, otherwise we assume that it is dynamically typed and thus return an * [AutoType]. */ - override fun typeOf(type: Python.AST?): Type { + override fun typeOf(type: Python.AST.AST?): Type { return when (type) { null -> { // No type information -> we return an autoType to infer things magically autoType() } - is Python.ASTName -> { - // We have some kind of name here; let's quickly check, if this is a primitive type - val id = type.id - if (id in language.primitiveTypeNames) { - return primitiveType(id) - } - - // Otherwise, this could already be a fully qualified type - val name = - if (language.namespaceDelimiter in id) { - // TODO: This might create problem with nested classes - parseName(id) - } else { - // If it is not, we want place it in the current namespace - scopeManager.currentNamespace.fqn(id) - } - - objectType(name) + is Python.AST.Name -> { + this.typeOf(type.id) } else -> { // The AST supplied us with some kind of type information, but we could not parse - // it, so we - // need to return the unknown type. + // it, so we need to return the unknown type. unknownType() } } } + /** Resolves a [Type] based on its string identifier. */ + fun typeOf(typeId: String): Type { + // Check if the typeId contains a namespace delimiter for qualified types + val name = + if (language.namespaceDelimiter in typeId) { + // TODO: This might create problem with nested classes + parseName(typeId) + } else { + // Unqualified name, resolved by the type resolver + typeId + } + + return objectType(name) + } + /** * This functions extracts the source code from the input file given a location. This is a bit * tricky in Python, as indents are part of the syntax. We also don't want to include leading @@ -180,16 +178,21 @@ class PythonLanguageFrontend(language: Language, ctx: Tr * 2) Delete extra code at the end of the last line that is not part of the provided location * 3) Remove trailing whitespaces / tabs */ - override fun codeOf(astNode: Python.AST): String? { - val location = locationOf(astNode) - if (location != null) { - var lines = getRelevantLines(location) - lines = removeExtraAtEnd(location, lines) - lines = fixStartColumn(location, lines) - - return lines.joinToString(separator = lineSeparator.toString()) + override fun codeOf(astNode: Python.AST.AST): String? { + return if (astNode is Python.AST.Module) { + fileContent + } else { + val location = locationOf(astNode) + if (location != null) { + var lines = getRelevantLines(location) + lines = removeExtraAtEnd(location, lines) + lines = fixStartColumn(location, lines) + + lines.joinToString(separator = lineSeparator.toString()) + } else { + null + } } - return null } private fun getRelevantLines(location: PhysicalLocation): MutableList { @@ -228,8 +231,8 @@ class PythonLanguageFrontend(language: Language, ctx: Tr return lines } - override fun locationOf(astNode: Python.AST): PhysicalLocation? { - return if (astNode is Python.WithPythonLocation) { + override fun locationOf(astNode: Python.AST.AST): PhysicalLocation? { + return if (astNode is Python.AST.WithLocation) { PhysicalLocation( uri, Region( @@ -244,13 +247,13 @@ class PythonLanguageFrontend(language: Language, ctx: Tr } } - override fun setComment(node: Node, astNode: Python.AST) { + override fun setComment(node: Node, astNode: Python.AST.AST) { // will be invoked by native function } private fun pythonASTtoCPG(pyAST: PyObject, path: String): TranslationUnitDeclaration { val pythonASTModule = - fromPython(pyAST) as? Python.ASTModule + fromPython(pyAST) as? Python.AST.Module ?: TODO( "Python ast of type ${fromPython(pyAST).javaClass} is not supported yet" ) // could be one of "ast.{Module,Interactive,Expression,FunctionType} @@ -273,29 +276,29 @@ class PythonLanguageFrontend(language: Language, ctx: Tr return tud } - fun operatorToString(op: Python.ASTBASEoperator) = + fun operatorToString(op: Python.AST.BaseOperator) = when (op) { - is Python.ASTAdd -> "+" - is Python.ASTSub -> "-" - is Python.ASTMult -> "*" - is Python.ASTMatMult -> "*" - is Python.ASTDiv -> "/" - is Python.ASTMod -> "%" - is Python.ASTPow -> "**" - is Python.ASTLShift -> "<<" - is Python.ASTRShift -> ">>" - is Python.ASTBitOr -> "|" - is Python.ASTBitXor -> "^" - is Python.ASTBitAnd -> "&" - is Python.ASTFloorDiv -> "//" + is Python.AST.Add -> "+" + is Python.AST.Sub -> "-" + is Python.AST.Mult -> "*" + is Python.AST.MatMult -> "*" + is Python.AST.Div -> "/" + is Python.AST.Mod -> "%" + is Python.AST.Pow -> "**" + is Python.AST.LShift -> "<<" + is Python.AST.RShift -> ">>" + is Python.AST.BitOr -> "|" + is Python.AST.BitXor -> "^" + is Python.AST.BitAnd -> "&" + is Python.AST.FloorDiv -> "//" } - fun operatorUnaryToString(op: Python.ASTBASEunaryop) = + fun operatorUnaryToString(op: Python.AST.BaseUnaryOp) = when (op) { - is Python.ASTInvert -> "~" - is Python.ASTNot -> "not" - is Python.ASTUAdd -> "+" - is Python.ASTUSub -> "-" + is Python.AST.Invert -> "~" + is Python.AST.Not -> "not" + is Python.AST.UAdd -> "+" + is Python.AST.USub -> "-" } } @@ -305,7 +308,7 @@ class PythonLanguageFrontend(language: Language, ctx: Tr * @param pyObject the Python object * @return our Kotlin view of the Python `ast` object */ -fun fromPython(pyObject: Any?): Python.AST { +fun fromPython(pyObject: Any?): Python.BaseObject { if (pyObject !is PyObject) { TODO("Expected a PyObject") } else { @@ -313,131 +316,134 @@ fun fromPython(pyObject: Any?): Python.AST { pyObject.getAttr("__class__").toString().substringAfter("'").substringBeforeLast("'") objectname = if (objectname.startsWith("_")) objectname.substringAfter("_") else objectname return when (objectname) { - "ast.Module" -> Python.ASTModule(pyObject) - - // statements - "ast.FunctionDef" -> Python.ASTFunctionDef(pyObject) - "ast.AsyncFunctionDef" -> Python.ASTAsyncFunctionDef(pyObject) - "ast.ClassDef" -> Python.ASTClassDef(pyObject) - "ast.Return" -> Python.ASTReturn(pyObject) - "ast.Delete" -> Python.ASTDelete(pyObject) - "ast.Assign" -> Python.ASTAssign(pyObject) - "ast.AugAssign" -> Python.ASTAugAssign(pyObject) - "ast.AnnAssign" -> Python.ASTAnnAssign(pyObject) - "ast.For" -> Python.ASTFor(pyObject) - "ast.AsyncFor" -> Python.ASTAsyncFor(pyObject) - "ast.While" -> Python.ASTWhile(pyObject) - "ast.If" -> Python.ASTIf(pyObject) - "ast.With" -> Python.ASTWith(pyObject) - "ast.AsyncWith" -> Python.ASTAsyncWith(pyObject) - "ast.Match" -> Python.ASTMatch(pyObject) - "ast.Raise" -> Python.ASTRaise(pyObject) - "ast.Try" -> Python.ASTTry(pyObject) - "ast.TryStar" -> Python.ASTTryStar(pyObject) - "ast.Assert" -> Python.ASTAssert(pyObject) - "ast.Import" -> Python.ASTImport(pyObject) - "ast.ImportFrom" -> Python.ASTImportFrom(pyObject) - "ast.Global" -> Python.ASTGlobal(pyObject) - "ast.Nonlocal" -> Python.ASTNonlocal(pyObject) - "ast.Expr" -> Python.ASTExpr(pyObject) - "ast.Pass" -> Python.ASTPass(pyObject) - "ast.Break" -> Python.ASTBreak(pyObject) - "ast.Continue" -> Python.ASTContinue(pyObject) - - // `"ast.expr` - "ast.BoolOp" -> Python.ASTBoolOp(pyObject) - "ast.NamedExpr" -> Python.ASTNamedExpr(pyObject) - "ast.BinOp" -> Python.ASTBinOp(pyObject) - "ast.UnaryOp" -> Python.ASTUnaryOp(pyObject) - "ast.Lambda" -> Python.ASTLambda(pyObject) - "ast.IfExp" -> Python.ASTIfExp(pyObject) - "ast.Dict" -> Python.ASTDict(pyObject) - "ast.Set" -> Python.ASTSet(pyObject) - "ast.ListComp" -> Python.ASTListComp(pyObject) - "ast.SetComp" -> Python.ASTSetComp(pyObject) - "ast.DictComp" -> Python.ASTDictComp(pyObject) - "ast.GeneratorExp" -> Python.ASTGeneratorExp(pyObject) - "ast.Await" -> Python.ASTAwait(pyObject) - "ast.Yield" -> Python.ASTYield(pyObject) - "ast.YieldFrom" -> Python.ASTYieldFrom(pyObject) - "ast.Compare" -> Python.ASTCompare(pyObject) - "ast.Call" -> Python.ASTCall(pyObject) - "ast.FormattedValue" -> Python.ASTFormattedValue(pyObject) - "ast.JoinedStr" -> Python.ASTJoinedStr(pyObject) - "ast.Constant" -> Python.ASTConstant(pyObject) - "ast.Attribute" -> Python.ASTAttribute(pyObject) - "ast.Subscript" -> Python.ASTSubscript(pyObject) - "ast.Starred" -> Python.ASTStarred(pyObject) - "ast.Name" -> Python.ASTName(pyObject) - "ast.List" -> Python.ASTList(pyObject) - "ast.Tuple" -> Python.ASTTuple(pyObject) - "ast.Slice" -> Python.ASTSlice(pyObject) - - // `"ast.boolop` - "ast.And" -> Python.ASTAnd(pyObject) - "ast.Or" -> Python.ASTOr(pyObject) - - // `"ast.cmpop` - "ast.Eq" -> Python.ASTEq(pyObject) - "ast.NotEq" -> Python.ASTNotEq(pyObject) - "ast.Lt" -> Python.ASTLt(pyObject) - "ast.LtE" -> Python.ASTLtE(pyObject) - "ast.Gt" -> Python.ASTGt(pyObject) - "ast.GtE" -> Python.ASTGtE(pyObject) - "ast.Is" -> Python.ASTIs(pyObject) - "ast.IsNot" -> Python.ASTIsNot(pyObject) - "ast.In" -> Python.ASTIn(pyObject) - "ast.NotIn" -> Python.ASTNotIn(pyObject) - - // `"ast.expr_context` - "ast.Load" -> Python.ASTLoad(pyObject) - "ast.Store" -> Python.ASTStore(pyObject) - "ast.Del" -> Python.ASTDel(pyObject) - - // `"ast.operator` - "ast.Add" -> Python.ASTAdd(pyObject) - "ast.Sub" -> Python.ASTSub(pyObject) - "ast.Mult" -> Python.ASTMult(pyObject) - "ast.MatMult" -> Python.ASTMatMult(pyObject) - "ast.Div" -> Python.ASTDiv(pyObject) - "ast.Mod" -> Python.ASTMod(pyObject) - "ast.Pow" -> Python.ASTPow(pyObject) - "ast.LShift" -> Python.ASTLShift(pyObject) - "ast.RShift" -> Python.ASTRShift(pyObject) - "ast.BitOr" -> Python.ASTBitOr(pyObject) - "ast.BitXor" -> Python.ASTBitXor(pyObject) - "ast.BitAnd" -> Python.ASTBitAnd(pyObject) - "ast.FloorDiv" -> Python.ASTFloorDiv(pyObject) - - // `"ast.pattern` - "ast.MatchValue" -> Python.ASTMatchValue(pyObject) - "ast.MatchSingleton" -> Python.ASTMatchSingleton(pyObject) - "ast.MatchSequence" -> Python.ASTMatchSequence(pyObject) - "ast.MatchMapping" -> Python.ASTMatchMapping(pyObject) - "ast.MatchClass" -> Python.ASTMatchClass(pyObject) - "ast.MatchStar" -> Python.ASTMatchStar(pyObject) - "ast.MatchAs" -> Python.ASTMatchAs(pyObject) - "ast.MatchOr" -> Python.ASTMatchOr(pyObject) - - // `"ast.unaryop` - "ast.Invert" -> Python.ASTInvert(pyObject) - "ast.Not" -> Python.ASTNot(pyObject) - "ast.UAdd" -> Python.ASTUAdd(pyObject) - "ast.USub" -> Python.ASTUSub(pyObject) + "ast.Module" -> Python.AST.Module(pyObject) + + // `ast.stmt` + "ast.FunctionDef" -> Python.AST.FunctionDef(pyObject) + "ast.AsyncFunctionDef" -> Python.AST.AsyncFunctionDef(pyObject) + "ast.ClassDef" -> Python.AST.ClassDef(pyObject) + "ast.Return" -> Python.AST.Return(pyObject) + "ast.Delete" -> Python.AST.Delete(pyObject) + "ast.Assign" -> Python.AST.Assign(pyObject) + "ast.AugAssign" -> Python.AST.AugAssign(pyObject) + "ast.AnnAssign" -> Python.AST.AnnAssign(pyObject) + "ast.For" -> Python.AST.For(pyObject) + "ast.AsyncFor" -> Python.AST.AsyncFor(pyObject) + "ast.While" -> Python.AST.While(pyObject) + "ast.If" -> Python.AST.If(pyObject) + "ast.With" -> Python.AST.With(pyObject) + "ast.AsyncWith" -> Python.AST.AsyncWith(pyObject) + "ast.Match" -> Python.AST.Match(pyObject) + "ast.Raise" -> Python.AST.Raise(pyObject) + "ast.Try" -> Python.AST.Try(pyObject) + "ast.TryStar" -> Python.AST.TryStar(pyObject) + "ast.Assert" -> Python.AST.Assert(pyObject) + "ast.Import" -> Python.AST.Import(pyObject) + "ast.ImportFrom" -> Python.AST.ImportFrom(pyObject) + "ast.Global" -> Python.AST.Global(pyObject) + "ast.Nonlocal" -> Python.AST.Nonlocal(pyObject) + "ast.Expr" -> Python.AST.Expr(pyObject) + "ast.Pass" -> Python.AST.Pass(pyObject) + "ast.Break" -> Python.AST.Break(pyObject) + "ast.Continue" -> Python.AST.Continue(pyObject) + + // `ast.expr` + "ast.BoolOp" -> Python.AST.BoolOp(pyObject) + "ast.NamedExpr" -> Python.AST.NamedExpr(pyObject) + "ast.BinOp" -> Python.AST.BinOp(pyObject) + "ast.UnaryOp" -> Python.AST.UnaryOp(pyObject) + "ast.Lambda" -> Python.AST.Lambda(pyObject) + "ast.IfExp" -> Python.AST.IfExp(pyObject) + "ast.Dict" -> Python.AST.Dict(pyObject) + "ast.Set" -> Python.AST.Set(pyObject) + "ast.ListComp" -> Python.AST.ListComp(pyObject) + "ast.SetComp" -> Python.AST.SetComp(pyObject) + "ast.DictComp" -> Python.AST.DictComp(pyObject) + "ast.GeneratorExp" -> Python.AST.GeneratorExp(pyObject) + "ast.Await" -> Python.AST.Await(pyObject) + "ast.Yield" -> Python.AST.Yield(pyObject) + "ast.YieldFrom" -> Python.AST.YieldFrom(pyObject) + "ast.Compare" -> Python.AST.Compare(pyObject) + "ast.Call" -> Python.AST.Call(pyObject) + "ast.FormattedValue" -> Python.AST.FormattedValue(pyObject) + "ast.JoinedStr" -> Python.AST.JoinedStr(pyObject) + "ast.Constant" -> Python.AST.Constant(pyObject) + "ast.Attribute" -> Python.AST.Attribute(pyObject) + "ast.Subscript" -> Python.AST.Subscript(pyObject) + "ast.Starred" -> Python.AST.Starred(pyObject) + "ast.Name" -> Python.AST.Name(pyObject) + "ast.List" -> Python.AST.List(pyObject) + "ast.Tuple" -> Python.AST.Tuple(pyObject) + "ast.Slice" -> Python.AST.Slice(pyObject) + + // `ast.boolop` + "ast.And" -> Python.AST.And(pyObject) + "ast.Or" -> Python.AST.Or(pyObject) + + // `ast.cmpop` + "ast.Eq" -> Python.AST.Eq(pyObject) + "ast.NotEq" -> Python.AST.NotEq(pyObject) + "ast.Lt" -> Python.AST.Lt(pyObject) + "ast.LtE" -> Python.AST.LtE(pyObject) + "ast.Gt" -> Python.AST.Gt(pyObject) + "ast.GtE" -> Python.AST.GtE(pyObject) + "ast.Is" -> Python.AST.Is(pyObject) + "ast.IsNot" -> Python.AST.IsNot(pyObject) + "ast.In" -> Python.AST.In(pyObject) + "ast.NotIn" -> Python.AST.NotIn(pyObject) + + // `ast.expr_context` + "ast.Load" -> Python.AST.Load(pyObject) + "ast.Store" -> Python.AST.Store(pyObject) + "ast.Del" -> Python.AST.Del(pyObject) + + // `ast.operator` + "ast.Add" -> Python.AST.Add(pyObject) + "ast.Sub" -> Python.AST.Sub(pyObject) + "ast.Mult" -> Python.AST.Mult(pyObject) + "ast.MatMult" -> Python.AST.MatMult(pyObject) + "ast.Div" -> Python.AST.Div(pyObject) + "ast.Mod" -> Python.AST.Mod(pyObject) + "ast.Pow" -> Python.AST.Pow(pyObject) + "ast.LShift" -> Python.AST.LShift(pyObject) + "ast.RShift" -> Python.AST.RShift(pyObject) + "ast.BitOr" -> Python.AST.BitOr(pyObject) + "ast.BitXor" -> Python.AST.BitXor(pyObject) + "ast.BitAnd" -> Python.AST.BitAnd(pyObject) + "ast.FloorDiv" -> Python.AST.FloorDiv(pyObject) + + // `ast.pattern` + "ast.MatchValue" -> Python.AST.MatchValue(pyObject) + "ast.MatchSingleton" -> Python.AST.MatchSingleton(pyObject) + "ast.MatchSequence" -> Python.AST.MatchSequence(pyObject) + "ast.MatchMapping" -> Python.AST.MatchMapping(pyObject) + "ast.MatchClass" -> Python.AST.MatchClass(pyObject) + "ast.MatchStar" -> Python.AST.MatchStar(pyObject) + "ast.MatchAs" -> Python.AST.MatchAs(pyObject) + "ast.MatchOr" -> Python.AST.MatchOr(pyObject) + + // `ast.unaryop` + "ast.Invert" -> Python.AST.Invert(pyObject) + "ast.Not" -> Python.AST.Not(pyObject) + "ast.UAdd" -> Python.AST.UAdd(pyObject) + "ast.USub" -> Python.AST.USub(pyObject) + + // `ast.excepthandler` + "ast.ExceptHandler" -> Python.AST.ExceptHandler(pyObject) // misc - "ast.alias" -> Python.ASTalias(pyObject) - "ast.arg" -> Python.ASTarg(pyObject) - "ast.arguments" -> Python.ASTarguments(pyObject) - "ast.comprehension" -> Python.ASTcomprehension(pyObject) - "ast.excepthandler" -> Python.ASTexcepthandler(pyObject) - "ast.keyword" -> Python.ASTkeyword(pyObject) - "ast.match_case" -> Python.ASTmatch_case(pyObject) - "ast.type_ignore" -> Python.ASTtype_ignore(pyObject) - "ast.withitem" -> Python.ASTwithitem(pyObject) + "ast.alias" -> Python.AST.alias(pyObject) + "ast.arg" -> Python.AST.arg(pyObject) + "ast.arguments" -> Python.AST.arguments(pyObject) + "ast.comprehension" -> Python.AST.comprehension(pyObject) + "ast.keyword" -> Python.AST.keyword(pyObject) + "ast.match_case" -> Python.AST.match_case(pyObject) + "ast.type_ignore" -> Python.AST.type_ignore(pyObject) + "ast.withitem" -> Python.AST.withitem(pyObject) // complex numbers - "complex" -> TODO("Complex numbers are not supported yet") + "complex" -> Python.Complex(pyObject) + "ellipsis" -> Python.Ellipsis(pyObject) else -> { TODO("Implement for ${pyObject.getAttr("__class__")}") } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 6959642678..244f5fc869 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -25,48 +25,64 @@ */ package de.fraunhofer.aisec.cpg.frontends.python +import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading +import de.fraunhofer.aisec.cpg.frontends.isKnownOperatorName +import de.fraunhofer.aisec.cpg.frontends.python.Python.AST.IsAsync +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_KEYWORD_ONLY_ARGUMENT +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_POSITIONAL_ONLY_ARGUMENT import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.statements.AssertStatement +import de.fraunhofer.aisec.cpg.graph.statements.CatchClause import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement +import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.statements.TryStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeleteExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.helpers.Util +import kotlin.collections.plusAssign class StatementHandler(frontend: PythonLanguageFrontend) : - PythonHandler(::ProblemExpression, frontend) { - override fun handleNode(node: Python.ASTBASEstmt): Statement { + PythonHandler(::ProblemExpression, frontend) { + + override fun handleNode(node: Python.AST.BaseStmt): Statement { return when (node) { - is Python.ASTClassDef -> handleClassDef(node) - is Python.ASTFunctionDef -> handleFunctionDef(node) - is Python.ASTAsyncFunctionDef -> handleAsyncFunctionDef(node) - is Python.ASTPass -> return newEmptyStatement(rawNode = node) - is Python.ASTImportFrom -> handleImportFrom(node) - is Python.ASTAssign -> handleAssign(node) - is Python.ASTAugAssign -> handleAugAssign(node) - is Python.ASTReturn -> handleReturn(node) - is Python.ASTIf -> handleIf(node) - is Python.ASTAnnAssign -> handleAnnAssign(node) - is Python.ASTExpr -> handleExpressionStatement(node) - is Python.ASTFor -> handleFor(node) - is Python.ASTAsyncFor -> handleAsyncFor(node) - is Python.ASTWhile -> handleWhile(node) - is Python.ASTImport -> handleImport(node) - is Python.ASTBreak -> newBreakStatement(rawNode = node) - is Python.ASTContinue -> newContinueStatement(rawNode = node) - is Python.ASTAssert, - is Python.ASTDelete, - is Python.ASTGlobal, - is Python.ASTMatch, - is Python.ASTNonlocal, - is Python.ASTRaise, - is Python.ASTTry, - is Python.ASTTryStar, - is Python.ASTWith, - is Python.ASTAsyncWith -> + is Python.AST.ClassDef -> handleClassDef(node) + is Python.AST.FunctionDef -> handleFunctionDef(node) + is Python.AST.AsyncFunctionDef -> handleFunctionDef(node) + is Python.AST.Pass -> return newEmptyStatement(rawNode = node) + is Python.AST.ImportFrom -> handleImportFrom(node) + is Python.AST.Assign -> handleAssign(node) + is Python.AST.AugAssign -> handleAugAssign(node) + is Python.AST.Return -> handleReturn(node) + is Python.AST.If -> handleIf(node) + is Python.AST.AnnAssign -> handleAnnAssign(node) + is Python.AST.Expr -> handleExpressionStatement(node) + is Python.AST.For -> handleFor(node) + is Python.AST.AsyncFor -> handleFor(node) + is Python.AST.While -> handleWhile(node) + is Python.AST.Import -> handleImport(node) + is Python.AST.Break -> newBreakStatement(rawNode = node) + is Python.AST.Continue -> newContinueStatement(rawNode = node) + is Python.AST.Assert -> handleAssert(node) + is Python.AST.Try -> handleTryStatement(node) + is Python.AST.Delete -> handleDelete(node) + is Python.AST.Global, + is Python.AST.Match, + is Python.AST.Nonlocal, + is Python.AST.Raise, + is Python.AST.TryStar, + is Python.AST.With, + is Python.AST.AsyncWith -> newProblemExpression( "The statement of class ${node.javaClass} is not supported yet", rawNode = node @@ -74,7 +90,92 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } } - private fun handleImport(node: Python.ASTImport): Statement { + /** + * Translates an [`excepthandler`] which can only be a + * [`ExceptHandler`](https://docs.python.org/3/library/ast.html#ast.ExceptHandler) to a + * [CatchClause]. + * + * It adds all the statements to the body and will set a parameter if it exists. For the + * catch-all clause, we do not set the [CatchClause.parameter]. + */ + private fun handleBaseExcepthandler(node: Python.AST.BaseExcepthandler): CatchClause { + return when (node) { + is Python.AST.ExceptHandler -> { + val catchClause = newCatchClause(rawNode = node) + catchClause.body = makeBlock(node.body, node) + // The parameter can have a type but if the type is None/null, it's the "catch-all" + // clause. + // In this case, it also cannot have a name, so we can skip the variable + // declaration. + if (node.type != null) { + // the parameter can have a name, or we use the anonymous identifier _ + catchClause.parameter = + newVariableDeclaration( + name = node.name ?: "", + type = frontend.typeOf(node.type), + rawNode = node + ) + } + catchClause + } + } + } + + /** + * Translates a Python [`Try`](https://docs.python.org/3/library/ast.html#ast.Try) into a + * [TryStatement]. + */ + private fun handleTryStatement(node: Python.AST.Try): TryStatement { + val tryStatement = newTryStatement(rawNode = node) + tryStatement.tryBlock = makeBlock(node.body, node) + tryStatement.catchClauses.addAll(node.handlers.map { handleBaseExcepthandler(it) }) + + if (node.orelse.isNotEmpty()) { + tryStatement.elseBlock = makeBlock(node.orelse, node) + } + + if (node.finalbody.isNotEmpty()) { + tryStatement.finallyBlock = makeBlock(node.finalbody, node) + } + + return tryStatement + } + + /** + * Translates a Python [`Delete`](https://docs.python.org/3/library/ast.html#ast.Delete) into a + * [DeleteExpression]. + */ + private fun handleDelete(node: Python.AST.Delete): DeleteExpression { + val delete = newDeleteExpression(rawNode = node) + node.targets.forEach { target -> + if (target is Python.AST.Subscript) { + delete.operands.add(frontend.expressionHandler.handle(target)) + } else { + delete.additionalProblems += + newProblemExpression( + problem = + "handleDelete: 'Name' and 'Attribute' deletions are not supported, as they removes them from the scope.", + rawNode = target + ) + } + } + return delete + } + + /** + * Translates a Python [`Assert`](https://docs.python.org/3/library/ast.html#ast.Assert) into a + * [AssertStatement]. + */ + private fun handleAssert(node: Python.AST.Assert): AssertStatement { + val assertStatement = newAssertStatement(rawNode = node) + val testExpression = frontend.expressionHandler.handle(node.test) + assertStatement.condition = testExpression + node.msg?.let { assertStatement.message = frontend.expressionHandler.handle(it) } + + return assertStatement + } + + private fun handleImport(node: Python.AST.Import): Statement { val declStmt = newDeclarationStatement(rawNode = node) for (imp in node.names) { val alias = imp.asname @@ -90,12 +191,12 @@ class StatementHandler(frontend: PythonLanguageFrontend) : newImportDeclaration(parseName(imp.name), false, rawNode = imp) } frontend.scopeManager.addDeclaration(decl) - declStmt.addToPropertyEdgeDeclaration(decl) + declStmt.declarationEdges += decl } return declStmt } - private fun handleImportFrom(node: Python.ASTImportFrom): Statement { + private fun handleImportFrom(node: Python.AST.ImportFrom): Statement { val declStmt = newDeclarationStatement(rawNode = node) val level = node.level if (level == null || level > 0) { @@ -110,7 +211,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // Wildcards luckily do not have aliases val decl = if (imp.name == "*") { - // In the wildcard case, our "import" is the module name and we set "wildcard" + // In the wildcard case, our "import" is the module name, and we set "wildcard" // to true newImportDeclaration(module, true, rawNode = imp) } else { @@ -127,81 +228,169 @@ class StatementHandler(frontend: PythonLanguageFrontend) : // Finally, add our declaration to the scope and the declaration statement frontend.scopeManager.addDeclaration(decl) - declStmt.addToPropertyEdgeDeclaration(decl) + declStmt.declarationEdges += decl } return declStmt } - private fun handleWhile(node: Python.ASTWhile): Statement { + private fun handleWhile(node: Python.AST.While): Statement { val ret = newWhileStatement(rawNode = node) ret.condition = frontend.expressionHandler.handle(node.test) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } + ret.statement = makeBlock(node.body, parentNode = node) + if (node.orelse.isNotEmpty()) { + ret.additionalProblems += + newProblemExpression( + problem = "Cannot handle \"orelse\" in while loops.", + rawNode = node + ) + } return ret } - private fun handleFor(node: Python.ASTFor): Statement { + /** + * Translates a Python [`For`](https://docs.python.org/3/library/ast.html#ast.For) into an + * [ForEachStatement]. + * + * Python supports implicit unpacking of multiple loop variables. To map this to CPG node, we + * translate the following implicit unpacking of code like this: + * ```python + * for a, b, c in someNestedList: + * pass + * ``` + * + * to have only one loop variable and add the unpacking statement to the top of the loop body + * like this: + * ```python + * for tempVar in someNestedList: + * a, b, c = tempVar + * pass + * ``` + */ + private fun handleFor(node: Python.AST.NormalOrAsyncFor): ForEachStatement { val ret = newForEachStatement(rawNode = node) + if (node is IsAsync) { + ret.addDeclaration( + newProblemDeclaration( + problem = "The \"async\" keyword is not yet supported.", + rawNode = node + ) + ) + } + ret.iterable = frontend.expressionHandler.handle(node.iter) - ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } + + when (val loopVar = frontend.expressionHandler.handle(node.target)) { + is InitializerListExpression -> { // unpacking + val (tempVarRef, unpackingAssignment) = getUnpackingNodes(loopVar) + + ret.variable = tempVarRef + + val body = makeBlock(node.body, parentNode = node) + body.statements.add( + 0, + unpackingAssignment + ) // add the unpacking instruction to the top of the loop body + ret.statement = body + } + is Reference -> { // only one var + ret.variable = loopVar + ret.statement = makeBlock(node.body, parentNode = node) + } + else -> { + ret.variable = + newProblemExpression( + problem = + "handleFor: cannot handle loop variable of type ${loopVar::class.simpleName}.", + rawNode = node.target + ) + ret.statement = makeBlock(node.body, parentNode = node) + } + } + + if (node.orelse.isNotEmpty()) { + ret.additionalProblems += + newProblemExpression( + problem = "handleFor: Cannot handle \"orelse\" in for loops.", + rawNode = node + ) + } return ret } - private fun handleAsyncFor(node: Python.ASTAsyncFor): Statement { - val ret = newForEachStatement(rawNode = node) - ret.iterable = frontend.expressionHandler.handle(node.iter) - ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body).codeAndLocationFromChildren(node) - node.orelse.firstOrNull()?.let { TODO("Not supported") } - return ret + /** + * This function creates two things: + * - A [Reference] to a variable with a random [Name] + * - An [AssignExpression] assigning the reference above to the [loopVar] input + * + * This is used in [handleFor] when loops have multiple loop variables to iterate over with + * automatic unpacking. We translate this implicit unpacking to multiple CPG nodes, as the CPG + * does not support automatic unpacking. + */ + private fun getUnpackingNodes( + loopVar: InitializerListExpression + ): Pair { + val tempVarName = Name.random(prefix = LOOP_VAR_PREFIX) + val tempRef = newReference(name = tempVarName).implicit().codeAndLocationFrom(loopVar) + val assign = + newAssignExpression( + operatorCode = "=", + lhs = (loopVar).initializers, + rhs = listOf(tempRef) + ) + .implicit() + .codeAndLocationFrom(loopVar) + return Pair(tempRef, assign) } - private fun handleExpressionStatement(node: Python.ASTExpr): Statement { + private fun handleExpressionStatement(node: Python.AST.Expr): Statement { return frontend.expressionHandler.handle(node.value) } - private fun handleAnnAssign(node: Python.ASTAnnAssign): Statement { - // TODO: annotations + /** + * Translates a Python [`AnnAssign`](https://docs.python.org/3/library/ast.html#ast.AnnAssign) + * into an [AssignExpression]. + */ + private fun handleAnnAssign(node: Python.AST.AnnAssign): AssignExpression { val lhs = frontend.expressionHandler.handle(node.target) - return if (node.value != null) { - newAssignExpression( - lhs = listOf(lhs), - rhs = listOf(frontend.expressionHandler.handle(node.value!!)), // TODO !! - rawNode = node - ) - } else { - lhs - } + lhs.type = frontend.typeOf(node.annotation) + val rhs = node.value?.let { listOf(frontend.expressionHandler.handle(it)) } ?: emptyList() + return newAssignExpression(lhs = listOf(lhs), rhs = rhs, rawNode = node) } - private fun handleIf(node: Python.ASTIf): Statement { + private fun handleIf(node: Python.AST.If): Statement { val ret = newIfStatement(rawNode = node) ret.condition = frontend.expressionHandler.handle(node.test) ret.thenStatement = if (node.body.isNotEmpty()) { - makeBlock(node.body).codeAndLocationFromChildren(node) + makeBlock(node.body, parentNode = node) } else { null } ret.elseStatement = if (node.orelse.isNotEmpty()) { - makeBlock(node.orelse).codeAndLocationFromChildren(node) + makeBlock(node.orelse, parentNode = node) } else { null } return ret } - private fun handleReturn(node: Python.ASTReturn): Statement { + private fun handleReturn(node: Python.AST.Return): Statement { val ret = newReturnStatement(rawNode = node) node.value?.let { ret.returnValue = frontend.expressionHandler.handle(it) } return ret } - private fun handleAssign(node: Python.ASTAssign): Statement { + /** + * Translates a Python [`Assign`](https://docs.python.org/3/library/ast.html#ast.Assign) into an + * [AssignExpression]. + */ + private fun handleAssign(node: Python.AST.Assign): AssignExpression { val lhs = node.targets.map { frontend.expressionHandler.handle(it) } + node.type_comment?.let { typeComment -> + val tpe = frontend.typeOf(typeComment) + lhs.forEach { it.type = tpe } + } val rhs = frontend.expressionHandler.handle(node.value) if (rhs is List<*>) newAssignExpression( @@ -219,7 +408,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return newAssignExpression(lhs = lhs, rhs = listOf(rhs), rawNode = node) } - private fun handleAugAssign(node: Python.ASTAugAssign): Statement { + private fun handleAugAssign(node: Python.AST.AugAssign): Statement { val lhs = frontend.expressionHandler.handle(node.target) val rhs = frontend.expressionHandler.handle(node.value) val op = frontend.operatorToString(node.op) + "=" @@ -231,18 +420,21 @@ class StatementHandler(frontend: PythonLanguageFrontend) : ) } - private fun handleClassDef(stmt: Python.ASTClassDef): Statement { + private fun handleClassDef(stmt: Python.AST.ClassDef): Statement { val cls = newRecordDeclaration(stmt.name, "class", rawNode = stmt) stmt.bases.map { cls.superClasses.add(frontend.typeOf(it)) } frontend.scopeManager.enterScope(cls) - stmt.keywords.map { TODO() } + stmt.keywords.forEach { + cls.additionalProblems += + newProblemExpression(problem = "could not parse keyword $it in class", rawNode = it) + } for (s in stmt.body) { when (s) { - is Python.ASTFunctionDef -> handleFunctionDef(s, cls) - else -> cls.addStatement(handleNode(s)) + is Python.AST.FunctionDef -> handleFunctionDef(s, cls) + else -> cls.statements += handleNode(s) } } @@ -254,9 +446,9 @@ class StatementHandler(frontend: PythonLanguageFrontend) : /** * We have to consider multiple things when matching Python's FunctionDef to the CPG: - * - A [Python.ASTFunctionDef] is a [Statement] from Python's point of view. The CPG sees it as + * - A [Python.AST.FunctionDef] is a [Statement] from Python's point of view. The CPG sees it as * a declaration -> we have to wrap the result in a [DeclarationStatement]. - * - A [Python.ASTFunctionDef] could be one of + * - A [Python.AST.FunctionDef] could be one of * - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__` * - a [MethodeDeclaration] if it appears in a record, and it isn't a * [ConstructorDeclaration] @@ -266,9 +458,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * `receiver` (most often called `self`). */ private fun handleFunctionDef( - s: Python.ASTFunctionDef, + s: Python.AST.NormalOrAsyncFunctionDef, recordDeclaration: RecordDeclaration? = null ): DeclarationStatement { + val language = language val result = if (recordDeclaration != null) { if (s.name == "__init__") { @@ -277,6 +470,23 @@ class StatementHandler(frontend: PythonLanguageFrontend) : recordDeclaration = recordDeclaration, rawNode = s ) + } else if (language is HasOperatorOverloading && s.name.isKnownOperatorName) { + var decl = + newOperatorDeclaration( + name = s.name, + recordDeclaration = recordDeclaration, + operatorCode = language.operatorCodeFor(s.name) ?: "", + rawNode = s + ) + if (decl.operatorCode == "") { + Util.warnWithFileLocation( + decl, + log, + "Could not find operator code for operator {}. This will most likely result in a failure", + s.name + ) + } + decl } else { newMethodDeclaration( name = s.name, @@ -290,70 +500,17 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } frontend.scopeManager.enterScope(result) - // Handle decorators (which are translated into CPG "annotations") - result.addAnnotations(handleAnnotations(s)) - - // Handle return type and calculate function type - if (result is ConstructorDeclaration) { - // Return type of the constructor is always its record declaration type - result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) - } else { - result.returnTypes = listOf(frontend.typeOf(s.returns)) - } - result.type = FunctionType.computeType(result) - - handleArguments(s.args, result, recordDeclaration) - - if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body).codeAndLocationFromChildren(s) + if (s is Python.AST.AsyncFunctionDef) { + result.addDeclaration( + newProblemDeclaration( + problem = "The \"async\" keyword is not yet supported.", + rawNode = s + ) + ) } - frontend.scopeManager.leaveScope(result) - frontend.scopeManager.addDeclaration(result) - - return wrapDeclarationToStatement(result) - } - - /** - * We have to consider multiple things when matching Python's FunctionDef to the CPG: - * - A [Python.ASTFunctionDef] is a [Statement] from Python's point of view. The CPG sees it as - * a declaration -> we have to wrap the result in a [DeclarationStatement]. - * - A [Python.ASTFunctionDef] could be one of - * - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__` - * - a [MethodeDeclaration] if it appears in a record, and it isn't a - * [ConstructorDeclaration] - * - a [FunctionDeclaration] if neither of the above apply - * - * In case of a [ConstructorDeclaration] or[MethodDeclaration]: the first argument is the - * `receiver` (most often called `self`). - */ - private fun handleAsyncFunctionDef( - s: Python.ASTAsyncFunctionDef, - recordDeclaration: RecordDeclaration? = null - ): DeclarationStatement { - val result = - if (recordDeclaration != null) { - if (s.name == "__init__") { - newConstructorDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - rawNode = s - ) - } else { - newMethodDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - isStatic = false, - rawNode = s - ) - } - } else { - newFunctionDeclaration(name = s.name, rawNode = s) - } - frontend.scopeManager.enterScope(result) - // Handle decorators (which are translated into CPG "annotations") - result.addAnnotations(handleAnnotations(s)) + result.annotations += handleAnnotations(s) // Handle return type and calculate function type if (result is ConstructorDeclaration) { @@ -367,7 +524,11 @@ class StatementHandler(frontend: PythonLanguageFrontend) : handleArguments(s.args, result, recordDeclaration) if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body).codeAndLocationFromChildren(s) + // Make sure we open a new (block) scope for the function body. This is not a 1:1 + // mapping to python scopes, since python only has a "function scope", but in the CPG + // the function scope only comprises the function arguments, and we need a block scope + // to hold all local variables within the function body. + result.body = makeBlock(s.body, parentNode = s, enterScope = true) } frontend.scopeManager.leaveScope(result) @@ -378,167 +539,203 @@ class StatementHandler(frontend: PythonLanguageFrontend) : /** Adds the arguments to [result] which might be located in a [recordDeclaration]. */ private fun handleArguments( - args: Python.ASTarguments, + args: Python.AST.arguments, result: FunctionDeclaration, recordDeclaration: RecordDeclaration? ) { - // Handle arguments - if (args.posonlyargs.isNotEmpty()) { - val problem = - newProblemDeclaration( - "`posonlyargs` are not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = args - ) - frontend.scopeManager.addDeclaration(problem) - } + // We can merge posonlyargs and args because both are positional arguments. We do not + // enforce that posonlyargs can ONLY be used in a positional style, whereas args can be used + // both in positional and keyword style. + var positionalArguments = args.posonlyargs + args.args + // Handle receiver if it exists if (recordDeclaration != null) { - // first argument is the `receiver` - if (args.args.isEmpty()) { - val problem = - newProblemDeclaration( - "Expected a receiver", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = args - ) - frontend.scopeManager.addDeclaration(problem) - } else { - val recvPythonNode = args.args.first() - val tpe = recordDeclaration.toType() - val recvNode = - newVariableDeclaration( - name = recvPythonNode.arg, - type = tpe, - implicitInitializerAllowed = false, - rawNode = recvPythonNode - ) - frontend.scopeManager.addDeclaration(recvNode) - when (result) { - is ConstructorDeclaration -> result.receiver = recvNode - is MethodDeclaration -> result.receiver = recvNode - else -> TODO() - } - } + handleReceiverArgument(positionalArguments, args, result, recordDeclaration) + // Skip the receiver argument for further processing + positionalArguments = positionalArguments.drop(1) } - if (recordDeclaration != null) { - // first argument is the receiver - for (arg in args.args.subList(1, args.args.size)) { - handleArgument(arg) - } - } else { - for (arg in args.args) { - handleArgument(arg) - } - } + // Handle remaining arguments + handlePositionalArguments(positionalArguments, args) - args.vararg?.let { - val problem = - newProblemDeclaration( - "`vararg` is not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = it - ) - frontend.scopeManager.addDeclaration(problem) - } + args.vararg?.let { handleArgument(it, isPosOnly = false, isVariadic = true) } + args.kwarg?.let { handleArgument(it, isPosOnly = false, isVariadic = false) } - if (args.kwonlyargs.isNotEmpty()) { - val problem = - newProblemDeclaration( - "`kwonlyargs` are not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = args - ) - frontend.scopeManager.addDeclaration(problem) - } + handleKeywordOnlyArguments(args) + } - if (args.kw_defaults.isNotEmpty()) { - val problem = - newProblemDeclaration( - "`kw_defaults` are not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = args + /** + * This method retrieves the first argument of the [positionalArguments], which is typically the + * receiver object. + * + * A receiver can also have a default value. However, this case is not handled and is therefore + * passed with a problem expression. + */ + private fun handleReceiverArgument( + positionalArguments: List, + args: Python.AST.arguments, + result: FunctionDeclaration, + recordDeclaration: RecordDeclaration + ) { + // first argument is the receiver + val recvPythonNode = positionalArguments.firstOrNull() + if (recvPythonNode == null) { + result.additionalProblems += newProblemExpression("Expected a receiver", rawNode = args) + } else { + val tpe = recordDeclaration.toType() + val recvNode = + newVariableDeclaration( + name = recvPythonNode.arg, + type = tpe, + implicitInitializerAllowed = false, + rawNode = recvPythonNode ) - frontend.scopeManager.addDeclaration(problem) - } - args.kwarg?.let { - val problem = - newProblemDeclaration( - "`kwarg` is not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = it - ) - frontend.scopeManager.addDeclaration(problem) + // If the number of defaults equals the number of positional arguments, the receiver has + // a default value + if (args.defaults.size == positionalArguments.size) { + val defaultValue = + args.defaults.getOrNull(0)?.let { frontend.expressionHandler.handle(it) } + defaultValue?.let { + frontend.scopeManager.addDeclaration(recvNode) + result.additionalProblems += + newProblemExpression("Receiver with default value", rawNode = args) + } + } + + when (result) { + is ConstructorDeclaration, + is MethodDeclaration -> result.receiver = recvNode + else -> + result.additionalProblems += + newProblemExpression( + problem = + "Expected a constructor or method declaration. Got something else.", + rawNode = result + ) + } } + } - if (args.defaults.isNotEmpty()) { - val problem = - newProblemDeclaration( - "`defaults` are not yet supported", - problemType = ProblemNode.ProblemType.TRANSLATION, - rawNode = args - ) - frontend.scopeManager.addDeclaration(problem) + /** + * This method extracts the [positionalArguments] including those that may have default values. + * + * In Python only the arguments with default values are stored in `args.defaults`. + * https://docs.python.org/3/library/ast.html#ast.arguments + * + * For example: `def my_func(a, b=1, c=2): pass` + * + * In this case, `args.defaults` contains only the defaults for `b` and `c`, while `args.args` + * includes all arguments (`a`, `b` and `c`). The number of arguments without defaults is + * determined by subtracting the size of `args.defaults` from the total number of arguments. + * This matches each default to its corresponding argument. + * + * From the Python docs: "If there are fewer defaults, they correspond to the last n arguments." + */ + private fun handlePositionalArguments( + positionalArguments: List, + args: Python.AST.arguments + ) { + val nonDefaultArgsCount = positionalArguments.size - args.defaults.size + + for (idx in positionalArguments.indices) { + val arg = positionalArguments[idx] + val defaultIndex = idx - nonDefaultArgsCount + val defaultValue = + if (defaultIndex >= 0) { + args.defaults.getOrNull(defaultIndex)?.let { + frontend.expressionHandler.handle(it) + } + } else { + null + } + handleArgument(arg, isPosOnly = arg in args.posonlyargs, defaultValue = defaultValue) } } - private fun handleAnnotations(node: Python.ASTAsyncFunctionDef): Collection { - return handleDeclaratorList(node, node.decorator_list) + /** + * This method extracts the keyword-only arguments from [args] and maps them to the + * corresponding function parameters. + */ + private fun handleKeywordOnlyArguments(args: Python.AST.arguments) { + for (idx in args.kwonlyargs.indices) { + val arg = args.kwonlyargs[idx] + val default = args.kw_defaults.getOrNull(idx) + handleArgument( + arg, + isPosOnly = false, + isKwoOnly = true, + defaultValue = default?.let { frontend.expressionHandler.handle(it) } + ) + } } - private fun handleAnnotations(node: Python.ASTFunctionDef): Collection { + private fun handleAnnotations( + node: Python.AST.NormalOrAsyncFunctionDef + ): Collection { return handleDeclaratorList(node, node.decorator_list) } fun handleDeclaratorList( - node: Python.AST, - decoratorList: List + node: Python.AST.WithLocation, + decoratorList: List ): List { val annotations = mutableListOf() for (decorator in decoratorList) { - if (decorator !is Python.ASTCall) { - log.warn( - "Decorator (${decorator::class}) is not ASTCall, cannot handle this (yet)." - ) - continue - } - - val decFuncParsed = frontend.expressionHandler.handle(decorator.func) - if (decFuncParsed !is MemberExpression) { - log.warn( - "parsed function expression (${decFuncParsed::class}) is not a member expression, cannot handle this (yet)." - ) - continue - } - - val annotation = - newAnnotation( - name = - Name( - localName = decFuncParsed.name.localName, - parent = decFuncParsed.base.name - ), - rawNode = node - ) - for (arg in decorator.args) { - val argParsed = frontend.expressionHandler.handle(arg) - annotation.members += - newAnnotationMember( - "annotationArg" + decorator.args.indexOf(arg), // TODO - argParsed, - rawNode = arg - ) - } - for (keyword in decorator.keywords) { - annotation.members += - newAnnotationMember( - name = keyword.arg, - value = frontend.expressionHandler.handle(keyword.value), - rawNode = keyword - ) - } + var annotation = + when (decorator) { + is Python.AST.Name -> { + val parsedDecorator = frontend.expressionHandler.handle(decorator) + newAnnotation(name = parsedDecorator.name, rawNode = decorator) + } + is Python.AST.Attribute -> { + val parsedDecorator = frontend.expressionHandler.handle(decorator) + val name = + if (parsedDecorator is MemberExpression) { + parsedDecorator.base.name.fqn(parsedDecorator.name.localName) + } else { + parsedDecorator.name + } + newAnnotation(name = name, rawNode = decorator) + } + is Python.AST.Call -> { + val parsedDecorator = frontend.expressionHandler.handle(decorator.func) + val name = + if (parsedDecorator is MemberExpression) { + parsedDecorator.base.name.fqn(parsedDecorator.name.localName) + } else { + parsedDecorator.name + } + val annotation = newAnnotation(name = name, rawNode = decorator) + for (arg in decorator.args) { + val argParsed = frontend.expressionHandler.handle(arg) + annotation.members += + newAnnotationMember( + "annotationArg" + decorator.args.indexOf(arg), // TODO + argParsed, + rawNode = arg + ) + } + for (keyword in decorator.keywords) { + annotation.members += + newAnnotationMember( + name = keyword.arg, + value = frontend.expressionHandler.handle(keyword.value), + rawNode = keyword + ) + } + annotation + } + else -> { + Util.warnWithFileLocation( + frontend, + decorator, + log, + "Decorator is of type ${decorator::class}, cannot handle this (yet)." + ) + continue + } + } annotations += annotation } @@ -546,17 +743,74 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return annotations } - private fun makeBlock(stmts: List, rawNode: Python.AST? = null): Block { - val result = newBlock(rawNode = rawNode) + /** + * This function "wraps" a list of [Python.AST.BaseStmt] nodes into a [Block]. Since the list + * itself does not have a code/location, we need to employ [codeAndLocationFromChildren] on the + * [parentNode]. + * + * Optionally, a new scope will be opened when [enterScope] is specified. This should be done + * VERY carefully, as Python has a very limited set of scopes and is most likely only to be used + * by [handleFunctionDef]. + */ + private fun makeBlock( + stmts: List, + parentNode: Python.AST.WithLocation, + enterScope: Boolean = false, + ): Block { + val result = newBlock() + if (enterScope) { + frontend.scopeManager.enterScope(result) + } + for (stmt in stmts) { - result.addStatement(handle(stmt)) + result.statements += handle(stmt) } + + if (enterScope) { + frontend.scopeManager.leaveScope(result) + } + + // Try to retrieve the code and location from the parent node, if it is a base stmt + val ast = parentNode as? Python.AST.AST + if (ast != null) { + // We need to scope the call to codeAndLocationFromChildren to our frontend, so that + // all Python.AST.AST nodes are accepted, otherwise it would be scoped to the handler + // and only Python.AST.BaseStmt nodes would be accepted. This would cause issues with + // other nodes that are not "statements", but also handled as part of this handler, + // e.g., the Python.AST.ExceptHandler. + with(frontend) { result.codeAndLocationFromChildren(ast) } + } + return result } - internal fun handleArgument(node: Python.ASTarg) { + /** + * This function creates a [newParameterDeclaration] for the argument, setting any modifiers + * (like positional-only or keyword-only) and [defaultValue] if applicable. + */ + internal fun handleArgument( + node: Python.AST.arg, + isPosOnly: Boolean = false, + isVariadic: Boolean = false, + isKwoOnly: Boolean = false, + defaultValue: Expression? = null + ) { val type = frontend.typeOf(node.annotation) - val arg = newParameterDeclaration(name = node.arg, type = type, rawNode = node) + val arg = + newParameterDeclaration( + name = node.arg, + type = type, + variadic = isVariadic, + rawNode = node + ) + defaultValue?.let { arg.default = it } + if (isPosOnly) { + arg.modifiers += MODIFIER_POSITIONAL_ONLY_ARGUMENT + } + + if (isKwoOnly) { + arg.modifiers += MODIFIER_KEYWORD_ONLY_ARGUMENT + } frontend.scopeManager.addDeclaration(arg) } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 28a6dec744..100791c74a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -159,14 +159,12 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { } } - // TODO document why this is necessary and implement for other possible places + // New variables can also be declared as `variable` in a [ForEachStatement] private fun handleForEach(node: ForEachStatement) { - when (node.variable) { + when (val forVar = node.variable) { is Reference -> { - val handled = handleReference(node.variable as Reference) - if (handled is Declaration) { - handled.let { node.addDeclaration(it) } - } + val handled = handleReference(forVar) + (handled as? Declaration)?.let { scopeManager.addDeclaration(it) } } } } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt new file mode 100644 index 0000000000..51da0d504d --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python + +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.test.analyze +import de.fraunhofer.aisec.cpg.test.assertLiteralValue +import de.fraunhofer.aisec.cpg.test.assertLocalName +import java.nio.file.Path +import kotlin.test.* + +class ExpressionHandlerTest { + @Test + fun testBoolOps() { + val topLevel = Path.of("src", "test", "resources", "python") + val result = + analyze(listOf(topLevel.resolve("boolop.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + + val twoBoolOpCondition = result.functions["twoBoolOp"]?.ifs?.singleOrNull()?.condition + assertIs(twoBoolOpCondition) + assertEquals("and", twoBoolOpCondition.operatorCode) + assertLocalName("a", twoBoolOpCondition.lhs) + assertLiteralValue(true, twoBoolOpCondition.rhs) + + // We expect that lhs comes first in the EOG and then the rhs. + assertContains(twoBoolOpCondition.lhs.nextEOG, twoBoolOpCondition.rhs) + + val threeBoolOpCondition = result.functions["threeBoolOp"]?.ifs?.singleOrNull()?.condition + assertIs(threeBoolOpCondition) + assertEquals("and", threeBoolOpCondition.operatorCode) + assertLocalName("a", threeBoolOpCondition.lhs) + val threeBoolOpConditionRhs = threeBoolOpCondition.rhs + assertIs(threeBoolOpConditionRhs) + assertEquals("and", threeBoolOpConditionRhs.operatorCode) + assertLiteralValue(true, threeBoolOpConditionRhs.lhs) + assertLocalName("b", threeBoolOpConditionRhs.rhs) + + val threeBoolOpNoBoolCondition = + result.functions["threeBoolOpNoBool"]?.ifs?.singleOrNull()?.condition + assertIs(threeBoolOpNoBoolCondition) + assertEquals("and", threeBoolOpNoBoolCondition.operatorCode) + assertLocalName("a", threeBoolOpNoBoolCondition.lhs) + val threeBoolOpNoBoolConditionRhs = threeBoolOpNoBoolCondition.rhs + assertIs(threeBoolOpNoBoolConditionRhs) + assertEquals("and", threeBoolOpNoBoolConditionRhs.operatorCode) + assertLiteralValue(true, threeBoolOpNoBoolConditionRhs.lhs) + assertLiteralValue("foo", threeBoolOpNoBoolConditionRhs.rhs) + + // We expect that lhs comes first in the EOG and then the lhs of the rhs and last the rhs of + // the rhs. + assertContains(threeBoolOpNoBoolCondition.lhs.nextEOG, threeBoolOpNoBoolConditionRhs.lhs) + assertContains(threeBoolOpNoBoolConditionRhs.lhs.nextEOG, threeBoolOpNoBoolConditionRhs.rhs) + + val nestedBoolOpDifferentOp = + result.functions["nestedBoolOpDifferentOp"]?.ifs?.singleOrNull()?.condition + + assertIs(nestedBoolOpDifferentOp) + assertEquals("or", nestedBoolOpDifferentOp.operatorCode) + assertLocalName("b", nestedBoolOpDifferentOp.rhs) + val nestedBoolOpDifferentOpLhs = nestedBoolOpDifferentOp.lhs + assertIs(nestedBoolOpDifferentOpLhs) + assertEquals("and", nestedBoolOpDifferentOpLhs.operatorCode) + assertLiteralValue(true, nestedBoolOpDifferentOpLhs.rhs) + assertLocalName("a", nestedBoolOpDifferentOpLhs.lhs) + + // We expect that lhs of the "and" comes first in the EOG and then the rhs of the "and", + // then we evaluate the whole "and" and last the rhs of the "or". + assertContains(nestedBoolOpDifferentOpLhs.lhs.nextEOG, nestedBoolOpDifferentOpLhs.rhs) + assertContains(nestedBoolOpDifferentOpLhs.rhs.nextEOG, nestedBoolOpDifferentOpLhs) + assertContains(nestedBoolOpDifferentOpLhs.nextEOG, nestedBoolOpDifferentOp.rhs) + + val nestedBoolOpDifferentOp2 = + result.functions["nestedBoolOpDifferentOp2"]?.ifs?.singleOrNull()?.condition + assertIs(nestedBoolOpDifferentOp2) + assertEquals("or", nestedBoolOpDifferentOp2.operatorCode) + assertLocalName("a", nestedBoolOpDifferentOp2.lhs) + val nestedBoolOpDifferentOp2Rhs = nestedBoolOpDifferentOp2.rhs + assertIs(nestedBoolOpDifferentOp2Rhs) + assertEquals("and", nestedBoolOpDifferentOp2Rhs.operatorCode) + assertLiteralValue(true, nestedBoolOpDifferentOp2Rhs.lhs) + assertLocalName("b", nestedBoolOpDifferentOp2Rhs.rhs) + + // We expect that lhs comes first in the EOG and then the lhs of the rhs and last the rhs of + // the rhs. + assertContains(nestedBoolOpDifferentOp2.lhs.nextEOG, nestedBoolOpDifferentOp2Rhs.lhs) + assertContains(nestedBoolOpDifferentOp2Rhs.lhs.nextEOG, nestedBoolOpDifferentOp2Rhs.rhs) + } +} diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 7a82e97a2a..17f8b0855d 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -29,7 +29,6 @@ import de.fraunhofer.aisec.cpg.analysis.ValueEvaluator import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.ObjectType @@ -58,19 +57,25 @@ class PythonFrontendTest : BaseTest() { assertNotNull(b) assertLocalName("b", b) assertEquals(assertResolvedType("bool"), b.type) - assertEquals(true, (b.firstAssignment as? Literal<*>)?.value) + val bFirstAssignment = b.firstAssignment + assertIs>(bFirstAssignment) + assertEquals(true, bFirstAssignment.value) val i = p.variables["i"] assertNotNull(i) assertLocalName("i", i) assertEquals(assertResolvedType("int"), i.type) - assertEquals(42L, (i.firstAssignment as? Literal<*>)?.value) + val iFirstAssignment = i.firstAssignment + assertIs>(iFirstAssignment) + assertEquals(42L, iFirstAssignment.value) val f = p.variables["f"] assertNotNull(f) assertLocalName("f", f) assertEquals(assertResolvedType("float"), f.type) - assertEquals(1.0, (f.firstAssignment as? Literal<*>)?.value) + val fFirstAssignment = f.firstAssignment + assertIs>(fFirstAssignment) + assertEquals(1.0, fFirstAssignment.value) val c = p.variables["c"] assertNotNull(c) @@ -83,13 +88,17 @@ class PythonFrontendTest : BaseTest() { assertNotNull(t) assertLocalName("t", t) assertEquals(assertResolvedType("str"), t.type) - assertEquals("Hello", (t.firstAssignment as? Literal<*>)?.value) + val tAssignment = t.firstAssignment + assertIs>(tAssignment) + assertEquals("Hello", tAssignment.value) val n = p.variables["n"] assertNotNull(n) assertLocalName("n", n) assertEquals(assertResolvedType("None"), n.type) - assertEquals(null, (n.firstAssignment as? Literal<*>)?.value) + val nAssignment = n.firstAssignment + assertIs>(nAssignment) + assertEquals(null, nAssignment.value) } } @@ -105,76 +114,82 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["function"] assertNotNull(p) - val foo = p.declarations.first() as? FunctionDeclaration - assertNotNull(foo) + val foo = p.declarations.firstOrNull() + assertIs(foo) - val bar = p.declarations[1] as? FunctionDeclaration - assertNotNull(bar) + val bar = p.declarations[1] + assertIs(bar) assertEquals(2, bar.parameters.size) - var callExpression = (foo.body as? Block)?.statements?.get(0) as? CallExpression - assertNotNull(callExpression) + val fooBody = foo.body + assertIs(fooBody) + var callExpression = fooBody.statements[0] + assertIs(callExpression) assertLocalName("bar", callExpression) - assertEquals(bar, callExpression.invokes.first()) + assertInvokes(callExpression, bar) val edge = callExpression.argumentEdges[1] assertNotNull(edge) - assertEquals("s2", edge.getProperty(Properties.NAME)) + assertEquals("s2", edge.name) - val s = bar.parameters.first() + val s = bar.parameters.firstOrNull() assertNotNull(s) assertLocalName("s", s) assertEquals(tu.primitiveType("str"), s.type) assertLocalName("bar", bar) - val compStmt = bar.body as? Block - assertNotNull(compStmt) + val compStmt = bar.body + assertIs(compStmt) assertNotNull(compStmt.statements) - callExpression = compStmt.statements[0] as? CallExpression - assertNotNull(callExpression) + callExpression = compStmt.statements[0] + assertIs(callExpression) assertFullName("print", callExpression) - val literal = callExpression.arguments.first() as? Literal<*> - assertNotNull(literal) + val literal = callExpression.arguments.firstOrNull() + assertIs>(literal) assertEquals("bar(s) here: ", literal.value) assertEquals(tu.primitiveType("str"), literal.type) - val ref = callExpression.arguments[1] as? Reference - assertNotNull(ref) + val ref = callExpression.arguments[1] + assertIs(ref) assertLocalName("s", ref) - assertEquals(s, ref.refersTo) + assertRefersTo(ref, s) - val stmt = compStmt.statements[1] as? AssignExpression - assertNotNull(stmt) + val stmt = compStmt.statements[1] + assertIs(stmt) - val a = stmt.declarations.first() as? VariableDeclaration + val a = stmt.declarations.firstOrNull() assertNotNull(a) assertLocalName("a", a) - val op = a.firstAssignment as? BinaryOperator - assertNotNull(op) + val op = a.firstAssignment + assertIs(op) assertEquals("+", op.operatorCode) - val lhs = op.lhs as? Literal<*> - assertNotNull(lhs) + val lhs = op.lhs + assertIs>(lhs) - assertEquals(1, (lhs.value as? Long)?.toInt()) + val lhsValue = lhs.value + assertIs(lhsValue) + assertEquals(1, lhsValue.toInt()) - val rhs = op.rhs as? Literal<*> - assertNotNull(rhs) + val rhs = op.rhs + assertIs>(rhs) - assertEquals(2, (rhs.value as? Long)?.toInt()) + val rhsValue = rhs.value + assertIs(rhsValue) + assertEquals(2, rhsValue.toInt()) - val r = compStmt.statements[3] as? ReturnStatement - assertNotNull(r) + val r = compStmt.statements[3] + assertIs(r) val s3 = tu.variables["s3"] assertNotNull(s3) @@ -198,21 +213,23 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val body = main.body as? Block - assertNotNull(body) + val body = main.body + assertIs(body) - val sel = (body.statements.first() as? AssignExpression)?.declarations?.first() + val bodyFirstStmt = body.statements.firstOrNull() + assertIs(bodyFirstStmt) + val sel = bodyFirstStmt.declarations.firstOrNull() assertNotNull(sel) assertLocalName("sel", sel) assertEquals(tu.primitiveType("bool"), sel.type) - val firstAssignment = sel.firstAssignment as? Literal<*> - assertNotNull(firstAssignment) + val firstAssignment = sel.firstAssignment + assertIs>(firstAssignment) assertEquals(tu.primitiveType("bool"), firstAssignment.type) assertEquals("True", firstAssignment.code) - val `if` = body.statements[1] as? IfStatement - assertNotNull(`if`) + val `if` = body.statements[1] + assertIs(`if`) } @Test @@ -240,31 +257,31 @@ class PythonFrontendTest : BaseTest() { assertLocalName("SomeClass", cls) assertEquals(1, cls.methods.size) - val clsfunc = cls.methods.first() + val clsfunc = cls.methods.firstOrNull() assertLocalName("someFunc", clsfunc) assertLocalName("foo", foo) - val body = foo.body as? Block - assertNotNull(body) + val body = foo.body + assertIs(body) assertNotNull(body.statements) assertEquals(2, body.statements.size) - val s1 = body.statements[0] as? AssignExpression - assertNotNull(s1) - val s2 = body.statements[1] as? MemberCallExpression - assertNotNull(s2) + val s1 = body.statements[0] + assertIs(s1) + val s2 = body.statements[1] + assertIs(s2) - val c1 = s1.declarations.first() as? VariableDeclaration + val c1 = s1.declarations.firstOrNull() assertNotNull(c1) assertLocalName("c1", c1) - val ctor = c1.firstAssignment as? ConstructExpression - assertNotNull(ctor) - assertEquals(ctor.constructor, cls.constructors.first() as? ConstructorDeclaration) + val ctor = c1.firstAssignment + assertIs(ctor) + assertEquals(ctor.constructor, cls.constructors.firstOrNull()) assertFullName("simple_class.SomeClass", c1.type) - assertEquals(c1, (s2.base as? Reference)?.refersTo) + assertRefersTo(s2.base, c1) assertEquals(1, s2.invokes.size) - assertEquals(clsfunc, s2.invokes.first()) + assertInvokes(s2, clsfunc) // member } @@ -282,33 +299,39 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val assignExpr = (main.body as? Block)?.statements?.first() as? AssignExpression - assertNotNull(assignExpr) + val mainBody = main.body + assertIs(mainBody) + val assignExpr = mainBody.statements.firstOrNull() + assertIs(assignExpr) - val foo = assignExpr.declarations.firstOrNull() as? VariableDeclaration + val foo = assignExpr.declarations.firstOrNull() assertNotNull(foo) assertLocalName("foo", foo) assertEquals(tu.primitiveType("int"), foo.type) - val initializer = foo.firstAssignment as? ConditionalExpression - assertNotNull(initializer) + val initializer = foo.firstAssignment + assertIs(initializer) assertEquals(tu.primitiveType("int"), initializer.type) - val ifCond = initializer.condition as? Literal<*> - assertNotNull(ifCond) - val thenExpr = initializer.thenExpression as? Literal<*> - assertNotNull(thenExpr) - val elseExpr = initializer.elseExpression as? Literal<*> - assertNotNull(elseExpr) + val ifCond = initializer.condition + assertIs>(ifCond) + val thenExpr = initializer.thenExpression + assertIs>(thenExpr) + val elseExpr = initializer.elseExpression + assertIs>(elseExpr) assertEquals(tu.primitiveType("bool"), ifCond.type) assertEquals(false, ifCond.value) assertEquals(tu.primitiveType("int"), thenExpr.type) - assertEquals(21, (thenExpr.value as? Long)?.toInt()) + val thenValue = thenExpr.value + assertIs(thenValue) + assertEquals(21, thenValue.toInt()) + val elseValue = elseExpr.value + assertIs(elseValue) assertEquals(tu.primitiveType("int"), elseExpr.type) - assertEquals(42, (elseExpr.value as? Long)?.toInt()) + assertEquals(42, elseValue.toInt()) } @Test @@ -361,14 +384,16 @@ class PythonFrontendTest : BaseTest() { assertNotNull(methBar) assertLocalName("bar", methBar) - val barZ = (methBar.body as? Block)?.statements?.get(0) as? MemberExpression - assertNotNull(barZ) - assertEquals(fieldZ, barZ.refersTo) + val methBarBody = methBar.body + assertIs(methBarBody) + val barZ = methBarBody.statements[0] + assertIs(barZ) + assertRefersTo(barZ, fieldZ) - val barBaz = (methBar.body as? Block)?.statements?.get(1) as? AssignExpression - assertNotNull(barBaz) - val barBazInner = barBaz.declarations[0] as? FieldDeclaration - assertNotNull(barBazInner) + val barBaz = methBarBody.statements[1] + assertIs(barBaz) + val barBazInner = barBaz.declarations[0] + assertIs(barBazInner) assertLocalName("baz", barBazInner) assertNotNull(barBazInner.firstAssignment) } @@ -419,26 +444,59 @@ class PythonFrontendTest : BaseTest() { // assertEquals(tu.primitiveType("int"), i.type) // self.somevar = i - val someVarDeclaration = - ((bar.body as? Block)?.statements?.get(0) as? AssignExpression)?.declarations?.first() - as? FieldDeclaration - assertNotNull(someVarDeclaration) + val barBody = bar.body + assertIs(barBody) + val barBodyFirstStmt = barBody.statements[0] + assertIs(barBodyFirstStmt) + val someVarDeclaration = barBodyFirstStmt.declarations.firstOrNull() + assertIs(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) - assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) + assertRefersTo(someVarDeclaration.firstAssignment, i) - val fooMemCall = (foo.body as? Block)?.statements?.get(0) as? MemberCallExpression - assertNotNull(fooMemCall) + val fooBody = foo.body + assertIs(fooBody) + val fooMemCall = fooBody.statements[0] + assertIs(fooMemCall) - val mem = fooMemCall.callee as? MemberExpression - assertNotNull(mem) + val mem = fooMemCall.callee + assertIs(mem) assertLocalName("bar", mem) assertEquals(".", fooMemCall.operatorCode) assertFullName("class_self.Foo.bar", fooMemCall) assertEquals(1, fooMemCall.invokes.size) - assertEquals(bar, fooMemCall.invokes[0]) + assertInvokes(fooMemCall, bar) assertLocalName("self", fooMemCall.base) } + @Test + fun testClassTypeAnnotations() { + val topLevel = Path.of("src", "test", "resources", "python") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("class_type_annotations.py").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val other = tu.records["Other"] + assertNotNull(other) + assertFullName("class_type_annotations.Other", other.toType()) + + val foo = tu.records["Foo"] + assertNotNull(foo) + assertFullName("class_type_annotations.Foo", foo.toType()) + + val fromOther = tu.functions["from_other"] + assertNotNull(fromOther) + + val paramType = fromOther.parameters.firstOrNull()?.type + assertNotNull(paramType) + assertEquals(other.toType(), paramType) + } + @Test fun testCtor() { val topLevel = Path.of("src", "test", "resources", "python") @@ -461,9 +519,9 @@ class PythonFrontendTest : BaseTest() { assertEquals(1, recordFoo.methods.size) assertEquals(1, recordFoo.constructors.size) - val fooCtor = recordFoo.constructors[0] as? ConstructorDeclaration + val fooCtor = recordFoo.constructors[0] assertNotNull(fooCtor) - val foobar = recordFoo.methods[0] as? MethodDeclaration + val foobar = recordFoo.methods[0] assertNotNull(foobar) assertLocalName("__init__", fooCtor) @@ -473,22 +531,27 @@ class PythonFrontendTest : BaseTest() { assertNotNull(bar) assertLocalName("bar", bar) - assertEquals(2, (bar.body as? Block)?.statements?.size) - val line1 = (bar.body as? Block)?.statements?.get(0) as? AssignExpression - assertNotNull(line1) - val line2 = (bar.body as? Block)?.statements?.get(1) as? MemberCallExpression - assertNotNull(line2) + val barBody = bar.body + assertIs(barBody) + + assertEquals(2, barBody.statements.size) + val line1 = barBody.statements[0] + assertIs(line1) + + val line2 = barBody.statements[1] + assertIs(line2) assertEquals(1, line1.declarations.size) - val fooDecl = line1.declarations[0] as? VariableDeclaration + val fooDecl = line1.declarations[0] assertNotNull(fooDecl) assertLocalName("foo", fooDecl) assertFullName("class_ctor.Foo", fooDecl.type) - val initializer = fooDecl.firstAssignment as? ConstructExpression - assertEquals(fooCtor, initializer?.constructor) + val initializer = fooDecl.firstAssignment + assertIs(initializer) + assertEquals(fooCtor, initializer.constructor) - assertEquals(fooDecl, (line2.base as? Reference)?.refersTo) - assertEquals(foobar, line2.invokes[0]) + assertRefersTo(line2.base, fooDecl) + assertInvokes(line2, foobar) } @Test @@ -525,27 +588,34 @@ class PythonFrontendTest : BaseTest() { assertNotNull(countParam) assertLocalName("c", countParam) - val countStmt = (methCount.body as? Block)?.statements?.get(0) as? IfStatement - assertNotNull(countStmt) + val methCountBody = methCount.body + assertIs(methCountBody) - val ifCond = countStmt.condition as? BinaryOperator - assertNotNull(ifCond) + val countStmt = methCountBody.statements[0] + assertIs(countStmt) - val lhs = ifCond.lhs as? MemberCallExpression - assertNotNull(lhs) - assertEquals(countParam, (lhs.base as? Reference)?.refersTo) + val ifCond = countStmt.condition + assertIs(ifCond) + + val lhs = ifCond.lhs + assertIs(lhs) + assertRefersTo(lhs.base, countParam) assertLocalName("inc", lhs) assertEquals(0, lhs.arguments.size) - val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) as? CallExpression - assertNotNull(ifThen) - assertEquals(methCount, ifThen.invokes.first()) - assertEquals(countParam, (ifThen.arguments.first() as? Reference)?.refersTo) + val ifThenBody = countStmt.thenStatement + assertIs(ifThenBody) + val ifThenFirstStmt = ifThenBody.statements[0] + assertIs(ifThenFirstStmt) + assertInvokes(ifThenFirstStmt, methCount) + assertRefersTo(ifThenFirstStmt.arguments.firstOrNull(), countParam) assertNull(countStmt.elseStatement) // class c1(counter) assertLocalName("c1", clsC1) - assertEquals(clsCounter, (clsC1.superClasses.first() as? ObjectType)?.recordDeclaration) + val cls1Super = clsC1.superClasses.firstOrNull() + assertIs(cls1Super) + assertEquals(clsCounter, cls1Super.recordDeclaration) assertEquals(1, clsC1.fields.size) val field = clsC1.fields[0] @@ -564,32 +634,29 @@ class PythonFrontendTest : BaseTest() { assertLocalName("self", selfReceiver) assertEquals(0, meth.parameters.size) // self is receiver and not a parameter - val methBody = meth.body as? Block - assertNotNull(methBody) + val methBody = meth.body + assertIs(methBody) - val assign = methBody.statements[0] as? AssignExpression - assertNotNull(assign) + val assign = methBody.statements[0] + assertIs(assign) val assignLhs = assign.lhs() val assignRhs = assign.rhs() assertEquals("=", assign.operatorCode) assertNotNull(assignLhs) assertNotNull(assignRhs) - assertEquals(selfReceiver, (assignLhs.base as? Reference)?.refersTo) + assertRefersTo(assignLhs.base, selfReceiver) assertEquals("+", assignRhs.operatorCode) - val assignRhsLhs = - assignRhs.lhs - as? MemberExpression // the second "self.total" in "self.total = self.total + 1" - assertNotNull(assignRhsLhs) - assertEquals(selfReceiver, (assignRhsLhs.base as? Reference)?.refersTo) + val assignRhsLhs = assignRhs.lhs // the second "self.total" in "self.total = self.total + 1" + assertIs(assignRhsLhs) + assertRefersTo(assignRhsLhs.base, selfReceiver) - val r = methBody.statements[1] as? ReturnStatement - assertNotNull(r) - assertEquals( - selfReceiver, - ((r.returnValue as? MemberExpression)?.base as? Reference)?.refersTo - ) + val r = methBody.statements[1] + assertIs(r) + val retVal = r.returnValue + assertIs(retVal) + assertRefersTo(retVal.base, selfReceiver) // TODO last line "count(c1())" } @@ -627,54 +694,52 @@ class PythonFrontendTest : BaseTest() { assertNotNull(classFieldWithInit) // classFieldNoInitializer = classFieldWithInit - val assignClsFieldOutsideFunc = clsFoo.statements[2] as? AssignExpression - assertNotNull(assignClsFieldOutsideFunc) - assertEquals( - classFieldNoInitializer, - (assignClsFieldOutsideFunc.lhs())?.refersTo - ) - assertEquals(classFieldWithInit, (assignClsFieldOutsideFunc.rhs())?.refersTo) + val assignClsFieldOutsideFunc = clsFoo.statements[2] + assertIs(assignClsFieldOutsideFunc) + assertRefersTo(assignClsFieldOutsideFunc.lhs(), classFieldNoInitializer) + assertRefersTo((assignClsFieldOutsideFunc.rhs()), classFieldWithInit) assertEquals("=", assignClsFieldOutsideFunc.operatorCode) - val barBody = methBar.body as? Block - assertNotNull(barBody) + val barBody = methBar.body + assertIs(barBody) // self.classFieldDeclaredInFunction = 456 - val barStmt0 = barBody.statements[0] as? AssignExpression - val decl0 = barStmt0?.declarations?.get(0) as? FieldDeclaration - assertNotNull(decl0) + val barStmt0 = barBody.statements[0] + assertIs(barStmt0) + val decl0 = barStmt0.declarations[0] + assertIs(decl0) assertLocalName("classFieldDeclaredInFunction", decl0) assertNotNull(decl0.firstAssignment) // self.classFieldNoInitializer = 789 - val barStmt1 = barBody.statements[1] as? AssignExpression - assertNotNull(barStmt1) - assertEquals(classFieldNoInitializer, (barStmt1.lhs())?.refersTo) + val barStmt1 = barBody.statements[1] + assertIs(barStmt1) + assertRefersTo((barStmt1.lhs()), classFieldNoInitializer) // self.classFieldWithInit = 12 - val barStmt2 = barBody.statements[2] as? AssignExpression - assertNotNull(barStmt2) - assertEquals(classFieldWithInit, (barStmt2.lhs())?.refersTo) + val barStmt2 = barBody.statements[2] + assertIs(barStmt2) + assertRefersTo((barStmt2.lhs()), classFieldWithInit) // classFieldNoInitializer = "shadowed" - val barStmt3 = barBody.statements[3] as? AssignExpression - assertNotNull(barStmt3) + val barStmt3 = barBody.statements[3] + assertIs(barStmt3) assertEquals("=", barStmt3.operatorCode) - assertEquals(classFieldNoInitializer, (barStmt3.lhs())?.refersTo) + assertRefersTo((barStmt3.lhs()), classFieldNoInitializer) assertEquals("shadowed", (barStmt3.rhs>())?.value) // classFieldWithInit = "shadowed" - val barStmt4 = barBody.statements[4] as? AssignExpression - assertNotNull(barStmt4) + val barStmt4 = barBody.statements[4] + assertIs(barStmt4) assertEquals("=", barStmt4.operatorCode) - assertEquals(classFieldWithInit, (barStmt4.lhs())?.refersTo) + assertRefersTo((barStmt4.lhs()), classFieldWithInit) assertEquals("shadowed", (barStmt4.rhs>())?.value) // classFieldDeclaredInFunction = "shadowed" - val barStmt5 = barBody.statements[5] as? AssignExpression - assertNotNull(barStmt5) + val barStmt5 = barBody.statements[5] + assertIs(barStmt5) assertEquals("=", barStmt5.operatorCode) - assertEquals(classFieldDeclaredInFunction, (barStmt5.lhs())?.refersTo) + assertRefersTo((barStmt5.lhs()), classFieldDeclaredInFunction) assertEquals("shadowed", (barStmt5.rhs>())?.value) /* TODO: @@ -725,19 +790,19 @@ class PythonFrontendTest : BaseTest() { val foo = p.variables["foo"] assertNotNull(foo) - val firstAssignment = foo.firstAssignment as? MemberCallExpression - assertNotNull(firstAssignment) + val firstAssignment = foo.firstAssignment + assertIs(firstAssignment) assertLocalName("zzz", firstAssignment) - val base = firstAssignment.base as? MemberExpression - assertNotNull(base) + val base = firstAssignment.base + assertIs(base) assertLocalName("baz", base) - val baseBase = base.base as? Reference - assertNotNull(baseBase) + val baseBase = base.base + assertIs(baseBase) assertLocalName("bar", baseBase) - val memberExpression = firstAssignment.callee as? MemberExpression - assertNotNull(memberExpression) + val memberExpression = firstAssignment.callee + assertIs(memberExpression) assertLocalName("zzz", memberExpression) } @@ -756,29 +821,27 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["main"] assertNotNull(main) - val mainBody = (main as? FunctionDeclaration)?.body as? Block - assertNotNull(mainBody) + val mainBody = main.body + assertIs(mainBody) - val whlStmt = mainBody.statements[3] as? WhileStatement - assertNotNull(whlStmt) + val whlStmt = mainBody.statements[3] + assertIs(whlStmt) - val whlBody = whlStmt.statement as? Block - assertNotNull(whlBody) + val whlBody = whlStmt.statement + assertIs(whlBody) - val xDeclaration = whlBody.statements[0] as? AssignExpression - assertNotNull(xDeclaration) + val xDeclaration = whlBody.statements[0] + assertIs(xDeclaration) - val ifStatement = whlBody.statements[1] as? IfStatement - assertNotNull(ifStatement) + val ifStatement = whlBody.statements[1] + assertIs(ifStatement) - val brk = ifStatement.elseStatement as? Block - assertNotNull(brk) - brk.statements[0] as? BreakStatement + val brk = ifStatement.elseStatement + assertIs(brk) + assertIs(brk.statements[0]) } @Test - @Ignore // TODO fix & re-enable this test once there is proper support for multiple variables in - // a loop fun testIssue615() { val topLevel = Path.of("src", "test", "resources", "python") val tu = @@ -790,58 +853,94 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["issue615"] assertNotNull(p) - assertEquals(1, p.declarations.size) + assertEquals( + 5, + p.variables.size + ) // including one dummy variable introduced for the loop var + assertEquals( + 4, + p.variables.filter { !it.name.localName.contains(PythonHandler.LOOP_VAR_PREFIX) }.size + ) assertEquals(2, p.statements.size) // test = [(1, 2, 3)] val testDeclaration = p.variables[0] assertNotNull(testDeclaration) assertLocalName("test", testDeclaration) - val testDeclStmt = p.statements[0] as? DeclarationStatement - assertNotNull(testDeclStmt) - assertEquals(1, testDeclStmt.declarations.size) - assertEquals(testDeclaration, testDeclStmt.variables[0]) + val testDeclStmt = p.statements[0] + assertIs(testDeclStmt) /* for loop: for t1, t2, t3 in test: print("bug ... {} {} {}".format(t1, t2, t3)) */ - val forStmt = p.statements[1] as? ForEachStatement - assertNotNull(forStmt) - - val forVariable = forStmt.variable as? InitializerListExpression - assertNotNull(forVariable) - assertEquals(3, forVariable.initializers.size) - val t1Decl = forVariable.initializers[0] as? Reference - val t2Decl = forVariable.initializers[1] as? Reference - val t3Decl = forVariable.initializers[2] as? Reference + val forStmt = p.statements[1] + assertIs(forStmt) + + val forVariable = forStmt.variable + assertIs(forVariable) + val forVarDecl = + p.declarations.firstOrNull { + it.name.localName.contains((PythonHandler.LOOP_VAR_PREFIX)) + } + assertNotNull(forVarDecl) + assertRefersTo(forVariable, forVarDecl) + + val iter = forStmt.iterable + assertIs(iter) + assertRefersTo(iter, testDeclaration) + + val forBody = forStmt.statement + assertIs(forBody) + assertEquals(2, forBody.statements.size) // loop var assign and print stmt + + /* + We model the 3 loop variables + + ``` + for t1, t2, t3 in ... + ``` + + implicitly as follows: + + ``` + for tempVar in ...: + t1, t2, t3 = tempVar + rest of the loop + ``` + */ + val forVariableImplicitStmt = forBody.statements.firstOrNull() + assertIs(forVariableImplicitStmt) + assertEquals("=", forVariableImplicitStmt.operatorCode) + assertEquals(forStmt.variable, forVariableImplicitStmt.rhs.firstOrNull()) + val (t1Decl, t2Decl, t3Decl) = forVariableImplicitStmt.declarations + val (t1RefAssign, t2RefAssign, t3RefAssign) = forVariableImplicitStmt.lhs assertNotNull(t1Decl) assertNotNull(t2Decl) assertNotNull(t3Decl) - // TODO no refersTo - - val iter = forStmt.iterable as? Reference - assertNotNull(iter) - assertEquals(testDeclaration, iter.refersTo) - - val forBody = forStmt.statement as? Block - assertNotNull(forBody) - assertEquals(1, forBody.statements.size) + assertIs(t1RefAssign) + assertIs(t2RefAssign) + assertIs(t3RefAssign) + assertRefersTo(t1RefAssign, t1Decl) + assertRefersTo(t2RefAssign, t2Decl) + assertRefersTo(t3RefAssign, t3Decl) // print("bug ... {} {} {}".format(t1, t2, t3)) - val forBodyStmt = forBody.statements[0] as? CallExpression + val forBodyStmt = forBody.statements(1) assertNotNull(forBodyStmt) assertLocalName("print", forBodyStmt) - val printArg = forBodyStmt.arguments[0] as? MemberCallExpression - assertNotNull(printArg) - val formatArgT1 = printArg.arguments[0] as? Reference - assertNotNull(formatArgT1) - val formatArgT2 = printArg.arguments[1] as? Reference - assertNotNull(formatArgT2) - val formatArgT3 = printArg.arguments[2] as? Reference - assertNotNull(formatArgT3) - // TODO check refersTo + val printArg = forBodyStmt.arguments[0] + assertIs(printArg) + val formatArgT1 = printArg.arguments[0] + assertIs(formatArgT1) + assertRefersTo(formatArgT1, t1Decl) + val formatArgT2 = printArg.arguments[1] + assertIs(formatArgT2) + assertRefersTo(formatArgT2, t2Decl) + val formatArgT3 = printArg.arguments[2] + assertIs(formatArgT3) + assertRefersTo(formatArgT3, t3Decl) } @Test @@ -856,31 +955,37 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["issue473"] assertNotNull(p) - val ifStatement = p.statements[0] as? IfStatement - assertNotNull(ifStatement) - val ifCond = ifStatement.condition as? BinaryOperator - assertNotNull(ifCond) - val ifThen = ifStatement.thenStatement as? Block - assertNotNull(ifThen) - val ifElse = ifStatement.elseStatement as? Block - assertNotNull(ifElse) + val ifStatement = p.statements[0] + assertIs(ifStatement) + val ifCond = ifStatement.condition + assertIs(ifCond) + val ifThen = ifStatement.thenStatement + assertIs(ifThen) + val ifElse = ifStatement.elseStatement + assertIs(ifElse) // sys.version_info.minor > 9 assertEquals(">", ifCond.operatorCode) - assertLocalName("minor", ifCond.lhs as? Reference) + assertIs(ifCond.lhs) + assertLocalName("minor", ifCond.lhs) // phr = {"user_id": user_id} | content - val phrDeclaration = (ifThen.statements[0] as? AssignExpression)?.declarations?.get(0) + val ifThenFirstStmt = ifThen.statements.firstOrNull() + assertIs(ifThenFirstStmt) + val phrDeclaration = ifThenFirstStmt.declarations[0] assertNotNull(phrDeclaration) assertLocalName("phr", phrDeclaration) - val phrInitializer = phrDeclaration.firstAssignment as? BinaryOperator - assertNotNull(phrInitializer) + val phrInitializer = phrDeclaration.firstAssignment + assertIs(phrInitializer) assertEquals("|", phrInitializer.operatorCode) - assertEquals(true, phrInitializer.lhs is InitializerListExpression) + val phrInitializerLhs = phrInitializer.lhs + assertIs(phrInitializerLhs) // z = {"user_id": user_id} - val elseStmt1 = (ifElse.statements[0] as? AssignExpression)?.declarations?.get(0) + val elseFirstStmt = ifElse.statements.firstOrNull() + assertIs(elseFirstStmt) + val elseStmt1 = elseFirstStmt.declarations[0] assertNotNull(elseStmt1) assertLocalName("z", elseStmt1) @@ -909,12 +1014,12 @@ class PythonFrontendTest : BaseTest() { assertEquals(1, functions.size) assertEquals( "# a function", - functions.first().comment, + functions.firstOrNull()?.comment, ) val literals = commentedNodes.filterIsInstance>() assertEquals(1, literals.size) - assertEquals("# comment start", literals.first().comment) + assertEquals("# comment start", literals.firstOrNull()?.comment) val params = commentedNodes.filterIsInstance() assertEquals(2, params.size) @@ -923,17 +1028,17 @@ class PythonFrontendTest : BaseTest() { val assignment = commentedNodes.filterIsInstance() assertEquals(2, assignment.size) - assertEquals("# A comment# a number", assignment.first().comment) + assertEquals("# A comment# a number", assignment.firstOrNull()?.comment) assertEquals("# comment end", assignment.last().comment) val block = commentedNodes.filterIsInstance() assertEquals(1, block.size) - assertEquals("# foo", block.first().comment) + assertEquals("# foo", block.firstOrNull()?.comment) val kvs = commentedNodes.filterIsInstance() assertEquals(2, kvs.size) - assertEquals("# a entry", kvs.first { it.code?.contains("a") ?: false }.comment) - assertEquals("# b entry", kvs.first { it.code?.contains("b") ?: false }.comment) + assertEquals("# a entry", kvs.first { it.code?.contains("a") == true }.comment) + assertEquals("# b entry", kvs.first { it.code?.contains("b") == true }.comment) } @Test @@ -950,8 +1055,10 @@ class PythonFrontendTest : BaseTest() { assertNotNull(tu) val annotations = tu.allChildren() - val route = annotations.firstOrNull() - assertFullName("app.route", route) + assertEquals( + listOf("app.route", "some.otherannotation", "annotations.other_func"), + annotations.map { it.name.toString() } + ) } @Test @@ -972,30 +1079,32 @@ class PythonFrontendTest : BaseTest() { val varDefinedInLoop = forloopFunc.variables["varDefinedInLoop"] assertNotNull(varDefinedInLoop) - val functionBody = forloopFunc.body as? Block - assertNotNull(functionBody) + val functionBody = forloopFunc.body + assertIs(functionBody) + + val firstLoop = functionBody.statements[1] + assertIs(firstLoop) - val firstLoop = functionBody.statements[1] as? ForEachStatement - assertNotNull(firstLoop) + val secondLoop = functionBody.statements[2] + assertIs(secondLoop) - val secondLoop = functionBody.statements[2] as? ForEachStatement - assertNotNull(secondLoop) + val fooCall = functionBody.statements[3] + assertIs(fooCall) - val fooCall = functionBody.statements[3] as? CallExpression - assertNotNull(fooCall) + val barCall = functionBody.statements[4] + assertIs(barCall) - val barCall = functionBody.statements[4] as? CallExpression - assertNotNull(barCall) + val bodyFirstStmt = functionBody.statements.firstOrNull() + assertIs(bodyFirstStmt) + val varDefinedBeforeLoopRef = bodyFirstStmt.lhs.firstOrNull() + assertIs(varDefinedBeforeLoopRef) - val varDefinedBeforeLoopRef = - (functionBody.statements.firstOrNull() as? AssignExpression)?.lhs?.firstOrNull() - as? Reference ?: TODO() // no dataflow from var declaration to loop variable because it's a write access assert((firstLoop.variable?.prevDFG?.contains(varDefinedBeforeLoopRef) == false)) // dataflow from range call to loop variable - val firstLoopIterable = firstLoop.iterable as? CallExpression - assertNotNull(firstLoopIterable) + val firstLoopIterable = firstLoop.iterable + assertIs(firstLoopIterable) assert((firstLoop.variable?.prevDFG?.contains((firstLoopIterable)) == true)) // dataflow from var declaration to loop iterable call @@ -1005,25 +1114,23 @@ class PythonFrontendTest : BaseTest() { ) // dataflow from first loop to foo call - val loopVar = firstLoop.variable as? Reference - assertNotNull(loopVar) - assert(fooCall.arguments.first().prevDFG.contains(loopVar)) + val loopVar = firstLoop.variable + assertIs(loopVar) + assertTrue(fooCall.arguments.firstOrNull()?.prevDFG?.contains(loopVar) == true) // dataflow from var declaration to foo call (in case for loop is not executed) - assert(fooCall.arguments.first().prevDFG.contains(varDefinedBeforeLoopRef)) + assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(varDefinedBeforeLoopRef) == true) // dataflow from range call to loop variable - val secondLoopIterable = secondLoop.iterable as? CallExpression - assertNotNull(secondLoopIterable) - assert( - ((secondLoop.variable as? Reference)?.prevDFG?.contains((secondLoopIterable)) == true) - ) + val secondLoopIterable = secondLoop.iterable + assertIs(secondLoopIterable) + + val secondLoopVar = secondLoop.variable + assertIs(secondLoopVar) + assert(secondLoopVar.prevDFG.contains(secondLoopIterable) == true) // dataflow from second loop var to bar call - assertEquals( - (secondLoop.variable as? Reference), - barCall.arguments.first().prevDFG.firstOrNull() - ) + assertEquals(secondLoopVar, barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull()) } @Test @@ -1043,73 +1150,96 @@ class PythonFrontendTest : BaseTest() { val bAugAssign = tu.allChildren().singleOrNull { - (it.lhs.singleOrNull() as? Reference)?.name?.localName == "b" && - it.location?.region?.startLine == 4 + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "b" && it.location?.region?.startLine == 4 } assertNotNull(bAugAssign) assertEquals("*=", bAugAssign.operatorCode) assertEquals("b", bAugAssign.lhs.singleOrNull()?.name?.localName) - assertEquals(2L, (bAugAssign.rhs.singleOrNull() as? Literal<*>)?.value) + val bAugAssignRhs = bAugAssign.rhs.singleOrNull() + assertIs>(bAugAssignRhs) + assertEquals(2L, bAugAssignRhs.value) // c = (not True and False) or True val cAssign = tu.allChildren() - .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "c" } + .singleOrNull { + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "c" + } ?.rhs - ?.singleOrNull() as? BinaryOperator - assertNotNull(cAssign) + ?.singleOrNull() + assertIs(cAssign) assertEquals("or", cAssign.operatorCode) - assertEquals(true, (cAssign.rhs as? Literal<*>)?.value) - assertEquals("and", (cAssign.lhs as? BinaryOperator)?.operatorCode) - assertEquals(false, ((cAssign.lhs as? BinaryOperator)?.rhs as? Literal<*>)?.value) - assertEquals("not", ((cAssign.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.operatorCode) - assertEquals( - true, - (((cAssign.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.input as? Literal<*>)?.value - ) + val cAssignRhs = cAssign.rhs + assertIs>(cAssignRhs) + assertEquals(true, cAssignRhs.value) + val cAssignLhs = cAssign.lhs + assertIs(cAssignLhs) + assertEquals("and", cAssignLhs.operatorCode) + val cAssignLhsRhs = cAssignLhs.rhs + assertIs>(cAssignLhsRhs) + assertEquals(false, cAssignLhsRhs.value) + val cAssignLhsLhs = cAssignLhs.lhs + assertIs(cAssignLhsLhs) + assertEquals("not", cAssignLhsLhs.operatorCode) + val cAssignLhsLhsInput = cAssignLhsLhs.input + assertIs>(cAssignLhsLhsInput) + assertEquals(true, cAssignLhsLhsInput.value) // d = ((-5 >> 2) & ~7 | (+4 << 1)) ^ 0xffff val dAssign = tu.allChildren() - .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "d" } + .singleOrNull { + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "d" + } ?.rhs - ?.singleOrNull() as? BinaryOperator - assertNotNull(dAssign) + ?.singleOrNull() + assertIs(dAssign) assertEquals("^", dAssign.operatorCode) - assertEquals(0xffffL, (dAssign.rhs as? Literal<*>)?.value) - assertEquals("|", (dAssign.lhs as? BinaryOperator)?.operatorCode) - assertEquals("<<", ((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.operatorCode) - assertEquals( - 1L, - (((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.rhs as? Literal<*>)?.value - ) - assertEquals( - "+", - (((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.lhs as? UnaryOperator) - ?.operatorCode - ) - assertEquals( - 4L, - ((((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.lhs as? UnaryOperator) - ?.input as? Literal<*>) - ?.value - ) - val dAssignLhsOfOr = (dAssign.lhs as? BinaryOperator)?.lhs as? BinaryOperator - assertNotNull(dAssignLhsOfOr) + val dAssignRhs = dAssign.rhs + assertIs>(dAssignRhs) + assertEquals(0xffffL, dAssignRhs.value) + val dAssignLhs = dAssign.lhs + assertIs(dAssignLhs) + assertEquals("|", dAssignLhs.operatorCode) + val dAssignLhsRhs = dAssignLhs.rhs + assertIs(dAssignLhsRhs) + assertEquals("<<", dAssignLhsRhs.operatorCode) + val dAssignLhsRhsRhs = dAssignLhsRhs.rhs + assertIs>(dAssignLhsRhsRhs) + assertEquals(1L, dAssignLhsRhsRhs.value) + val dAssignLhsRhsLhs = dAssignLhsRhs.lhs + assertIs(dAssignLhsRhsLhs) + assertEquals("+", dAssignLhsRhsLhs.operatorCode) + val dAssignLhsRhsLhsInput = dAssignLhsRhsLhs.input + assertIs>(dAssignLhsRhsLhsInput) + assertEquals(4L, dAssignLhsRhsLhsInput.value) + val dAssignLhsOfOr = dAssignLhs.lhs + assertIs(dAssignLhsOfOr) assertEquals("&", dAssignLhsOfOr.operatorCode) - assertEquals("~", (dAssignLhsOfOr.rhs as? UnaryOperator)?.operatorCode) - assertEquals(7L, ((dAssignLhsOfOr.rhs as? UnaryOperator)?.input as? Literal<*>)?.value) - assertEquals(">>", (dAssignLhsOfOr.lhs as? BinaryOperator)?.operatorCode) - assertEquals(2L, ((dAssignLhsOfOr.lhs as? BinaryOperator)?.rhs as? Literal<*>)?.value) - assertEquals( - "-", - ((dAssignLhsOfOr.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.operatorCode - ) - assertEquals( - 5L, - (((dAssignLhsOfOr.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.input as? Literal<*>) - ?.value - ) + val dAssignLhsOfOrRhs = dAssignLhsOfOr.rhs + assertIs(dAssignLhsOfOrRhs) + assertEquals("~", dAssignLhsOfOrRhs.operatorCode) + val dAssignLhsOfOrRhsInput = dAssignLhsOfOrRhs.input + assertIs>(dAssignLhsOfOrRhsInput) + assertEquals(7L, dAssignLhsOfOrRhsInput.value) + val dAssignLhsOfOrLhs = dAssignLhsOfOr.lhs + assertIs(dAssignLhsOfOrLhs) + assertEquals(">>", dAssignLhsOfOrLhs.operatorCode) + val dAssignLhsOfOrLhsRhs = dAssignLhsOfOrLhs.rhs + assertIs>(dAssignLhsOfOrLhsRhs) + assertEquals(2L, dAssignLhsOfOrLhsRhs.value) + val dAssignLhsOfOrLhsLhs = dAssignLhsOfOrLhs.lhs + assertIs(dAssignLhsOfOrLhsLhs) + assertEquals("-", dAssignLhsOfOrLhsLhs.operatorCode) + val dAssignLhsOfOrLhsLhsInput = dAssignLhsOfOrLhsLhs.input + assertIs>(dAssignLhsOfOrLhsLhsInput) + assertEquals(5L, dAssignLhsOfOrLhsLhsInput.value) } @Test @@ -1126,63 +1256,70 @@ class PythonFrontendTest : BaseTest() { assertNotNull(tu) val namespace = tu.namespaces.singleOrNull() assertNotNull(namespace) - val aStmt = namespace.statements[0] as? AssignExpression - assertNotNull(aStmt) - assertEquals( - "list", - (aStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) - val bStmt = namespace.statements[1] as? AssignExpression - assertNotNull(bStmt) - assertEquals( - "set", - (bStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) - val cStmt = namespace.statements[2] as? AssignExpression - assertNotNull(cStmt) - assertEquals( - "tuple", - (cStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) - val dStmt = namespace.statements[3] as? AssignExpression - assertNotNull(dStmt) - assertEquals( - "dict", - (dStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) - val eStmtRhs = - (namespace.statements[4] as? AssignExpression)?.rhs?.singleOrNull() as? BinaryOperator - assertNotNull(eStmtRhs) - assertEquals("Values of a: ", (eStmtRhs.lhs as? Literal<*>)?.value) - val eStmtRhsRhs = (eStmtRhs.rhs as? BinaryOperator) + val aStmt = namespace.statements[0] + assertIs(aStmt) + val aStmtRhs = aStmt.rhs.singleOrNull() + assertIs(aStmtRhs) + assertEquals("list", aStmtRhs.type.name.localName) + + val bStmt = namespace.statements[1] + assertIs(bStmt) + val bStmtRhs = bStmt.rhs.singleOrNull() + assertIs(bStmtRhs) + assertEquals("set", bStmtRhs.type.name.localName) + + val cStmt = namespace.statements[2] + assertIs(cStmt) + val cStmtRhs = cStmt.rhs.singleOrNull() + assertIs(cStmtRhs) + assertEquals("tuple", cStmtRhs.type.name.localName) + + val dStmt = namespace.statements[3] + assertIs(dStmt) + val dStmtRhs = dStmt.rhs.singleOrNull() + assertIs(dStmtRhs) + assertEquals("dict", dStmtRhs.type.name.localName) + + val fourthStmt = namespace.statements[4] + assertIs(fourthStmt) + val eStmtRhs = fourthStmt.rhs.singleOrNull() + assertIs(eStmtRhs) + val eStmtRhsLhs = eStmtRhs.lhs + assertIs>(eStmtRhsLhs) + assertEquals("Values of a: ", eStmtRhsLhs.value) + val eStmtRhsRhs = eStmtRhs.rhs + assertIs(eStmtRhsRhs) assertNotNull(eStmtRhsRhs) - val aRef = eStmtRhsRhs.lhs as? Reference - assertEquals("a", aRef?.name?.localName) - val eStmtRhsRhsRhs = (eStmtRhsRhs.rhs as? BinaryOperator) - assertEquals(" and b: ", (eStmtRhsRhsRhs?.lhs as? Literal<*>)?.value) - val bCall = eStmtRhsRhsRhs?.rhs as? CallExpression - assertEquals("str", bCall?.name?.localName) - assertEquals("b", bCall?.arguments?.singleOrNull()?.name?.localName) - - val fStmtRhs = - (namespace.statements[5] as? AssignExpression)?.rhs?.singleOrNull() - as? SubscriptExpression - assertNotNull(fStmtRhs) + val aRef = eStmtRhsRhs.lhs + assertEquals("a", aRef.name.localName) + val eStmtRhsRhsRhs = eStmtRhsRhs.rhs + assertIs(eStmtRhsRhsRhs) + val eStmtRhsRhsRhsLhs = eStmtRhsRhsRhs.lhs + assertIs>(eStmtRhsRhsRhsLhs) + assertEquals(" and b: ", eStmtRhsRhsRhsLhs.value) + val bCall = eStmtRhsRhsRhs.rhs + assertIs(bCall) + assertEquals("str", bCall.name.localName) + assertEquals("b", bCall.arguments.singleOrNull()?.name?.localName) + + val fifthStmt = namespace.statements[5] + assertIs(fifthStmt) + val fStmtRhs = fifthStmt.rhs.singleOrNull() + + assertIs(fStmtRhs) assertEquals("a", fStmtRhs.arrayExpression.name.localName) - assertTrue(fStmtRhs.subscriptExpression is RangeExpression) - assertEquals( - 1L, - ((fStmtRhs.subscriptExpression as RangeExpression).floor as? Literal<*>)?.value - ) - assertEquals( - 3L, - ((fStmtRhs.subscriptExpression as RangeExpression).ceiling as? Literal<*>)?.value - ) - assertEquals( - 2L, - ((fStmtRhs.subscriptExpression as RangeExpression).third as? Literal<*>)?.value - ) + val subscriptExpression = fStmtRhs.subscriptExpression + assertIs(subscriptExpression) + val fStmtRhsFloor = subscriptExpression.floor + assertIs>(fStmtRhsFloor) + assertEquals(1L, fStmtRhsFloor.value) + val fStmtRhsCeiling = subscriptExpression.ceiling + assertIs>(fStmtRhsCeiling) + assertEquals(3L, fStmtRhsCeiling.value) + val fStmtRhsThird = subscriptExpression.third + assertIs>(fStmtRhsThird) + assertEquals(2L, fStmtRhsThird.value) } @Test @@ -1257,6 +1394,67 @@ class PythonFrontendTest : BaseTest() { assertInvokes(call, cCompletelyDifferentFunc) } + @Test + fun testInterfaceStubs() { + val topLevel = Path.of("src", "test", "resources", "python") + val result = + analyze( + listOf( + topLevel.resolve("complex_class.pyi").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + with(result) { + val foo = records["Foo"] + assertNotNull(foo) + + val bar = foo.methods["bar"] + assertNotNull(bar) + + assertEquals(assertResolvedType("int"), bar.returnTypes.singleOrNull()) + assertEquals(assertResolvedType("int"), bar.parameters.firstOrNull()?.type) + assertEquals(assertResolvedType("complex_class.Foo"), bar.receiver?.type) + } + } + + @Test + fun testNamedExpression() { + val topLevel = Path.of("src", "test", "resources", "python") + val result = + analyze( + listOf( + topLevel.resolve("named_expressions.py").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + val namedExpression = result.functions["named_expression"] + assertNotNull(namedExpression) + + val assignExpression = result.statements[1] + assertIs(assignExpression) + assertEquals(":=", assignExpression.operatorCode) + assertEquals(true, assignExpression.usedAsExpression) + + val lhs = assignExpression.lhs.firstOrNull() + assertIs(lhs) + + val lhsVariable = lhs.refersTo + assertIs(lhsVariable) + assertLocalName("x", lhsVariable) + + val rhs = assignExpression.rhs.firstOrNull() + assertIs>(rhs) + + assertEquals(4.toLong(), rhs.evaluate()) + } + class PythonValueEvaluator : ValueEvaluator() { override fun computeBinaryOpEffect( lhsValue: Any?, diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/ArgumentsHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/ArgumentsHandlerTest.kt new file mode 100644 index 0000000000..62152213ce --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/ArgumentsHandlerTest.kt @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python.statementHandler + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.get +import de.fraunhofer.aisec.cpg.graph.parameters +import de.fraunhofer.aisec.cpg.test.analyze +import java.nio.file.Path +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.iterator +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class ArgumentsHandlerTest { + + private lateinit var topLevel: Path + private lateinit var result: TranslationResult + + @BeforeAll + fun setup() { + topLevel = Path.of("src", "test", "resources", "python") + analyzeFile() + } + + fun analyzeFile() { + result = + analyze(listOf(topLevel.resolve("arguments.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + } + + @Test + fun testPosOnlyArguments() { + val func = result.functions["pos_only_and_args"] + assertNotNull(func) + + val list = mapOf("a" to true, "b" to true, "c" to false) + list.keys.forEachIndexed { idx, name -> + val param = func.parameterEdges.firstOrNull { it.end.name.localName == name } + assertNotNull(param, "$name should not be empty") + if (list[name] == true) { + assertContains( + param.end.modifiers, + PythonLanguage.Companion.MODIFIER_POSITIONAL_ONLY_ARGUMENT + ) + } + assertEquals(idx, param.index) + } + } + + @Test + fun testVarargsArguments() { + val func = result.functions["test_varargs"] + assertNotNull(func, "Function 'test_varargs' should be found") + + val variadicArg = func.parameters["args"] + assertNotNull(variadicArg, "Failed to find variadic args") + assertEquals(true, variadicArg.isVariadic) + } + + @Test + fun testKwOnlyArguments() { + val func = result.functions["kwd_only_arg"] + assertNotNull(func, "Function 'kwd_only_arg' should be found") + + val kwOnlyArg = func.parameters["arg"] + assertNotNull(kwOnlyArg, "Failed to find keyword only args") + assertContains(kwOnlyArg.modifiers, PythonLanguage.Companion.MODIFIER_KEYWORD_ONLY_ARGUMENT) + } + + @Test + fun testKwDefaultArguments() { + val func = result.functions["kw_defaults"] + assertNotNull(func, "Function 'kw_defaults' should be found") + + assertEquals(4, func.parameters.size) + val kwOnlyParams = mapOf("c" to 2, "d" to null, "e" to 3) + + for ((paramName, expectedDefaultValue) in kwOnlyParams) { + val param = func.parameters[paramName] + assertNotNull(param, "Failed to find keyword-only argument '$paramName'") + + assertContains(param.modifiers, PythonLanguage.MODIFIER_KEYWORD_ONLY_ARGUMENT) + + if (expectedDefaultValue != null) { + assertNotNull(param.default, "Parameter '$paramName' should have a default value") + assertEquals( + expectedDefaultValue.toLong(), + param.default?.evaluate(), + "Default value for parameter '$paramName' is incorrect" + ) + } else { + assertNull(param.default, "Parameter '$paramName' should not have a default value") + } + } + } + + @Test + fun testDefaultsArguments() { + val func = result.functions["defaults"] + assertNotNull(func, "Function 'defaults' should be found") + + assertEquals(4, func.parameters.size) + val defaults = mapOf("b" to 1, "c" to 2) + for ((paramName, expectedDefaultValue) in defaults) { + val param = func.parameters[paramName] + assertNotNull(param, "Failed to find keyword-only argument '$paramName'") + + assertNotNull(param.default, "Parameter '$paramName' should have a default value") + assertEquals( + expectedDefaultValue.toLong(), + param.default?.evaluate(), + "Default value for parameter '$paramName' is incorrect" + ) + } + } + + @Test + fun testReceiverWithDefault() { + val func = result.functions["my_method"] + assertNotNull(func, "Function 'my_method' should be found") + + assertEquals(2, func.parameters.size) + assertNotNull(func.methods[0].receiver) + + val parameterD = func.parameters["d"] + assertNotNull(parameterD?.default, "Expected the parameter `d` to have a default value.") + assertEquals(3.toLong(), parameterD.default?.evaluate()) + + val parameterE = func.parameters["e"] + assertNotNull(parameterE?.default, "Expected the parameter `e` to have a default value.") + assertEquals(1.toLong(), parameterE.default?.evaluate()) + } + + @Test + fun testKwArguments() { + val func = result.functions["kw_args"] + assertNotNull(func, "Function 'kw_args' should be found") + + val kwArgs = func.parameters["kwargs"] + assertNotNull(kwArgs, "Failed to find kw args") + assertEquals(false, kwArgs.isVariadic) + } + + @Test + fun testMethodDefaults() { + val func = result.functions["method_with_some_defaults"] + + assertEquals(3, func.parameters.size) + val parameterA = func.parameters["a"] + val parameterB = func.parameters["b"] + val parameterC = func.parameters["c"] + assertNotNull( + parameterA, + "Failed to find parameter `a` -> cannot test for non-existing default value." + ) + assertNull(parameterA.default, "Expected the parameter `a` to not have a default value.") + + assertNotNull(parameterB?.default, "Expected the parameter `b` to have a default value.") + assertEquals(1.toLong(), parameterB.default?.evaluate()) + assertNotNull(parameterC?.default, "Expected the parameter `c` to have a default value.") + assertEquals(2.toLong(), parameterC.default?.evaluate()) + } +} diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt new file mode 100644 index 0000000000..f13e054207 --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python.statementHandler + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.frontends.python.* +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.test.* +import de.fraunhofer.aisec.cpg.test.analyze +import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.test.assertResolvedType +import java.nio.file.Path +import kotlin.test.* +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class StatementHandlerTest : BaseTest() { + + private lateinit var topLevel: Path + private lateinit var result: TranslationResult + + @BeforeAll + fun setup() { + topLevel = Path.of("src", "test", "resources", "python") + } + + fun analyzeFile(fileName: String) { + result = + analyze(listOf(topLevel.resolve(fileName).toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + } + + @Test + fun testTry() { + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("try.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val tryAll = tu.functions["tryAll"]?.trys?.singleOrNull() + assertNotNull(tryAll) + + assertEquals(1, tryAll.tryBlock?.statements?.size) + + assertEquals(3, tryAll.catchClauses.size) + assertLocalName("", tryAll.catchClauses[0].parameter) + assertLocalName("e", tryAll.catchClauses[1].parameter) + assertNull(tryAll.catchClauses[2].parameter) + + assertEquals(1, tryAll.elseBlock?.statements?.size) + assertEquals(1, tryAll.finallyBlock?.statements?.size) + + val tryOnlyFinally = tu.functions["tryOnlyFinally"]?.trys?.singleOrNull() + assertNotNull(tryOnlyFinally) + + assertEquals(1, tryOnlyFinally.tryBlock?.statements?.size) + + assertEquals(0, tryOnlyFinally.catchClauses.size) + + assertNull(tryOnlyFinally.elseBlock) + assertEquals(1, tryOnlyFinally.finallyBlock?.statements?.size) + + val tryOnlyExcept = tu.functions["tryOnlyExcept"]?.trys?.singleOrNull() + assertNotNull(tryOnlyExcept) + + assertEquals(1, tryOnlyExcept.tryBlock?.statements?.size) + + assertEquals(1, tryOnlyExcept.catchClauses.size) + assertNull(tryOnlyExcept.catchClauses.single().parameter) + + assertNull(tryOnlyExcept.elseBlock) + assertNull(tryOnlyExcept.finallyBlock) + + // Test EOG integrity with else block + + // All entries to the else block must come from the try block + assertTrue( + Util.eogConnect( + n = tryAll.elseBlock, + en = Util.Edge.ENTRIES, + refs = listOf(tryAll.tryBlock) + ) + ) + + // All exits from the else block must go to the entries of the non-empty finals block + assertTrue( + Util.eogConnect( + n = tryAll.elseBlock, + en = Util.Edge.EXITS, + refs = listOf(tryAll.finallyBlock) + ) + ) + } + + @Test + fun testAsync() { + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("async.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val myFunc = tu.functions["my_func"] + assertNotNull(myFunc) + assertEquals(1, myFunc.parameters.size) + + val myOtherFunc = tu.functions["my_other_func"] + assertNotNull(myOtherFunc) + assertEquals(1, myOtherFunc.parameters.size) + } + + @Test + fun testOperatorOverload() { + analyzeFile("operator.py") + + with(result) { + val numberType = assertResolvedType("operator.Number") + val strType = assertResolvedType("str") + + // we should have an operator call to __add__ (+) now + var opCall = result.operatorCalls("+").getOrNull(0) + assertNotNull(opCall) + assertEquals(numberType, opCall.type) + + val add = result.operators["__add__"] + assertNotNull(add) + assertInvokes(opCall, add) + + // ... and one to __pos__ (+) + opCall = result.operatorCalls("+").getOrNull(1) + assertNotNull(opCall) + assertEquals(strType, opCall.type) + + val pos = result.operators["__pos__"] + assertNotNull(pos) + assertInvokes(opCall, pos) + } + } + + @Test + fun testAssert() { + analyzeFile("assert.py") + + val func = result.functions["test_assert"] + assertNotNull(func, "Function 'test_assert' should be found") + + val assertStatement = func.body.statements.firstOrNull { it is AssertStatement } + assertIs(assertStatement, "Assert statement should be found") + + val condition = assertStatement.condition + assertNotNull(condition, "Assert statement should have a condition") + assertEquals("1 == 1", condition.code, "The condition is incorrect") + + val message = assertStatement.message + assertIs>(message, "Assert statement should have a message") + assertEquals("Test message", message.value, "The assert message is incorrect") + } + + @Test + fun testDeleteStatements() { + analyzeFile("delete.py") + + val deleteExpressions = result.statements.filterIsInstance() + assertEquals(4, deleteExpressions.size) + + // Test for `del a` + val deleteStmt1 = deleteExpressions[0] + assertEquals(0, deleteStmt1.operands.size) + assertEquals(1, deleteStmt1.additionalProblems.size) + + // Test for `del my_list[2]` + val deleteStmt2 = deleteExpressions[1] + assertEquals(1, deleteStmt2.operands.size) + assertIs(deleteStmt2.operands.firstOrNull()) + assertTrue(deleteStmt2.additionalProblems.isEmpty()) + + // Test for `del my_dict['b']` + val deleteStmt3 = deleteExpressions[2] + assertEquals(1, deleteStmt3.operands.size) + assertIs(deleteStmt3.operands.firstOrNull()) + assertTrue(deleteStmt3.additionalProblems.isEmpty()) + + // Test for `del obj.d` + val deleteStmt4 = deleteExpressions[3] + assertEquals(0, deleteStmt4.operands.size) + assertEquals(1, deleteStmt4.additionalProblems.size) + } + + @Test + fun testTypeHints() { + analyzeFile("type_hints.py") + with(result) { + // type comments + val a = result.refs["a"] + assertNotNull(a) + assertEquals(assertResolvedType("int"), a.type) + + // type annotation + val b = result.refs["b"] + assertNotNull(b) + assertEquals(assertResolvedType("str"), b.type) + } + } +} diff --git a/cpg-language-python/src/test/resources/python/annotations.py b/cpg-language-python/src/test/resources/python/annotations.py index 422340ffde..e22dfea5c9 100644 --- a/cpg-language-python/src/test/resources/python/annotations.py +++ b/cpg-language-python/src/test/resources/python/annotations.py @@ -8,5 +8,15 @@ def collect_data(): return "OK", 200 +@some.otherannotation +def other_func(func): + pass + + +@other_func +def other_other_func(): + pass + + if __name__ == '__main__': app.run(host='0.0.0.0', port=8080, debug=True, threaded=True) diff --git a/cpg-language-python/src/test/resources/python/arguments.py b/cpg-language-python/src/test/resources/python/arguments.py new file mode 100644 index 0000000000..2ed9761ad1 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/arguments.py @@ -0,0 +1,23 @@ +def pos_only_and_args(a, b, /, c): + pass + +def test_varargs(*args): + pass + +def kwd_only_arg(*, arg): + pass + +def kw_defaults(b=1, *, c=2, d, e=3): + pass + +def kw_args(**kwargs): + pass + +def defaults(b=1, c=2, *d, e): + pass + +class MyClass: + def my_method(self=5, d=3, e=1): + pass + def method_with_some_defaults(self, a, b = 1, c = 2): + pass diff --git a/cpg-language-python/src/test/resources/python/assert.py b/cpg-language-python/src/test/resources/python/assert.py new file mode 100644 index 0000000000..8fcf3d766e --- /dev/null +++ b/cpg-language-python/src/test/resources/python/assert.py @@ -0,0 +1,2 @@ +def test_assert(): + assert 1 == 1, "Test message" \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/async.py b/cpg-language-python/src/test/resources/python/async.py new file mode 100644 index 0000000000..5db305451e --- /dev/null +++ b/cpg-language-python/src/test/resources/python/async.py @@ -0,0 +1,11 @@ +import asyncio + + +async def my_func(i: int): + async for obj in generator: + pass + + await asyncio.sleep(i) + +def my_other_func(i: int): + pass diff --git a/cpg-language-python/src/test/resources/python/boolop.py b/cpg-language-python/src/test/resources/python/boolop.py new file mode 100644 index 0000000000..4433b876bc --- /dev/null +++ b/cpg-language-python/src/test/resources/python/boolop.py @@ -0,0 +1,24 @@ +def twoBoolOp(a): + if a and True: + print(a) + return a + +def threeBoolOp(a, b): + if a and True and b: + print(a) + return a + +def nestedBoolOpDifferentOp(a, b): + if a and True or b: + print(a) + return a + +def nestedBoolOpDifferentOp2(a, b): + if a or True and b: + print(a) + return a + +def threeBoolOpNoBool(a): + if a and True and "foo": + print(a) + return a \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/class_type_annotations.py b/cpg-language-python/src/test/resources/python/class_type_annotations.py new file mode 100644 index 0000000000..60f53bcd5d --- /dev/null +++ b/cpg-language-python/src/test/resources/python/class_type_annotations.py @@ -0,0 +1,8 @@ +class Other: + j: int + +class Foo: + i: int + + def from_other(self, other: Other): + self.i = other.j diff --git a/cpg-language-python/src/test/resources/python/complex_class.py b/cpg-language-python/src/test/resources/python/complex_class.py new file mode 100644 index 0000000000..018bd5d720 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/complex_class.py @@ -0,0 +1,8 @@ +class Foo: + x = 1 + s: str + + def bar(self, y: int): + z = self.x + y + self.x = z + return z diff --git a/cpg-language-python/src/test/resources/python/complex_class.pyi b/cpg-language-python/src/test/resources/python/complex_class.pyi new file mode 100644 index 0000000000..d1fd5d5bb2 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/complex_class.pyi @@ -0,0 +1,4 @@ +class Foo: + x: int + s: str + def bar(self: Foo, y: int) -> int: ... diff --git a/cpg-language-python/src/test/resources/python/delete.py b/cpg-language-python/src/test/resources/python/delete.py new file mode 100644 index 0000000000..6181c76681 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/delete.py @@ -0,0 +1,16 @@ +a = 5 +b = a +del a + +my_list = [1, 2, 3] +del my_list[2] + +my_dict = {'a': 1, 'b': 2} +del my_dict['b'] + +class MyClass: + def __init__(self): + self.d = 1 + +obj = MyClass() +del obj.d \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/named_expressions.py b/cpg-language-python/src/test/resources/python/named_expressions.py new file mode 100644 index 0000000000..38171b8f83 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/named_expressions.py @@ -0,0 +1,2 @@ +def named_expression(): + (x := 4) \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/operator.py b/cpg-language-python/src/test/resources/python/operator.py new file mode 100644 index 0000000000..646ec75b29 --- /dev/null +++ b/cpg-language-python/src/test/resources/python/operator.py @@ -0,0 +1,32 @@ +from __future__ import annotations + + +class Number: + i: int + + def __init__(self, i): + self.i = i + + def __add__(self, other: Number) -> Number: + return Number(self.i + other.i) + + def __pos__(self) -> str: + return "python is quite crazy" + + +def main(): + a = Number(5) + b = Number(10) + c = a + b + print(c) + + d = +b + print(d) + + # the following unfortunately does not work yet because the types of a and b does not seem to propagate to c + # currently yet. + d = +c + + +if __name__ == "__main__": + main() diff --git a/cpg-language-python/src/test/resources/python/try.py b/cpg-language-python/src/test/resources/python/try.py new file mode 100644 index 0000000000..dbf91aa85e --- /dev/null +++ b/cpg-language-python/src/test/resources/python/try.py @@ -0,0 +1,25 @@ +def tryAll(a): + try: + b = a+2 + except Exception1: + print("There was an occurrence of Exception1") + except OtherException as e: + print("We saw exception" + e) + except: + print("Catch all!") + else: + print("All good, got " + b) + finally: + print("It's over") + +def tryOnlyFinally(a): + try: + b = a+2 + finally: + print("It's over") + +def tryOnlyExcept(a): + try: + b = a+2 + except: + print("Fail") \ No newline at end of file diff --git a/cpg-language-python/src/test/resources/python/type_hints.py b/cpg-language-python/src/test/resources/python/type_hints.py new file mode 100644 index 0000000000..49032d1caa --- /dev/null +++ b/cpg-language-python/src/test/resources/python/type_hints.py @@ -0,0 +1,3 @@ +a = 1 #type: int + +b: str \ No newline at end of file diff --git a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/ExpressionHandler.kt b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/ExpressionHandler.kt index 7c85bbc6d5..c9089dd6ee 100644 --- a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/ExpressionHandler.kt +++ b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/ExpressionHandler.kt @@ -94,8 +94,8 @@ class ExpressionHandler(lang: RubyLanguageFrontend) : val assign = newAssignExpression("=") // we need to build a reference out of the assignment node itself for our LHS - assign.lhs = listOf(handleINameNode(node)) - assign.rhs = listOf(this.handle(node.valueNode)) + assign.lhs = mutableListOf(handleINameNode(node)) + assign.rhs = mutableListOf(this.handle(node.valueNode)) return assign } @@ -104,8 +104,8 @@ class ExpressionHandler(lang: RubyLanguageFrontend) : val assign = newAssignExpression("=") // we need to build a reference out of the assignment node itself for our LHS - assign.lhs = listOf(handleINameNode(node)) - assign.rhs = listOf(this.handle(node.valueNode)) + assign.lhs = mutableListOf(handleINameNode(node)) + assign.rhs = mutableListOf(this.handle(node.valueNode)) return assign } diff --git a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/RubyLanguage.kt b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/RubyLanguage.kt index 93969276d9..a6d14d4be5 100644 --- a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/RubyLanguage.kt +++ b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/RubyLanguage.kt @@ -75,8 +75,8 @@ class RubyLanguage() : "^=" // Bitwise XOR assignment ) - override fun handleSuperCall( - callee: MemberExpression, + override fun handleSuperExpression( + memberExpression: MemberExpression, curClass: RecordDeclaration, scopeManager: ScopeManager ): Boolean { diff --git a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/StatementHandler.kt b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/StatementHandler.kt index be81d708f8..fd67899cd2 100644 --- a/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/StatementHandler.kt +++ b/cpg-language-ruby/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ruby/StatementHandler.kt @@ -52,7 +52,7 @@ class StatementHandler(lang: RubyLanguageFrontend) : val compoundStatement = newBlock() for (node in blockNode.filterNotNull()) { - compoundStatement.addStatement(handle(node)) + compoundStatement.statements += handle(node) } return compoundStatement diff --git a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/DeclarationHandler.kt b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/DeclarationHandler.kt index d60bb9aac7..19f5e446d7 100644 --- a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/DeclarationHandler.kt +++ b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/DeclarationHandler.kt @@ -116,7 +116,7 @@ class DeclarationHandler(lang: TypeScriptLanguageFrontend) : if (childNode.type.endsWith("Statement")) { val statement = this.frontend.statementHandler.handle(childNode) - statement?.let { tu.addStatement(it) } + statement?.let { tu.statements += it } } else { val decl = this.handle(childNode) diff --git a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt index 54adeb603d..635bc1e739 100644 --- a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt +++ b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt @@ -62,8 +62,10 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : } private fun handleJsxAttribute(node: TypeScriptNode): KeyValueExpression { - val key = node.children?.first()?.let { this.handle(it) } - val value = node.children?.last()?.let { this.handle(it) } + val key = + node.children?.first()?.let { this.handle(it) } ?: newProblemExpression("missing key") + val value = + node.children?.last()?.let { this.handle(it) } ?: newProblemExpression("missing value") return newKeyValueExpression(key, value, rawNode = node) } @@ -93,8 +95,11 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : // and a container named JsxAttributes, with JsxAttribute nodes tag.expressions = - node.firstChild("JsxAttributes")?.children?.mapNotNull { this.handle(it) } - ?: emptyList() + node + .firstChild("JsxAttributes") + ?.children + ?.mapNotNull { this.handle(it) } + ?.toMutableList() ?: mutableListOf() return tag } @@ -102,7 +107,8 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : private fun handeJsxElement(node: TypeScriptNode): ExpressionList { val jsx = newExpressionList(rawNode = node) - jsx.expressions = node.children?.mapNotNull { this.handle(it) } ?: emptyList() + jsx.expressions = + node.children?.mapNotNull { this.handle(it) }?.toMutableList() ?: mutableListOf() return jsx } @@ -136,8 +142,10 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : } private fun handlePropertyAssignment(node: TypeScriptNode): KeyValueExpression { - val key = node.children?.first()?.let { this.handle(it) } - val value = node.children?.last()?.let { this.handle(it) } + val key = + node.children?.first()?.let { this.handle(it) } ?: newProblemExpression("missing key") + val value = + node.children?.last()?.let { this.handle(it) } ?: newProblemExpression("missing value") return newKeyValueExpression(key, value, rawNode = node) } @@ -145,7 +153,8 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : private fun handleObjectLiteralExpression(node: TypeScriptNode): InitializerListExpression { val ile = newInitializerListExpression(unknownType(), rawNode = node) - ile.initializers = node.children?.mapNotNull { this.handle(it) } ?: emptyList() + ile.initializers = + node.children?.mapNotNull { this.handle(it) }?.toMutableList() ?: mutableListOf() return ile } diff --git a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/StatementHandler.kt b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/StatementHandler.kt index 6b56d7acb3..eb87e75654 100644 --- a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/StatementHandler.kt +++ b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/StatementHandler.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import kotlin.collections.plusAssign class StatementHandler(lang: TypeScriptLanguageFrontend) : Handler(::ProblemExpression, lang) { @@ -63,7 +64,7 @@ class StatementHandler(lang: TypeScriptLanguageFrontend) : val decl = this.frontend.declarationHandler.handle(node) if (decl != null) { - statement.addToPropertyEdgeDeclaration(decl) + statement.declarationEdges += decl } this.frontend.scopeManager.addDeclaration(decl) @@ -84,7 +85,7 @@ class StatementHandler(lang: TypeScriptLanguageFrontend) : private fun handleBlock(node: TypeScriptNode): Block { val block = newBlock(rawNode = node) - node.children?.forEach { this.handle(it)?.let { it1 -> block.addStatement(it1) } } + node.children?.forEach { this.handle(it)?.let { it1 -> block.statements += it1 } } return block } @@ -107,7 +108,7 @@ class StatementHandler(lang: TypeScriptLanguageFrontend) : val decl = this.frontend.declarationHandler.handle(variableNode) if (decl != null) { - statement.addToPropertyEdgeDeclaration(decl) + statement.declarationEdges += decl } this.frontend.scopeManager.addDeclaration(decl) diff --git a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/TypeScriptLanguageFrontend.kt b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/TypeScriptLanguageFrontend.kt index ba528f6ba2..17a42895c0 100644 --- a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/TypeScriptLanguageFrontend.kt +++ b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/TypeScriptLanguageFrontend.kt @@ -209,7 +209,7 @@ class TypeScriptLanguageFrontend( astNode.children ?.filter { it.type == "Decorator" } ?.map { handleDecorator(it) } - ?.let { node.addAnnotations(it) } + ?.let { node.annotations += it } } private fun handleDecorator(node: TypeScriptNode): Annotation { diff --git a/cpg-language-typescript/src/main/nodejs/package-lock.json b/cpg-language-typescript/src/main/nodejs/package-lock.json index a7d450b7d9..0e38314b0c 100644 --- a/cpg-language-typescript/src/main/nodejs/package-lock.json +++ b/cpg-language-typescript/src/main/nodejs/package-lock.json @@ -7,64 +7,36 @@ "license": "Apache-2.0", "dependencies": { "@types/node": "^20.0.0", - "typescript": "5.5.2" + "typescript": "5.6.2" }, "devDependencies": { - "@rollup/plugin-commonjs": "^26.0.0", + "@rollup/plugin-commonjs": "^28.0.0", "@rollup/plugin-node-resolve": "^15.1.0", - "@rollup/plugin-typescript": "^11.1.2", + "@rollup/plugin-typescript": "^12.0.0", "rollup": "^4.0.0", "tslib": "^2.6.0" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", "dev": true }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "dev": true, - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@rollup/plugin-commonjs": { - "version": "26.0.1", - "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-26.0.1.tgz", - "integrity": "sha512-UnsKoZK6/aGIH6AdkptXhNvhaqftcjq3zZdT+LY5Ftms6JR06nADcDsYp5hTU9E2lbJUEOhdlY5J4DNTneM+jQ==", + "version": "28.0.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-28.0.0.tgz", + "integrity": "sha512-BJcu+a+Mpq476DMXG+hevgPSl56bkUoi88dKT8t3RyUp8kGuOh+2bU8Gs7zXDlu+fyZggnJ+iOBGrb/O1SorYg==", "dev": true, "license": "MIT", "dependencies": { "@rollup/pluginutils": "^5.0.1", "commondir": "^1.0.1", "estree-walker": "^2.0.2", - "glob": "^10.4.1", + "fdir": "^6.1.1", "is-reference": "1.2.1", - "magic-string": "^0.30.3" + "magic-string": "^0.30.3", + "picomatch": "^2.3.1" }, "engines": { "node": ">=16.0.0 || 14 >= 14.17" @@ -78,16 +50,29 @@ } } }, + "node_modules/@rollup/plugin-commonjs/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/@rollup/plugin-node-resolve": { - "version": "15.2.3", - "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.2.3.tgz", - "integrity": "sha512-j/lym8nf5E21LwBT4Df1VD6hRO2L2iwUeUmP7litikRsVp1H6NWx20NEp0Y7su+7XGc476GnXXc4kFeZNGmaSQ==", + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.3.0.tgz", + "integrity": "sha512-9eO5McEICxMzJpDW9OnMYSv4Sta3hmt7VtBFz5zR9273suNOydOyq/FrGeGy+KsTRFm8w0SLVhzig2ILFT63Ag==", "dev": true, + "license": "MIT", "dependencies": { "@rollup/pluginutils": "^5.0.1", "@types/resolve": "1.20.2", "deepmerge": "^4.2.2", - "is-builtin-module": "^3.2.1", "is-module": "^1.0.0", "resolve": "^1.22.1" }, @@ -104,12 +89,13 @@ } }, "node_modules/@rollup/plugin-typescript": { - "version": "11.1.5", - "resolved": "https://registry.npmjs.org/@rollup/plugin-typescript/-/plugin-typescript-11.1.5.tgz", - "integrity": "sha512-rnMHrGBB0IUEv69Q8/JGRD/n4/n6b3nfpufUu26axhUcboUzv/twfZU8fIBbTOphRAe0v8EyxzeDpKXqGHfyDA==", + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-typescript/-/plugin-typescript-12.1.0.tgz", + "integrity": "sha512-Kzs8KGJofe7cfTRODsnG1jNGxSvU8gVoNNd7Z/QaY25AYwe2LSSUpx/kPxqF38NYkpR8de3m51r9uwJpDlz6dg==", "dev": true, + "license": "MIT", "dependencies": { - "@rollup/pluginutils": "^5.0.1", + "@rollup/pluginutils": "^5.1.0", "resolve": "^1.22.1" }, "engines": { @@ -130,10 +116,11 @@ } }, "node_modules/@rollup/pluginutils": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.5.tgz", - "integrity": "sha512-6aEYR910NyP73oHiJglti74iRyOwgFU4x3meH/H8OJx6Ry0j6cOVZ5X/wTvub7G7Ao6qaHBEaNsV3GLJkSsF+Q==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.1.tgz", + "integrity": "sha512-bVRmQqBIyGD+VMihdEV2IBurfIrdW9tD9yzJUL3CBRDbyPBVzQnBSMSgyUZHl1E335rpMRj7r4o683fXLYw8iw==", "dev": true, + "license": "MIT", "dependencies": { "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", @@ -151,10 +138,23 @@ } } }, + "node_modules/@rollup/pluginutils/node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz", - "integrity": "sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.23.0.tgz", + "integrity": "sha512-8OR+Ok3SGEMsAZispLx8jruuXw0HVF16k+ub2eNXKHDmdxL4cf9NlNpAzhlOhNyXzKDEJuFeq0nZm+XlNb1IFw==", "cpu": [ "arm" ], @@ -166,9 +166,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz", - "integrity": "sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.23.0.tgz", + "integrity": "sha512-rEFtX1nP8gqmLmPZsXRMoLVNB5JBwOzIAk/XAcEPuKrPa2nPJ+DuGGpfQUR0XjRm8KjHfTZLpWbKXkA5BoFL3w==", "cpu": [ "arm64" ], @@ -180,9 +180,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz", - "integrity": "sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.23.0.tgz", + "integrity": "sha512-ZbqlMkJRMMPeapfaU4drYHns7Q5MIxjM/QeOO62qQZGPh9XWziap+NF9fsqPHT0KzEL6HaPspC7sOwpgyA3J9g==", "cpu": [ "arm64" ], @@ -194,9 +194,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz", - "integrity": "sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.23.0.tgz", + "integrity": "sha512-PfmgQp78xx5rBCgn2oYPQ1rQTtOaQCna0kRaBlc5w7RlA3TDGGo7m3XaptgitUZ54US9915i7KeVPHoy3/W8tA==", "cpu": [ "x64" ], @@ -208,9 +208,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz", - "integrity": "sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.23.0.tgz", + "integrity": "sha512-WAeZfAAPus56eQgBioezXRRzArAjWJGjNo/M+BHZygUcs9EePIuGI1Wfc6U/Ki+tMW17FFGvhCfYnfcKPh18SA==", "cpu": [ "arm" ], @@ -222,9 +222,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz", - "integrity": "sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.23.0.tgz", + "integrity": "sha512-v7PGcp1O5XKZxKX8phTXtmJDVpE20Ub1eF6w9iMmI3qrrPak6yR9/5eeq7ziLMrMTjppkkskXyxnmm00HdtXjA==", "cpu": [ "arm" ], @@ -236,9 +236,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz", - "integrity": "sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.23.0.tgz", + "integrity": "sha512-nAbWsDZ9UkU6xQiXEyXBNHAKbzSAi95H3gTStJq9UGiS1v+YVXwRHcQOQEF/3CHuhX5BVhShKoeOf6Q/1M+Zhg==", "cpu": [ "arm64" ], @@ -250,9 +250,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz", - "integrity": "sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.23.0.tgz", + "integrity": "sha512-5QT/Di5FbGNPaVw8hHO1wETunwkPuZBIu6W+5GNArlKHD9fkMHy7vS8zGHJk38oObXfWdsuLMogD4sBySLJ54g==", "cpu": [ "arm64" ], @@ -264,9 +264,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz", - "integrity": "sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.23.0.tgz", + "integrity": "sha512-Sefl6vPyn5axzCsO13r1sHLcmPuiSOrKIImnq34CBurntcJ+lkQgAaTt/9JkgGmaZJ+OkaHmAJl4Bfd0DmdtOQ==", "cpu": [ "ppc64" ], @@ -278,9 +278,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz", - "integrity": "sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.23.0.tgz", + "integrity": "sha512-o4QI2KU/QbP7ZExMse6ULotdV3oJUYMrdx3rBZCgUF3ur3gJPfe8Fuasn6tia16c5kZBBw0aTmaUygad6VB/hQ==", "cpu": [ "riscv64" ], @@ -292,9 +292,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz", - "integrity": "sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.23.0.tgz", + "integrity": "sha512-+bxqx+V/D4FGrpXzPGKp/SEZIZ8cIW3K7wOtcJAoCrmXvzRtmdUhYNbgd+RztLzfDEfA2WtKj5F4tcbNPuqgeg==", "cpu": [ "s390x" ], @@ -306,9 +306,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz", - "integrity": "sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.23.0.tgz", + "integrity": "sha512-I/eXsdVoCKtSgK9OwyQKPAfricWKUMNCwJKtatRYMmDo5N859tbO3UsBw5kT3dU1n6ZcM1JDzPRSGhAUkxfLxw==", "cpu": [ "x64" ], @@ -320,9 +320,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz", - "integrity": "sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.23.0.tgz", + "integrity": "sha512-4ZoDZy5ShLbbe1KPSafbFh1vbl0asTVfkABC7eWqIs01+66ncM82YJxV2VtV3YVJTqq2P8HMx3DCoRSWB/N3rw==", "cpu": [ "x64" ], @@ -334,9 +334,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz", - "integrity": "sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.23.0.tgz", + "integrity": "sha512-+5Ky8dhft4STaOEbZu3/NU4QIyYssKO+r1cD3FzuusA0vO5gso15on7qGzKdNXnc1gOrsgCqZjRw1w+zL4y4hQ==", "cpu": [ "arm64" ], @@ -348,9 +348,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz", - "integrity": "sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.23.0.tgz", + "integrity": "sha512-0SPJk4cPZQhq9qA1UhIRumSE3+JJIBBjtlGl5PNC///BoaByckNZd53rOYD0glpTkYFBQSt7AkMeLVPfx65+BQ==", "cpu": [ "ia32" ], @@ -362,9 +362,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz", - "integrity": "sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.23.0.tgz", + "integrity": "sha512-lqCK5GQC8fNo0+JvTSxcG7YB1UKYp8yrNLhsArlvPWN+16ovSZgoehlVHg6X0sSWPUkpjRBR5TuR12ZugowZ4g==", "cpu": [ "x64" ], @@ -376,18 +376,19 @@ ] }, "node_modules/@types/estree": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", - "dev": true + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "dev": true, + "license": "MIT" }, "node_modules/@types/node": { - "version": "20.14.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.0.tgz", - "integrity": "sha512-5cHBxFGJx6L4s56Bubp4fglrEpmyJypsqI6RgzMfBHWUJQGWAAi8cWcgetEbZXHYXo9C2Fa4EEds/uSyS4cxmA==", + "version": "20.16.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.16.0.tgz", + "integrity": "sha512-vDxceJcoZhIVh67S568bm1UGZO0DX0hpplJZxzeXMKwIPLn190ec5RRxQ69BKhX44SUGIxxgMdDY557lGLKprQ==", "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.19.2" } }, "node_modules/@types/resolve": { @@ -396,102 +397,12 @@ "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", "dev": true }, - "node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/builtin-modules": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", - "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", - "dev": true, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", @@ -501,41 +412,25 @@ "node": ">=0.10.0" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, - "license": "MIT" - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, - "license": "MIT" - }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "dev": true }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "node_modules/fdir": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.3.0.tgz", + "integrity": "sha512-QOnuT+BOtivR77wYvCWHfGt9s4Pz1VIMbD463vegT5MLqNXy8rYFT/lPVEqf/bhYeT6qmqrNHhsX+rWwe3rOCQ==", "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } } }, "node_modules/fsevents": { @@ -561,29 +456,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/glob": { - "version": "10.4.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", - "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/hasown": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", @@ -596,21 +468,6 @@ "node": ">= 0.4" } }, - "node_modules/is-builtin-module": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", - "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", - "dev": true, - "dependencies": { - "builtin-modules": "^3.3.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -623,16 +480,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", @@ -648,42 +495,6 @@ "@types/estree": "*" } }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/jackspeak": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.0.tgz", - "integrity": "sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "dev": true, - "license": "ISC", - "engines": { - "node": "14 || >=16.14" - } - }, "node_modules/magic-string": { "version": "0.30.5", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", @@ -696,72 +507,22 @@ "node": ">=12" } }, - "node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", "dev": true, + "license": "MIT", + "optional": true, + "peer": true, "engines": { - "node": ">=8.6" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" @@ -785,13 +546,13 @@ } }, "node_modules/rollup": { - "version": "4.18.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.18.0.tgz", - "integrity": "sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.23.0.tgz", + "integrity": "sha512-vXB4IT9/KLDrS2WRXmY22sVB2wTsTwkpxjB8Q3mnakTENcYw3FRmfdYDy/acNmls+lHmDazgrRjK/yQ6hQAtwA==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.5" + "@types/estree": "1.0.6" }, "bin": { "rollup": "dist/bin/rollup" @@ -801,165 +562,25 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.18.0", - "@rollup/rollup-android-arm64": "4.18.0", - "@rollup/rollup-darwin-arm64": "4.18.0", - "@rollup/rollup-darwin-x64": "4.18.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.18.0", - "@rollup/rollup-linux-arm-musleabihf": "4.18.0", - "@rollup/rollup-linux-arm64-gnu": "4.18.0", - "@rollup/rollup-linux-arm64-musl": "4.18.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.18.0", - "@rollup/rollup-linux-riscv64-gnu": "4.18.0", - "@rollup/rollup-linux-s390x-gnu": "4.18.0", - "@rollup/rollup-linux-x64-gnu": "4.18.0", - "@rollup/rollup-linux-x64-musl": "4.18.0", - "@rollup/rollup-win32-arm64-msvc": "4.18.0", - "@rollup/rollup-win32-ia32-msvc": "4.18.0", - "@rollup/rollup-win32-x64-msvc": "4.18.0", + "@rollup/rollup-android-arm-eabi": "4.23.0", + "@rollup/rollup-android-arm64": "4.23.0", + "@rollup/rollup-darwin-arm64": "4.23.0", + "@rollup/rollup-darwin-x64": "4.23.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.23.0", + "@rollup/rollup-linux-arm-musleabihf": "4.23.0", + "@rollup/rollup-linux-arm64-gnu": "4.23.0", + "@rollup/rollup-linux-arm64-musl": "4.23.0", + "@rollup/rollup-linux-powerpc64le-gnu": "4.23.0", + "@rollup/rollup-linux-riscv64-gnu": "4.23.0", + "@rollup/rollup-linux-s390x-gnu": "4.23.0", + "@rollup/rollup-linux-x64-gnu": "4.23.0", + "@rollup/rollup-linux-x64-musl": "4.23.0", + "@rollup/rollup-win32-arm64-msvc": "4.23.0", + "@rollup/rollup-win32-ia32-msvc": "4.23.0", + "@rollup/rollup-win32-x64-msvc": "4.23.0", "fsevents": "~2.3.2" } }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", @@ -973,15 +594,16 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "dev": true + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.7.0.tgz", + "integrity": "sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==", + "dev": true, + "license": "0BSD" }, "node_modules/typescript": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz", - "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -992,123 +614,10 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, + "version": "6.19.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.6.tgz", + "integrity": "sha512-e/vggGopEfTKSvj4ihnOLTsqhrKRN3LeO6qSN/GxohhuRv8qH9bNQ4B8W7e/vFL+0XTnmHPB4/kegunZGA4Org==", "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } } } } diff --git a/cpg-language-typescript/src/main/nodejs/package.json b/cpg-language-typescript/src/main/nodejs/package.json index 00c91e0f82..5edee969eb 100644 --- a/cpg-language-typescript/src/main/nodejs/package.json +++ b/cpg-language-typescript/src/main/nodejs/package.json @@ -6,13 +6,13 @@ }, "dependencies": { "@types/node": "^20.0.0", - "typescript": "5.5.2" + "typescript": "5.6.2" }, "license": "Apache-2.0", "devDependencies": { - "@rollup/plugin-commonjs": "^26.0.0", + "@rollup/plugin-commonjs": "^28.0.0", "@rollup/plugin-node-resolve": "^15.1.0", - "@rollup/plugin-typescript": "^11.1.2", + "@rollup/plugin-typescript": "^12.0.0", "rollup": "^4.0.0", "tslib": "^2.6.0" } diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index 270efaa6a6..3063b5e33c 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -522,6 +522,8 @@ class Application : Callable { if (!noDefaultPasses) { translationConfiguration.defaultPasses() + translationConfiguration.registerPass() + translationConfiguration.registerPass() } if (customPasses != "DEFAULT") { val pieces = customPasses.split(",") diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index fbe9cddb86..bf40cdb2a8 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -28,8 +28,8 @@ package de.fraunhofer.aisec.cpg_vis_neo4j import com.fasterxml.jackson.databind.ObjectMapper import de.fraunhofer.aisec.cpg.TranslationManager import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import java.io.File import java.nio.file.Files @@ -62,6 +62,7 @@ class ApplicationTest { @Test fun testSerializeCpgViaOGM() { val (application, translationResult) = createTranslationResult() + // 22 inferred functions, 1 inferred method, 2 inferred constructors, 11 regular functions assertEquals(36, translationResult.functions.size) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index efedc540d8..96f477d58b 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg_vis_neo4j import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.builder.translationResult -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.functions import kotlin.test.Test @@ -47,23 +46,6 @@ class Neo4JTest { assertEquals(36, translationResult.functions.size) application.pushToNeo4j(translationResult) - - val sessionAndSessionFactoryPair = application.connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - val functions = session.loadAll(FunctionDeclaration::class.java) - assertNotNull(functions) - - // 22 inferred functions, 1 inferred method, 2 inferred constructors, 11 regular - // functions - assertEquals(36, functions.size) - - transaction.commit() - } - - session.clear() - sessionAndSessionFactoryPair.second.close() } @Test diff --git a/docs/docs/CPG/impl/index.md b/docs/docs/CPG/impl/index.md index 9359f148d5..dc0b49f7db 100755 --- a/docs/docs/CPG/impl/index.md +++ b/docs/docs/CPG/impl/index.md @@ -24,3 +24,4 @@ the graph. These two stages are strictly separated one from each other. * [Languages and Language Frontends](./language) * [Scopes](./scopes) * [Passes](./passes) +* [Symbol Resolution](./symbol-resolver.md) diff --git a/docs/docs/CPG/impl/scopes.md b/docs/docs/CPG/impl/scopes.md index 9fafc1ea83..c5ab985104 100755 --- a/docs/docs/CPG/impl/scopes.md +++ b/docs/docs/CPG/impl/scopes.md @@ -1,6 +1,6 @@ --- -title: "Implementation and Concepts - Scopes" -linkTitle: "Implementation and Concepts - Scopes" +title: "Implementation and Concepts - Scopes and Symbols" +linkTitle: "Implementation and Concepts - Scopes and Symbols" weight: 20 no_list: false menu: @@ -11,5 +11,96 @@ description: > --- -# Implementation and Concepts: Scopes and Scope Manger +# Scopes and Symbols +The concept of scopes and symbols are at the heart of every programming language and thus are also the core of static analysis. Both concepts consist in the CPG library through the types `Scope` and `Symbol` respectively. + +A "symbol" can be seen as an identifier in most programming languages, referring to variables or functions. Symbols are often grouped in scopes, which defines the visibility of a symbol, e.g. a slice of a program that can "see" the symbol. Often this is also synonymous with the life-time of a variable, e.g., that its memory will be freed (or collected by a garbage collector) once it goes "out of scope". + +```c +// This defines a symbol "a" in the global/file scope. +// Its visibility is global within the file. +int a = 1; + +int main() { + // this defines another symbol "a" in a function/block scope. + // Its visibility is limited to the block it is defined in. + int a = 1; +} +``` + +Usually symbols declared in a local scope override the declaration of a symbol in a higher (e.g., global scope), which is also referred to as "shadowing". This needs to be taken into account when resolving symbols to their declarations. + +The `Scope` class holds all its symbols in the `Scope::symbols` property. More specifically, this property is a `SymbolMap`, which is a type alias to a map, whose key type is a `Symbol` and whose value type is a list of `Declaration` nodes. This is basically a symbol lookup table for all symbols in its scope. It is a map of a list because some programming languages have concepts like function overloading, which leads to the declaration of multiple `FunctionDeclaration` nodes under the same symbol in one scope. In the current implementation, a `Symbol` is just a typealias for a string, and it is always "local" to the scope, meaning that it MUST NOT contain any qualifier. If you want to refer to a fully qualified identifier, a `Name` must be used. In the future, we might consider merging the concepts of `Symbol` and `Name`. + +For a frontend or pass developer, the main interaction point with scopes and symbols is through the `ScopeManager`. The scope manager is available to all nodes via the `TranslationContext` and also injected in frontend, handlers and passes. + +## Hierarchy of Scopes + +Each scope (except the `GlobalScope`) can have a parent and possible child scopes. This can be used to model a hierarchy of scopes within a program. For example using the snippet above, the following scopes are defined in the CPG: + +* A `GlobalScope` that comprises the whole file +* A `FunctionScope` that comprises the function `main` +* A `BlockScope` that comprises the function body + +Note, that each programming language is different when it comes to scoping and this needs to be thought of by a frontend developer. For example in C/C++ each block introduced by `{}` introduces a new scope and variables can be declared only for such a block, meaning that each `for`, `if` and other statements also introduce a new scope. In contrast, Python only differentiates between a global scope, function and class scope. + +## Defining Scopes and Declaring Symbols + +In order to define new scopes, the `ScopeManager` offers two main APIs: + +* `enterScope(node)`, which specifies that `node` will declare a new scope and that an appropriate `Scope` (or derived type) will be created +* `leaveScope(node)`, which closes the scope again + +It is important that every opened scope must also be closed again. When scopes are nested, they also need to be closed in reverse order. + +```Kotlin +// We are inside the global scope here and want to create a new function +var func = newFunctionDeclaration("main") + +// Create a function scope +scopeManager.enterScope(func) + +// Create a block scope for the body because our language works this way +var body = newBlock() +func.body = body +scopeManager.enterScope(body) + +// Add statements here +body.statements += /* ... */ + +// Leave block scope +scopeManager.leaveScope(body) + +// Back to global scope, add the function to global scope +scopeManager.leaveScope(func) +scopeManager.addDeclaration(func) +``` + +Inside the scope, declarations can be added with `ScopeManager::addDeclaration`. This takes care of adding the declaration to an appropriate place in the AST (which beyond the scope of this document) and also adds the `Declaration` to the `Scope` under the appropriate `Symbol`. + + +## Looking up Symbols + +During different analysis steps, e.g., in different passes, we want to find certain symbols or lookup the declaration(s) belonging to a particular symbol. There are two functions in order to do so - a "higher" level concept in the `ScopeManager` and a "lower" level function on the `Scope` itself. + +The lower level one is called `Scope::lookupSymbol` and can be used to retrieve a list of `Declaration` nodes that belong to a particular `Symbol` that is "visible" the scope. It does so by first looking through its own `Scope::symbols`. If no match was found, the scope is traversed upwards to its `Scope::parent`, until a match is found. Furthermore, additional logic is needed to resolve symbol that are pointing to another scope, e.g., because they represent an `ImportDeclaration`. + +```Kotlin +var scope = /* ... */ +var declarations = scope.lookupSymbol("a") { + // Some additional predicate if we want +} +``` + +Additionally, the lookup can be fine-tuned by an additional predicate. However, this should be used carefully as it restricts the possible list of symbols very early. In most cases the list of symbols should be quite exhaustive at first to find all possible candidates and then selecting the best candidate in a second step (e.g., based on argument types for a function call). + +While the aforementioned API works great if we already have a specific start scope and local `Symbol`, we often start our resolution process with a `Name` -- which could potentially be qualified, such as `std::string`. Therefore, the "higher level" function `ScopeManager::lookupSymbolByName` can be used to retrieve a list of candidate declarations by a given `Name`. In a first step, the name is checked for a potential scope qualifier (`std` in this example). If present, it is extracted and the search scope is set to it. This is what is usually referred to as a "qualified lookup". Otherwise, the local part of the name is used to start the lookup, in what is called an "unqualified lookup". In both cases, the actual lookup is delegated to `ScopeManager::lookupSymbols`, but with different parameters. + +```Kotlin +var name = parseName("std::string") +// This will return all the 'string' symbols within the 'std' name scope +var stringSymbols = scopeManager.lookupSymbolByName(name) +``` + +Developers should avoid symbol lookup during frontend parsing, since often during parsing, only a limited view of all symbols is available. Instead, a dedicated pass that is run on the complete translation result is the preferred option. Apart from that, the main usage of this API is in the [SymbolResolver](symbol-resolver.md). \ No newline at end of file diff --git a/docs/docs/CPG/impl/symbol-resolver.md b/docs/docs/CPG/impl/symbol-resolver.md new file mode 100644 index 0000000000..c052e6b69f --- /dev/null +++ b/docs/docs/CPG/impl/symbol-resolver.md @@ -0,0 +1,38 @@ +--- +title: "Implementation and Concepts - Symbol Resolution" +linkTitle: "Implementation and Concepts - Symbol Resolution" +weight: 20 +no_list: false +menu: + main: + weight: 20 +description: > + The CPG library is a language-agnostic graph representation of source code. +--- + + +# Symbol Resolution + +This pages describes the main functionality behind symbol resolution in the CPG library. This is mostly done by the `SymbolResolver` pass, in combination with the symbol lookup API (see [Scopes and Symbols](scopes.md#looking-up-symbols)). In addition to the *lookup* of a symbol, the *resolution* takes the input of the lookup and provides a "definite" decision which symbol is used. This mostly referred to symbols / names used in a `Reference` or a `CallExpression` (which also has a reference as its `CallExpression::callee`). + +## The `SymbolResolver` Pass + +The `SymbolResolver` pass takes care of the heavy lifting of symbol (or rather reference) resolving: + +* It sets the `Reference::refersTo` property, +* and sets the `CallExpression::invokes` property, +* and finally takes cares of operator overloading (if the language supports it). + +In a way, it can be compared to a linker step in a compiler. The pass operates on a single `Component` and starts by identifying EOG starter nodes within the component. These node "start" an EOG sub-graph, i.e., they do not have any previous EOG edges. The symbol resolver uses the `ScopedWalker` with a special set-up that traverses the EOG starting with each EOG starter node until it reaches the end. This ensures that symbols are resolved in the correct order of "evaluation", e.g., that a base of a member expression is resolved before the expression itself. This ensures that necessary type information on the base are available in order to resolve appropriate fields of the member expression. + +The symbol resolver itself has gone through many re-writes over the years and there is still some code left that we consider *legacy*. These functions are marked as such, and we aim to remove them slowly. + +## Resolving References + +The main functionality lies in `ScopeManager::handleReference`. For all `Reference` nodes (that are not `MemberExpression` nodes) we use the symbol lookup API to find declaration candidates for the name the reference is referring to. This candidate list is then stored in `Reference::candidates`. If the reference is the `CallExpression::callee` property of a call, we abort here and jump to [Resolve Calls](#resolve-calls). + +Otherwise, we currently take the first entry of the candidate list and set the `Reference::refersTo` property to it. + +## Resolve Calls + +Prequisite: The `CallExpression::callee` reference must have been resolved (see [Resolving References](#resolving-references)). \ No newline at end of file diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index b6a535732d..e73c12e752 100755 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -167,8 +167,9 @@ nav: - "Implementation": - CPG/impl/index.md - "Language Frontends": CPG/impl/language.md - - "Scopes": CPG/impl/scopes.md + - "Scopes and Symbols": CPG/impl/scopes.md - "Passes": CPG/impl/passes.md + - "Symbol Resolution": CPG/impl/symbol-resolver.md - "Contributing": - "Contributing to the CPG library": Contributing/index.md # This assumes that the most recent dokka build was generated with the "main" tag! diff --git a/gradle.properties.example b/gradle.properties.example index c9e3c651dc..f956f34730 100644 --- a/gradle.properties.example +++ b/gradle.properties.example @@ -7,4 +7,5 @@ enableGoFrontend=true enablePythonFrontend=true enableLLVMFrontend=true enableTypeScriptFrontend=true -enableRubyFrontend=true \ No newline at end of file +enableRubyFrontend=true +enableJVMFrontend=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 82f837c64f..432afe9e87 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,11 +1,11 @@ [versions] -kotlin = "2.0.0" +kotlin = "2.0.20" kotlin19 = "1.9.10" neo4j = "4.0.10" -log4j = "2.23.0" -sonarqube = "5.1.0.4882" +log4j = "2.24.0" spotless = "6.25.0" nexus-publish = "2.0.0" +sootup = "1.3.0" [libraries] kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin"} @@ -15,20 +15,20 @@ kotlin-scripting-jvm = { module = "org.jetbrains.kotlin:kotlin-scripting-jvm", v kotlin-scripting-dependencies = { module = "org.jetbrains.kotlin:kotlin-scripting-dependencies", version.ref = "kotlin19"} kotlin-scripting-dependencies-maven-all = { module = "org.jetbrains.kotlin:kotlin-scripting-dependencies-maven-all", version.ref = "kotlin19"} kotlin-scripting-ide-services = { module = "org.jetbrains.kotlin:kotlin-scripting-ide-services", version.ref = "kotlin19"} -kotlin-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.8.0"} +kotlin-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.9.0"} kotlin-ki-shell = { module = "com.github.Kotlin:kotlin-interactive-shell", version = "5b1ff4d821"} kotlin-test-junit5 = { module = "org.jetbrains.kotlin:kotlin-test-junit5", version.ref = "kotlin"} # this is only needed for the testFixtures in cpg-core, everywhere else kotlin("test") is used log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" } -apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.14.0"} +apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0"} neo4j-ogm-core = { module = "org.neo4j:neo4j-ogm-core", version.ref = "neo4j"} neo4j-ogm-bolt-driver = { module = "org.neo4j:neo4j-ogm-bolt-driver", version.ref = "neo4j"} javaparser = { module = "com.github.javaparser:javaparser-symbol-solver-core", version = "3.26.0"} -jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.17.0"} -jacksonyml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version = "2.17.0"} +jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.18.0"} +jacksonyml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version = "2.18.0"} eclipse-runtime = { module = "org.eclipse.platform:org.eclipse.core.runtime", version = "3.31.0"} osgi-service = { module = "org.osgi:org.osgi.service.prefs", version = "1.1.2"} icu4j = { module = "com.ibm.icu:icu4j", version = "75.1"} @@ -40,22 +40,27 @@ picocli-codegen = { module = "info.picocli:picocli-codegen", version = "4.7.0"} jep = { module = "black.ninia:jep", version = "4.2.0" } # build.yml uses grep to extract the jep verison number for CI/CD purposes llvm = { module = "org.bytedeco:llvm-platform", version = "16.0.4-1.5.9"} jruby = { module = "org.jruby:jruby-core", version = "9.4.3.0" } -jline = { module = "org.jline:jline", version = "3.26.0" } +jline = { module = "org.jline:jline", version = "3.27.0" } antlr-runtime = { module = "org.antlr:antlr4-runtime", version = "4.8-1" } # we cannot upgrade until ki-shell upgrades this! # test -junit-params = { module = "org.junit.jupiter:junit-jupiter-params", version = "5.10.0"} -mockito = { module = "org.mockito:mockito-core", version = "5.12.0"} +junit-params = { module = "org.junit.jupiter:junit-jupiter-params", version = "5.11.0"} +mockito = { module = "org.mockito:mockito-core", version = "5.14.0"} # plugins needed for build.gradle.kts in buildSrc kotlin-gradle = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } dokka-gradle = { module = "org.jetbrains.dokka:dokka-gradle-plugin", version = "1.9.0" } # the dokka plugin is slightly behind the main Kotlin release cycle dokka-versioning = { module = "org.jetbrains.dokka:versioning-plugin", version = "1.9.0"} kover-gradle = { module = "org.jetbrains.kotlinx:kover-gradle-plugin", version = "0.8.0" } -sonarqube-gradle = { module = "org.sonarsource.scanner.gradle:sonarqube-gradle-plugin", version.ref = "sonarqube" } spotless-gradle = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } nexus-publish-gradle = { module = "io.github.gradle-nexus:publish-plugin", version.ref = "nexus-publish" } +sootup-core = { module = "org.soot-oss:sootup.core", version.ref = "sootup" } +sootup-java-core = { module = "org.soot-oss:sootup.java.core", version.ref = "sootup" } +sootup-java-sourcecode = { module = "org.soot-oss:sootup.java.sourcecode", version.ref = "sootup" } +sootup-java-bytecode = { module = "org.soot-oss:sootup.java.bytecode", version.ref = "sootup" } +sootup-jimple-parser = { module = "org.soot-oss:sootup.jimple.parser", version.ref = "sootup" } + [bundles] log4j = ["log4j-impl", "log4j-core"] neo4j = ["neo4j-ogm-core", "neo4j-ogm-bolt-driver"] @@ -66,10 +71,10 @@ kotlin-scripting = [ "kotlin-scripting-dependencies", "kotlin-scripting-dependencies-maven-all", ] +sootup = ["sootup-core", "sootup-java-core", "sootup-java-sourcecode", "sootup-java-bytecode", "sootup-jimple-parser"] [plugins] kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"} dokka = { id = "org.jetbrains.dokka", version.ref = "kotlin" } -sonarqube = { id = "org.sonarqube", version.ref = "sonarqube" } spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } -node = { id = "com.github.node-gradle.node", version = "7.0.0"} +node = { id = "com.github.node-gradle.node", version = "7.1.0"} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 2c3521197d..a4b76b9530 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 09523c0e54..9355b41557 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/settings.gradle.kts b/settings.gradle.kts index b7a66ec8a1..afe927cdee 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -37,6 +37,10 @@ val enableRubyFrontend: Boolean by extra { val enableRubyFrontend: String? by settings enableRubyFrontend.toBoolean() } +val enableJVMFrontend: Boolean by extra { + val enableJVMFrontend: String? by settings + enableJVMFrontend.toBoolean() +} if (enableJavaFrontend) include(":cpg-language-java") if (enableCXXFrontend) include(":cpg-language-cxx") @@ -44,4 +48,5 @@ if (enableGoFrontend) include(":cpg-language-go") if (enableLLVMFrontend) include(":cpg-language-llvm") if (enablePythonFrontend) include(":cpg-language-python") if (enableTypeScriptFrontend) include(":cpg-language-typescript") -if (enableRubyFrontend) include(":cpg-language-ruby") \ No newline at end of file +if (enableRubyFrontend) include(":cpg-language-ruby") +if (enableJVMFrontend) include(":cpg-language-jvm")