From ef6249ff9ef554180f8e7f83000ce39d900265d7 Mon Sep 17 00:00:00 2001 From: Rdeisenroth Date: Tue, 20 Aug 2024 01:02:22 +0200 Subject: [PATCH 1/5] Exporter for sourcegrade/lab exporting individual test details --- .../org/sourcegrade/jagr/core/CommonModule.kt | 2 + .../jagr/core/export/rubric/LabExporter.kt | 68 +++++++++++++++++++ .../jagr/launcher/io/GradedRubricExporter.kt | 1 + .../org/sourcegrade/jagr/StandardGrading.kt | 3 + 4 files changed, 74 insertions(+) create mode 100644 core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/CommonModule.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/CommonModule.kt index a1060dc6..513f034c 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/CommonModule.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/CommonModule.kt @@ -34,6 +34,7 @@ import org.sourcegrade.jagr.core.executor.GradingQueueFactoryImpl import org.sourcegrade.jagr.core.executor.TimeoutHandler import org.sourcegrade.jagr.core.export.rubric.BasicHTMLExporter import org.sourcegrade.jagr.core.export.rubric.GermanCSVExporter +import org.sourcegrade.jagr.core.export.rubric.LabExporter import org.sourcegrade.jagr.core.export.rubric.MoodleJSONExporter import org.sourcegrade.jagr.core.export.submission.EclipseSubmissionExporter import org.sourcegrade.jagr.core.export.submission.GradleSubmissionExporter @@ -78,6 +79,7 @@ class CommonModule(private val configuration: LaunchConfiguration) : AbstractMod bind(GradedRubricExporter.CSV::class.java).to(GermanCSVExporter::class.java) bind(GradedRubricExporter.HTML::class.java).to(BasicHTMLExporter::class.java) bind(GradedRubricExporter.Moodle::class.java).to(MoodleJSONExporter::class.java) + bind(GradedRubricExporter.Lab::class.java).to(LabExporter::class.java) bind(Grader.Factory::class.java).to(GraderFactoryImpl::class.java) bind(GradeResult.Factory::class.java).to(GradeResultFactoryImpl::class.java) bind(GradingQueue.Factory::class.java).to(GradingQueueFactoryImpl::class.java) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt new file mode 100644 index 00000000..83ae05af --- /dev/null +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt @@ -0,0 +1,68 @@ +package org.sourcegrade.jagr.core.export.rubric + +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.sourcegrade.jagr.api.rubric.GradedRubric +import org.sourcegrade.jagr.launcher.io.GradedRubricExporter +import org.sourcegrade.jagr.launcher.io.Resource +import org.sourcegrade.jagr.launcher.io.buildResource + +class LabExporter : GradedRubricExporter.Lab { + + override fun export(gradedRubric: GradedRubric): Resource { + val jUnitResult = gradedRubric.testCycle.jUnitResult + + if (jUnitResult != null) { + val testPlan = jUnitResult.testPlan + val statusListener = jUnitResult.statusListener + + // Gather detailed test results + val testResults = testPlan.roots.flatMap { root -> + // Collect detailed information about each test + testPlan.getDescendants(root).mapNotNull { testIdentifier -> + val testExecutionResult = statusListener.testResults[testIdentifier] + + // If the test has a result, collect the information + testExecutionResult?.let { + TestResult( + id = testIdentifier.uniqueId, + name = testIdentifier.displayName, + type = testIdentifier.type.toString(), + status = testExecutionResult.status.toString(), +// duration = Duration.between(it.startTime, it.endTime).toMillis(), + stackTrace = testExecutionResult.throwable.orElse(null)?.stackTraceToString() + ) + } + } + } + + // Serialize the results to JSON + val testResultsJson = TestResults(testResults) + val jsonString = Json.encodeToString(testResultsJson) + + // Build the Resource with the JSON string + return buildResource { + name = "${gradedRubric.testCycle.submission.info}.json" + outputStream.bufferedWriter().use { it.write(jsonString) } + } + } else { + throw IllegalArgumentException("No JUnitResult present in the test cycle.") + } + } + + @Serializable + data class TestResult( + val id: String, + val name: String, + val type: String, + val status: String, +// val duration: Long, + val stackTrace: String? = null + ) + + @Serializable + data class TestResults( + val tests: List + ) +} diff --git a/launcher/src/main/kotlin/org/sourcegrade/jagr/launcher/io/GradedRubricExporter.kt b/launcher/src/main/kotlin/org/sourcegrade/jagr/launcher/io/GradedRubricExporter.kt index 6e384f8f..b65b5fac 100644 --- a/launcher/src/main/kotlin/org/sourcegrade/jagr/launcher/io/GradedRubricExporter.kt +++ b/launcher/src/main/kotlin/org/sourcegrade/jagr/launcher/io/GradedRubricExporter.kt @@ -26,4 +26,5 @@ interface GradedRubricExporter { interface CSV : GradedRubricExporter interface HTML : GradedRubricExporter interface Moodle : GradedRubricExporter + interface Lab : GradedRubricExporter } diff --git a/src/main/kotlin/org/sourcegrade/jagr/StandardGrading.kt b/src/main/kotlin/org/sourcegrade/jagr/StandardGrading.kt index bb47b1a8..3b388619 100644 --- a/src/main/kotlin/org/sourcegrade/jagr/StandardGrading.kt +++ b/src/main/kotlin/org/sourcegrade/jagr/StandardGrading.kt @@ -55,8 +55,10 @@ class StandardGrading( private val rubricsFile = File(config.dir.rubrics).ensure(jagr.logger)!! private val csvDir = checkNotNull(rubricsFile.resolve("csv").ensure(jagr.logger)) { "rubrics/csv directory" } private val moodleDir = checkNotNull(rubricsFile.resolve("moodle").ensure(jagr.logger)) { "rubrics/moodle directory" } + private val labDir = checkNotNull(rubricsFile.resolve("lab").ensure(jagr.logger)) { "rubrics/lab directory" } private val csvExporter = jagr.injector.getInstance(GradedRubricExporter.CSV::class.java) private val moodleExporter = jagr.injector.getInstance(GradedRubricExporter.Moodle::class.java) + private val labExporter = jagr.injector.getInstance(GradedRubricExporter.Lab::class.java) fun grade(noExport: Boolean, exportOnly: Boolean) = runBlocking { jagr.logger.info("Starting Jagr v${Jagr.version}") @@ -123,6 +125,7 @@ class StandardGrading( for ((gradedRubric, _) in result.rubrics) { csvExporter.exportSafe(gradedRubric, csvDir) moodleExporter.exportSafe(gradedRubric, moodleDir) + labExporter.exportSafe(gradedRubric, labDir) } } From efc84a23073f13a93c3d9bded2084a979d564cc4 Mon Sep 17 00:00:00 2001 From: Rdeisenroth Date: Tue, 20 Aug 2024 01:54:15 +0200 Subject: [PATCH 2/5] more information --- .../jagr/core/export/rubric/LabExporter.kt | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt index 83ae05af..71182837 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt @@ -4,8 +4,10 @@ import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import org.sourcegrade.jagr.api.rubric.GradedRubric +import org.sourcegrade.jagr.core.testing.JavaSubmission import org.sourcegrade.jagr.launcher.io.GradedRubricExporter import org.sourcegrade.jagr.launcher.io.Resource +import org.sourcegrade.jagr.launcher.io.SubmissionInfo import org.sourcegrade.jagr.launcher.io.buildResource class LabExporter : GradedRubricExporter.Lab { @@ -31,14 +33,40 @@ class LabExporter : GradedRubricExporter.Lab { type = testIdentifier.type.toString(), status = testExecutionResult.status.toString(), // duration = Duration.between(it.startTime, it.endTime).toMillis(), - stackTrace = testExecutionResult.throwable.orElse(null)?.stackTraceToString() + message = testExecutionResult.throwable.orElse(null)?.message, + stackTrace = testExecutionResult.throwable.orElse(null)?.stackTraceToString(), ) } } } +// // recursive function to get all test results with children +// fun getTestResults(testIdentifier: TestIdentifier): TestResult { +// val testExecutionResult = statusListener.testResults[testIdentifier] +// return testExecutionResult?.let { +// TestResult( +// id = testIdentifier.uniqueId, +// name = testIdentifier.displayName, +// type = testIdentifier.type.toString(), +// status = testExecutionResult.status.toString(), +// message = testExecutionResult.throwable.orElse(null)?.message, +// stackTrace = testExecutionResult.throwable.orElse(null)?.stackTraceToString(), +// children = testPlan.getDescendants(testIdentifier).map { getTestResults(it) }, +// ) +// } ?: throw IllegalArgumentException("No testExecutionResult found for $testIdentifier") +// } +// +//// val testResults = +//// jUnitResult.testPlan.roots.flatMap { t -> testPlan.getDescendants(t).map { getTestResults(it) } } +// +// val testResults = jUnitResult.testPlan.roots.map { getTestResults(it) } // Serialize the results to JSON - val testResultsJson = TestResults(testResults) + val testResultsJson = LabRubric( + submissionInfo = (gradedRubric.testCycle.submission as JavaSubmission).submissionInfo, + totalPointsMin = gradedRubric.grade.minPoints, + totalPointsMax = gradedRubric.grade.maxPoints, + tests = testResults, + ) val jsonString = Json.encodeToString(testResultsJson) // Build the Resource with the JSON string @@ -58,11 +86,27 @@ class LabExporter : GradedRubricExporter.Lab { val type: String, val status: String, // val duration: Long, - val stackTrace: String? = null + val message: String? = null, + val stackTrace: String? = null, + val children: List = emptyList(), ) @Serializable - data class TestResults( - val tests: List + data class Criterion( + val name: String, + val archivedPointsMin: Int, + val archivedPointsMax: Int, + val message: String? = null, + val relevantTests: List = emptyList(), + val children: List = emptyList(), + ) + + @Serializable + data class LabRubric( + val submissionInfo: SubmissionInfo, + val totalPointsMin: Int, + val totalPointsMax: Int, + val criteria: List = emptyList(), + val tests: List = emptyList(), ) } From f1774b6ed5f77781c33137a4cff8dcb696a92664 Mon Sep 17 00:00:00 2001 From: Rdeisenroth Date: Tue, 20 Aug 2024 03:35:29 +0200 Subject: [PATCH 3/5] export relevant tests and criteria --- .../jagr/core/export/rubric/LabExporter.kt | 69 +++++++++++++------ .../jagr/core/rubric/CriterionImpl.kt | 1 + .../core/rubric/JUnitTestRefFactoryImpl.kt | 2 +- .../rubric/grader/DescendingPriorityGrader.kt | 2 +- .../core/rubric/grader/TestAwareGraderImpl.kt | 4 +- .../jagr/api/rubric/Criterion.java | 7 ++ 6 files changed, 60 insertions(+), 25 deletions(-) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt index 71182837..75cfca90 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt @@ -3,7 +3,13 @@ package org.sourcegrade.jagr.core.export.rubric import kotlinx.serialization.Serializable import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import org.sourcegrade.jagr.api.rubric.GradedCriterion import org.sourcegrade.jagr.api.rubric.GradedRubric +import org.sourcegrade.jagr.api.rubric.Grader +import org.sourcegrade.jagr.api.rubric.JUnitTestRef +import org.sourcegrade.jagr.core.rubric.JUnitTestRefFactoryImpl +import org.sourcegrade.jagr.core.rubric.grader.DescendingPriorityGrader +import org.sourcegrade.jagr.core.rubric.grader.TestAwareGraderImpl import org.sourcegrade.jagr.core.testing.JavaSubmission import org.sourcegrade.jagr.launcher.io.GradedRubricExporter import org.sourcegrade.jagr.launcher.io.Resource @@ -40,31 +46,52 @@ class LabExporter : GradedRubricExporter.Lab { } } -// // recursive function to get all test results with children -// fun getTestResults(testIdentifier: TestIdentifier): TestResult { -// val testExecutionResult = statusListener.testResults[testIdentifier] -// return testExecutionResult?.let { -// TestResult( -// id = testIdentifier.uniqueId, -// name = testIdentifier.displayName, -// type = testIdentifier.type.toString(), -// status = testExecutionResult.status.toString(), -// message = testExecutionResult.throwable.orElse(null)?.message, -// stackTrace = testExecutionResult.throwable.orElse(null)?.stackTraceToString(), -// children = testPlan.getDescendants(testIdentifier).map { getTestResults(it) }, -// ) -// } ?: throw IllegalArgumentException("No testExecutionResult found for $testIdentifier") -// } -// -//// val testResults = -//// jUnitResult.testPlan.roots.flatMap { t -> testPlan.getDescendants(t).map { getTestResults(it) } } -// -// val testResults = jUnitResult.testPlan.roots.map { getTestResults(it) } + // Get all relevant tests for a grader + fun getRelevantTests(grader:Grader): List { + return when (grader) { + is TestAwareGraderImpl -> { + val testRefs: MutableSet = mutableSetOf() + testRefs.addAll(grader.requirePass.keys) + testRefs.addAll(grader.requireFail.keys) + + testRefs.mapNotNull { ref -> + when (ref) { + is JUnitTestRefFactoryImpl.Default -> testPlan.roots.flatMap { testPlan.getDescendants(it) }.firstOrNull { + it.source.isPresent && it.source.orElse(null) == ref.testSource + }?.uniqueId + else -> null + } + } + } + is DescendingPriorityGrader -> grader.graders.flatMap { getRelevantTests(it) } + else -> emptyList() + } + } + + // recursive function to get all criteria with children + fun getCriteria(criterion: GradedCriterion): Criterion { + val children = criterion.childCriteria.map { getCriteria(it) } +// gradedRubric.grade.comments + val relevantTests = children.flatMap { it.relevantTests ?: emptyList() }.toMutableSet() + if(criterion.criterion.grader != null) { + relevantTests.addAll(getRelevantTests(criterion.criterion.grader!!)) + } + return Criterion( + name = criterion.criterion.shortDescription, + archivedPointsMin = criterion.grade.minPoints, + archivedPointsMax = criterion.grade.maxPoints, + message = criterion.grade.comments.joinToString("
") { "

$it

" }, + relevantTests = relevantTests.toList(), + children = children, + ) + } + // Serialize the results to JSON val testResultsJson = LabRubric( submissionInfo = (gradedRubric.testCycle.submission as JavaSubmission).submissionInfo, totalPointsMin = gradedRubric.grade.minPoints, totalPointsMax = gradedRubric.grade.maxPoints, + criteria = gradedRubric.childCriteria.map { getCriteria(it) }, tests = testResults, ) val jsonString = Json.encodeToString(testResultsJson) @@ -97,7 +124,7 @@ class LabExporter : GradedRubricExporter.Lab { val archivedPointsMin: Int, val archivedPointsMax: Int, val message: String? = null, - val relevantTests: List = emptyList(), + val relevantTests: List? = emptyList(), val children: List = emptyList(), ) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/CriterionImpl.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/CriterionImpl.kt index a06462e2..7c57cbb0 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/CriterionImpl.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/CriterionImpl.kt @@ -90,6 +90,7 @@ class CriterionImpl( override fun getParentCriterion(): Criterion? = parentCriterionKt override fun getPeers(): List = peersKt override fun getChildCriteria(): List = childCriteria + override fun getGrader(): Grader? = grader override fun grade(testCycle: TestCycle): GradedCriterion { val graderResult = GradeResult.clamped( grader?.grade(testCycle, this) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/JUnitTestRefFactoryImpl.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/JUnitTestRefFactoryImpl.kt index 7230806c..b66c24c2 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/JUnitTestRefFactoryImpl.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/JUnitTestRefFactoryImpl.kt @@ -68,7 +68,7 @@ class JUnitTestRefFactoryImpl @Inject constructor( TestExecutionResult.aborted(NoOpFailedError()) } - class Default(private val testSource: TestSource) : JUnitTestRef { + class Default(val testSource: TestSource) : JUnitTestRef { inner class TestNotFoundError : AssertionFailedError("Test result not found") override operator fun get(testResults: Map): TestExecutionResult { diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/DescendingPriorityGrader.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/DescendingPriorityGrader.kt index f831561f..ccd3c332 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/DescendingPriorityGrader.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/DescendingPriorityGrader.kt @@ -28,7 +28,7 @@ import org.sourcegrade.jagr.core.rubric.GradeResultImpl class DescendingPriorityGrader( private val logger: Logger, - private vararg val graders: Grader, + vararg val graders: Grader, ) : Grader { override fun grade(testCycle: TestCycle, criterion: Criterion): GradeResult { diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/TestAwareGraderImpl.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/TestAwareGraderImpl.kt index 245a602e..2fcd06b6 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/TestAwareGraderImpl.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/rubric/grader/TestAwareGraderImpl.kt @@ -32,8 +32,8 @@ import org.sourcegrade.jagr.core.rubric.message class TestAwareGraderImpl( private val graderPassed: Grader, private val graderFailed: Grader, - private val requirePass: Map, - private val requireFail: Map, + val requirePass: Map, + val requireFail: Map, private val commentIfFailed: String?, ) : Grader { diff --git a/grader-api/src/main/java/org/sourcegrade/jagr/api/rubric/Criterion.java b/grader-api/src/main/java/org/sourcegrade/jagr/api/rubric/Criterion.java index 1516a9c2..0ca34ba5 100644 --- a/grader-api/src/main/java/org/sourcegrade/jagr/api/rubric/Criterion.java +++ b/grader-api/src/main/java/org/sourcegrade/jagr/api/rubric/Criterion.java @@ -90,6 +90,13 @@ static Builder builder() { */ @Nullable Criterion getParentCriterion(); + /** + * The {@link Grader} that will be used to calculate the points for the criterion. + * + * @return The {@link Grader} to use for this {@link Criterion} + */ + @Nullable Grader getGrader(); + /** * The peers of this criterion. * From ea249ca0a874dea17913017872aa29bccf647f49 Mon Sep 17 00:00:00 2001 From: Rdeisenroth Date: Tue, 20 Aug 2024 13:03:38 +0200 Subject: [PATCH 4/5] ktlint warnings --- .../org/sourcegrade/jagr/core/export/rubric/LabExporter.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt index 75cfca90..0628aade 100644 --- a/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt +++ b/core/src/main/kotlin/org/sourcegrade/jagr/core/export/rubric/LabExporter.kt @@ -47,7 +47,7 @@ class LabExporter : GradedRubricExporter.Lab { } // Get all relevant tests for a grader - fun getRelevantTests(grader:Grader): List { + fun getRelevantTests(grader: Grader): List { return when (grader) { is TestAwareGraderImpl -> { val testRefs: MutableSet = mutableSetOf() @@ -73,7 +73,7 @@ class LabExporter : GradedRubricExporter.Lab { val children = criterion.childCriteria.map { getCriteria(it) } // gradedRubric.grade.comments val relevantTests = children.flatMap { it.relevantTests ?: emptyList() }.toMutableSet() - if(criterion.criterion.grader != null) { + if (criterion.criterion.grader != null) { relevantTests.addAll(getRelevantTests(criterion.criterion.grader!!)) } return Criterion( From a674ffc167bf0b9773a08d106745d6e005d19635 Mon Sep 17 00:00:00 2001 From: Rdeisenroth Date: Tue, 20 Aug 2024 16:29:46 +0200 Subject: [PATCH 5/5] jank temp fix for jagr not terminating --- src/main/kotlin/org/sourcegrade/jagr/Main.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/org/sourcegrade/jagr/Main.kt b/src/main/kotlin/org/sourcegrade/jagr/Main.kt index 05213bd9..d07dfe9c 100644 --- a/src/main/kotlin/org/sourcegrade/jagr/Main.kt +++ b/src/main/kotlin/org/sourcegrade/jagr/Main.kt @@ -27,6 +27,7 @@ import com.github.ajalt.clikt.parameters.types.choice import org.sourcegrade.jagr.launcher.env.Environment import org.sourcegrade.jagr.launcher.env.Jagr import org.sourcegrade.jagr.launcher.env.logger +import kotlin.system.exitProcess fun main(vararg args: String) { try { @@ -35,6 +36,7 @@ fun main(vararg args: String) { Jagr.logger.error("A fatal error occurred", e) throw e } + exitProcess(0) } class MainCommand : CliktCommand() {