Adding exit code native flag that indicates the origin of the exit co…

…de in a test result.

app/helpers/Evaluation/EvaluationResults/SandboxResults/ISandboxResults.php

@@ -7,12 +7,14 @@
  */
 interface ISandboxResults
 {
+    public const EXIT_CODE_OK = 0;
+    public const EXIT_CODE_UNKNOWN = -1;
 
-    const EXIT_CODE_OK = 0;
-    const EXIT_CODE_UNKNOWN = -1;
-
-    const STATUS_OK = "OK";
-    const STATUS_TO = "TO";
+    public const STATUS_OK = "OK";
+    public const STATUS_RE = "RE"; // run-time error, i.e., exited with a non-zero exit code
+    public const STATUS_SG = "SG"; // program died on a signal
+    public const STATUS_TO = "TO"; // timed out
+    public const STATUS_XX = "XX"; // internal error of the sandbox
 
     /**
      * Get total amount of consumed wall time

app/helpers/Evaluation/EvaluationResults/SandboxResults/SandboxResults.php

@@ -129,7 +129,10 @@ public function getUsedMemory(): int
      */
     public function getExitCode(): int
     {
-        if ($this->status !== self::STATUS_OK && $this->exitcode === self::EXIT_CODE_OK) {
+        if (
+            $this->status !== self::STATUS_OK && $this->status !== self::STATUS_RE
+            && $this->exitcode === self::EXIT_CODE_OK
+        ) {
             return self::EXIT_CODE_UNKNOWN;
         }
 

app/helpers/Evaluation/EvaluationResults/SandboxResults/SkippedSandboxResults.php

@@ -7,7 +7,6 @@
  */
 class SkippedSandboxResults implements ISandboxResults
 {
-
     /**
      * Get total amount of consumed time
      * @return float The time for which the process ran in seconds

app/helpers/Evaluation/EvaluationResults/TaskResults/ExecutionTaskResult.php

@@ -9,7 +9,7 @@
  */
 class ExecutionTaskResult extends TaskResult
 {
-    const SANDBOX_RESULTS_KEY = "sandbox_results";
+    private const SANDBOX_RESULTS_KEY = "sandbox_results";
 
     /** @var ISandboxResults Statistics of the execution */
     private $stats;

app/helpers/Evaluation/EvaluationResults/TestResult.php

@@ -266,6 +266,24 @@ public function getExitCode(): int
         return 0;
     }
 
+    /**
+     * Check if the exit code is the native result of the executable
+     * (not interrupted by signal, timeout, or sandbox error).
+     * @return bool True if all tasks ended in OK or RE states.
+     */
+    public function isExitCodeNative(): bool
+    {
+        foreach ($this->sandboxResultsList as $results) {
+            if (
+                $results->getStatus() !== ISandboxResults::STATUS_OK
+                && $results->getStatus() !== ISandboxResults::STATUS_RE
+            ) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * Get the termination signal number or null, if no executed task was terminated.
      * @return int|null

app/model/entity/TestResult.php

@@ -20,6 +20,7 @@ public function __construct(
         $this->status = $result->getStatus();
         $this->score = $result->getScore();
         $this->exitCode = $result->getExitCode();
+        $this->exitCodeNative = $result->isExitCodeNative();
         $this->exitSignal = $result->getExitSignal();
         $this->usedMemory = $result->getUsedMemory();
         $this->usedMemoryLimit = $result->getUsedMemoryLimit();
@@ -114,6 +115,12 @@ public function __construct(
      */
     protected $exitCode;
 
+    /**
+     * @ORM\Column(type="boolean")
+     * Whether the exit code was produced by regular termination (not by signal, timeout, or sandbox error).
+     */
+    protected $exitCodeNative;
+
     /**
      * @ORM\Column(type="integer", nullable=true)
      */
@@ -186,6 +193,7 @@ public function getDataForView(SubmissionViewOptions $options)
             "wallTimeExceeded" => $this->wallTimeExceeded,
             "cpuTimeExceeded" => $this->cpuTimeExceeded,
             "exitCode" => $this->exitCode,
+            "exitCodeNative" => $this->exitCodeNative,
             "exitSignal" => $this->exitSignal,
             "message" => $this->message,
             "wallTimeRatio" => $wallTimeRatio,
@@ -266,6 +274,11 @@ public function getExitCode(): int
         return $this->exitCode;
     }
 
+    public function isExitCodeNative(): bool
+    {
+        return $this->exitCodeNative;
+    }
+
     public function getExitSignal(): ?int
     {
         return $this->exitSignal;

migrations/Version20240716161149.php

@@ -41,6 +41,7 @@ public function up(Schema $schema): void
     {
         // this up() migration is auto-generated, please modify it to your needs
         $this->addSql('ALTER TABLE assignment ADD can_view_measured_values TINYINT(1) NOT NULL');
+        $this->addSql('ALTER TABLE test_result ADD exit_code_native TINYINT(1) NOT NULL');
 
         $this->addSql('ALTER TABLE assignment_localized_exercise DROP FOREIGN KEY FK_9C8F78CD19302F8');
         $this->addSql('ALTER TABLE assignment_localized_exercise ADD CONSTRAINT FK_9DF069D6D19302F8 FOREIGN KEY (assignment_id) REFERENCES assignment (id) ON DELETE CASCADE');
@@ -68,6 +69,15 @@ public function up(Schema $schema): void
 
     public function postUp(Schema $schema): void
     {
+        /*
+         * fill newly created exit_code_native column
+         */
+        $this->connection->executeQuery(
+            "UPDATE test_result SET exit_code_native = 1 WHERE `status` = 'OK' OR
+            (`status` = 'FAILED' AND memory_exceeded = 0 AND wall_time_exceeded = 0 AND
+            exit_signal IS NULL AND exit_code >= 0 AND message NOT LIKE '%Caught%')"
+        );
+
         /*
          * scan all pipelines
          */
@@ -173,6 +183,8 @@ public function down(Schema $schema): void
     {
         // this down() migration is auto-generated, please modify it to your needs
         $this->addSql('ALTER TABLE assignment DROP can_view_measured_values');
+        $this->addSql('ALTER TABLE test_result DROP exit_code_native');
+
         $this->addSql('ALTER TABLE assignment_localized_exercise DROP FOREIGN KEY FK_9DF069D6D19302F8');
         $this->addSql('ALTER TABLE assignment_localized_exercise ADD CONSTRAINT FK_9C8F78CD19302F8 FOREIGN KEY (assignment_id) REFERENCES assignment (id) ON DELETE CASCADE');
         $this->addSql('ALTER TABLE assignment_solution DROP FOREIGN KEY FK_5B315D2ED19302F8');