Rescan files on missing class storage items

In case not all class storage items were available during
a given file scan, those failed files will be rescanned
later with more populated details in the codebase.

There's a maximum of 10 rescans to avoid endless recursions.

src/Psalm/Codebase.php

@@ -501,6 +501,18 @@ public function addFilesToAnalyze(array $files_to_analyze): void
      * Scans all files their related files
      */
     public function scanFiles(int $threads = 1): void
+    {
+        $this->scanAndPopulateFiles($threads);
+        // in case the previous scan failed, let's try again since previously found
+        // references (class-like storage items and dependencies) were populated
+        $names = $this->scanner->prepareRescanning();
+        if ($names !== []) {
+            $this->progress->debug('Rescanning for missing classlike names [' . implode(', ', $names) . ']');
+            $this->scanAndPopulateFiles($threads);
+        }
+    }
+
+    private function scanAndPopulateFiles(int $threads = 1): void
     {
         $has_changes = $this->scanner->scanFiles($this->classlikes, $threads);
 

src/Psalm/Internal/Codebase/Scanner.php

@@ -5,6 +5,7 @@
 use Closure;
 use Psalm\Codebase;
 use Psalm\Config;
+use Psalm\Exception\ClassStorageNotFoundException;
 use Psalm\Internal\Analyzer\IssueData;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\ErrorHandler;
@@ -23,9 +24,13 @@
 use Throwable;
 use UnexpectedValueException;
 
+use function array_combine;
 use function array_filter;
+use function array_keys;
 use function array_merge;
 use function array_pop;
+use function array_unique;
+use function array_values;
 use function ceil;
 use function count;
 use function error_reporting;
@@ -102,6 +107,14 @@ final class Scanner
      */
     private array $files_to_scan = [];
 
+    /**
+     * Class-like names per failed file that could not be resolved (yet).
+     * Those names are collected for another rescanning approach.
+     *
+     * @var array<string, ?string>
+     */
+    private array $missing_classlikes_per_file = [];
+
     /**
      * @var array<string, string>
      */
@@ -288,6 +301,30 @@ public function scanFiles(ClassLikes $classlikes, int $pool_size = 1): bool
         return $has_changes;
     }
 
+    /**
+     * @return list<string> class-like names to be rescanned
+     */
+    public function prepareRescanning(): array
+    {
+        $classlike_storage_provider = $this->codebase->classlike_storage_provider;
+        $classlikes_per_file = $this->missing_classlikes_per_file;
+        $this->missing_classlikes_per_file = [];
+
+        // filter files that actually can be rescanned
+        // (since the class-like name does exist now)
+        $classlikes_per_file = array_filter(
+            $classlikes_per_file,
+            fn($classlike): bool => $classlike === null || $classlike_storage_provider->has($classlike)
+        );
+        if ($classlikes_per_file === []) {
+            return [];
+        }
+
+        $files = array_keys($classlikes_per_file);
+        $this->files_to_scan = array_combine($files, $files);
+        return array_values(array_filter($classlikes_per_file));
+    }
+
     private function shouldScan(string $file_path): bool
     {
         return $this->file_provider->fileExists($file_path)
@@ -347,6 +384,7 @@ function (): void {
 
                     $this->progress->debug('Have initialised forked process for scanning' . PHP_EOL);
                 },
+                // @todo how are failures processed in pooled sub processes?! (see Pool::$task_closure)
                 Closure::fromCallable([$this, 'scanAPath']),
                 /**
                  * @return PoolData
@@ -423,10 +461,13 @@ function () {
             }
         } else {
             $i = 0;
-
             foreach ($files_to_scan as $file_path => $_) {
-                $this->scanAPath($i, $file_path);
-                ++$i;
+                try {
+                    $this->scanAPath($i++, $file_path);
+                } catch (ClassStorageNotFoundException $e) {
+                    // @todo there might be more failures besides `ClassStorageNotFoundException`
+                    $this->missing_classlikes_per_file[$file_path] = $e->getName();
+                }
             }
         }
 
@@ -547,8 +588,6 @@ private function scanFile(
             $this->file_storage_provider->create($file_path);
         }
 
-        $this->scanned_files[$file_path] = $will_analyze;
-
         $file_storage = $this->file_storage_provider->get($file_path);
 
         $file_scanner->scan(
@@ -558,6 +597,9 @@ private function scanFile(
             $this->progress,
         );
 
+        // only mark scanned files, after they were actually scanned without any exception
+        $this->scanned_files[$file_path] = $will_analyze;
+
         if (!$from_cache) {
             if (!$file_storage->has_visitor_issues && $this->file_storage_provider->cache) {
                 $this->file_storage_provider->cache->writeToCache($file_storage, $file_contents);

src/Psalm/Internal/Fork/Pool.php

@@ -376,6 +376,7 @@ private function readResultsFromChildren(): array
                                  */
                                 posix_kill($child_pid, SIGTERM);
                             }
+                            // @todo add more meta-data why the process failed for a potential auto-recovery
                             throw new Exception($message->message);
                         } else {
                             error_log('Child should return ForkMessage - response type=' . gettype($message));

src/Psalm/Internal/Provider/ClassLikeStorageProvider.php

@@ -43,9 +43,12 @@ public function get(string $fq_classlike_name): ClassLikeStorage
         $fq_classlike_name_lc = strtolower($fq_classlike_name);
         /** @psalm-suppress ImpureStaticProperty Used only for caching */
         if (!isset(self::$storage[$fq_classlike_name_lc])) {
-            throw (new ClassStorageNotFoundException(
-                'Could not get class storage for ' . $fq_classlike_name_lc)
-            )->setName($fq_classlike_name_lc);
+            throw new ClassStorageNotFoundException(
+                'Could not get class storage for ' . $fq_classlike_name_lc,
+                0,
+                null,
+                $fq_classlike_name_lc,
+            );
         }
 
         /** @psalm-suppress ImpureStaticProperty Used only for caching */