Refactor Error Handling

phpunit.xml

@@ -13,4 +13,4 @@
             <directory>./tests/Migration/E2E</directory>
         </testsuite>
     </testsuites>
-</phpunit>
+</phpunit>

playground.php

@@ -11,7 +11,6 @@
 use Utopia\CLI\Console;
 use Utopia\Migration\Destinations\Appwrite as AppwriteDestination;
 use Utopia\Migration\Destinations\Local;
-use Utopia\Migration\Resource;
 use Utopia\Migration\Sources\Appwrite;
 use Utopia\Migration\Sources\Firebase;
 use Utopia\Migration\Sources\NHost;
@@ -88,13 +87,16 @@ function (array $resources) use ($transfer) {
 
 $cache = $transfer->getCache()->getAll();
 
-foreach ($cache as $type => $resources) {
-    foreach ($resources as $resource) {
-        /** @var resource $resource */
-        if ($resource->getStatus() !== Resource::STATUS_ERROR) {
-            continue;
-        }
+if (count($sourceAppwrite->getErrors()) > 0) {
+    foreach ($sourceAppwrite->getErrors() as $error) {
+        /* @var \Utopia\Migration\Exception $error */
+        Console::error('[Source] ['.$error->getResourceType().'] '.$error->getMessage());
+    }
+}
 
-        Console::error($resource->getName().' '.$resource->getInternalId().' '.$resource->getMessage());
+if (count($destinationAppwrite->getErrors()) > 0) {
+    foreach ($destinationAppwrite->getErrors() as $error) {
+        /* @var \Utopia\Migration\Exception $error */
+        Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage());
     }
 }

src/Migration/Destinations/Appwrite.php

@@ -11,6 +11,7 @@
 use Appwrite\Services\Teams;
 use Appwrite\Services\Users;
 use Utopia\Migration\Destination;
+use Utopia\Migration\Exception;
 use Utopia\Migration\Resource;
 use Utopia\Migration\Resources\Auth\Hash;
 use Utopia\Migration\Resources\Auth\Membership;
@@ -204,7 +205,7 @@ public function report(array $resources = []): array
             }
 
             return [];
-        } catch (\Exception $exception) {
+        } catch (\Throwable $exception) {
             if ($exception->getCode() === 403) {
                 throw new \Exception('Missing permission: '.$currentPermission);
             } else {
@@ -238,11 +239,17 @@ protected function import(array $resources, callable $callback): void
                         $responseResource = $this->importFunctionResource($resource);
                         break;
                 }
-            } catch (\Exception $e) {
+            } catch (\Throwable $e) {
                 if ($e->getCode() === 409) {
                     $resource->setStatus(Resource::STATUS_SKIPPED, $e->getMessage());
                 } else {
                     $resource->setStatus(Resource::STATUS_ERROR, $e->getMessage());
+                    $this->addError(new Exception(
+                        resourceType: $resource->getGroup(),
+                        resourceId: $resource->getId(),
+                        message: $e->getMessage(),
+                        code: $e->getCode()
+                    ));
                 }
 
                 $responseResource = $resource;

src/Migration/Exception.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace Utopia\Migration;
+
+class Exception extends \Exception
+{
+    public string $resourceType;
+
+    public string $resourceId;
+
+    public function __construct(string $resourceType, string $message, int $code = 0, ?\Throwable $previous = null, string $resourceId = '')
+    {
+        $this->resourceId = $resourceId;
+        $this->resourceType = $resourceType;
+
+        parent::__construct($message, $code, $previous);
+    }
+
+    public function getResourceType(): string
+    {
+        return $this->resourceType;
+    }
+
+    public function getResourceId(): string
+    {
+        return $this->resourceId;
+    }
+}

src/Migration/Sources/Appwrite.php

@@ -9,6 +9,8 @@
 use Appwrite\Services\Storage;
 use Appwrite\Services\Teams;
 use Appwrite\Services\Users;
+use Utopia\Migration\Exception;
+use Utopia\Migration\Exception;
 use Utopia\Migration\Resource;
 use Utopia\Migration\Resources\Auth\Hash;
 use Utopia\Migration\Resources\Auth\Membership;
@@ -265,7 +267,7 @@ public function report(array $resources = []): array
             $this->previousReport = $report;
 
             return $report;
-        } catch (\Exception $e) {
+        } catch (\Throwable $e) {
             if ($e->getCode() === 403) {
                 throw new \Exception("Missing Permission: {$currentPermission}.");
             } else {
@@ -977,15 +979,24 @@ private function exportFiles(int $batchSize)
                 );
 
                 foreach ($response['files'] as $file) {
-                    $this->exportFileData(new File(
-                        $file['$id'],
-                        $bucket,
-                        $file['name'],
-                        $file['signature'],
-                        $file['mimeType'],
-                        $file['$permissions'],
-                        $file['sizeOriginal'],
-                    ));
+                    try {
+                        $this->exportFileData(new File(
+                            $file['$id'],
+                            $bucket,
+                            $file['name'],
+                            $file['signature'],
+                            $file['mimeType'],
+                            $file['$permissions'],
+                            $file['sizeOriginal'],
+                        ));
+                    } catch (\Throwable $e) {
+                        $this->addError(new Exception(
+                            resourceType: Resource::TYPE_FILE,
+                            message: $e->getMessage(),
+                            code: $e->getCode(),
+                            resourceId: $file['$id']
+                        ));
+                    }
 
                     $lastDocument = $file['$id'];
                 }
@@ -1106,7 +1117,7 @@ private function exportDeployments(int $batchSize, bool $exportOnlyActive = fals
 
                 try {
                     $this->exportDeploymentData($func, $deployment);
-                } catch (\Exception $e) {
+                } catch (\Throwable $e) {
                     $func->setStatus(Resource::STATUS_ERROR, $e->getMessage());
                 }
 
@@ -1128,7 +1139,7 @@ private function exportDeployments(int $batchSize, bool $exportOnlyActive = fals
                 foreach ($response['deployments'] as $deployment) {
                     try {
                         $this->exportDeploymentData($func, $deployment);
-                    } catch (\Exception $e) {
+                    } catch (\Throwable $e) {
                         $func->setStatus(Resource::STATUS_ERROR, $e->getMessage());
                     }
 

src/Migration/Sources/Firebase.php

@@ -89,7 +89,7 @@ private function authenticate()
             $this->currentToken = $response['access_token'];
             $this->tokenExpires = time() + $response['expires_in'];
             $this->headers['Authorization'] = 'Bearer '.$this->currentToken;
-        } catch (\Exception $e) {
+        } catch (\Throwable $e) {
             throw new \Exception('Failed to authenticate with Firebase: '.$e->getMessage());
         }
     }
@@ -155,7 +155,7 @@ protected function exportGroupAuth(int $batchSize, array $resources)
         // Check if Auth is enabled
         try {
             $this->call('GET', 'https://identitytoolkit.googleapis.com/v1/projects');
-        } catch (\Exception $e) {
+        } catch (\Throwable $e) {
             $message = json_decode($e->getMessage(), true);
 
             if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') {

src/Migration/Target.php

@@ -20,6 +20,13 @@ abstract class Target
      */
     public $cache;
 
+    /**
+     * Errors
+     *
+     * @var array
+     */
+    public $errors = [];
+
     /**
      * Endpoint
      *
@@ -168,4 +175,32 @@ protected function flatten(array $data, string $prefix = ''): array
 
         return $output;
     }
+
+    /**
+     * Get Errors
+     *
+     * @returns Error[]
+     */
+    public function getErrors(): array
+    {
+        return $this->errors;
+    }
+
+    /**
+     * Set Errors
+     *
+     * @param  Error[]  $errors
+     */
+    public function setErrors(array $errors): void
+    {
+        $this->errors = $errors;
+    }
+
+    /**
+     * Push Error
+     */
+    public function addError(Exception $error): void
+    {
+        $this->errors[] = $error;
+    }
 }

src/Migration/Transfer.php

@@ -99,13 +99,29 @@ public function getStatusCounters()
         foreach ($this->cache->getAll() as $resources) {
             foreach ($resources as $resource) {
                 /** @var resource $resource */
-                $status[$resource->getName()][$resource->getStatus()]++;
-                if ($status[$resource->getName()]['pending'] > 0) {
-                    $status[$resource->getName()]['pending']--;
+                if (isset($status[$resource->getName()])) {
+                    $status[$resource->getName()][$resource->getStatus()]++;
+                    if ($status[$resource->getName()]['pending'] > 0) {
+                        $status[$resource->getName()]['pending']--;
+                    }
                 }
             }
         }
 
+        // Process Destination Errors
+        foreach ($this->destination->getErrors() as $error) {
+            if (isset($status[$error->getResourceType()])) {
+                $status[$error->getResourceType()][Resource::STATUS_ERROR]++;
+            }
+        }
+
+        // Process Source Errprs
+        foreach ($this->source->getErrors() as $error) {
+            if (isset($status[$error->getResourceType()])) {
+                $status[$error->getResourceType()][Resource::STATUS_ERROR]++;
+            }
+        }
+
         // Remove all empty resources
         foreach ($status as $resource => $data) {
             $allEmpty = true;

tests/Migration/resources/supabase/api.json

@@ -1336,4 +1336,4 @@
     }
   ],
   "data": []
-}
+}