From 3e13d044c49c59238499d994e10bbc64a6760014 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 19 Feb 2024 11:55:04 +0000 Subject: [PATCH 1/5] Refactor Error Handling --- playground.php | 18 +++++++------ src/Migration/Destinations/Appwrite.php | 7 +++++ src/Migration/Error.php | 28 ++++++++++++++++++++ src/Migration/Resource.php | 2 +- src/Migration/Sources/Appwrite.php | 28 +++++++++++++------- src/Migration/Target.php | 35 +++++++++++++++++++++++++ src/Migration/Transfer.php | 14 ++++++++++ 7 files changed, 114 insertions(+), 18 deletions(-) create mode 100644 src/Migration/Error.php diff --git a/playground.php b/playground.php index 34517ae..82d14c9 100644 --- a/playground.php +++ b/playground.php @@ -11,7 +11,6 @@ use Utopia\CLI\Console; use Utopia\Migration\Destinations\Appwrite as AppwriteDestination; use Utopia\Migration\Destinations\Local; -use Utopia\Migration\Resource; use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; @@ -88,13 +87,16 @@ function (array $resources) use ($transfer) { $cache = $transfer->getCache()->getAll(); -foreach ($cache as $type => $resources) { - foreach ($resources as $resource) { - /** @var resource $resource */ - if ($resource->getStatus() !== Resource::STATUS_ERROR) { - continue; - } +if (count($sourceAppwrite->getErrors()) > 0) { + foreach ($sourceAppwrite->getErrors() as $error) { + /* @var \Utopia\Migration\Error $error */ + Console::error('[Source] ['.$error->getResourceType().'] '.$error->getMessage()); + } +} - Console::error($resource->getName().' '.$resource->getInternalId().' '.$resource->getMessage()); +if (count($destinationAppwrite->getErrors()) > 0) { + foreach ($destinationAppwrite->getErrors() as $error) { + /* @var \Utopia\Migration\Error $error */ + Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage()); } } diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index cb3431b..a86bfec 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -11,6 +11,7 @@ use Appwrite\Services\Teams; use Appwrite\Services\Users; use Utopia\Migration\Destination; +use Utopia\Migration\Error; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; use Utopia\Migration\Resources\Auth\Membership; @@ -243,6 +244,12 @@ protected function import(array $resources, callable $callback): void $resource->setStatus(Resource::STATUS_SKIPPED, $e->getMessage()); } else { $resource->setStatus(Resource::STATUS_ERROR, $e->getMessage()); + $this->pushError(new Error( + resourceType: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode() + )); } $responseResource = $resource; diff --git a/src/Migration/Error.php b/src/Migration/Error.php new file mode 100644 index 0000000..11a187e --- /dev/null +++ b/src/Migration/Error.php @@ -0,0 +1,28 @@ +resourceId = $resourceId; + $this->resourceType = $resourceType; + + parent::__construct($message, $code, $previous); + } + + public function getResourceType(): string + { + return $this->resourceType; + } + + public function getResourceId(): string + { + return $this->resourceId; + } +} diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index bd294fa..b1f460c 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -52,7 +52,7 @@ abstract class Resource public const TYPE_HASH = 'hash'; - public const TYPE_ENVVAR = 'envvar'; + public const TYPE_ENVVAR = 'environment variable'; public const ALL_RESOURCES = [ self::TYPE_ATTRIBUTE, diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 622155a..56ca1c4 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -9,6 +9,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Utopia\Migration\Error; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; use Utopia\Migration\Resources\Auth\Membership; @@ -977,15 +978,24 @@ private function exportFiles(int $batchSize) ); foreach ($response['files'] as $file) { - $this->exportFileData(new File( - $file['$id'], - $bucket, - $file['name'], - $file['signature'], - $file['mimeType'], - $file['$permissions'], - $file['sizeOriginal'], - )); + try { + $this->exportFileData(new File( + $file['$id'], + $bucket, + $file['name'], + $file['signature'], + $file['mimeType'], + $file['$permissions'], + $file['sizeOriginal'], + )); + } catch (\Exception $e) { + $this->pushError(new Error( + resourceType: Resource::TYPE_FILE, + message: $e->getMessage(), + code: $e->getCode(), + resourceId: $file['$id'] + )); + } $lastDocument = $file['$id']; } diff --git a/src/Migration/Target.php b/src/Migration/Target.php index 8149e52..fdfbdc6 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -20,6 +20,13 @@ abstract class Target */ public $cache; + /** + * Errors + * + * @var array + */ + public $errors = []; + /** * Endpoint * @@ -168,4 +175,32 @@ protected function flatten(array $data, string $prefix = ''): array return $output; } + + /** + * Get Errors + * + * @returns Error[] + */ + public function getErrors(): array + { + return $this->errors; + } + + /** + * Set Errors + * + * @param Error[] $errors + */ + public function setErrors(array $errors): void + { + $this->errors = $errors; + } + + /** + * Push Error + */ + public function pushError(Error $error): void + { + $this->errors[] = $error; + } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 06af5ba..4703190 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -106,6 +106,20 @@ public function getStatusCounters() } } + // Process Destination Errors + foreach ($this->destination->getErrors() as $error) { + if (isset($status[$error->getResourceType()])) { + $status[$error->getResourceType()][Resource::STATUS_ERROR]++; + } + } + + // Process Source Errprs + foreach ($this->source->getErrors() as $error) { + if (isset($status[$error->getResourceType()])) { + $status[$error->getResourceType()][Resource::STATUS_ERROR]++; + } + } + // Remove all empty resources foreach ($status as $resource => $data) { $allEmpty = true; From 8be4aba3d157af3a7791cdf2435960c3fb3bd883 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 19 Feb 2024 12:25:38 +0000 Subject: [PATCH 2/5] Add cache resource check --- src/Migration/Transfer.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 4703190..b88386f 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -98,10 +98,12 @@ public function getStatusCounters() foreach ($this->cache->getAll() as $resources) { foreach ($resources as $resource) { - /** @var resource $resource */ - $status[$resource->getName()][$resource->getStatus()]++; - if ($status[$resource->getName()]['pending'] > 0) { - $status[$resource->getName()]['pending']--; + /** @var Resource $resource */ + if (isset($status[$resource->getName()])) { + $status[$resource->getName()][$resource->getStatus()]++; + if ($status[$resource->getName()]['pending'] > 0) { + $status[$resource->getName()]['pending']--; + } } } } From 7bc6af9a18d5d401433115b995ad5009306f672b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 19 Feb 2024 14:05:52 +0000 Subject: [PATCH 3/5] Update Supabase and NHost Tests --- src/Migration/Transfer.php | 2 +- tests/Migration/E2E/Sources/NHostTest.php | 2 +- tests/Migration/E2E/Sources/SupabaseTest.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index b88386f..5b34221 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -98,7 +98,7 @@ public function getStatusCounters() foreach ($this->cache->getAll() as $resources) { foreach ($resources as $resource) { - /** @var Resource $resource */ + /** @var resource $resource */ if (isset($status[$resource->getName()])) { $status[$resource->getName()][$resource->getStatus()]++; if ($status[$resource->getName()]['pending'] > 0) { diff --git a/tests/Migration/E2E/Sources/NHostTest.php b/tests/Migration/E2E/Sources/NHostTest.php index 410216f..4412b3f 100644 --- a/tests/Migration/E2E/Sources/NHostTest.php +++ b/tests/Migration/E2E/Sources/NHostTest.php @@ -157,7 +157,7 @@ public function testValidateUserTransfer($state): void $this->assertEquals('$2a$10$ARQ/f.K6OmCjZ8XF0U.6fezPMlxDqsmcl0Rs6xQVkvj62u7gcSzOW', $foundUser->getPasswordHash()->getHash()); $this->assertEquals('bcrypt', $foundUser->getPasswordHash()->getAlgorithm()); $this->assertEquals('test@test.com', $foundUser->getUsername()); - $this->assertEquals(['email'], $foundUser->getTypes()); + $this->assertEquals(['password'], $foundUser->getTypes()); } /** diff --git a/tests/Migration/E2E/Sources/SupabaseTest.php b/tests/Migration/E2E/Sources/SupabaseTest.php index 832e783..66b8025 100644 --- a/tests/Migration/E2E/Sources/SupabaseTest.php +++ b/tests/Migration/E2E/Sources/SupabaseTest.php @@ -136,7 +136,7 @@ public function testValidateUserTransfer($state): void $this->assertEquals('success', $foundUser->getStatus()); $this->assertEquals('$2a$10$NGZAAOfXeheUoH9V3dnRoeR.r3J5ynnSZ6KjvHxOUlV8XUrulJzQa', $foundUser->getPasswordHash()->getHash()); $this->assertEquals('bcrypt', $foundUser->getPasswordHash()->getAlgorithm()); - $this->assertEquals(['email'], $foundUser->getTypes()); + $this->assertEquals(['password'], $foundUser->getTypes()); } /** From 1624f6663c6b48c71c45c1741b235177358daae9 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 21 Feb 2024 14:15:53 +0000 Subject: [PATCH 4/5] Change Error to Exception and catch Throwables --- phpunit.xml | 2 +- playground.php | 4 ++-- src/Migration/Destinations/Appwrite.php | 8 ++++---- src/Migration/{Error.php => Exception.php} | 2 +- src/Migration/Sources/Appwrite.php | 13 +++++++------ src/Migration/Sources/Firebase.php | 4 ++-- src/Migration/Target.php | 2 +- tests/Migration/resources/supabase/api.json | 2 +- 8 files changed, 19 insertions(+), 18 deletions(-) rename src/Migration/{Error.php => Exception.php} (94%) diff --git a/phpunit.xml b/phpunit.xml index b5dd6f8..9a35000 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -13,4 +13,4 @@ ./tests/Migration/E2E - \ No newline at end of file + diff --git a/playground.php b/playground.php index 82d14c9..571a768 100644 --- a/playground.php +++ b/playground.php @@ -89,14 +89,14 @@ function (array $resources) use ($transfer) { if (count($sourceAppwrite->getErrors()) > 0) { foreach ($sourceAppwrite->getErrors() as $error) { - /* @var \Utopia\Migration\Error $error */ + /* @var \Utopia\Migration\Exception $error */ Console::error('[Source] ['.$error->getResourceType().'] '.$error->getMessage()); } } if (count($destinationAppwrite->getErrors()) > 0) { foreach ($destinationAppwrite->getErrors() as $error) { - /* @var \Utopia\Migration\Error $error */ + /* @var \Utopia\Migration\Exception $error */ Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage()); } } diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index a86bfec..87fccbc 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -11,7 +11,7 @@ use Appwrite\Services\Teams; use Appwrite\Services\Users; use Utopia\Migration\Destination; -use Utopia\Migration\Error; +use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; use Utopia\Migration\Resources\Auth\Membership; @@ -205,7 +205,7 @@ public function report(array $resources = []): array } return []; - } catch (\Exception $exception) { + } catch (\Throwable $exception) { if ($exception->getCode() === 403) { throw new \Exception('Missing permission: '.$currentPermission); } else { @@ -239,12 +239,12 @@ protected function import(array $resources, callable $callback): void $responseResource = $this->importFunctionResource($resource); break; } - } catch (\Exception $e) { + } catch (\Throwable $e) { if ($e->getCode() === 409) { $resource->setStatus(Resource::STATUS_SKIPPED, $e->getMessage()); } else { $resource->setStatus(Resource::STATUS_ERROR, $e->getMessage()); - $this->pushError(new Error( + $this->pushError(new Exception( resourceType: $resource->getGroup(), resourceId: $resource->getId(), message: $e->getMessage(), diff --git a/src/Migration/Error.php b/src/Migration/Exception.php similarity index 94% rename from src/Migration/Error.php rename to src/Migration/Exception.php index 11a187e..d90f50c 100644 --- a/src/Migration/Error.php +++ b/src/Migration/Exception.php @@ -2,7 +2,7 @@ namespace Utopia\Migration; -class Error extends \Exception +class Exception extends \Exception { public string $resourceType; diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 56ca1c4..7204e4e 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -9,7 +9,8 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; -use Utopia\Migration\Error; +use Utopia\Migration\Exception; +use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; use Utopia\Migration\Resources\Auth\Membership; @@ -266,7 +267,7 @@ public function report(array $resources = []): array $this->previousReport = $report; return $report; - } catch (\Exception $e) { + } catch (\Throwable $e) { if ($e->getCode() === 403) { throw new \Exception("Missing Permission: {$currentPermission}."); } else { @@ -988,8 +989,8 @@ private function exportFiles(int $batchSize) $file['$permissions'], $file['sizeOriginal'], )); - } catch (\Exception $e) { - $this->pushError(new Error( + } catch (\Throwable $e) { + $this->pushError(new Exception( resourceType: Resource::TYPE_FILE, message: $e->getMessage(), code: $e->getCode(), @@ -1116,7 +1117,7 @@ private function exportDeployments(int $batchSize, bool $exportOnlyActive = fals try { $this->exportDeploymentData($func, $deployment); - } catch (\Exception $e) { + } catch (\Throwable $e) { $func->setStatus(Resource::STATUS_ERROR, $e->getMessage()); } @@ -1138,7 +1139,7 @@ private function exportDeployments(int $batchSize, bool $exportOnlyActive = fals foreach ($response['deployments'] as $deployment) { try { $this->exportDeploymentData($func, $deployment); - } catch (\Exception $e) { + } catch (\Throwable $e) { $func->setStatus(Resource::STATUS_ERROR, $e->getMessage()); } diff --git a/src/Migration/Sources/Firebase.php b/src/Migration/Sources/Firebase.php index 18c8bf4..4816875 100644 --- a/src/Migration/Sources/Firebase.php +++ b/src/Migration/Sources/Firebase.php @@ -89,7 +89,7 @@ private function authenticate() $this->currentToken = $response['access_token']; $this->tokenExpires = time() + $response['expires_in']; $this->headers['Authorization'] = 'Bearer '.$this->currentToken; - } catch (\Exception $e) { + } catch (\Throwable $e) { throw new \Exception('Failed to authenticate with Firebase: '.$e->getMessage()); } } @@ -155,7 +155,7 @@ protected function exportGroupAuth(int $batchSize, array $resources) // Check if Auth is enabled try { $this->call('GET', 'https://identitytoolkit.googleapis.com/v1/projects'); - } catch (\Exception $e) { + } catch (\Throwable $e) { $message = json_decode($e->getMessage(), true); if (isset($message['error']['details']) && $message['error']['details'][1]['reason'] == 'SERVICE_DISABLED') { diff --git a/src/Migration/Target.php b/src/Migration/Target.php index fdfbdc6..d6df46c 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -199,7 +199,7 @@ public function setErrors(array $errors): void /** * Push Error */ - public function pushError(Error $error): void + public function pushError(Exception $error): void { $this->errors[] = $error; } diff --git a/tests/Migration/resources/supabase/api.json b/tests/Migration/resources/supabase/api.json index 60ffed5..2816e90 100644 --- a/tests/Migration/resources/supabase/api.json +++ b/tests/Migration/resources/supabase/api.json @@ -1336,4 +1336,4 @@ } ], "data": [] -} \ No newline at end of file +} From 4d464e7ac592536c92fc745025cdcf43594e4781 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Thu, 22 Feb 2024 16:58:36 +0000 Subject: [PATCH 5/5] Rename pushError to addError --- src/Migration/Destinations/Appwrite.php | 2 +- src/Migration/Sources/Appwrite.php | 2 +- src/Migration/Target.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 87fccbc..bd3d78b 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -244,7 +244,7 @@ protected function import(array $resources, callable $callback): void $resource->setStatus(Resource::STATUS_SKIPPED, $e->getMessage()); } else { $resource->setStatus(Resource::STATUS_ERROR, $e->getMessage()); - $this->pushError(new Exception( + $this->addError(new Exception( resourceType: $resource->getGroup(), resourceId: $resource->getId(), message: $e->getMessage(), diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 7204e4e..3b0102a 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -990,7 +990,7 @@ private function exportFiles(int $batchSize) $file['sizeOriginal'], )); } catch (\Throwable $e) { - $this->pushError(new Exception( + $this->addError(new Exception( resourceType: Resource::TYPE_FILE, message: $e->getMessage(), code: $e->getCode(), diff --git a/src/Migration/Target.php b/src/Migration/Target.php index d6df46c..9771456 100644 --- a/src/Migration/Target.php +++ b/src/Migration/Target.php @@ -199,7 +199,7 @@ public function setErrors(array $errors): void /** * Push Error */ - public function pushError(Exception $error): void + public function addError(Exception $error): void { $this->errors[] = $error; }