Merge pull request #12 from keboola/roman-pst-1796

PST-1796: Add dry-run mode
keboola · Jul 11, 2024 · 8862504 · 8862504
2 parents efd9fe4 + 6d1b3a3
commit 8862504
Show file tree

Hide file tree

Showing 13 changed files with 65 additions and 18 deletions.
diff --git a/phpstan.neon b/phpstan.neon
@@ -1,4 +1,4 @@
 parameters:
     checkMissingIterableValueType: false
     ignoreErrors:
-        - '#Cannot call method scalarNode\(\) on Symfony\\Component\\Config\\Definition\\Builder\\NodeParentInterface\|null\.#'
+        - '#Cannot call method (scalar|boolean)Node\(\) on Symfony\\Component\\Config\\Definition\\Builder\\NodeParentInterface\|null\.#'
diff --git a/src/Component.php b/src/Component.php
@@ -36,6 +36,7 @@ protected function run(): void
                     $sourceSapiClient,
                     $targetSapiClient,
                     $this->getLogger(),
+                    $this->getConfig()->isDryRun(),
                 );
                 break;
             case 'database':
@@ -83,6 +84,7 @@ protected function run(): void
                     $sourceDatabase,
                     $replicaDatabase,
                     $targetDatabase,
+                    $this->getConfig()->isDryRun(),
                 );
                 break;
             default:

diff --git a/src/Config.php b/src/Config.php
@@ -154,4 +154,9 @@ private function getDbConfigNode(): array
         }
         return $this->getImageParameters()['db'];
     }
+
+    public function isDryRun(): bool
+    {
+        return (bool) $this->getValue(['parameters', 'dryRun']);
+    }
 }
diff --git a/src/Configuration/ConfigDefinition.php b/src/Configuration/ConfigDefinition.php
@@ -17,6 +17,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
         $parametersNode
             ->children()
                 ->enumNode('mode')->values(['sapi', 'database'])->defaultValue('sapi')->end()
+                ->booleanNode('dryRun')->defaultFalse()->end()
                 ->scalarNode('sourceKbcUrl')->isRequired()->cannotBeEmpty()->end()
                 ->scalarNode('#sourceKbcToken')->isRequired()->cannotBeEmpty()->end()
                 ->arrayNode('tables')->prototype('scalar')->end()->end()

diff --git a/src/Strategy/DatabaseMigrate.php b/src/Strategy/DatabaseMigrate.php
@@ -27,6 +27,7 @@ public function __construct(
         private readonly string $sourceDatabase,
         private readonly string $replicaDatabase,
         private readonly string $targetDatabase,
+        private readonly bool $dryRun = false,
     ) {
     }
 
@@ -95,12 +96,21 @@ private function migrateSchema(array $tablesWhiteList, string $schemaName): void
             $this->migrateTable($schemaName, $table['name']);
         }
 
-        $this->logger->info(sprintf('Refreshing table information in bucket %s', $schemaName));
-        $this->targetSapiClient->refreshTableInformationInBucket($schemaName);
+        if ($this->dryRun === false) {
+            $this->logger->info(sprintf('Refreshing table information in bucket %s', $schemaName));
+            $this->targetSapiClient->refreshTableInformationInBucket($schemaName);
+        } else {
+            $this->logger->info(sprintf('[dry-run] Refreshing table information in bucket %s', $schemaName));
+        }
     }
 
     private function migrateTable(string $schemaName, string $tableName): void
     {
+        if ($this->dryRun) {
+            $this->logger->info(sprintf('[dry-run] Migrating table %s.%s', $schemaName, $tableName));
+            return;
+        }
+
         $this->logger->info(sprintf('Migrating table %s.%s', $schemaName, $tableName));
         $tableRole = $this->getSourceRole(
             $this->targetConnection,

diff --git a/src/Strategy/SapiMigrate.php b/src/Strategy/SapiMigrate.php
@@ -18,6 +18,7 @@ public function __construct(
         private readonly Client $sourceClient,
         private readonly Client $targetClient,
         private readonly LoggerInterface $logger,
+        private readonly bool $dryRun = false,
     ) {
     }
 
@@ -70,27 +71,41 @@ private function migrateTable(array $sourceTableInfo): void
             $this->logger->info(sprintf('Downloading table %s', $sourceTableInfo['id']));
             $slices = $this->sourceClient->downloadSlicedFile($sourceFileId, $tmp->getTmpFolder());
 
-            $this->logger->info(sprintf('Uploading table %s', $sourceTableInfo['id']));
-            $destinationFileId = $this->targetClient->uploadSlicedFile($slices, $optionUploadedFile);
+            if ($this->dryRun === false) {
+                $this->logger->info(sprintf('Uploading table %s', $sourceTableInfo['id']));
+                $destinationFileId = $this->targetClient->uploadSlicedFile($slices, $optionUploadedFile);
+            } else {
+                $this->logger->info(sprintf('[dry-run] Uploading table %s', $sourceTableInfo['id']));
+                $destinationFileId = null;
+            }
         } else {
             $fileName = $tmp->getTmpFolder() . '/' . $sourceFileInfo['name'];
 
             $this->logger->info(sprintf('Downloading table %s', $sourceTableInfo['id']));
             $this->sourceClient->downloadFile($sourceFileId, $fileName);
 
-            $this->logger->info(sprintf('Uploading table %s', $sourceTableInfo['id']));
-            $destinationFileId = $this->targetClient->uploadFile($fileName, $optionUploadedFile);
+            if ($this->dryRun === false) {
+                $this->logger->info(sprintf('Uploading table %s', $sourceTableInfo['id']));
+                $destinationFileId = $this->targetClient->uploadFile($fileName, $optionUploadedFile);
+            } else {
+                $this->logger->info(sprintf('[dry-run] Uploading table %s', $sourceTableInfo['id']));
+                $destinationFileId = null;
+            }
         }
 
-        // Upload data to table
-        $this->targetClient->writeTableAsyncDirect(
-            $sourceTableInfo['id'],
-            [
-                'name' => $sourceTableInfo['name'],
-                'dataFileId' => $destinationFileId,
-                'columns' => $sourceTableInfo['columns'],
-            ],
-        );
+        if ($this->dryRun === false) {
+            // Upload data to table
+            $this->targetClient->writeTableAsyncDirect(
+                $sourceTableInfo['id'],
+                [
+                    'name' => $sourceTableInfo['name'],
+                    'dataFileId' => $destinationFileId,
+                    'columns' => $sourceTableInfo['columns'],
+                ],
+            );
+        } else {
+            $this->logger->info(sprintf('[dry-run] Import data to table "%s"', $sourceTableInfo['name']));
+        }
     }
 
     private function getAllTables(): array
@@ -106,9 +121,9 @@ private function getAllTables(): array
                 fn($v) => $v['rowsCount'] === 0 || is_null($v['rowsCount']),
             );
 
-            $listTables = array_merge(
-                array_map(fn($v) => $v['id'], $filteredBucketTables),
+            array_unshift(
                 $listTables,
+                ...array_map(fn($v) => $v['id'], $filteredBucketTables),
             );
         }
         return $listTables;

diff --git a/tests/functional/dry-run/expected-code b/tests/functional/dry-run/expected-code
@@ -0,0 +1 @@
+0
diff --git a/tests/functional/dry-run/expected-stdout b/tests/functional/dry-run/expected-stdout
@@ -0,0 +1,4 @@
+Exporting table in.c-test.simple
+Downloading table in.c-test.simple
+[dry-run] Uploading table in.c-test.simple
+[dry-run] Import data to table "simple"
diff --git a/tests/functional/dry-run/expected/data/out/files/.gitkeep b/tests/functional/dry-run/expected/data/out/files/.gitkeep
diff --git a/tests/functional/dry-run/expected/data/out/tables/.gitkeep b/tests/functional/dry-run/expected/data/out/tables/.gitkeep
diff --git a/tests/functional/dry-run/source/data/config.json b/tests/functional/dry-run/source/data/config.json
@@ -0,0 +1,7 @@
+{
+  "parameters": {
+    "sourceKbcUrl": "%env(string:SOURCE_CLIENT_URL)%",
+    "#sourceKbcToken": "%env(string:SOURCE_CLIENT_TOKEN)%",
+    "dryRun": true
+  }
+}
diff --git a/tests/functional/dry-run/source/in/destination/in.test.simple.csv b/tests/functional/dry-run/source/in/destination/in.test.simple.csv
@@ -0,0 +1 @@
+"id","name","glasses","age"
diff --git a/tests/functional/dry-run/source/in/source/in.test.simple.csv b/tests/functional/dry-run/source/in/source/in.test.simple.csv
@@ -0,0 +1 @@
+"id","name","glasses","age"