diff --git a/composer.json b/composer.json index 597708d..42d3977 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,7 @@ "php": "^8.2", "keboola/db-adapter-snowflake": "^1.4", "keboola/php-component": "^9.4", - "keboola/storage-api-client": "^14.16" + "keboola/storage-api-client": "^14.21" }, "require-dev": { "cweagans/composer-patches": "^1.7", diff --git a/composer.lock b/composer.lock index 64f5117..3544072 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "7263ad8b18469f87c541fa7b58514984", + "content-hash": "c635a27ba32e3b3600e6bea62f892d9b", "packages": [ { "name": "aws/aws-crt-php", @@ -1354,16 +1354,16 @@ }, { "name": "keboola/storage-api-client", - "version": "v14.16.0", + "version": "v14.21.0", "source": { "type": "git", "url": "https://github.com/keboola/storage-api-php-client.git", - "reference": "11e9e160f619787660e5bbf834a141bd415d9028" + "reference": "2a232b51cc0c0229187d44a168f0d842c21c540a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/keboola/storage-api-php-client/zipball/11e9e160f619787660e5bbf834a141bd415d9028", - "reference": "11e9e160f619787660e5bbf834a141bd415d9028", + "url": "https://api.github.com/repos/keboola/storage-api-php-client/zipball/2a232b51cc0c0229187d44a168f0d842c21c540a", + "reference": "2a232b51cc0c0229187d44a168f0d842c21c540a", "shasum": "" }, "require": { @@ -1386,7 +1386,7 @@ "ext-pdo_pgsql": "*", "keboola/coding-standard": "^15.0", "keboola/php-csv-db-import": "^6", - "keboola/phpunit-retry-annotations": "^0.4", + "keboola/phpunit-retry-annotations": "^0.5", "keboola/retry": "^0.5.0", "keboola/table-backend-utils": "^1", "phpcompatibility/php-compatibility": "*", @@ -1395,7 +1395,7 @@ "phpunit/phpunit": "^7.0|^8.0|^9.0", "rector/rector": "^0.12.23", "squizlabs/php_codesniffer": "^3", - "tomasfejfar/phpstan-phpunit": "~0.0.1" + "tomasfejfar/phpstan-phpunit": "^0.1.0" }, "type": "library", "autoload": { @@ -1414,9 +1414,9 @@ "homepage": "http://keboola.com", "support": { "issues": "https://github.com/keboola/storage-api-php-client/issues", - "source": "https://github.com/keboola/storage-api-php-client/tree/v14.16.0" + "source": "https://github.com/keboola/storage-api-php-client/tree/v14.21.0" }, - "time": "2024-02-26T10:07:39+00:00" + "time": "2024-06-06T11:36:21+00:00" }, { "name": "microsoft/azure-storage-blob", diff --git a/src/Component.php b/src/Component.php index 1a2d71d..06f08a7 100644 --- a/src/Component.php +++ b/src/Component.php @@ -13,7 +13,9 @@ use Keboola\Component\BaseComponent; use Keboola\Component\UserException; use Keboola\SnowflakeDbAdapter\Exception\SnowflakeDbAdapterException; +use Keboola\StorageApi\BranchAwareClient; use Keboola\StorageApi\Client; +use Keboola\StorageApi\DevBranches; class Component extends BaseComponent { @@ -55,6 +57,7 @@ protected function run(): void $this->getConfig()->getTargetDatabasePrefix(), $targetSapiClient->verifyToken()['owner']['id'], ); + $defaultBranch = (new DevBranches($targetSapiClient))->getDefaultBranch(); try { $targetConnection = new Connection([ @@ -70,7 +73,13 @@ protected function run(): void $strategy = new DatabaseMigrate( $this->getLogger(), $targetConnection, - $targetSapiClient, + new BranchAwareClient( + $defaultBranch['id'], + [ + 'url' => $this->getConfig()->getEnvKbcUrl(), + 'token' => $this->getConfig()->getEnvKbcToken(), + ], + ), $sourceDatabase, $replicaDatabase, $targetDatabase, diff --git a/src/Strategy/DatabaseMigrate.php b/src/Strategy/DatabaseMigrate.php index 51604d6..ea18361 100644 --- a/src/Strategy/DatabaseMigrate.php +++ b/src/Strategy/DatabaseMigrate.php @@ -7,7 +7,6 @@ use Keboola\AppProjectMigrateLargeTables\Config; use Keboola\AppProjectMigrateLargeTables\MigrateInterface; use Keboola\AppProjectMigrateLargeTables\Snowflake\Connection; -use Keboola\Csv\CsvFile; use Keboola\SnowflakeDbAdapter\Exception\RuntimeException; use Keboola\SnowflakeDbAdapter\QueryBuilder; use Keboola\StorageApi\Client; @@ -92,6 +91,9 @@ private function migrateSchema(array $tablesWhiteList, string $schemaName): void } $this->migrateTable($schemaName, $table['name']); } + + $this->logger->info(sprintf('Refreshing table information in bucket %s', $schemaName)); + $this->targetSapiClient->refreshTableInformationInBucket($schemaName); } private function migrateTable(string $schemaName, string $tableName): void @@ -159,15 +161,6 @@ private function migrateTable(string $schemaName, string $tableName): void )); return; } - - $this->logger->info(sprintf('Refreshing table %s.%s metadata', $schemaName, $tableName)); - $columns = array_filter($columns, fn($v) => $v !== '_timestamp'); - $csv = $this->createDataFile($columns); - $tableId = sprintf('%s.%s', $schemaName, $tableName); - if (!$this->targetSapiClient->tableExists($tableId)) { - $this->logger->warning(sprintf('Table %s does not exist in Storage API', $tableId)); - } - $this->targetSapiClient->writeTableAsync($tableId, $csv, ['incremental' => true]); } private function getSourceRole(Connection $connection, string $showGrantsOn, string $targetSourceName): string @@ -184,13 +177,6 @@ private function getSourceRole(Connection $connection, string $showGrantsOn, str return current($ownershipOnDatabase)['grantee_name']; } - private function createDataFile(array $columns): CsvFile - { - $csv = new CsvFile('/tmp/tempDataFile.csv'); - $csv->writeRow($columns); - return $csv; - } - private function createReplicaDatabase(Config $config): void { // Migration database sqls