Skip to content

Commit

Permalink
Merge pull request #315 from keboola/odin-KAB-257
Browse files Browse the repository at this point in the history 
 Refactor staging provider
  • Loading branch information
@odinuv
odinuv authored Sep 17, 2024
2 parents bb47c00 + c2d7d3a commit 4f9c6a7
Show file tree
Hide file tree
Showing 54 changed files with 1,355 additions and 1,681 deletions.
2 changes: 1 addition & 1 deletion libs/staging-provider/phpstan.neon
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
parameters:
level: max
checkMissingIterableValueType: false
paths:
- src
- tests
ignoreErrors:
- identifier: missingType.iterableValue

includes:
- vendor/phpstan/phpstan-phpunit/extension.neon
93 changes: 80 additions & 13 deletions libs/staging-provider/src/AbstractProviderInitializer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,41 +6,108 @@

use Keboola\InputMapping\Staging\AbstractStrategyFactory;
use Keboola\InputMapping\Staging\Scope;
use Keboola\StagingProvider\Provider\AbstractWorkspaceProvider;
use Keboola\StagingProvider\Provider\LocalStagingProvider;
use Keboola\StagingProvider\Staging\LocalStaging;
use Keboola\StagingProvider\Staging\Workspace\WorkspaceStagingInterface;
use Keboola\StagingProvider\WorkspaceProviderFactory\WorkspaceProviderFactoryInterface;

abstract class AbstractProviderInitializer
{
public function __construct(
private readonly AbstractStrategyFactory $stagingFactory,
private readonly WorkspaceProviderFactoryInterface $workspaceProviderFactory,
private readonly string $dataDirectory,
private readonly AbstractWorkspaceProvider $workspaceStagingProvider,
private readonly LocalStagingProvider $localStagingProvider,
) {
}

abstract public function initializeProviders(string $stagingType, array $tokenInfo): void;

/**
* @param class-string<WorkspaceStagingInterface> $workspaceClass
* @param array<string, Scope> $scopes
*/
protected function addWorkspaceProvider(string $workspaceClass, array $scopes): void
protected function addWorkspaceProvider(array $scopes): void
{
$stagingProvider = $this->workspaceProviderFactory->getProvider($workspaceClass);
$this->stagingFactory->addProvider($stagingProvider, $scopes);
$this->stagingFactory->addProvider($this->workspaceStagingProvider, $scopes);
}

/**
* @param array<string, Scope> $scopes
*/
protected function addLocalProvider(array $scopes): void
{
$stagingProvider = new LocalStagingProvider(function () {
return new LocalStaging($this->dataDirectory);
});
$this->stagingFactory->addProvider($this->localStagingProvider, $scopes);
}

protected function addWorkspaceProviders(string $stagingType, array $tokenInfo): void
{
if ($stagingType === AbstractStrategyFactory::WORKSPACE_REDSHIFT &&
$tokenInfo['owner']['hasRedshift']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_REDSHIFT => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SNOWFLAKE &&
$tokenInfo['owner']['hasSnowflake']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_SNOWFLAKE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SYNAPSE &&
$tokenInfo['owner']['hasSynapse']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_SYNAPSE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_ABS &&
$tokenInfo['owner']['fileStorageProvider'] === 'azure'
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_ABS => new Scope(
[Scope::FILE_DATA, Scope::FILE_METADATA, Scope::TABLE_DATA],
),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_EXASOL &&
$tokenInfo['owner']['hasExasol']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_EXASOL => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_TERADATA &&
$tokenInfo['owner']['hasTeradata']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_TERADATA => new Scope([Scope::TABLE_DATA]),
],
);
}

$this->stagingFactory->addProvider($stagingProvider, $scopes);
if ($stagingType === AbstractStrategyFactory::WORKSPACE_BIGQUERY &&
$tokenInfo['owner']['hasBigquery']
) {
$this->addWorkspaceProvider(
[
AbstractStrategyFactory::WORKSPACE_BIGQUERY => new Scope([Scope::TABLE_DATA]),
],
);
}
}
}
105 changes: 17 additions & 88 deletions libs/staging-provider/src/InputProviderInitializer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,98 +6,27 @@

use Keboola\InputMapping\Staging\AbstractStrategyFactory;
use Keboola\InputMapping\Staging\Scope;
use Keboola\StagingProvider\Staging\Workspace\AbsWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\BigQueryWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\ExasolWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\RedshiftWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\SnowflakeWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\SynapseWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\TeradataWorkspaceStaging;
use Keboola\InputMapping\Staging\StrategyFactory as InputStrategyFactory;
use Keboola\StagingProvider\Provider\AbstractWorkspaceProvider;
use Keboola\StagingProvider\Provider\LocalStagingProvider;

class InputProviderInitializer extends AbstractProviderInitializer
{
public function initializeProviders(
string $stagingType,
array $tokenInfo,
): void {
if ($stagingType === AbstractStrategyFactory::WORKSPACE_REDSHIFT &&
$tokenInfo['owner']['hasRedshift']
) {
$this->addWorkspaceProvider(
RedshiftWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_REDSHIFT => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SNOWFLAKE &&
$tokenInfo['owner']['hasSnowflake']
) {
$this->addWorkspaceProvider(
SnowflakeWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_SNOWFLAKE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SYNAPSE &&
$tokenInfo['owner']['hasSynapse']
) {
$this->addWorkspaceProvider(
SynapseWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_SYNAPSE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_ABS &&
$tokenInfo['owner']['fileStorageProvider'] === 'azure'
) {
$this->addWorkspaceProvider(
AbsWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_ABS => new Scope(
[Scope::FILE_DATA, Scope::FILE_METADATA, Scope::TABLE_DATA],
),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_EXASOL &&
$tokenInfo['owner']['hasExasol']
) {
$this->addWorkspaceProvider(
ExasolWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_EXASOL => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_TERADATA &&
$tokenInfo['owner']['hasTeradata']
) {
$this->addWorkspaceProvider(
TeradataWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_TERADATA => new Scope([Scope::TABLE_DATA]),
],
);
}
public function __construct(
InputStrategyFactory $stagingFactory, // This is just to make sure the correct StrategyFactory is injected
private readonly AbstractWorkspaceProvider $workspaceStagingProvider,
private readonly LocalStagingProvider $localStagingProvider,
) {
parent::__construct(
$stagingFactory,
$this->workspaceStagingProvider,
$this->localStagingProvider,
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_BIGQUERY &&
$tokenInfo['owner']['hasBigquery']
) {
$this->addWorkspaceProvider(
BigQueryWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_BIGQUERY => new Scope([Scope::TABLE_DATA]),
],
);
}
public function initializeProviders(string $stagingType, array $tokenInfo): void
{
$this->addWorkspaceProviders($stagingType, $tokenInfo);

$this->addLocalProvider(
[
Expand Down
105 changes: 10 additions & 95 deletions libs/staging-provider/src/OutputProviderInitializer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,111 +7,26 @@
use Keboola\InputMapping\Staging\AbstractStrategyFactory;
use Keboola\InputMapping\Staging\Scope;
use Keboola\OutputMapping\Staging\StrategyFactory as OutputStrategyFactory;
use Keboola\StagingProvider\Staging\Workspace\AbsWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\BigQueryWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\ExasolWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\RedshiftWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\SnowflakeWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\SynapseWorkspaceStaging;
use Keboola\StagingProvider\Staging\Workspace\TeradataWorkspaceStaging;
use Keboola\StagingProvider\WorkspaceProviderFactory\WorkspaceProviderFactoryInterface;
use Keboola\StagingProvider\Provider\AbstractWorkspaceProvider;
use Keboola\StagingProvider\Provider\LocalStagingProvider;

class OutputProviderInitializer extends AbstractProviderInitializer
{
public function __construct(
OutputStrategyFactory $stagingFactory,
WorkspaceProviderFactoryInterface $workspaceProviderFactory,
string $dataDirectory,
OutputStrategyFactory $stagingFactory, // This is just to make sure the correct StrategyFactory is injected
private readonly AbstractWorkspaceProvider $workspaceStagingProvider,
private readonly LocalStagingProvider $localStagingProvider,
) {
parent::__construct(
$stagingFactory,
$workspaceProviderFactory,
$dataDirectory,
$this->workspaceStagingProvider,
$this->localStagingProvider,
);
}

public function initializeProviders(
string $stagingType,
array $tokenInfo,
): void {
if ($stagingType === AbstractStrategyFactory::WORKSPACE_REDSHIFT &&
$tokenInfo['owner']['hasRedshift']
) {
$this->addWorkspaceProvider(
RedshiftWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_REDSHIFT => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SNOWFLAKE &&
$tokenInfo['owner']['hasSnowflake']
) {
$this->addWorkspaceProvider(
SnowflakeWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_SNOWFLAKE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_SYNAPSE &&
$tokenInfo['owner']['hasSynapse']
) {
$this->addWorkspaceProvider(
SynapseWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_SYNAPSE => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_ABS &&
$tokenInfo['owner']['fileStorageProvider'] === 'azure'
) {
$this->addWorkspaceProvider(
AbsWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_ABS => new Scope(
[Scope::FILE_DATA, Scope::FILE_METADATA, Scope::TABLE_DATA],
),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_EXASOL &&
$tokenInfo['owner']['hasExasol']
) {
$this->addWorkspaceProvider(
ExasolWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_EXASOL => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_TERADATA &&
$tokenInfo['owner']['hasTeradata']
) {
$this->addWorkspaceProvider(
TeradataWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_TERADATA => new Scope([Scope::TABLE_DATA]),
],
);
}

if ($stagingType === AbstractStrategyFactory::WORKSPACE_BIGQUERY &&
$tokenInfo['owner']['hasBigquery']
) {
$this->addWorkspaceProvider(
BigQueryWorkspaceStaging::class,
[
AbstractStrategyFactory::WORKSPACE_BIGQUERY => new Scope([Scope::TABLE_DATA]),
],
);
}
public function initializeProviders(string $stagingType, array $tokenInfo): void
{
$this->addWorkspaceProviders($stagingType, $tokenInfo);

$this->addLocalProvider(
[
Expand Down
Loading

0 comments on commit 4f9c6a7

Please sign in to comment.