Skip to content

Commit

Permalink
implements config-rows configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
@ondrajodas
ondrajodas committed Jan 4, 2021
1 parent 6894a0e commit fce9804
Show file tree
Hide file tree
Showing 13 changed files with 246 additions and 191 deletions.
2 changes: 1 addition & 1 deletion phpstan.neon
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
parameters:
checkMissingIterableValueType: false
ignoreErrors:
- '#Call to an undefined method Symfony\\Component\\Config\\Definition\\Builder\\NodeDefinition::children\(\)\.#'
- '#Cannot call method scalarNode\(\) on Symfony\\Component\\Config\\Definition\\Builder\\NodeParentInterface\|null\.#'
2 changes: 1 addition & 1 deletion phpunit.xml.dist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
convertWarningsToExceptions="true"
processIsolation="false"
bootstrap="tests/phpunit/bootstrap.php"
stopOnFailure="false"
stopOnFailure="true"
xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd"
>
<coverage processUncoveredFiles="true">
Expand Down
4 changes: 2 additions & 2 deletions src/Component.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ protected function run(): void
{
$extractor = new Extractor(
new Client($this->getGoogleRestApi()),
new Output($this->getDataDir()),
new Output($this->getDataDir(), $this->getConfig()),
$this->getLogger()
);

try {
$extractor->run($this->getConfig()->getSheets());
$extractor->run($this->getConfig());
} catch (RequestException $exception) {
$this->handleException($exception);
}
Expand Down
32 changes: 27 additions & 5 deletions src/Configuration/Config.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,33 @@

class Config extends BaseConfig
{
public function getSheets(): array
public function getFileId(): string
{
return $this->getValue(
['parameters', 'sheets'],
[]
);
return $this->getValue(['parameters', 'fileId']);
}

public function getFileTitle(): string
{
return $this->getValue(['parameters', 'fileTitle']);
}

public function getSheetId(): int
{
return (int) $this->getValue(['parameters', 'sheetId']);
}

public function getSheetTitle(): string
{
return $this->getValue(['parameters', 'sheetTitle']);
}

public function getHeader(): array
{
return $this->getValue(['parameters', 'header']);
}

public function getOutputTable(): string
{
return $this->getValue(['parameters', 'outputTable']);
}
}
101 changes: 47 additions & 54 deletions src/Configuration/ConfigDefinition.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,64 +14,57 @@ protected function getParametersDefinition(): ArrayNodeDefinition
$parametersNode = parent::getParametersDefinition();
$parametersNode
->children()
->arrayNode('sheets')
->integerNode('id')
->isRequired()
->prototype('array')
->children()
->integerNode('id')
->isRequired()
->min(0)
->end()
->scalarNode('fileId')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('fileTitle')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('sheetId')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('sheetTitle')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('outputTable')
->isRequired()
->cannotBeEmpty()
->min(0)
->end()
->scalarNode('fileId')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('fileTitle')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('sheetId')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('sheetTitle')
->isRequired()
->cannotBeEmpty()
->end()
->scalarNode('outputTable')
->isRequired()
->cannotBeEmpty()
->end()
->arrayNode('header')
->addDefaultsIfNotSet()
->children()
->integerNode('rows')
->defaultValue(1)
->end()
->arrayNode('columns')
->prototype('scalar')
->end()
->arrayNode('header')
->addDefaultsIfNotSet()
->children()
->integerNode('rows')
->defaultValue(1)
->end()
->arrayNode('columns')
->prototype('scalar')
->end()
->end()
->arrayNode('transpose')
->children()
->integerNode('row')
->end()
->scalarNode('name')
->end()
->end()
->end()
->booleanNode('sanitize')
->end()
->end()
->arrayNode('transpose')
->children()
->integerNode('row')
->end()
->scalarNode('name')
->end()
->end()
->arrayNode('transform')
->children()
->arrayNode('transpose')
->children()
->scalarNode('from')
->end()
->end()
->end()
->end()
->booleanNode('sanitize')
->end()
->end()
->end()
->arrayNode('transform')
->children()
->arrayNode('transpose')
->children()
->scalarNode('from')
->end()
->end()
->end()
Expand Down
19 changes: 12 additions & 7 deletions src/Extractor/ExceptionHandler.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,20 @@

use GuzzleHttp\Exception\RequestException;
use Keboola\Component\UserException;
use Keboola\GoogleDriveExtractor\Configuration\Config;
use Keboola\GoogleDriveExtractor\Exception\ApplicationException;

class ExceptionHandler
{
public function handleGetSpreadsheetException(\Throwable $e, array $sheet): void
public function handleGetSpreadsheetException(\Throwable $e, Config $config): void
{
if (($e instanceof RequestException) && ($e->getResponse() !== null)) {
if ($e->getResponse()->getStatusCode() === 404) {
throw new UserException(sprintf('File "%s" not found in Google Drive', $sheet['sheetTitle']), 404, $e);
throw new UserException(
sprintf('File "%s" not found in Google Drive', $config->getSheetTitle()),
404,
$e
);
}

$errorSpec = json_decode((string) $e->getResponse()->getBody()->getContents(), true);
Expand All @@ -24,7 +29,7 @@ public function handleGetSpreadsheetException(\Throwable $e, array $sheet): void
throw new UserException(
sprintf(
'Invalid OAuth grant when fetching "%s", try reauthenticating the extractor',
$sheet['fileTitle']
$config->getFileTitle()
),
0,
$e
Expand All @@ -40,7 +45,7 @@ public function handleGetSpreadsheetException(\Throwable $e, array $sheet): void
'"%s" (%s) for "%s"',
$errorSpec['error']['message'],
$errorSpec['error']['status'],
$sheet['sheetTitle']
$config->getSheetTitle()
),
0,
$e
Expand Down Expand Up @@ -69,14 +74,14 @@ public function handleGetSpreadsheetException(\Throwable $e, array $sheet): void
);
}

public function handleExportException(\Throwable $e, array $sheet): void
public function handleExportException(\Throwable $e, Config $config): void
{
if (($e instanceof RequestException) && ($e->getResponse() !== null)) {
throw new UserException(
sprintf(
"Error importing file - sheet: '%s - %s'",
$sheet['fileTitle'],
$sheet['sheetTitle']
$config->getFileTitle(),
$config->getSheetTitle()
),
400,
$e
Expand Down
39 changes: 19 additions & 20 deletions src/Extractor/Extractor.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace Keboola\GoogleDriveExtractor\Extractor;

use Keboola\Component\UserException;
use Keboola\GoogleDriveExtractor\Configuration\Config;
use Keboola\GoogleDriveExtractor\GoogleDrive\Client;
use Psr\Http\Message\ResponseInterface;
use Psr\Log\LoggerInterface;
Expand Down Expand Up @@ -45,39 +46,37 @@ public function getBackoffCallback403(): callable
};
}

public function run(array $sheets): array
public function run(Config $config): array
{
$status = [];
$exceptionHandler = new ExceptionHandler();

foreach ($sheets as $sheet) {
try {
$spreadsheet = $this->driveApi->getSpreadsheet($config->getFileId());
$this->logger->info('Obtained spreadsheet metadata');

try {
$spreadsheet = $this->driveApi->getSpreadsheet($sheet['fileId']);
$this->logger->info('Obtained spreadsheet metadata');

try {
$this->logger->info('Extracting sheet ' . $sheet['sheetTitle']);
$this->export($spreadsheet, $sheet);
} catch (UserException $e) {
throw new UserException($e->getMessage(), 0, $e);
} catch (\Throwable $e) {
$exceptionHandler->handleExportException($e, $sheet);
}
$this->logger->info('Extracting sheet ' . $config->getSheetTitle());
$this->export($spreadsheet, $config);
} catch (UserException $e) {
throw new UserException($e->getMessage(), 0, $e);
} catch (\Throwable $e) {
$exceptionHandler->handleGetSpreadsheetException($e, $sheet);
$exceptionHandler->handleExportException($e, $config);
}

$status[$sheet['fileTitle']][$sheet['sheetTitle']] = 'success';
} catch (UserException $e) {
throw new UserException($e->getMessage(), 0, $e);
} catch (\Throwable $e) {
$exceptionHandler->handleGetSpreadsheetException($e, $config);
}

$status[$config->getFileTitle()][$config->getSheetTitle()] = 'success';

return $status;
}

private function export(array $spreadsheet, array $sheetCfg): void
private function export(array $spreadsheet, Config $config): void
{
$sheet = $this->getSheetById($spreadsheet['sheets'], (string) $sheetCfg['sheetId']);
$sheet = $this->getSheetById($spreadsheet['sheets'], (string) $config->getSheetId());
$rowCount = $sheet['properties']['gridProperties']['rowCount'];
$columnCount = $sheet['properties']['gridProperties']['columnCount'];
$offset = 1;
Expand All @@ -95,8 +94,8 @@ private function export(array $spreadsheet, array $sheetCfg): void
if (!empty($response['values'])) {
if ($offset === 1) {
// it is a first run
$csvFilename = $this->output->createCsv($sheetCfg);
$this->output->createManifest($csvFilename, $sheetCfg['outputTable']);
$csvFilename = $this->output->createCsv();
$this->output->createManifest($csvFilename);
}

$this->output->write($response['values'], $offset);
Expand Down
20 changes: 11 additions & 9 deletions src/Extractor/Output.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace Keboola\GoogleDriveExtractor\Extractor;

use Keboola\Csv\CsvWriter;
use Keboola\GoogleDriveExtractor\Configuration\Config;

class Output
{
Expand All @@ -14,26 +15,26 @@ class Output

private ?array $header;

private array $sheetCfg;
private Config $config;

public function __construct(string $dataDir)
public function __construct(string $dataDir, Config $config)
{
$this->dataDir = $dataDir;
$this->config = $config;
}

public function createCsv(array $sheet): string
public function createCsv(): string
{
$outTablesDir = $this->dataDir . '/out/tables';
if (!is_dir($outTablesDir)) {
mkdir($outTablesDir, 0777, true);
}

$filename = $outTablesDir . '/' . $sheet['fileId'] . '_' . $sheet['sheetId'] . '.csv';
$filename = $outTablesDir . '/' . $this->config->getFileId() . '_' . $this->config->getSheetId() . '.csv';
touch($filename);

$this->csv = new CsvWriter($filename);
$this->header = null;
$this->sheetCfg = $sheet;

return $filename;
}
Expand All @@ -44,8 +45,9 @@ public function write(array $data, int $offset): void
return;
}

$headerConfig = $this->config->getHeader();
if ($this->header === null) {
$headerRowNum = $this->sheetCfg['header']['rows'] - 1;
$headerRowNum = $headerConfig['rows'] - 1;
$this->header = $data[$headerRowNum];
$headerLength = $this->getHeaderLength($data, (int) $headerRowNum);
} else {
Expand All @@ -54,8 +56,8 @@ public function write(array $data, int $offset): void

foreach ($data as $k => $row) {
// backward compatibility fix
if ($this->sheetCfg['header']['rows'] === 1 && $k === 0 && $offset === 1) {
if (!isset($this->sheetCfg['header']['sanitize']) || $this->sheetCfg['header']['sanitize'] !== false) {
if ($headerConfig['rows'] === 1 && $k === 0 && $offset === 1) {
if (!isset($headerConfig['sanitize']) || $headerConfig['sanitize'] !== false) {
$row = $this->normalizeCsvHeader($row);
}
}
Expand All @@ -67,7 +69,7 @@ public function write(array $data, int $offset): void
}
}

public function createManifest(string $filename, string $outputTable): bool
public function createManifest(string $filename): bool
{
$outFilename = $filename . '.manifest';

Expand Down
2 changes: 1 addition & 1 deletion tests/phpunit/BaseTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ protected function makeConfig(array $testFile): array
]
),
];
$config['parameters']['sheets'][0] = [
$config['parameters'] = [
'id' => 0,
'fileId' => $testFile['spreadsheetId'],
'fileTitle' => $testFile['properties']['title'],
Expand Down
Loading

0 comments on commit fce9804

Please sign in to comment.