From a8db5d2a76723aafda7f274a826682edee6efe20 Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Thu, 3 Dec 2020 09:46:12 +0100 Subject: [PATCH 1/3] fix composer version --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f16406e..afd89fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM php:7.4-cli -ENV COMPOSER_ALLOW_SUPERUSER 1 +ENV COMPOSER_ALLOW_SUPERUSER=1 +ARG COMPOSER_FLAGS="--prefer-dist --no-interaction" WORKDIR /code @@ -11,7 +12,9 @@ RUN apt-get update && apt-get install -y \ COPY ./docker/php/php.ini /usr/local/etc/php/php.ini -RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer +ADD https://getcomposer.org/installer composer-setup.php +RUN php composer-setup.php --1 \ + && mv composer.phar /usr/local/bin/composer COPY composer.* ./ RUN composer install $COMPOSER_FLAGS --no-scripts --no-autoloader From 9be4ceb6b0802c0c2873c960520b8a87716dd380 Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Thu, 3 Dec 2020 09:49:38 +0100 Subject: [PATCH 2/3] test for bug --- tests/FunctionalTest.php | 40 ++++++++++++++++++++ tests/data/expectedFiles/multiple_header.csv | 5 +++ 2 files changed, 45 insertions(+) create mode 100644 tests/data/expectedFiles/multiple_header.csv diff --git a/tests/FunctionalTest.php b/tests/FunctionalTest.php index 09ccdbf..e5ceaa8 100644 --- a/tests/FunctionalTest.php +++ b/tests/FunctionalTest.php @@ -4,6 +4,7 @@ namespace Keboola\GoogleDriveExtractor\Tests; +use Keboola\Csv\CsvWriter; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\Process\Process; use Symfony\Component\Yaml\Yaml; @@ -102,6 +103,36 @@ public function testDoNotSanitizeHeader(): void unlink($filePath); } + public function testMultipleHeader(): void + { + $filePath = __DIR__ . '/data/in/multiple_header.csv'; + touch($filePath); + $csv = new CsvWriter($filePath); + $this->writeLineToCsvFile($csv, 1, 7); + $this->writeLineToCsvFile($csv, 2, 5); + $this->writeLineToCsvFile($csv, 3, 20); + $this->writeLineToCsvFile($csv, 4, 2); + $this->writeLineToCsvFile($csv, 5, 7); + + $this->testFileName = 'multiple_header'; + $this->testFile = $this->prepareTestFile($filePath, 'multiple_header'); + $this->config = $this->makeConfig($this->testFile); + + // leave the header as is + $this->config['parameters']['sheets'][0]['header']['rows'] = 2; + + $process = $this->runProcess(); + $this->assertEquals(0, $process->getExitCode(), $process->getErrorOutput()); + + $fileId = $this->config['parameters']['sheets'][0]['fileId']; + $sheetId = $this->config['parameters']['sheets'][0]['sheetId']; + + $this->assertFileEquals( + __DIR__ . '/data/expectedFiles/multiple_header.csv', + $this->dataPath . '/out/tables/' . $this->getOutputFileName($fileId, $sheetId) + ); + } + private function runProcess(): Process { $fs = new Filesystem(); @@ -117,4 +148,13 @@ private function runProcess(): Process return $process; } + + private function writeLineToCsvFile(CsvWriter $csvWriter, int $rowNumber, int $countColumns): void + { + $headerLine = []; + for ($j = 0; $j < $countColumns; $j++) { + $headerLine[] = sprintf('Col_%s_%s', $rowNumber, $j); + } + $csvWriter->writeRow($headerLine); + } } diff --git a/tests/data/expectedFiles/multiple_header.csv b/tests/data/expectedFiles/multiple_header.csv new file mode 100644 index 0000000..95bb9ce --- /dev/null +++ b/tests/data/expectedFiles/multiple_header.csv @@ -0,0 +1,5 @@ +"Col_1_0","Col_1_1","Col_1_2","Col_1_3","Col_1_4","Col_1_5","Col_1_6" +"Col_2_0","Col_2_1","Col_2_2","Col_2_3","Col_2_4","","" +"Col_3_0","Col_3_1","Col_3_2","Col_3_3","Col_3_4","Col_3_5","Col_3_6" +"Col_4_0","Col_4_1","","","","","" +"Col_5_0","Col_5_1","Col_5_2","Col_5_3","Col_5_4","Col_5_5","Col_5_6" From 74052bb2f15a98c6c447adb2aabff316b75cf09b Mon Sep 17 00:00:00 2001 From: Ondra Jodas Date: Thu, 3 Dec 2020 09:57:31 +0100 Subject: [PATCH 3/3] fix count columns in multiple headers --- .../GoogleDriveExtractor/Extractor/Output.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Keboola/GoogleDriveExtractor/Extractor/Output.php b/src/Keboola/GoogleDriveExtractor/Extractor/Output.php index 62566b8..007621a 100644 --- a/src/Keboola/GoogleDriveExtractor/Extractor/Output.php +++ b/src/Keboola/GoogleDriveExtractor/Extractor/Output.php @@ -56,10 +56,11 @@ public function write(array $data, int $offset): void if ($this->header === null) { $headerRowNum = $this->sheetCfg['header']['rows'] - 1; $this->header = $data[$headerRowNum]; + $headerLength = $this->getHeaderLength($data, (int) $headerRowNum); + } else { + $headerLength = count($this->header); } - $headerLength = count($this->header); - foreach ($data as $k => $row) { // backward compatibility fix if ($this->sheetCfg['header']['rows'] === 1 && $k === 0 && $offset === 1) { @@ -94,4 +95,13 @@ protected function normalizeCsvHeader(array $header): array } return $header; } + + private function getHeaderLength(array $data, int $headerRowNum): int + { + $headerLength = 0; + for ($i = 0; $i <= $headerRowNum; $i++) { + $headerLength = max($headerLength, count($data[$i])); + } + return $headerLength; + } }