Merge pull request #28 from keboola/COM-556-ondra-fill-empty-values

fill empty value to merged header rows
keboola · Dec 3, 2020 · 94735a2 · 94735a2
2 parents a7fface + 74052bb
commit 94735a2
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 4 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,7 @@
 FROM php:7.4-cli
 
-ENV COMPOSER_ALLOW_SUPERUSER 1
+ENV COMPOSER_ALLOW_SUPERUSER=1
+ARG COMPOSER_FLAGS="--prefer-dist --no-interaction"
 
 WORKDIR /code
 
@@ -11,7 +12,9 @@ RUN apt-get update && apt-get install -y \
 
 COPY ./docker/php/php.ini /usr/local/etc/php/php.ini
 
-RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin/ --filename=composer
+ADD https://getcomposer.org/installer composer-setup.php
+RUN php composer-setup.php --1 \
+  && mv composer.phar /usr/local/bin/composer
 
 COPY composer.* ./
 RUN composer install $COMPOSER_FLAGS --no-scripts --no-autoloader

diff --git a/src/Keboola/GoogleDriveExtractor/Extractor/Output.php b/src/Keboola/GoogleDriveExtractor/Extractor/Output.php
@@ -56,10 +56,11 @@ public function write(array $data, int $offset): void
         if ($this->header === null) {
             $headerRowNum = $this->sheetCfg['header']['rows'] - 1;
             $this->header = $data[$headerRowNum];
+            $headerLength = $this->getHeaderLength($data, (int) $headerRowNum);
+        } else {
+            $headerLength = count($this->header);
         }
 
-        $headerLength = count($this->header);
-
         foreach ($data as $k => $row) {
             // backward compatibility fix
             if ($this->sheetCfg['header']['rows'] === 1 && $k === 0 && $offset === 1) {
@@ -94,4 +95,13 @@ protected function normalizeCsvHeader(array $header): array
         }
         return $header;
     }
+
+    private function getHeaderLength(array $data, int $headerRowNum): int
+    {
+        $headerLength = 0;
+        for ($i = 0; $i <= $headerRowNum; $i++) {
+            $headerLength = max($headerLength, count($data[$i]));
+        }
+        return $headerLength;
+    }
 }
diff --git a/tests/FunctionalTest.php b/tests/FunctionalTest.php
@@ -4,6 +4,7 @@
 
 namespace Keboola\GoogleDriveExtractor\Tests;
 
+use Keboola\Csv\CsvWriter;
 use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Process\Process;
 use Symfony\Component\Yaml\Yaml;
@@ -102,6 +103,36 @@ public function testDoNotSanitizeHeader(): void
         unlink($filePath);
     }
 
+    public function testMultipleHeader(): void
+    {
+        $filePath = __DIR__ . '/data/in/multiple_header.csv';
+        touch($filePath);
+        $csv = new CsvWriter($filePath);
+        $this->writeLineToCsvFile($csv, 1, 7);
+        $this->writeLineToCsvFile($csv, 2, 5);
+        $this->writeLineToCsvFile($csv, 3, 20);
+        $this->writeLineToCsvFile($csv, 4, 2);
+        $this->writeLineToCsvFile($csv, 5, 7);
+
+        $this->testFileName = 'multiple_header';
+        $this->testFile = $this->prepareTestFile($filePath, 'multiple_header');
+        $this->config = $this->makeConfig($this->testFile);
+
+        // leave the header as is
+        $this->config['parameters']['sheets'][0]['header']['rows'] = 2;
+
+        $process = $this->runProcess();
+        $this->assertEquals(0, $process->getExitCode(), $process->getErrorOutput());
+
+        $fileId = $this->config['parameters']['sheets'][0]['fileId'];
+        $sheetId = $this->config['parameters']['sheets'][0]['sheetId'];
+
+        $this->assertFileEquals(
+            __DIR__ . '/data/expectedFiles/multiple_header.csv',
+            $this->dataPath . '/out/tables/' . $this->getOutputFileName($fileId, $sheetId)
+        );
+    }
+
     private function runProcess(): Process
     {
         $fs = new Filesystem();
@@ -117,4 +148,13 @@ private function runProcess(): Process
 
         return $process;
     }
+
+    private function writeLineToCsvFile(CsvWriter $csvWriter, int $rowNumber, int $countColumns): void
+    {
+        $headerLine = [];
+        for ($j = 0; $j < $countColumns; $j++) {
+            $headerLine[] = sprintf('Col_%s_%s', $rowNumber, $j);
+        }
+        $csvWriter->writeRow($headerLine);
+    }
 }
diff --git a/tests/data/expectedFiles/multiple_header.csv b/tests/data/expectedFiles/multiple_header.csv
@@ -0,0 +1,5 @@
+"Col_1_0","Col_1_1","Col_1_2","Col_1_3","Col_1_4","Col_1_5","Col_1_6"
+"Col_2_0","Col_2_1","Col_2_2","Col_2_3","Col_2_4","",""
+"Col_3_0","Col_3_1","Col_3_2","Col_3_3","Col_3_4","Col_3_5","Col_3_6"
+"Col_4_0","Col_4_1","","","","",""
+"Col_5_0","Col_5_1","Col_5_2","Col_5_3","Col_5_4","Col_5_5","Col_5_6"