Merge pull request #33 from keboola/COM-595-ondra-implements-datadir-…

…tests

Implements datadir tests

composer.json

@@ -19,6 +19,7 @@
   },
   "require-dev": {
     "keboola/coding-standard": ">=7.0.2",
+    "keboola/datadir-tests": "^5.2",
     "php-parallel-lint/php-parallel-lint": "^1.2",
     "phpstan/phpstan": "^0.12.64",
     "phpunit/phpunit": "^9.5"
@@ -30,13 +31,16 @@
   },
   "autoload-dev": {
     "psr-4": {
-      "Keboola\\GoogleDriveExtractor\\Tests\\": "tests/phpunit/"
+      "Keboola\\GoogleDriveExtractor\\Tests\\": "tests/phpunit/",
+      "Keboola\\GoogleDriveExtractor\\FunctionalTests\\": "tests/functional/"
     }
   },
   "scripts": {
     "tests-phpunit": "phpunit",
+    "tests-datadir": "phpunit tests/functional",
     "tests": [
-      "@tests-phpunit"
+      "@tests-phpunit",
+      "@tests-datadir"
     ],
 
     "phpstan": "phpstan analyse ./src ./tests --level=max --no-progress -c phpstan.neon",

src/Configuration/ConfigDefinition.php

@@ -14,10 +14,6 @@ protected function getParametersDefinition(): ArrayNodeDefinition
         $parametersNode = parent::getParametersDefinition();
         $parametersNode
             ->children()
-                ->integerNode('id')
-                    ->isRequired()
-                    ->min(0)
-                ->end()
                 ->scalarNode('fileId')
                     ->isRequired()
                     ->cannotBeEmpty()

src/Extractor/Output.php

@@ -30,7 +30,7 @@ public function createCsv(): string
             mkdir($outTablesDir, 0777, true);
         }
 
-        $filename = $outTablesDir . '/' . $this->config->getFileId() . '_' . $this->config->getSheetId() . '.csv';
+        $filename = $outTablesDir . '/' . $this->config->getOutputTable() . '.csv';
         touch($filename);
 
         $this->csv = new CsvWriter($filename);

tests/functional/DatadirTest.php

@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\GoogleDriveExtractor\FunctionalTests;
+
+use Keboola\DatadirTests\DatadirTestCase;
+use Keboola\DatadirTests\DatadirTestSpecificationInterface;
+use Keboola\Google\ClientBundle\Google\RestApi;
+use Keboola\GoogleDriveExtractor\GoogleDrive\Client;
+use Keboola\Temp\Temp;
+use Symfony\Component\Finder\Finder;
+use \Throwable;
+
+class DatadirTest extends DatadirTestCase
+{
+    /**
+     * @dataProvider provideDatadirSpecifications
+     */
+    public function testDatadir(DatadirTestSpecificationInterface $specification): void
+    {
+        $tempDatadir = $this->getTempDatadir($specification);
+
+        $googleDriveApi = new Client(
+            new RestApi(
+                (string) getenv('CLIENT_ID'),
+                (string) getenv('CLIENT_SECRET'),
+                (string) getenv('ACCESS_TOKEN'),
+                (string) getenv('REFRESH_TOKEN')
+            )
+        );
+
+        $testFileInfo = $this->prepareTestFile(
+            $googleDriveApi,
+            (string) $specification->getSourceDatadirDirectory()
+        );
+
+        $this->prepareConfigFile($tempDatadir, $testFileInfo);
+
+        $process = $this->runScript($tempDatadir->getTmpFolder());
+
+        $this->removeTestFile($googleDriveApi, $testFileInfo['spreadsheetId']);
+
+        $this->assertMatchesSpecification($specification, $process, $tempDatadir->getTmpFolder());
+    }
+
+    private function prepareTestFile(Client $googleDriveApi, string $sourceDatadir): array
+    {
+        $finder = new Finder();
+        $finder->in(sprintf('%s/setup/files/', $sourceDatadir));
+        $finder->files();
+
+        $items = iterator_to_array($finder->getIterator());
+
+        /** @var \SplFileInfo $item */
+        $item = current($items);
+        $file = $googleDriveApi->createFile(
+            $item->getPathname(),
+            $item->getBasename('.' . $item->getExtension())
+        );
+        return $googleDriveApi->getSpreadsheet($file['id']);
+    }
+
+    private function prepareConfigFile(Temp $tempDatadir, array $testFileInfo): void
+    {
+        $configPath = $tempDatadir->getTmpFolder() . '/config.json';
+
+        $content = (string) file_get_contents($configPath);
+
+        $jsonContent = json_decode($content, true);
+
+        $jsonContent['authorization']['oauth_api']['credentials'] = [
+            'appKey' => (string) getenv('CLIENT_ID'),
+            '#appSecret' => (string) getenv('CLIENT_SECRET'),
+            '#data' => json_encode(
+                [
+                    'access_token' => (string) getenv('ACCESS_TOKEN'),
+                    'refresh_token' => (string) getenv('REFRESH_TOKEN'),
+                ]
+            ),
+        ];
+
+        $parameters = $jsonContent['parameters'];
+
+        $jsonContent['parameters'] = array_merge(
+            $parameters,
+            [
+                'fileId' => $parameters['fileId'] ?? $testFileInfo['spreadsheetId'],
+                'fileTitle' => $parameters['fileTitle'] ?? $testFileInfo['properties']['title'],
+                'sheetId' => $parameters['sheetId'] ?? $testFileInfo['sheets'][0]['properties']['sheetId'],
+                'sheetTitle' => $parameters['sheetTitle'] ?? $testFileInfo['sheets'][0]['properties']['title'],
+            ]
+        );
+
+        file_put_contents($configPath, json_encode($jsonContent));
+    }
+
+    private function removeTestFile(Client $googleDriveApi, string $spreadsheetId): void
+    {
+        try {
+            $googleDriveApi->deleteFile($spreadsheetId);
+        } catch (Throwable $e) {
+        }
+    }
+}

tests/functional/correct-run/expected-code

@@ -0,0 +1 @@
+0

tests/functional/correct-run/expected-stdout

@@ -0,0 +1,3 @@
+Obtained spreadsheet metadata
+Extracting sheet titanic
+Extracting rows 1 to 1001

tests/functional/correct-run/expected/data/out/tables/titanic-output.csv.manifest

@@ -0,0 +1 @@
+{"incremental":false}

tests/phpunit/data/config.json → tests/functional/correct-run/source/data/config.json

@@ -1,13 +1,12 @@
 {
   "parameters": {
-    "id": 0,
     "fileId": null,
     "fileTitle": null,
     "sheetId": null,
     "sheetTitle": null,
-    "outputTable": null,
     "header": {
       "rows": 1
-    }
+    },
+    "outputTable": "titanic-output"
   }
 }

tests/functional/correct-run/source/data/setup/files/titanic.csv

@@ -0,0 +1,33 @@
+"Id","Class","Sex","Age","Survived","Freq"
+"1","1st","Male","Child","No","0"
+"2","2nd","Male","Child","No","0"
+"3","3rd","Male","Child","No","35"
+"4","Crew","Male","Child","No","0"
+"5","1st","Female","Child","No","0"
+"6","2nd","Female","Child","No","0"
+"7","3rd","Female","Child","No","17"
+"8","Crew","Female","Child","No","0"
+"9","1st","Male","Adult","No","118"
+"10","2nd","Male","Adult","No","154"
+"11","3rd","Male","Adult","No","387"
+"12","Crew","Male","Adult","No","670"
+"13","1st","Female","Adult","No","4"
+"14","2nd","Female","Adult","No","13"
+"15","3rd","Female","Adult","No","89"
+"16","Crew","Female","Adult","No","3"
+"17","1st","Male","Child","Yes","5"
+"18","2nd","Male","Child","Yes","11"
+"19","3rd","Male","Child","Yes","13"
+"20","Crew","Male","Child","Yes","0"
+"21","1st","Female","Child","Yes","1"
+"22","2nd","Female","Child","Yes","13"
+"23","3rd","Female","Child","Yes","14"
+"24","Crew","Female","Child","Yes","0"
+"25","1st","Male","Adult","Yes","57"
+"26","2nd","Male","Adult","Yes","14"
+"27","3rd","Male","Adult","Yes","75"
+"28","Crew","Male","Adult","Yes","192"
+"29","1st","Female","Adult","Yes","140"
+"30","2nd","Female","Adult","Yes","80"
+"31","3rd","Female","Adult","Yes","76"
+"32","Crew","Female","Adult","Yes","20"

tests/functional/empty-file/expected-code

@@ -0,0 +1 @@
+0

tests/functional/empty-file/expected-stdout

@@ -0,0 +1,3 @@
+Obtained spreadsheet metadata
+Extracting sheet empty
+Extracting rows 1 to 1001

tests/functional/empty-file/source/data/config.json

@@ -0,0 +1,12 @@
+{
+  "parameters": {
+    "fileId": null,
+    "fileTitle": null,
+    "sheetId": null,
+    "sheetTitle": null,
+    "header": {
+      "rows": 1
+    },
+    "outputTable": "empty"
+  }
+}

tests/functional/invalid-spreadsheet-id/expected-code

@@ -0,0 +1 @@
+1

tests/functional/invalid-spreadsheet-id/expected-stderr

@@ -0,0 +1 @@
+Sheet id "18293729" not found

tests/functional/invalid-spreadsheet-id/expected-stdout

@@ -0,0 +1,2 @@
+Obtained spreadsheet metadata
+Extracting sheet titanic

tests/functional/invalid-spreadsheet-id/source/data/config.json

@@ -0,0 +1,12 @@
+{
+  "parameters": {
+    "fileId": null,
+    "fileTitle": null,
+    "sheetId": 18293729,
+    "sheetTitle": null,
+    "header": {
+      "rows": 1
+    },
+    "outputTable": "titanic-output"
+  }
+}

tests/functional/invalid-spreadsheet-id/source/data/setup/files/titanic.csv

@@ -0,0 +1,33 @@
+"Id","Class","Sex","Age","Survived","Freq"
+"1","1st","Male","Child","No","0"
+"2","2nd","Male","Child","No","0"
+"3","3rd","Male","Child","No","35"
+"4","Crew","Male","Child","No","0"
+"5","1st","Female","Child","No","0"
+"6","2nd","Female","Child","No","0"
+"7","3rd","Female","Child","No","17"
+"8","Crew","Female","Child","No","0"
+"9","1st","Male","Adult","No","118"
+"10","2nd","Male","Adult","No","154"
+"11","3rd","Male","Adult","No","387"
+"12","Crew","Male","Adult","No","670"
+"13","1st","Female","Adult","No","4"
+"14","2nd","Female","Adult","No","13"
+"15","3rd","Female","Adult","No","89"
+"16","Crew","Female","Adult","No","3"
+"17","1st","Male","Child","Yes","5"
+"18","2nd","Male","Child","Yes","11"
+"19","3rd","Male","Child","Yes","13"
+"20","Crew","Male","Child","Yes","0"
+"21","1st","Female","Child","Yes","1"
+"22","2nd","Female","Child","Yes","13"
+"23","3rd","Female","Child","Yes","14"
+"24","Crew","Female","Child","Yes","0"
+"25","1st","Male","Adult","Yes","57"
+"26","2nd","Male","Adult","Yes","14"
+"27","3rd","Male","Adult","Yes","75"
+"28","Crew","Male","Adult","Yes","192"
+"29","1st","Female","Adult","Yes","140"
+"30","2nd","Female","Adult","Yes","80"
+"31","3rd","Female","Adult","Yes","76"
+"32","Crew","Female","Adult","Yes","20"

tests/functional/multiple-header/expected-code

@@ -0,0 +1 @@
+0

tests/functional/multiple-header/expected-stdout

@@ -0,0 +1,3 @@
+Obtained spreadsheet metadata
+Extracting sheet multiple_header
+Extracting rows 1 to 1001

tests/functional/multiple-header/expected/data/out/tables/multiple_header.csv.manifest

@@ -0,0 +1 @@
+{"incremental":false}

tests/functional/multiple-header/source/data/config.json

@@ -0,0 +1,12 @@
+{
+  "parameters": {
+    "fileId": null,
+    "fileTitle": null,
+    "sheetId": null,
+    "sheetTitle": null,
+    "header": {
+      "rows": 2
+    },
+    "outputTable": "multiple_header"
+  }
+}

tests/functional/multiple-header/source/data/setup/files/multiple_header.csv

@@ -0,0 +1,5 @@
+"Col_1_0","Col_1_1","Col_1_2","Col_1_3","Col_1_4","Col_1_5","Col_1_6"
+"Col_2_0","Col_2_1","Col_2_2","Col_2_3","Col_2_4"
+"Col_3_0","Col_3_1","Col_3_2","Col_3_3","Col_3_4","Col_3_5","Col_3_6","Col_3_7","Col_3_8","Col_3_9","Col_3_10","Col_3_11","Col_3_12","Col_3_13","Col_3_14","Col_3_15","Col_3_16","Col_3_17","Col_3_18","Col_3_19"
+"Col_4_0","Col_4_1"
+"Col_5_0","Col_5_1","Col_5_2","Col_5_3","Col_5_4","Col_5_5","Col_5_6"

tests/functional/not-sanitize-header/expected-code

@@ -0,0 +1 @@
+0

tests/functional/not-sanitize-header/expected-stdout

@@ -0,0 +1,3 @@
+Obtained spreadsheet metadata
+Extracting sheet not-sanitize
+Extracting rows 1 to 1001

tests/functional/not-sanitize-header/expected/data/out/tables/not-sanitize.csv

@@ -0,0 +1 @@
+"Weird-[]./;-_*Chars", "Second column", "# poops per day"

tests/functional/not-sanitize-header/expected/data/out/tables/not-sanitize.csv.manifest

@@ -0,0 +1 @@
+{"incremental":false}