detect delimiter for csv files

jgrygierek · Mar 27, 2024 · ecfbe84 · ecfbe84
1 parent 4abde34
commit ecfbe84
Show file tree

Hide file tree

Showing 18 changed files with 334 additions and 47 deletions.
diff --git a/src/Enums/CsvDelimiterEnum.php b/src/Enums/CsvDelimiterEnum.php
@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JG\BatchEntityImportBundle\Enums;
+
+enum CsvDelimiterEnum: string
+{
+    case SEMICOLON = ';';
+    case COMMA = ',';
+
+    public static function asValues(): array
+    {
+        return [
+            self::SEMICOLON->value,
+            self::COMMA->value,
+        ];
+    }
+}
diff --git a/src/Model/Matrix/MatrixFactory.php b/src/Model/Matrix/MatrixFactory.php
@@ -5,7 +5,9 @@
 namespace JG\BatchEntityImportBundle\Model\Matrix;
 
 use InvalidArgumentException;
+use JG\BatchEntityImportBundle\Service\CsvDelimiterDetector;
 use PhpOffice\PhpSpreadsheet\Reader\BaseReader;
+use PhpOffice\PhpSpreadsheet\Reader\Csv;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 
 class MatrixFactory
@@ -36,7 +38,7 @@ private static function addKeysToRows(array $header, array &$data): void
             $data,
             static function (array &$row) use ($header): void {
                 $row = array_combine($header, $row);
-            }
+            },
         );
     }
 
@@ -48,6 +50,12 @@ private static function getReader(UploadedFile $file): BaseReader
             throw new InvalidArgumentException("Reader for extension $extension is not supported by PhpOffice.");
         }
 
-        return new $readerClass();
+        $reader = new $readerClass();
+        if ($reader instanceof Csv) {
+            $detectedDelimiter = (new CsvDelimiterDetector())->detect($file->getContent());
+            $reader->setDelimiter($detectedDelimiter->value);
+        }
+
+        return $reader;
     }
 }
diff --git a/src/Service/CsvDelimiterDetector.php b/src/Service/CsvDelimiterDetector.php
@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JG\BatchEntityImportBundle\Service;
+
+use JG\BatchEntityImportBundle\Enums\CsvDelimiterEnum;
+
+class CsvDelimiterDetector
+{
+    public function detect(string $csvContent): CsvDelimiterEnum
+    {
+        $delimiter = $this->detectDelimiter($csvContent);
+
+        return ',' === $delimiter
+            ? CsvDelimiterEnum::COMMA
+            : CsvDelimiterEnum::SEMICOLON;
+    }
+
+    private function detectDelimiter(string $csvContent): string
+    {
+        $delimiters = CsvDelimiterEnum::asValues();
+        $delimiterCount = array_fill_keys($delimiters, 0);
+
+        foreach ($delimiters as $delimiter) {
+            $delimiterCount[$delimiter] = substr_count($csvContent, $delimiter);
+        }
+
+        return array_search(max($delimiterCount), $delimiterCount, true);
+    }
+}
diff --git a/src/Validator/Constraints/DatabaseEntityUnique.php b/src/Validator/Constraints/DatabaseEntityUnique.php
@@ -16,7 +16,7 @@ class DatabaseEntityUnique extends Constraint
     public string $entityClassName;
     public array $fields;
 
-    public function __construct(mixed $options = null, array $groups = null, mixed $payload = null)
+    public function __construct(mixed $options = null, ?array $groups = null, mixed $payload = null)
     {
         parent::__construct($options, $groups, $payload);
 

diff --git a/src/Validator/Constraints/MatrixRecordUnique.php b/src/Validator/Constraints/MatrixRecordUnique.php
@@ -15,7 +15,7 @@ class MatrixRecordUnique extends Constraint
     public string $message = 'validation.matrix.record.unique';
     public array $fields = [];
 
-    public function __construct(mixed $options = null, array $groups = null, mixed $payload = null)
+    public function __construct(mixed $options = null, ?array $groups = null, mixed $payload = null)
     {
         parent::__construct($options, $groups, $payload);
 

diff --git a/tests/Controller/ImportControllerTraitTest.php b/tests/Controller/ImportControllerTraitTest.php
@@ -5,6 +5,7 @@
 namespace JG\BatchEntityImportBundle\Tests\Controller;
 
 use Doctrine\ORM\EntityRepository;
+use Generator;
 use JG\BatchEntityImportBundle\Tests\DatabaseLoader;
 use JG\BatchEntityImportBundle\Tests\Fixtures\Entity\TranslatableEntity;
 use Symfony\Bundle\FrameworkBundle\KernelBrowser;
@@ -26,17 +27,26 @@ protected function setUp(): void
         $databaseLoader->loadFixtures();
     }
 
-    public function testControllerWorksOk(): void
+    public function testInsertNewData(): void
     {
         $importUrl = '/jg_batch_entity_import_bundle/import';
         $updatedEntityId = self::DEFAULT_RECORDS_NUMBER + 2;
         self::assertCount(self::DEFAULT_RECORDS_NUMBER, $this->getRepository()->findAll());
-        // insert new data
+
         $this->submitSelectFileForm(__DIR__ . '/../Fixtures/Resources/test.csv', $importUrl);
         $this->client->submitForm('btn-submit');
         $this->checkData(['test2', 'lorem ipsum 2', 'qwerty2', 'test2_en', 'test2_pl'], $updatedEntityId, $importUrl);
+    }
+
+    /**
+     * @dataProvider updateRecordDataProvider
+     */
+    public function testUpdateExistingRecord(int $updatedEntityId, array $expectedDefaultValues, array $expectedValuesAfterChange): void
+    {
+        $importUrl = '/jg_batch_entity_import_bundle/import_base_translatable';
+        self::assertCount(self::DEFAULT_RECORDS_NUMBER, $this->getRepository()->findAll());
+        $this->assertEntityValues($expectedDefaultValues, $updatedEntityId);
 
-        // update existing data
         $this->submitSelectFileForm(__DIR__ . '/../Fixtures/Resources/test_updated_data.csv', $importUrl);
         $this->client->submitForm('btn-submit', [
             'matrix' => [
@@ -52,7 +62,49 @@ public function testControllerWorksOk(): void
                 ],
             ],
         ]);
-        $this->checkData(['new_value', 'new_value2', 'new_value3', 'new_value4', 'new_value5'], $updatedEntityId, $importUrl);
+        $this->checkData($expectedValuesAfterChange, $updatedEntityId, $importUrl, 0);
+    }
+
+    public function updateRecordDataProvider(): Generator
+    {
+        yield 'record with all fields filled' => [
+            10,
+            ['abcd_9', '', '', 'qwerty_en_9', 'qwerty_pl_9'],
+            ['new_value', 'new_value2', 'new_value3', 'new_value4', 'new_value5'],
+        ];
+
+        yield 'record without en field filled' => [
+            20,
+            ['abcd_19', '', '', '', 'qwerty_pl_19'],
+            ['new_value', 'new_value2', 'new_value3', 'new_value4', 'new_value5'],
+        ];
+
+        yield 'record without pl field filled' => [
+            1,
+            ['abcd_0', '', '', 'qwerty_en_0', 'qwerty_en_0'],
+            ['new_value', 'new_value2', 'new_value3', 'new_value5', 'new_value5'], // todo: it is a bug, it should be new_value4
+        ];
+    }
+
+    public function testUpdateOnlySingleTranslatableColumn(): void
+    {
+        $importUrl = '/jg_batch_entity_import_bundle/import_base_translatable';
+        $updatedEntityId = 10;
+        self::assertCount(self::DEFAULT_RECORDS_NUMBER, $this->getRepository()->findAll());
+        $this->assertEntityValues(['abcd_9', '', '', 'qwerty_en_9', 'qwerty_pl_9'], $updatedEntityId);
+
+        $this->submitSelectFileForm(__DIR__ . '/../Fixtures/Resources/test_update_single_column.csv', $importUrl);
+        $this->client->submitForm('btn-submit', [
+            'matrix' => [
+                'records' => [
+                    [
+                        'entity' => $updatedEntityId,
+                        'testTranslationProperty:pl' => 'Lorem Ipsum',
+                    ],
+                ],
+            ],
+        ]);
+        $this->checkData(['abcd_9', '', '', 'qwerty_en_9', 'Lorem Ipsum'], $updatedEntityId, $importUrl, 0);
     }
 
     public function testDuplicationFoundInDatabase(): void
@@ -124,17 +176,26 @@ private function submitSelectFileForm(string $uploadedFile, string $importUrl =
         self::assertEquals($importUrl, $this->client->getRequest()->getRequestUri());
     }
 
-    private function checkData(array $expectedValues, int $entityId, string $importUrl = '/jg_batch_entity_import_bundle/import'): void
-    {
+    private function checkData(
+        array $expectedValues,
+        int $entityId,
+        string $importUrl = '/jg_batch_entity_import_bundle/import',
+        int $newRecordsNumber = self::NEW_RECORDS_NUMBER,
+    ): void {
         $repository = $this->getRepository();
         self::assertTrue($this->client->getResponse()->isRedirect($importUrl));
         $this->client->followRedirect();
         self::assertTrue($this->client->getResponse()->isSuccessful());
         self::assertStringContainsString('Data has been imported', $this->client->getResponse()->getContent());
-        self::assertCount(self::DEFAULT_RECORDS_NUMBER + self::NEW_RECORDS_NUMBER, $repository->findAll());
+        self::assertCount(self::DEFAULT_RECORDS_NUMBER + $newRecordsNumber, $repository->findAll());
 
+        $this->assertEntityValues($expectedValues, $entityId);
+    }
+
+    private function assertEntityValues(array $expectedValues, int $entityId): void
+    {
         /** @var TranslatableEntity|null $item */
-        $item = $repository->find($entityId);
+        $item = $this->getRepository()->find($entityId);
         self::assertNotEmpty($item);
         self::assertSame($expectedValues[0], $item->getTestPrivateProperty());
         self::assertSame($expectedValues[1], $item->getTestPrivateProperty2());

diff --git a/tests/Enum/CsvDelimiterEnumTest.php b/tests/Enum/CsvDelimiterEnumTest.php
@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JG\BatchEntityImportBundle\Tests\Enum;
+
+use JG\BatchEntityImportBundle\Enums\CsvDelimiterEnum;
+use PHPUnit\Framework\TestCase;
+
+class CsvDelimiterEnumTest extends TestCase
+{
+    public function testEnum(): void
+    {
+        $this->assertCount(2, CsvDelimiterEnum::cases());
+        $this->assertSame([';', ','], CsvDelimiterEnum::asValues());
+        $this->assertSame(';', CsvDelimiterEnum::SEMICOLON->value);
+        $this->assertSame(',', CsvDelimiterEnum::COMMA->value);
+    }
+}
diff --git a/tests/Fixtures/Configuration/TranslatableEntityBaseConfiguration.php b/tests/Fixtures/Configuration/TranslatableEntityBaseConfiguration.php
@@ -6,7 +6,6 @@
 
 use JG\BatchEntityImportBundle\Model\Configuration\AbstractImportConfiguration;
 use JG\BatchEntityImportBundle\Tests\Fixtures\Entity\TranslatableEntity;
-use JG\BatchEntityImportBundle\Validator\Constraints\DatabaseEntityUnique;
 
 class TranslatableEntityBaseConfiguration extends AbstractImportConfiguration
 {
@@ -19,17 +18,4 @@ public function getEntityTranslationRelationName(): ?string
     {
         return 'translations';
     }
-
-    public function getMatrixConstraints(): array
-    {
-        return [
-            new DatabaseEntityUnique(['entityClassName' => $this->getEntityClassName(), 'fields' => [
-                'test_private_property',
-                'test_public_property',
-            ]]),
-            new DatabaseEntityUnique(['entityClassName' => $this->getEntityClassName(), 'fields' => [
-                'test-private-property2',
-            ]]),
-        ];
-    }
 }
diff --git a/tests/Fixtures/Configuration/TranslatableEntityConfiguration.php b/tests/Fixtures/Configuration/TranslatableEntityConfiguration.php
@@ -0,0 +1,35 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JG\BatchEntityImportBundle\Tests\Fixtures\Configuration;
+
+use JG\BatchEntityImportBundle\Model\Configuration\AbstractImportConfiguration;
+use JG\BatchEntityImportBundle\Tests\Fixtures\Entity\TranslatableEntity;
+use JG\BatchEntityImportBundle\Validator\Constraints\DatabaseEntityUnique;
+
+class TranslatableEntityConfiguration extends AbstractImportConfiguration
+{
+    public function getEntityClassName(): string
+    {
+        return TranslatableEntity::class;
+    }
+
+    public function getEntityTranslationRelationName(): ?string
+    {
+        return 'translations';
+    }
+
+    public function getMatrixConstraints(): array
+    {
+        return [
+            new DatabaseEntityUnique(['entityClassName' => $this->getEntityClassName(), 'fields' => [
+                'test_private_property',
+                'test_public_property',
+            ]]),
+            new DatabaseEntityUnique(['entityClassName' => $this->getEntityClassName(), 'fields' => [
+                'test-private-property2',
+            ]]),
+        ];
+    }
+}
diff --git a/tests/Fixtures/Controller/ControllerWithBaseTranslatableConfiguration.php b/tests/Fixtures/Controller/ControllerWithBaseTranslatableConfiguration.php
@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JG\BatchEntityImportBundle\Tests\Fixtures\Controller;
+
+use JG\BatchEntityImportBundle\Controller\ImportConfigurationAutoInjectInterface;
+use JG\BatchEntityImportBundle\Controller\ImportConfigurationAutoInjectTrait;
+use JG\BatchEntityImportBundle\Controller\ImportControllerTrait;
+use JG\BatchEntityImportBundle\Tests\Fixtures\Configuration\TranslatableEntityBaseConfiguration;
+use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\HttpFoundation\RedirectResponse;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Validator\Validator\ValidatorInterface;
+use Symfony\Contracts\Translation\TranslatorInterface;
+
+class ControllerWithBaseTranslatableConfiguration extends AbstractController implements ImportConfigurationAutoInjectInterface
+{
+    use ImportConfigurationAutoInjectTrait;
+    use ImportControllerTrait;
+
+    public function import(Request $request, ValidatorInterface $validator): Response
+    {
+        return $this->doImport($request, $validator);
+    }
+
+    public function importSave(Request $request, TranslatorInterface $translator): Response
+    {
+        return $this->doImportSave($request, $translator);
+    }
+
+    protected function redirectToImport(): RedirectResponse
+    {
+        return $this->redirectToRoute('jg.batch_entity_import_bundle.test_controller.base_translatable_config.import');
+    }
+
+    protected function getMatrixSaveActionUrl(): string
+    {
+        return $this->generateUrl('jg.batch_entity_import_bundle.test_controller.base_translatable_config.import_save');
+    }
+
+    protected function getImportConfigurationClassName(): string
+    {
+        return TranslatableEntityBaseConfiguration::class;
+    }
+}
diff --git a/tests/Fixtures/Controller/ControllerWithInterface.php b/tests/Fixtures/Controller/ControllerWithInterface.php
@@ -7,7 +7,7 @@
 use JG\BatchEntityImportBundle\Controller\ImportConfigurationAutoInjectInterface;
 use JG\BatchEntityImportBundle\Controller\ImportConfigurationAutoInjectTrait;
 use JG\BatchEntityImportBundle\Controller\ImportControllerTrait;
-use JG\BatchEntityImportBundle\Tests\Fixtures\Configuration\TranslatableEntityBaseConfiguration;
+use JG\BatchEntityImportBundle\Tests\Fixtures\Configuration\TranslatableEntityConfiguration;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use Symfony\Component\HttpFoundation\RedirectResponse;
 use Symfony\Component\HttpFoundation\Request;
@@ -42,6 +42,6 @@ protected function getMatrixSaveActionUrl(): string
 
     protected function getImportConfigurationClassName(): string
     {
-        return TranslatableEntityBaseConfiguration::class;
+        return TranslatableEntityConfiguration::class;
     }
 }
diff --git a/tests/Fixtures/Data/TranslatableEntityFixtures.php b/tests/Fixtures/Data/TranslatableEntityFixtures.php
@@ -17,9 +17,16 @@ public function load(ObjectManager $manager): void
             $entity = new TranslatableEntity();
             $entity->setTestPrivateProperty('abcd_' . $i);
 
-            /** @var TranslatableEntityTranslation $translatedEntity */
-            $translatedEntity = $entity->translate('en');
-            $translatedEntity->setTestTranslationProperty('qwerty_' . $i);
+            if ($i < 15) {
+                /** @var TranslatableEntityTranslation $translatedEntity */
+                $translatedEntity = $entity->translate('en');
+                $translatedEntity->setTestTranslationProperty('qwerty_en_' . $i);
+            }
+
+            if ($i > 4) {
+                $translatedEntity = $entity->translate('pl');
+                $translatedEntity->setTestTranslationProperty('qwerty_pl_' . $i);
+            }
 
             $manager->persist($entity);
             $entity->mergeNewTranslations();

diff --git a/tests/Fixtures/Resources/test_update_single_column.csv b/tests/Fixtures/Resources/test_update_single_column.csv
@@ -0,0 +1,2 @@
+testTranslationProperty:pl
+Lorem Ipsum