From 9e25f6d7bd7ee20c6fdedddc1dd5a633261af125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Mon, 20 Sep 2021 16:42:17 +0200 Subject: [PATCH 1/6] Let item reader to drive item indexes --- src/batch/src/Job/Item/ItemJob.php | 6 +- .../src/Job/Item/Reader/IndexWithReader.php | 48 +++++++++++++++ src/batch/tests/Job/Item/ItemJobTest.php | 8 +++ .../Job/Item/Reader/IndexWithReaderTest.php | 60 +++++++++++++++++++ 4 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 src/batch/src/Job/Item/Reader/IndexWithReader.php create mode 100644 src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php diff --git a/src/batch/src/Job/Item/ItemJob.php b/src/batch/src/Job/Item/ItemJob.php index a37b6cfa..36eb9070 100644 --- a/src/batch/src/Job/Item/ItemJob.php +++ b/src/batch/src/Job/Item/ItemJob.php @@ -77,9 +77,7 @@ final protected function doExecute(JobExecution $jobExecution): void $writeCount = 0; $itemsToWrite = []; - $lineNumber = 1; - foreach ($this->reader->read() as $readItem) { - $lineNumber++; + foreach ($this->reader->read() as $readIndex => $readItem) { $summary->increment('read'); try { @@ -87,7 +85,7 @@ final protected function doExecute(JobExecution $jobExecution): void } catch (InvalidItemException $exception) { $summary->increment('invalid'); $jobExecution->addWarning( - new Warning($exception->getMessage(), $exception->getParameters(), ['line_number' => $lineNumber]) + new Warning($exception->getMessage(), $exception->getParameters(), ['itemIndex' => $readIndex]) ); continue; diff --git a/src/batch/src/Job/Item/Reader/IndexWithReader.php b/src/batch/src/Job/Item/Reader/IndexWithReader.php new file mode 100644 index 00000000..d16a5333 --- /dev/null +++ b/src/batch/src/Job/Item/Reader/IndexWithReader.php @@ -0,0 +1,48 @@ +reader = $reader; + $this->getIndex = $getIndex; + } + + public static function withArrayKey(ItemReaderInterface $reader, string $key): self + { + return new self($reader, fn(array $item) => $item[$key]); + } + + public static function withProperty(ItemReaderInterface $reader, string $property): self + { + return new self($reader, fn(object $item) => $item->$property); + } + + public static function withGetter(ItemReaderInterface $reader, string $getter): self + { + return new self($reader, fn(object $item) => $item->$getter()); + } + + /** + * @inheritdoc + */ + public function read(): iterable + { + foreach ($this->reader->read() as $item) { + yield ($this->getIndex)($item) => $item; + } + } +} diff --git a/src/batch/tests/Job/Item/ItemJobTest.php b/src/batch/tests/Job/Item/ItemJobTest.php index 140fe32a..1d2195d4 100644 --- a/src/batch/tests/Job/Item/ItemJobTest.php +++ b/src/batch/tests/Job/Item/ItemJobTest.php @@ -86,6 +86,14 @@ public function testExecute(): void self::assertSame(3, $jobExecution->getSummary()->get('invalid'), '3 items were invalid'); self::assertSame(9, $jobExecution->getSummary()->get('write'), '9 items were write'); + $warnings = $jobExecution->getWarnings(); + self::assertCount(3, $warnings); + foreach ([[0, 9, 10], [1, 10, 11], [2, 11, 12]] as [$warningIdx, $itemIdx, $paramValue]) { + self::assertSame('Item is greater than 9 got {value}', $warnings[$warningIdx]->getMessage()); + self::assertSame(['{value}' => $paramValue], $warnings[$warningIdx]->getParameters()); + self::assertSame(['itemIndex' => $itemIdx], $warnings[$warningIdx]->getContext()); + } + $expectedLogs = <<read() as $index => $item) { + $actual[$index] = $item; + } + + self::assertSame($expected, $actual); + } + + public function provider(): Generator + { + $john = ['name' => 'John', 'location' => 'Washington']; + $marie = ['name' => 'Marie', 'location' => 'London']; + yield 'Index with array key' => [ + IndexWithReader::withArrayKey( + new StaticIterableReader([$john, $marie]), + 'name' + ), + ['John' => $john, 'Marie' => $marie], + ]; + + $john = (object)$john; + $marie = (object)$marie; + yield 'Index with object property' => [ + IndexWithReader::withProperty( + new StaticIterableReader([$john, $marie]), + 'name' + ), + ['John' => $john, 'Marie' => $marie], + ]; + + $three = new ArrayIterator([1, 2, 3]); + $six = new ArrayIterator([1, 2, 3, 4, 5, 6]); + yield 'Index with object method' => [ + IndexWithReader::withGetter( + new StaticIterableReader([$three, $six]), + 'count' + ), + [3 => $three, 6 => $six], + ]; + } +} From c0a4f85c6c49fe26c42ac23e739badb8d3825080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Mon, 20 Sep 2021 18:09:23 +0200 Subject: [PATCH 2/6] Add documentation & Force Closure type instead of callable --- .../src/Job/Item/Reader/IndexWithReader.php | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/batch/src/Job/Item/Reader/IndexWithReader.php b/src/batch/src/Job/Item/Reader/IndexWithReader.php index d16a5333..1639853a 100644 --- a/src/batch/src/Job/Item/Reader/IndexWithReader.php +++ b/src/batch/src/Job/Item/Reader/IndexWithReader.php @@ -4,21 +4,25 @@ namespace Yokai\Batch\Job\Item\Reader; +use Closure; use Yokai\Batch\Job\Item\ItemReaderInterface; +/** + * An {@see ItemReaderInterface} that decorates another {@see ItemReaderInterface} + * and extract item index of each item using a {@see Closure}. + * + * Provided {@see Closure} must accept a single argument (the read item) + * and must return a value (preferably unique) that will be item index. + */ final class IndexWithReader implements ItemReaderInterface { private ItemReaderInterface $reader; + private Closure $extractItemIndex; - /** - * @var callable - */ - private $getIndex; - - public function __construct(ItemReaderInterface $reader, callable $getIndex) + public function __construct(ItemReaderInterface $reader, Closure $extractItemIndex) { $this->reader = $reader; - $this->getIndex = $getIndex; + $this->extractItemIndex = $extractItemIndex; } public static function withArrayKey(ItemReaderInterface $reader, string $key): self @@ -42,7 +46,7 @@ public static function withGetter(ItemReaderInterface $reader, string $getter): public function read(): iterable { foreach ($this->reader->read() as $item) { - yield ($this->getIndex)($item) => $item; + yield ($this->extractItemIndex)($item) => $item; } } } From 07c9fe505c6c43d290a0eaf276199369806ae36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Mon, 20 Sep 2021 18:09:45 +0200 Subject: [PATCH 3/6] Add test with direct constructor call --- src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php index e277ffe9..ed75da52 100644 --- a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php +++ b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php @@ -56,5 +56,13 @@ public function provider(): Generator ), [3 => $three, 6 => $six], ]; + + yield 'Index with arbitrary closure' => [ + new IndexWithReader( + new StaticIterableReader([1, 2, 3]), + fn(int $value) => $value * $value + ), + [1 => 1, 4 => 2, 9 => 3], + ]; } } From a86ca797b202475588983873f9c88c83a54d9b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Mon, 20 Sep 2021 18:12:44 +0200 Subject: [PATCH 4/6] Implement decorator missing features --- .../src/Job/Item/Reader/IndexWithReader.php | 31 ++++++++++++++++++- .../Job/Item/Reader/IndexWithReaderTest.php | 6 ++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/batch/src/Job/Item/Reader/IndexWithReader.php b/src/batch/src/Job/Item/Reader/IndexWithReader.php index 1639853a..528556ac 100644 --- a/src/batch/src/Job/Item/Reader/IndexWithReader.php +++ b/src/batch/src/Job/Item/Reader/IndexWithReader.php @@ -5,7 +5,12 @@ namespace Yokai\Batch\Job\Item\Reader; use Closure; +use Yokai\Batch\Job\Item\ElementConfiguratorTrait; +use Yokai\Batch\Job\Item\FlushableInterface; +use Yokai\Batch\Job\Item\InitializableInterface; use Yokai\Batch\Job\Item\ItemReaderInterface; +use Yokai\Batch\Job\JobExecutionAwareInterface; +use Yokai\Batch\Job\JobExecutionAwareTrait; /** * An {@see ItemReaderInterface} that decorates another {@see ItemReaderInterface} @@ -14,8 +19,15 @@ * Provided {@see Closure} must accept a single argument (the read item) * and must return a value (preferably unique) that will be item index. */ -final class IndexWithReader implements ItemReaderInterface +final class IndexWithReader implements + ItemReaderInterface, + InitializableInterface, + FlushableInterface, + JobExecutionAwareInterface { + use ElementConfiguratorTrait; + use JobExecutionAwareTrait; + private ItemReaderInterface $reader; private Closure $extractItemIndex; @@ -40,6 +52,15 @@ public static function withGetter(ItemReaderInterface $reader, string $getter): return new self($reader, fn(object $item) => $item->$getter()); } + /** + * @inheritdoc + */ + public function initialize(): void + { + $this->configureElementJobContext($this->reader, $this->jobExecution); + $this->initializeElement($this->reader); + } + /** * @inheritdoc */ @@ -49,4 +70,12 @@ public function read(): iterable yield ($this->extractItemIndex)($item) => $item; } } + + /** + * @inheritdoc + */ + public function flush(): void + { + $this->flushElement($this->reader); + } } diff --git a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php index ed75da52..3da5ea2e 100644 --- a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php +++ b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php @@ -9,6 +9,7 @@ use PHPUnit\Framework\TestCase; use Yokai\Batch\Job\Item\Reader\IndexWithReader; use Yokai\Batch\Job\Item\Reader\StaticIterableReader; +use Yokai\Batch\JobExecution; class IndexWithReaderTest extends TestCase { @@ -17,11 +18,16 @@ class IndexWithReaderTest extends TestCase */ public function test(IndexWithReader $reader, array $expected): void { + $reader->setJobExecution(JobExecution::createRoot('123456', 'testing')); + $reader->initialize(); + $actual = []; foreach ($reader->read() as $index => $item) { $actual[$index] = $item; } + $reader->flush(); + self::assertSame($expected, $actual); } From 8937739b28aaa734ac88366caa4af38fcd350a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Mon, 20 Sep 2021 18:17:06 +0200 Subject: [PATCH 5/6] Add documentation to static constructors --- .../src/Job/Item/Reader/IndexWithReader.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/batch/src/Job/Item/Reader/IndexWithReader.php b/src/batch/src/Job/Item/Reader/IndexWithReader.php index 528556ac..6d9ea199 100644 --- a/src/batch/src/Job/Item/Reader/IndexWithReader.php +++ b/src/batch/src/Job/Item/Reader/IndexWithReader.php @@ -37,16 +37,34 @@ public function __construct(ItemReaderInterface $reader, Closure $extractItemInd $this->extractItemIndex = $extractItemIndex; } + /** + * Uses item array value as the item index. + * + * Example, IndexWithReader::withArrayKey(..., 'name') + * will use 'name' array index of each read item as the item index. + */ public static function withArrayKey(ItemReaderInterface $reader, string $key): self { return new self($reader, fn(array $item) => $item[$key]); } + /** + * Uses object property value as the item index. + * + * Example, IndexWithReader::withProperty(..., 'name') + * will use 'name' object property of each read item as the item index. + */ public static function withProperty(ItemReaderInterface $reader, string $property): self { return new self($reader, fn(object $item) => $item->$property); } + /** + * Uses object method return value as the item index. + * + * Example, IndexWithReader::withProperty(..., 'getName') + * will call 'getName()' method of each read item and uses the result as the item index. + */ public static function withGetter(ItemReaderInterface $reader, string $getter): self { return new self($reader, fn(object $item) => $item->$getter()); From eb20ceebfdde8d2b69c5a9fcfa49c1de236d6089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yann=20Eugon=C3=A9?= Date: Tue, 21 Sep 2021 14:29:38 +0200 Subject: [PATCH 6/6] Refactored test so coverage of constructor is evaluated --- .../tests/Job/Item/Reader/IndexWithReaderTest.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php index 3da5ea2e..867f090d 100644 --- a/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php +++ b/src/batch/tests/Job/Item/Reader/IndexWithReaderTest.php @@ -16,8 +16,10 @@ class IndexWithReaderTest extends TestCase /** * @dataProvider provider */ - public function test(IndexWithReader $reader, array $expected): void + public function test(callable $factory, array $expected): void { + /** @var IndexWithReader $reader */ + $reader = $factory(); $reader->setJobExecution(JobExecution::createRoot('123456', 'testing')); $reader->initialize(); @@ -36,7 +38,7 @@ public function provider(): Generator $john = ['name' => 'John', 'location' => 'Washington']; $marie = ['name' => 'Marie', 'location' => 'London']; yield 'Index with array key' => [ - IndexWithReader::withArrayKey( + fn() => IndexWithReader::withArrayKey( new StaticIterableReader([$john, $marie]), 'name' ), @@ -46,7 +48,7 @@ public function provider(): Generator $john = (object)$john; $marie = (object)$marie; yield 'Index with object property' => [ - IndexWithReader::withProperty( + fn() => IndexWithReader::withProperty( new StaticIterableReader([$john, $marie]), 'name' ), @@ -56,7 +58,7 @@ public function provider(): Generator $three = new ArrayIterator([1, 2, 3]); $six = new ArrayIterator([1, 2, 3, 4, 5, 6]); yield 'Index with object method' => [ - IndexWithReader::withGetter( + fn() => IndexWithReader::withGetter( new StaticIterableReader([$three, $six]), 'count' ), @@ -64,7 +66,7 @@ public function provider(): Generator ]; yield 'Index with arbitrary closure' => [ - new IndexWithReader( + fn() => new IndexWithReader( new StaticIterableReader([1, 2, 3]), fn(int $value) => $value * $value ),