Skip to content

Commit b4d335b

Browse files
authored
Added json lines reader and writer (#29)
* Added reader for jsonl format * Added writer for jsonl format * Fixed missing phpstan return type * Fixed checkstyle
1 parent ed0e083 commit b4d335b

File tree

5 files changed

+232
-0
lines changed

5 files changed

+232
-0
lines changed
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Yokai\Batch\Job\Item\Reader\Filesystem;
6+
7+
use Generator;
8+
use Yokai\Batch\Exception\RuntimeException;
9+
use Yokai\Batch\Job\Item\ItemReaderInterface;
10+
use Yokai\Batch\Job\JobExecutionAwareInterface;
11+
use Yokai\Batch\Job\JobExecutionAwareTrait;
12+
use Yokai\Batch\Job\Parameters\JobParameterAccessorInterface;
13+
14+
final class JsonLinesReader implements
15+
ItemReaderInterface,
16+
JobExecutionAwareInterface
17+
{
18+
use JobExecutionAwareTrait;
19+
20+
private JobParameterAccessorInterface $filePath;
21+
22+
public function __construct(JobParameterAccessorInterface $filePath)
23+
{
24+
$this->filePath = $filePath;
25+
}
26+
27+
/**
28+
* @inheritdoc
29+
* @phpstan-return Generator<mixed>
30+
*/
31+
public function read(): Generator
32+
{
33+
$path = (string)$this->filePath->get($this->jobExecution);
34+
$file = @\fopen($path, 'r');
35+
if ($file === false) {
36+
throw new RuntimeException(\sprintf('Cannot open %s for reading.', $path));
37+
}
38+
39+
while ($line = \fgets($file)) {
40+
yield \json_decode($line, true);
41+
}
42+
43+
\fclose($file);
44+
}
45+
}
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Yokai\Batch\Job\Item\Writer\Filesystem;
6+
7+
use Yokai\Batch\Exception\RuntimeException;
8+
use Yokai\Batch\Job\Item\FlushableInterface;
9+
use Yokai\Batch\Job\Item\InitializableInterface;
10+
use Yokai\Batch\Job\Item\ItemWriterInterface;
11+
use Yokai\Batch\Job\JobExecutionAwareInterface;
12+
use Yokai\Batch\Job\JobExecutionAwareTrait;
13+
use Yokai\Batch\Job\Parameters\JobParameterAccessorInterface;
14+
15+
final class JsonLinesWriter implements
16+
ItemWriterInterface,
17+
InitializableInterface,
18+
FlushableInterface,
19+
JobExecutionAwareInterface
20+
{
21+
use JobExecutionAwareTrait;
22+
23+
private JobParameterAccessorInterface $filePath;
24+
25+
/**
26+
* @var resource
27+
*/
28+
private $file;
29+
30+
public function __construct(JobParameterAccessorInterface $filePath)
31+
{
32+
$this->filePath = $filePath;
33+
}
34+
35+
/**
36+
* @inheritdoc
37+
*/
38+
public function initialize(): void
39+
{
40+
$path = (string)$this->filePath->get($this->jobExecution);
41+
$file = @\fopen($path, 'w+');
42+
if ($file === false) {
43+
throw new RuntimeException(\sprintf('Cannot open %s for writing.', $path));
44+
}
45+
46+
$this->file = $file;
47+
}
48+
49+
/**
50+
* @inheritdoc
51+
*/
52+
public function write(iterable $items): void
53+
{
54+
foreach ($items as $json) {
55+
if (!\is_string($json)) {
56+
$json = \json_encode($json);
57+
}
58+
\fwrite($this->file, $json . \PHP_EOL);
59+
}
60+
}
61+
62+
/**
63+
* @inheritdoc
64+
*/
65+
public function flush(): void
66+
{
67+
if (isset($this->file)) {
68+
\fclose($this->file);
69+
}
70+
}
71+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Yokai\Batch\Tests\Job\Item\Reader\Filesystem;
6+
7+
use PHPUnit\Framework\TestCase;
8+
use Yokai\Batch\Exception\RuntimeException;
9+
use Yokai\Batch\Job\Item\Reader\Filesystem\JsonLinesReader;
10+
use Yokai\Batch\Job\Parameters\StaticValueParameterAccessor;
11+
use Yokai\Batch\JobExecution;
12+
13+
class JsonLinesReaderTest extends TestCase
14+
{
15+
public function testRead(): void
16+
{
17+
$reader = new JsonLinesReader(new StaticValueParameterAccessor(__DIR__ . '/fixtures/lines.jsonl'));
18+
$reader->setJobExecution(JobExecution::createRoot('123456', 'test'));
19+
20+
self::assertSame(
21+
[
22+
['object' => 'foo'],
23+
['array', 'value'],
24+
'string',
25+
false,
26+
null,
27+
0,
28+
],
29+
\iterator_to_array($reader->read())
30+
);
31+
}
32+
33+
public function testReadUnknownFile(): void
34+
{
35+
$this->expectException(RuntimeException::class);
36+
$reader = new JsonLinesReader(new StaticValueParameterAccessor(__DIR__ . '/fixtures/unknown-file.ext'));
37+
$reader->setJobExecution(JobExecution::createRoot('123456', 'test'));
38+
\iterator_to_array($reader->read());
39+
}
40+
}
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{"object":"foo"}
2+
["array","value"]
3+
"string"
4+
false
5+
null
6+
0
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Yokai\Batch\Tests\Job\Item\Writer\Filesystem;
6+
7+
use PHPUnit\Framework\TestCase;
8+
use Yokai\Batch\Exception\RuntimeException;
9+
use Yokai\Batch\Job\Item\Writer\Filesystem\JsonLinesWriter;
10+
use Yokai\Batch\Job\Parameters\StaticValueParameterAccessor;
11+
use Yokai\Batch\JobExecution;
12+
13+
class JsonLinesWriterTest extends TestCase
14+
{
15+
private const WRITE_DIR = ARTIFACT_DIR . '/json-lines-writer';
16+
17+
public static function setUpBeforeClass(): void
18+
{
19+
if (!\is_dir(self::WRITE_DIR)) {
20+
\mkdir(self::WRITE_DIR, 0777, true);
21+
}
22+
}
23+
24+
public function testWrite(): void
25+
{
26+
$filename = self::WRITE_DIR . '/lines.jsonl';
27+
$writer = new JsonLinesWriter(new StaticValueParameterAccessor($filename));
28+
29+
self::assertFileDoesNotExist($filename);
30+
31+
$writer->setJobExecution(JobExecution::createRoot('123456', 'test'));
32+
$writer->initialize();
33+
$writer->write([
34+
['object' => 'foo'],
35+
['array', 'value'],
36+
]);
37+
$writer->write([
38+
'"string"',
39+
false,
40+
null,
41+
0,
42+
]);
43+
$writer->flush();
44+
45+
self::assertFileExists($filename);
46+
self::assertSame(
47+
<<<JSONL
48+
{"object":"foo"}
49+
["array","value"]
50+
"string"
51+
false
52+
null
53+
0
54+
JSONL,
55+
\trim(\file_get_contents($filename))
56+
);
57+
}
58+
59+
public function testWriteUnknownDir(): void
60+
{
61+
$this->expectException(RuntimeException::class);
62+
$filename = '/path/to/unknown/dir/lines.jsonl';
63+
$writer = new JsonLinesWriter(new StaticValueParameterAccessor($filename));
64+
65+
self::assertFileDoesNotExist($filename);
66+
67+
$writer->setJobExecution(JobExecution::createRoot('123456', 'test'));
68+
$writer->initialize();
69+
}
70+
}

0 commit comments

Comments
 (0)