Skip to content
This repository was archived by the owner on Jan 9, 2025. It is now read-only.

Commit fdd30b0

Browse files
mzurmzur
authored
Merge pull request #133 from biigle/issue-127
Fix out of memory error with CSV report
2 parents 1363f5f + 98e8556 commit fdd30b0

File tree

3 files changed

+54
-29
lines changed

3 files changed

+54
-29
lines changed

‎src/Http/Controllers/Api/ReportsController.php

+11-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,17 @@ public function show($id)
4040
abort(Response::HTTP_NOT_FOUND);
4141
}
4242

43-
return $disk->download($report->getStorageFilename(), $report->filename);
43+
$path = $report->getStorageFilename();
44+
return $disk->download($path, $report->filename)
45+
// Use a custom fallback with fread() because the default fpassthru() could
46+
// lead to an out of memory error with large reports.
47+
->setCallback(function () use ($disk, $path) {
48+
$stream = $disk->readStream($path);
49+
while (!feof($stream)) {
50+
echo fread($stream, 8192);
51+
}
52+
fclose($stream);
53+
});
4454
}
4555

4656
/**

‎src/Support/Reports/Volumes/ImageAnnotations/CsvReportGenerator.php

+22-14
Original file line numberDiff line numberDiff line change
@@ -40,31 +40,39 @@ class CsvReportGenerator extends AnnotationReportGenerator
4040
*/
4141
public function generateReport($path)
4242
{
43-
$rows = $this->query()->get();
43+
$exists = $this->initQuery()->exists();
4444
$toZip = [];
4545

46-
if ($this->shouldSeparateLabelTrees() && $rows->isNotEmpty()) {
47-
$rows = $rows->groupBy('label_tree_id');
48-
$trees = LabelTree::whereIn('id', $rows->keys())->pluck('name', 'id');
46+
if ($this->shouldSeparateLabelTrees() && $exists) {
47+
$treeIds = $this->initQuery()
48+
->select('labels.label_tree_id')
49+
->distinct()
50+
->pluck('label_tree_id');
51+
52+
$trees = LabelTree::whereIn('id', $treeIds)->pluck('name', 'id');
4953

5054
foreach ($trees as $id => $name) {
51-
$csv = $this->createCsv($rows->get($id));
55+
$csv = $this->createCsv($this->query()->where('labels.label_tree_id', $id));
5256
$this->tmpFiles[] = $csv;
5357
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$id}-{$name}", 'csv');
5458
}
55-
} elseif ($this->shouldSeparateUsers() && $rows->isNotEmpty()) {
56-
$rows = $rows->groupBy('user_id');
57-
$users = User::whereIn('id', $rows->keys())
59+
} elseif ($this->shouldSeparateUsers() && $exists) {
60+
$userIds = $this->initQuery()
61+
->select('user_id')
62+
->distinct()
63+
->pluck('user_id');
64+
65+
$users = User::whereIn('id', $userIds)
5866
->selectRaw("id, concat(firstname, ' ', lastname) as name")
5967
->pluck('name', 'id');
6068

6169
foreach ($users as $id => $name) {
62-
$csv = $this->createCsv($rows->get($id));
70+
$csv = $this->createCsv($this->query()->where('user_id', $id));
6371
$this->tmpFiles[] = $csv;
6472
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$id}-{$name}", 'csv');
6573
}
6674
} else {
67-
$csv = $this->createCsv($rows);
75+
$csv = $this->createCsv($this->query());
6876
$this->tmpFiles[] = $csv;
6977
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$this->source->id}-{$this->source->name}", 'csv');
7078
}
@@ -107,10 +115,10 @@ protected function query()
107115
/**
108116
* Create a CSV file for this report.
109117
*
110-
* @param \Illuminate\Support\Collection $rows The rows for the CSV
118+
* @param \Illuminate\Database\QueryBuilder $query The query for the CSV rows
111119
* @return CsvFile
112120
*/
113-
protected function createCsv($rows)
121+
protected function createCsv($query)
114122
{
115123
$csv = CsvFile::makeTmp();
116124
// column headers
@@ -134,7 +142,7 @@ protected function createCsv($rows)
134142
'created_at',
135143
]);
136144

137-
foreach ($rows as $row) {
145+
$query->eachById(function ($row) use ($csv) {
138146
$csv->put([
139147
$row->annotation_label_id,
140148
$row->label_id,
@@ -154,7 +162,7 @@ protected function createCsv($rows)
154162
$row->annotation_id,
155163
$row->created_at,
156164
]);
157-
}
165+
}, column: 'image_annotation_labels.id', alias: 'annotation_label_id');
158166

159167
$csv->close();
160168

‎src/Support/Reports/Volumes/VideoAnnotations/CsvReportGenerator.php

+21-14
Original file line numberDiff line numberDiff line change
@@ -98,31 +98,38 @@ public function getFilename()
9898
*/
9999
public function generateReport($path)
100100
{
101-
$rows = $this->query()->get();
101+
$exists = $this->initQuery()->exists();
102102
$toZip = [];
103103

104-
if ($this->shouldSeparateLabelTrees() && $rows->isNotEmpty()) {
105-
$rows = $rows->groupBy('label_tree_id');
106-
$trees = LabelTree::whereIn('id', $rows->keys())->pluck('name', 'id');
104+
if ($this->shouldSeparateLabelTrees() && $exists) {
105+
$treeIds = $this->initQuery()
106+
->select('labels.label_tree_id')
107+
->distinct()
108+
->pluck('label_tree_id');
109+
$trees = LabelTree::whereIn('id', $treeIds)->pluck('name', 'id');
107110

108111
foreach ($trees as $id => $name) {
109-
$csv = $this->createCsv($rows->get($id));
112+
$csv = $this->createCsv($this->query()->where('labels.label_tree_id', $id));
110113
$this->tmpFiles[] = $csv;
111114
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$id}-{$name}", 'csv');
112115
}
113-
} elseif ($this->shouldSeparateUsers() && $rows->isNotEmpty()) {
114-
$rows = $rows->groupBy('user_id');
115-
$users = User::whereIn('id', $rows->keys())
116+
} elseif ($this->shouldSeparateUsers() && $exists) {
117+
$userIds = $this->initQuery()
118+
->select('user_id')
119+
->distinct()
120+
->pluck('user_id');
121+
122+
$users = User::whereIn('id', $userIds)
116123
->selectRaw("id, concat(firstname, ' ', lastname) as name")
117124
->pluck('name', 'id');
118125

119126
foreach ($users as $id => $name) {
120-
$csv = $this->createCsv($rows->get($id));
127+
$csv = $this->createCsv($this->query()->where('user_id', $id));
121128
$this->tmpFiles[] = $csv;
122129
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$id}-{$name}", 'csv');
123130
}
124131
} else {
125-
$csv = $this->createCsv($rows);
132+
$csv = $this->createCsv($this->query());
126133
$this->tmpFiles[] = $csv;
127134
$toZip[$csv->getPath()] = $this->sanitizeFilename("{$this->source->id}-{$this->source->name}", 'csv');
128135
}
@@ -218,10 +225,10 @@ protected function query()
218225
/**
219226
* Create a CSV file for this report.
220227
*
221-
* @param \Illuminate\Support\Collection $rows The rows for the CSV
228+
* @param \Illuminate\Database\QueryBuilder $query The query for the CSV rows
222229
* @return CsvFile
223230
*/
224-
protected function createCsv($rows)
231+
protected function createCsv($query)
225232
{
226233
$csv = CsvFile::makeTmp();
227234
// column headers
@@ -244,7 +251,7 @@ protected function createCsv($rows)
244251
'attributes',
245252
]);
246253

247-
foreach ($rows as $row) {
254+
$query->eachById(function ($row) use ($csv) {
248255
$csv->put([
249256
$row->video_annotation_label_id,
250257
$row->label_id,
@@ -263,7 +270,7 @@ protected function createCsv($rows)
263270
$row->created_at,
264271
$row->attrs,
265272
]);
266-
}
273+
}, column: 'video_annotation_labels.id', alias: 'video_annotation_label_id');
267274

268275
$csv->close();
269276

0 commit comments

Comments
 (0)