-
Notifications
You must be signed in to change notification settings - Fork 59
/
DishwashersCrawler.php
190 lines (169 loc) · 5.29 KB
/
DishwashersCrawler.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
<?php
namespace App\Services\Wishlist;
use App\Repositories\Frontend\Appliance\ApplianceRepository;
use Illuminate\Support\Facades\Storage;
use JildertMiedema\LaravelTactician\DispatchesCommands;
use Mfa\Application\Command\UpdateOrCreateApplianceCommand;
use Mfa\Domain\ApplianceCategory;
use Mfa\Domain\ApplianceId;
use PHPHtmlParser\Dom;
use PHPHtmlParser\Dom\Collection;
class DishwashersCrawler
{
use DispatchesCommands;
const BASE_DOMAIN = 'https://www.kainos.lt';
const DISHWASHERS_URL = self::BASE_DOMAIN . '/indaploves';
/**
* @var ApplianceRepository
*/
private $applianceRepository;
/**
* DishwashersCrawler constructor.
* @param ApplianceRepository $applianceRepository
*/
public function __construct(ApplianceRepository $applianceRepository)
{
$this->applianceRepository = $applianceRepository;
}
/**
* @param callable $callbackAfterPageLoaded
* @param callable $callbackAfterItemProcessed
*/
public function importData(callable $callbackAfterPageLoaded, callable $callbackAfterItemProcessed)
{
$dom = new Dom;
$pageIndex = 1;
do {
$this->loadPage($callbackAfterPageLoaded, $pageIndex, $dom);
$products = $this->parseProductList($callbackAfterItemProcessed, $dom);
$pageIndex++;
} while ($products->count() > 0);
}
/**
* @param callable $callbackAfterPageLoaded
* @param $pageIndex
* @param Dom $dom
*/
private function loadPage(callable $callbackAfterPageLoaded, $pageIndex, Dom $dom)
{
$pageUrl = $this->buildResultsPageUrl($pageIndex);
$dom->loadFromUrl($pageUrl);
$callbackAfterPageLoaded($pageUrl);
}
/**
* @param callable $callbackAfterItemProcessed
* @param Dom $dom
* @return array|Collection
*/
private function parseProductList(callable $callbackAfterItemProcessed, $dom)
{
/** @noinspection PhpUndefinedMethodInspection */
$products = $dom->find('.product-tile');
/** @var Collection $product */
foreach ($products as $product) {
$data = $this->parseProduct($product);
$this->importProductImage($data);
$this->dispatch(UpdateOrCreateApplianceCommand::fromData($data));
$callbackAfterItemProcessed($data);
}
return $products;
}
/**
* @param $product
* @return mixed
*/
private function parseProduct($product)
{
$productId = $this->parseProductId($product);
$data['appliance_id'] = ApplianceId::generate();
$data['external_id'] = $productId;
$data['title'] = $this->parseProductTile($product);
$data['description'] = $this->parseProductDescription($product);
$data['product_url'] = $this->parseProductUrl($product);
$data['image'] = $productId;
$data['image_url'] = $this->parseProductImageUrl($product);
$data['category'] = ApplianceCategory::DISHWASHER;
$data['price_amount'] = $this->parseProductPriceAmount($product);
$data['price_currency'] = 'EUR'; // All product prices are in euros.
return $data;
}
/**
* @param $product
* @return mixed
*/
private function parseProductTile($product)
{
/** @noinspection PhpUndefinedMethodInspection */
return $product->find('h3.title')->innerHtml();
}
/**
* @param $product
* @return mixed
*/
private function parseProductDescription($product)
{
return '';
}
/**
* @param $product
* @return mixed
*/
private function parseProductUrl($product)
{
/** @noinspection PhpUndefinedMethodInspection */
return self::BASE_DOMAIN . $product->find('a')->getAttribute('href');
}
/**
* @param $product
* @return mixed
*/
private function parseProductId($product)
{
$productUrl = $this->parseProductUrl($product);
if (preg_match("/(\d+)$/", $productUrl, $matches)) {
return $matches[1];
}
return null;
}
/**
* @param $product
* @return mixed
*/
private function parseProductImageUrl($product)
{
/** @noinspection PhpUndefinedMethodInspection */
return $product->find('.image-container img')->getAttribute('src');
}
/**
* @param $product
* @return mixed
*/
private function parseProductPriceAmount($product)
{
/** @noinspection PhpUndefinedMethodInspection */
$price = $product->find('.price')->innerHtml();
$price = str_replace('€', '', $price);
$price = str_replace(',', '', $price);
$price = str_replace('nuo', '', $price);
$price = trim($price);
$priceInCents = round($price * 100);
return $priceInCents;
}
/**
* @param array $data
*/
private function importProductImage(array $data)
{
$imageName = $data['external_id'];
/** @noinspection PhpUndefinedMethodInspection */
Storage::put("public/appliances/$imageName", file_get_contents($data['image_url']), 'public');
}
/**
* @param $pageIndex
* @return string
*/
private function buildResultsPageUrl($pageIndex)
{
return self::DISHWASHERS_URL . "?page=$pageIndex";
}
}