From f3980bdd0e039aa67019031c940f7f73a8f2e998 Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Thu, 10 Aug 2023 13:20:13 +0200 Subject: [PATCH 1/3] added Tier importer --- app/Importers/TierDataImporter.php | 74 ++++++++++++++++++++++++++++ app/Jobs/TierDataImporterJob.php | 15 ++++++ app/Services/DataImporterService.php | 2 + database/seeders/ProviderSeeder.php | 2 + 4 files changed, 93 insertions(+) create mode 100644 app/Importers/TierDataImporter.php create mode 100644 app/Jobs/TierDataImporterJob.php diff --git a/app/Importers/TierDataImporter.php b/app/Importers/TierDataImporter.php new file mode 100644 index 00000000..39ef200b --- /dev/null +++ b/app/Importers/TierDataImporter.php @@ -0,0 +1,74 @@ +client->get("https://www.tier.app/en/where-to-find-us"); + $html = $response->getBody()->getContents(); + } catch (GuzzleException) { + $this->createImportInfoDetails("400", self::getProviderName()); + + $this->stopExecution = true; + + return $this; + } + + $crawler = new Crawler($html); + $this->sections = $crawler->filter("main div.relative section.Accordion__AccordionWrapper-sc-ehu24-0.fvluSp>li > div.items-center"); + + if (count($this->sections) === 0) { + $this->createImportInfoDetails("204", self::getProviderName()); + + $this->stopExecution = true; + } + + return $this; + } + + public function transform(): void + { + if ($this->stopExecution) { + return; + } + + $existingCityProviders = []; + + foreach ($this->sections as $section) { + foreach ($section->childNodes as $node) { + if ($node->nodeName === "h5") { + $this->countryName = trim($node->nodeValue); + + continue 2; + } + + foreach ($node->childNodes as $childNode) { + if (trim($childNode->nodeValue) === "Klopein" || trim($childNode->nodeValue) === "Gaiserwald"){ + continue; } + + if (trim($childNode->nodeValue) !== ""){ + $cityName = trim($childNode->nodeValue); + $provider = $this->load($cityName, $this->countryName); + + if ($provider !== "") { + $existingCityProviders[] = $provider; + } + } + } + } + } + + $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); + } +} diff --git a/app/Jobs/TierDataImporterJob.php b/app/Jobs/TierDataImporterJob.php new file mode 100644 index 00000000..5999bf3c --- /dev/null +++ b/app/Jobs/TierDataImporterJob.php @@ -0,0 +1,15 @@ +setImportInfo($this->importInfoId)->extract()->transform(); + } +} diff --git a/app/Services/DataImporterService.php b/app/Services/DataImporterService.php index b2821304..0bfe91bb 100644 --- a/app/Services/DataImporterService.php +++ b/app/Services/DataImporterService.php @@ -14,6 +14,7 @@ use App\Jobs\QuickDataImporterJob; use App\Jobs\RydeDataImporterJob; use App\Jobs\SpinDataImporterJob; +use App\Jobs\TierDataImporterJob; use App\Jobs\UrentDataImporterJob; use App\Jobs\VoiDataImporterJob; use App\Jobs\ZwingsDataImporterJob; @@ -44,6 +45,7 @@ public function run(string $whoRunsIt = "admin"): void new QuickDataImporterJob($this->importInfoId), new RydeDataImporterJob($this->importInfoId), new SpinDataImporterJob($this->importInfoId), + new TierDataImporterJob($this->importInfoId), new UrentDataImporterJob($this->importInfoId), new VoiDataImporterJob($this->importInfoId), new ZwingsDataImporterJob($this->importInfoId), diff --git a/database/seeders/ProviderSeeder.php b/database/seeders/ProviderSeeder.php index bea4f0cd..59b4648f 100644 --- a/database/seeders/ProviderSeeder.php +++ b/database/seeders/ProviderSeeder.php @@ -14,6 +14,7 @@ use App\Importers\QuickDataImporter; use App\Importers\RydeDataImporter; use App\Importers\SpinDataImporter; +use App\Importers\TierDataImporter; use App\Importers\UrentDataImporter; use App\Importers\VoiDataImporter; use App\Importers\ZwingsDataImporter; @@ -34,6 +35,7 @@ public function run(): void ["name" => NeuronDataImporter::getProviderName(), "color" => "#445261"], ["name" => QuickDataImporter::getProviderName(), "color" => "#009ac7"], ["name" => SpinDataImporter::getProviderName(), "color" => "#ff5436"], + ["name" => TierDataImporter::getProviderName(), "color" => "#0E1A50"], ["name" => VoiDataImporter::getProviderName(), "color" => "#f46c63"], ["name" => UrentDataImporter::getProviderName(), "color" => "#9400FF"], ["name" => ZwingsDataImporter::getProviderName(), "color" => "#abb8c3"], From 28621fb4a9f56a227c6f8ff09a48ffec57a89b24 Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Thu, 10 Aug 2023 13:23:07 +0200 Subject: [PATCH 2/3] edit readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index c0e4f37d..efee97ad 100644 --- a/readme.md +++ b/readme.md @@ -54,7 +54,7 @@ npm run lintf ``` Runs a queue that processes importers' jobs: ``` -php artisan queue:work --queue="importers" --timeout=0 +php artisan queue:work ``` ### Containers From 59da21f7f816e60d355f8601bd41cb805f7f2c8d Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Thu, 10 Aug 2023 14:15:56 +0200 Subject: [PATCH 3/3] fix --- app/Importers/DataImporter.php | 1 - app/Importers/TierDataImporter.php | 3 --- app/Services/MapboxGeocodingService.php | 5 ++--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index ed83994f..e705ea3a 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -118,7 +118,6 @@ protected function load(string $cityName, string $countryName, string $lat = "", if (!$countCoordinates) { $this->createImportInfoDetails("419", self::getProviderName()); } - $city = City::query()->create([ "name" => $cityName, "latitude" => ($countCoordinates > 0) ? $coordinates[0] : null, diff --git a/app/Importers/TierDataImporter.php b/app/Importers/TierDataImporter.php index 39ef200b..d25c28a1 100644 --- a/app/Importers/TierDataImporter.php +++ b/app/Importers/TierDataImporter.php @@ -54,9 +54,6 @@ public function transform(): void } foreach ($node->childNodes as $childNode) { - if (trim($childNode->nodeValue) === "Klopein" || trim($childNode->nodeValue) === "Gaiserwald"){ - continue; } - if (trim($childNode->nodeValue) !== ""){ $cityName = trim($childNode->nodeValue); $provider = $this->load($cityName, $this->countryName); diff --git a/app/Services/MapboxGeocodingService.php b/app/Services/MapboxGeocodingService.php index 17844e25..64385f42 100644 --- a/app/Services/MapboxGeocodingService.php +++ b/app/Services/MapboxGeocodingService.php @@ -30,11 +30,10 @@ public function getCoordinatesFromApi(string $cityName, string $countryName): ar ); $coordinates = json_decode($response->getBody()->getContents(), associative: true)["features"][0]["center"]; - - return [$coordinates[1], $coordinates[0]]; } catch (Throwable $exception) { - throw new MapboxGeocodingServiceException(previous: $exception); } + + return [$coordinates[1] ?? null, $coordinates[0] ?? null]; } /**