From bfa5c549f210ab22b537eed66e462533aec8e2ab Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Thu, 4 Apr 2024 20:54:43 +0200 Subject: [PATCH 01/14] Add new types for mobility --- .../Controllers/CityProviderController.php | 3 +++ app/Importers/BirdDataImporter.php | 2 +- app/Importers/DataImporter.php | 17 +++++++++----- app/Models/CityProvider.php | 8 +++++++ app/Models/Service.php | 19 +++++++++++++++ ..._15_132727_create_city_providers_table.php | 1 - ...024_04_04_152429_create_services_table.php | 23 +++++++++++++++++++ ...create_city_providers_service_relation.php | 20 ++++++++++++++++ database/seeders/DatabaseSeeder.php | 1 + database/seeders/ServiceSeeder.php | 23 +++++++++++++++++++ 10 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 app/Models/Service.php create mode 100644 database/migrations/2024_04_04_152429_create_services_table.php create mode 100644 database/migrations/2024_04_04_162723_create_city_providers_service_relation.php create mode 100644 database/seeders/ServiceSeeder.php diff --git a/app/Http/Controllers/CityProviderController.php b/app/Http/Controllers/CityProviderController.php index 0796f65e..1ab50564 100644 --- a/app/Http/Controllers/CityProviderController.php +++ b/app/Http/Controllers/CityProviderController.php @@ -11,6 +11,7 @@ use App\Models\City; use App\Models\Country; use App\Models\Provider; +use App\Models\Service; use App\Services\CityProviderService; use App\Services\DataImporterService; @@ -36,11 +37,13 @@ public function index(): array ->sortBy("name"); $countries = CountryResource::collection($countries); + $services = Service::all(); return [ "cities" => $cities, "providers" => $providers, "countries" => $countries, + "services" => $services, ]; } diff --git a/app/Importers/BirdDataImporter.php b/app/Importers/BirdDataImporter.php index d09442d2..cde6066e 100644 --- a/app/Importers/BirdDataImporter.php +++ b/app/Importers/BirdDataImporter.php @@ -57,7 +57,7 @@ public function transform(): void protected function parseData(string $html): array { - $pattern = '/let features = \[([\s\S]*?)\];/'; + $pattern = '/let features = \[([\s\S]*?)];/'; if (preg_match($pattern, $html, $matches)) { $fetchedData = $matches[1]; diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index 0d33a03a..c7cefa0f 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -10,6 +10,7 @@ use App\Models\CityWithoutAssignedCountry; use App\Models\Country; use App\Models\ImportInfoDetail; +use App\Models\Service; use App\Services\MapboxGeocodingService; use GuzzleHttp\Client; use Stichoza\GoogleTranslate\GoogleTranslate; @@ -74,13 +75,17 @@ protected function countryNotFound(string $cityName, string $countryName): void } } - protected function createProvider(int $cityId, string $providerName): void + protected function createProvider(int $cityId, string $providerName, array $services = ["escooter"]): void { - CityProvider::query()->updateOrCreate([ - "city_id" => $cityId, - "provider_name" => $providerName, - "created_by" => "scrapper", - ]); + foreach ($services as $service) { + $serviceId = Service::query()->where("type", $service)->first()->id; + CityProvider::query()->updateOrCreate([ + "city_id" => $cityId, + "provider_name" => $providerName, + "service_id" => $serviceId, + "created_by" => "scrapper", + ]); + } } protected function deleteMissingProviders(string $providerName, array $existingCityProviders): void diff --git a/app/Models/CityProvider.php b/app/Models/CityProvider.php index 45212f16..e3c6675a 100644 --- a/app/Models/CityProvider.php +++ b/app/Models/CityProvider.php @@ -12,6 +12,8 @@ * @property int $id * @property string $provider_name * @property int $city_id + * @property string $created_by + * @property int $service_id */ class CityProvider extends Model { @@ -19,6 +21,7 @@ class CityProvider extends Model "provider_name", "city_id", "created_by", + "service_id", ]; public function city(): BelongsTo @@ -30,4 +33,9 @@ public function providers(): HasMany { return $this->hasMany(Provider::class); } + + public function services(): HasMany + { + return $this->hasMany(Service::class); + } } diff --git a/app/Models/Service.php b/app/Models/Service.php new file mode 100644 index 00000000..961dcad8 --- /dev/null +++ b/app/Models/Service.php @@ -0,0 +1,19 @@ +references("name") ->on("providers") ->onDelete("cascade"); - $table->unsignedBigInteger("city_id"); $table->foreign("city_id") ->references("id") diff --git a/database/migrations/2024_04_04_152429_create_services_table.php b/database/migrations/2024_04_04_152429_create_services_table.php new file mode 100644 index 00000000..ede5b805 --- /dev/null +++ b/database/migrations/2024_04_04_152429_create_services_table.php @@ -0,0 +1,23 @@ +id(); + $table->string("type"); + $table->timestamps(); + }); + } + + public function down(): void + { + Schema::dropIfExists("services"); + } +}; diff --git a/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php b/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php new file mode 100644 index 00000000..1b2e4974 --- /dev/null +++ b/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php @@ -0,0 +1,20 @@ +unsignedBigInteger("service_id")->nullable(); + $table->foreign("service_id") + ->references("id") + ->on("services") + ->onDelete("cascade"); + }); + } +}; diff --git a/database/seeders/DatabaseSeeder.php b/database/seeders/DatabaseSeeder.php index a7557ce7..34277522 100644 --- a/database/seeders/DatabaseSeeder.php +++ b/database/seeders/DatabaseSeeder.php @@ -16,6 +16,7 @@ public function run(): void CitiesAndCountriesSeeder::class, ProviderSeeder::class, CodeSeeder::class, + ServiceSeeder::class, ]); } } diff --git a/database/seeders/ServiceSeeder.php b/database/seeders/ServiceSeeder.php new file mode 100644 index 00000000..328f6708 --- /dev/null +++ b/database/seeders/ServiceSeeder.php @@ -0,0 +1,23 @@ + "bike"]); + Service::create(["type" => "cargo"]); + Service::create(["type" => "emoped"]); + Service::create(["type" => "escooter"]); + Service::create(["type" => "motorscooter"]); + } +} From 1dc6b0fb40c4b16f783b363468d5730364c50db0 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 27 Apr 2024 15:44:31 +0200 Subject: [PATCH 02/14] update service assigning --- app/Exceptions/ExceptionHandler.php | 3 +++ app/Importers/BirdDataImporter.php | 6 +++--- app/Importers/DataImporter.php | 19 ++++++++++--------- app/Importers/LimeDataImporter.php | 6 +++--- app/Importers/SpinDataImporter.php | 6 +++--- ...create_city_providers_service_relation.php | 8 ++++++++ 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/app/Exceptions/ExceptionHandler.php b/app/Exceptions/ExceptionHandler.php index 9c349d94..dd152938 100644 --- a/app/Exceptions/ExceptionHandler.php +++ b/app/Exceptions/ExceptionHandler.php @@ -26,6 +26,9 @@ public function register(): void }); } + /** + * @noinspection PhpParameterNameChangedDuringInheritanceInspection + */ public function render($request, Throwable $exception) { if ($exception instanceof ValidationException) { diff --git a/app/Importers/BirdDataImporter.php b/app/Importers/BirdDataImporter.php index cde6066e..21ff8bf3 100644 --- a/app/Importers/BirdDataImporter.php +++ b/app/Importers/BirdDataImporter.php @@ -75,7 +75,7 @@ protected function parseData(string $html): array return array_map("trim", $coordinates); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = ""): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); @@ -83,7 +83,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", if ($city || $alternativeCityName) { $cityId = $city ? $city->id : $alternativeCityName->city_id; - $this->createProvider($cityId, self::getProviderName()); + $this->createProvider($cityId, self::getProviderName(), $services); return strval($cityId); } @@ -97,7 +97,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", "country_id" => $country->id, ]); - $this->createProvider($city->id, self::getProviderName()); + $this->createProvider($city->id, self::getProviderName(), $services); return strval($city->id); } diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index 1fd52009..91342338 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -75,15 +75,16 @@ protected function countryNotFound(string $cityName, string $countryName): void } } - protected function createProvider(int $cityId, string $providerName, array $services = ["escooter"]): void + protected function createProvider(int $cityId, string $providerName, array $services): void { foreach ($services as $service) { - $serviceId = Service::query()->where("type", $service)->first()->id; + $service = Service::query()->where("type", $service)->first(); + CityProvider::query()->updateOrCreate([ - "city_id" => $cityId, "provider_name" => $providerName, - "service_id" => $serviceId, - "created_by" => "scrapper", + "city_id" => $cityId, + "created_by" => "scraper", + "service_id" => $service->id, ]); } } @@ -115,7 +116,7 @@ protected function createImportInfoDetails(string $code, string $providerName): ); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = ""): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $country = Country::query()->where("name", $countryName)->orWhere("alternative_name", $countryName)->first(); @@ -125,7 +126,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", if ($city) { $cityId = $city->id; - $this->createProvider($cityId, self::getProviderName()); + $this->createProvider($cityId, self::getProviderName(), $services); return strval($cityId); } elseif ($alternativeCityName) { @@ -133,7 +134,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", $city = City::query()->where("id", $cityId)->first(); if ($city->country_id === $country->id) { - $this->createProvider($cityId, self::getProviderName()); + $this->createProvider($cityId, self::getProviderName(), $services); } } @@ -151,7 +152,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", "country_id" => $country->id, ]); - $this->createProvider($city->id, self::getProviderName()); + $this->createProvider($city->id, self::getProviderName(), $services); return strval($city->id); } diff --git a/app/Importers/LimeDataImporter.php b/app/Importers/LimeDataImporter.php index ba884fbd..9a3eaead 100644 --- a/app/Importers/LimeDataImporter.php +++ b/app/Importers/LimeDataImporter.php @@ -59,7 +59,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = ""): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); @@ -67,7 +67,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", if ($city || $alternativeCityName) { $cityId = $city ? $city->id : $alternativeCityName->city_id; - $this->createProvider($cityId, self::getProviderName()); + $this->createProvider($cityId, self::getProviderName(), $services); return strval($cityId); } @@ -98,7 +98,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", "country_id" => $country->id, ]); - $this->createProvider($city->id, self::getProviderName()); + $this->createProvider($city->id, self::getProviderName(), $services); return strval($city->id); } diff --git a/app/Importers/SpinDataImporter.php b/app/Importers/SpinDataImporter.php index 74d1c619..563e0770 100644 --- a/app/Importers/SpinDataImporter.php +++ b/app/Importers/SpinDataImporter.php @@ -76,7 +76,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = ""): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); @@ -84,7 +84,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", if ($city || $alternativeCityName) { $cityId = $city ? $city->id : $alternativeCityName->city_id; - $this->createProvider($cityId, self::getProviderName()); + $this->createProvider($cityId, self::getProviderName(), $services); return strval($cityId); } @@ -115,7 +115,7 @@ protected function load(string $cityName, string $countryName, string $lat = "", "country_id" => $country->id, ]); - $this->createProvider($city->id, self::getProviderName()); + $this->createProvider($city->id, self::getProviderName(), $services); return strval($city->id); } diff --git a/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php b/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php index 1b2e4974..9937a61a 100644 --- a/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php +++ b/database/migrations/2024_04_04_162723_create_city_providers_service_relation.php @@ -17,4 +17,12 @@ public function up(): void ->onDelete("cascade"); }); } + + public function down(): void + { + Schema::table("city_providers", function (Blueprint $table): void { + $table->dropForeign(["service_id"]); + $table->dropColumn("service_id"); + }); + } }; From 986b34bd138e0c8ef5ea8d3a81ac9ff2a20fecb5 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 4 May 2024 10:43:00 +0200 Subject: [PATCH 03/14] Add example importer (Beam) --- .../Api/CityProviderController.php | 3 +- .../Controllers/CityProviderController.php | 3 -- app/Importers/BeamDataImporter.php | 36 ++++++++++++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/app/Http/Controllers/Api/CityProviderController.php b/app/Http/Controllers/Api/CityProviderController.php index d4990c6a..1fe98c09 100644 --- a/app/Http/Controllers/Api/CityProviderController.php +++ b/app/Http/Controllers/Api/CityProviderController.php @@ -10,6 +10,7 @@ use App\Http\Resources\CountryResource; use App\Http\Resources\ProviderResource; use App\Models\City; +use App\Models\CityProvider; use App\Models\Country; use App\Models\Provider; use App\Services\CityProviderService; @@ -30,7 +31,7 @@ public function index(): JsonResponse ->sortByDesc(fn(City $city): int => $city->cityProviders->count()), ); - $providers = ProviderResource::collection(Provider::all()->sortBy("name")); + $providers = ProviderResource::collection(Provider::all()); $countries = Country::whereHas("cities.cityProviders") ->with(["cities.cityAlternativeNames", "cities.cityProviders"]) ->get() diff --git a/app/Http/Controllers/CityProviderController.php b/app/Http/Controllers/CityProviderController.php index 1ab50564..0796f65e 100644 --- a/app/Http/Controllers/CityProviderController.php +++ b/app/Http/Controllers/CityProviderController.php @@ -11,7 +11,6 @@ use App\Models\City; use App\Models\Country; use App\Models\Provider; -use App\Models\Service; use App\Services\CityProviderService; use App\Services\DataImporterService; @@ -37,13 +36,11 @@ public function index(): array ->sortBy("name"); $countries = CountryResource::collection($countries); - $services = Service::all(); return [ "cities" => $cities, "providers" => $providers, "countries" => $countries, - "services" => $services, ]; } diff --git a/app/Importers/BeamDataImporter.php b/app/Importers/BeamDataImporter.php index 4dd82e3e..2735e05b 100644 --- a/app/Importers/BeamDataImporter.php +++ b/app/Importers/BeamDataImporter.php @@ -37,7 +37,8 @@ public function extract(): static public function transform(): void { - $url = "https://assets-global.website-files.com/63c4acbedbab5dea8b1b98cd/63d8a5b60da91e7d71298637_map-vehicle-saturn.png"; + $escooterImageUrl = "https://assets-global.website-files.com/63c4acbedbab5dea8b1b98cd/63d8a5b60da91e7d71298637_map-vehicle-saturn.png"; + $bikeImageUrl = "https://assets-global.website-files.com/63c4acbedbab5dea8b1b98cd/63d8a5b7f06e4a42de02241b_map-vehicle-saturn-apollo.png"; if ($this->stopExecution) { return; @@ -55,22 +56,31 @@ public function transform(): void foreach ($node->childNodes as $div) { if ($div->nodeName === "div") { foreach ($div->childNodes as $city) { - if ($city->nodeName === "img" && $city->getAttribute("src") === $url) { + if (!$city->nodeName) { + continue; + } + + $hasEscooters = false; + $hasBikes = false; + + if ($city->getAttribute("src") === $escooterImageUrl) { $hasEscooters = true; - } elseif ($city->nodeName === "img" && $city->getAttribute("src") !== $url) { - $hasEscooters = false; } - if ($city->nodeName === "p" && $hasEscooters === true) { + if ($city->getAttribute("src") === $bikeImageUrl) { + $hasBikes = true; + } + + if ($city->nodeName === "p") { $search = ["\u{00A0}", "\u{200D}", "Prefecture"]; $valueToDelete = "Selangor"; $cityName = preg_replace('/[\p{Hiragana}\p{Katakana}\p{Han}]+/u', "", $city->nodeValue); $cityName = str_replace($search, "", $cityName); - $cityName = preg_replace('/(?<=[^\s_\-])(?=[A-Z])/', " ", $city->nodeValue); + $cityName = preg_replace('/(?<=[^\s_\-])(?=[A-Z])/', " ", $cityName); $arrayOfCitiesNames = explode(" ", $cityName); $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($value) => $value !== $valueToDelete); $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => strlen($record) > 1); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => strpos($record, "•") === false); + $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => !str_contains($record, "•")); foreach ($arrayOfCitiesNames as $cityName) { if ($cityName !== "Selangor") { @@ -79,7 +89,17 @@ public function transform(): void if ($countryName === "Korea") { $countryName = "South Korea"; } - $provider = $this->load($cityName, $countryName); + $services = []; + + if ($hasBikes) { + $services[] = "bike"; + } + + if ($hasEscooters) { + $services[] = "escooter"; + } + $services = ["escooter", "bike"]; + $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); if ($provider !== "") { $existingCityProviders[] = $provider; From 186de3f0549f1cd19c6f9b44e66977a7460f63ff Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 4 May 2024 11:00:10 +0200 Subject: [PATCH 04/14] csf --- app/Http/Controllers/Api/CityProviderController.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Http/Controllers/Api/CityProviderController.php b/app/Http/Controllers/Api/CityProviderController.php index 1fe98c09..979b925f 100644 --- a/app/Http/Controllers/Api/CityProviderController.php +++ b/app/Http/Controllers/Api/CityProviderController.php @@ -10,7 +10,6 @@ use App\Http\Resources\CountryResource; use App\Http\Resources\ProviderResource; use App\Models\City; -use App\Models\CityProvider; use App\Models\Country; use App\Models\Provider; use App\Services\CityProviderService; From 65472375a94799e244e15ef82abc692ba1d59c5d Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 4 May 2024 11:07:36 +0200 Subject: [PATCH 05/14] Fix ImporterTest.php --- tests/Unit/ImporterTest.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Unit/ImporterTest.php b/tests/Unit/ImporterTest.php index be236c25..9ce444cc 100644 --- a/tests/Unit/ImporterTest.php +++ b/tests/Unit/ImporterTest.php @@ -8,8 +8,10 @@ use App\Models\City; use App\Models\CityProvider; use App\Models\Country; +use App\Models\Service; use App\Services\MapboxGeocodingService; use Database\Seeders\ProviderSeeder; +use Database\Seeders\ServiceSeeder; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; use GuzzleHttp\Handler\MockHandler; @@ -27,7 +29,10 @@ class ImporterTest extends TestCase protected function setUp(): void { parent::setUp(); - $this->seed(ProviderSeeder::class); + $this->seed([ + ProviderSeeder::class, + ServiceSeeder::class + ]); $mockResponseBody = "let features = [ { position: new google.maps.LatLng(-31.9523123, 115.861309), From 0167c2109e73d089e4bbaf9f08d7a343e244066e Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 4 May 2024 11:07:52 +0200 Subject: [PATCH 06/14] Fix ImporterTest.php --- tests/Unit/ImporterTest.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Unit/ImporterTest.php b/tests/Unit/ImporterTest.php index 9ce444cc..fe2afdf6 100644 --- a/tests/Unit/ImporterTest.php +++ b/tests/Unit/ImporterTest.php @@ -8,7 +8,6 @@ use App\Models\City; use App\Models\CityProvider; use App\Models\Country; -use App\Models\Service; use App\Services\MapboxGeocodingService; use Database\Seeders\ProviderSeeder; use Database\Seeders\ServiceSeeder; @@ -31,7 +30,7 @@ protected function setUp(): void parent::setUp(); $this->seed([ ProviderSeeder::class, - ServiceSeeder::class + ServiceSeeder::class, ]); $mockResponseBody = "let features = [ { From 9c5d272d3d9dca288e97a7be4c6931d556dea9ff Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Mon, 6 May 2024 16:45:02 +0200 Subject: [PATCH 07/14] Add enum for services --- app/Enums/ServicesEnum.php | 14 ++++++++++++++ app/Models/CityProvider.php | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 app/Enums/ServicesEnum.php diff --git a/app/Enums/ServicesEnum.php b/app/Enums/ServicesEnum.php new file mode 100644 index 00000000..0a89607c --- /dev/null +++ b/app/Enums/ServicesEnum.php @@ -0,0 +1,14 @@ +hasMany(Provider::class); } - public function services(): HasMany + public function services(): HasOne { - return $this->hasMany(Service::class); + return $this->hasOne(Service::class); } } From c17896b5c8fd8db39cb11c2380b628ef101bbcd8 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 11 May 2024 11:29:01 +0200 Subject: [PATCH 08/14] change seeder to work with enum --- database/seeders/ServiceSeeder.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/database/seeders/ServiceSeeder.php b/database/seeders/ServiceSeeder.php index 328f6708..c1725c20 100644 --- a/database/seeders/ServiceSeeder.php +++ b/database/seeders/ServiceSeeder.php @@ -6,6 +6,7 @@ use App\Models\Service; use Illuminate\Database\Seeder; +use App\Enums\ServicesEnum; class ServiceSeeder extends Seeder { @@ -14,10 +15,10 @@ class ServiceSeeder extends Seeder */ public function run(): void { - Service::create(["type" => "bike"]); - Service::create(["type" => "cargo"]); - Service::create(["type" => "emoped"]); - Service::create(["type" => "escooter"]); - Service::create(["type" => "motorscooter"]); + foreach (ServicesEnum::cases() as $service) { + Service::query()->create([ + "type" => $service->value, + ]); + } } } From 76ce603b257bb04ad1edaf8c5d13c86908ee60e6 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 11 May 2024 11:29:25 +0200 Subject: [PATCH 09/14] csf --- database/seeders/ServiceSeeder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/seeders/ServiceSeeder.php b/database/seeders/ServiceSeeder.php index c1725c20..54b8230f 100644 --- a/database/seeders/ServiceSeeder.php +++ b/database/seeders/ServiceSeeder.php @@ -4,9 +4,9 @@ namespace Database\Seeders; +use App\Enums\ServicesEnum; use App\Models\Service; use Illuminate\Database\Seeder; -use App\Enums\ServicesEnum; class ServiceSeeder extends Seeder { From 0ce9d630f9e581cfb431e968438a3de102be85a4 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 11 May 2024 12:21:32 +0200 Subject: [PATCH 10/14] fix example importer(beam) and rewrite loading to use enum --- app/Importers/BeamDataImporter.php | 88 +++++++--------------- app/Importers/BerylDataImporter.php | 110 +++++++++++++++------------- app/Importers/BirdDataImporter.php | 3 +- app/Importers/DataImporter.php | 7 +- app/Importers/LimeDataImporter.php | 3 +- app/Importers/SpinDataImporter.php | 3 +- 6 files changed, 99 insertions(+), 115 deletions(-) diff --git a/app/Importers/BeamDataImporter.php b/app/Importers/BeamDataImporter.php index 2735e05b..1c6eb46c 100644 --- a/app/Importers/BeamDataImporter.php +++ b/app/Importers/BeamDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use GuzzleHttp\Exception\GuzzleException; use Symfony\Component\DomCrawler\Crawler; @@ -47,68 +48,35 @@ public function transform(): void $existingCityProviders = []; foreach ($this->sections as $section) { - foreach ($section->childNodes as $node) { - if ($node->nodeName === "h4") { - $countryName = $node->nodeValue; + $crawler = new Crawler($section); + $countryName = $crawler->filter("h4")->text(); + + $columns = $crawler->filter('div[class="beam-col"]'); + + foreach ($columns as $column) { + $columnCrawler = new Crawler($column); + $services = []; + $images = $columnCrawler->filter("img"); + + foreach ($images as $image) { + if ($image->getAttribute("src") === $escooterImageUrl) { + $services[] = ServicesEnum::Escooter; + } else if ($image->getAttribute("src") === $bikeImageUrl) { + $services[] = ServicesEnum::Bike; + } } + $cityNamesHtml = $columnCrawler->filter("p")->html(); + $cityNamesArray = explode("
", $cityNamesHtml); + + foreach ($cityNamesArray as $cityName) { + if ($countryName === "Korea") { + $countryName = "South Korea"; + } + + $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); - if ($node->nodeName === "div") { - foreach ($node->childNodes as $div) { - if ($div->nodeName === "div") { - foreach ($div->childNodes as $city) { - if (!$city->nodeName) { - continue; - } - - $hasEscooters = false; - $hasBikes = false; - - if ($city->getAttribute("src") === $escooterImageUrl) { - $hasEscooters = true; - } - - if ($city->getAttribute("src") === $bikeImageUrl) { - $hasBikes = true; - } - - if ($city->nodeName === "p") { - $search = ["\u{00A0}", "\u{200D}", "Prefecture"]; - $valueToDelete = "Selangor"; - $cityName = preg_replace('/[\p{Hiragana}\p{Katakana}\p{Han}]+/u', "", $city->nodeValue); - $cityName = str_replace($search, "", $cityName); - $cityName = preg_replace('/(?<=[^\s_\-])(?=[A-Z])/', " ", $cityName); - $arrayOfCitiesNames = explode(" ", $cityName); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($value) => $value !== $valueToDelete); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => strlen($record) > 1); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => !str_contains($record, "•")); - - foreach ($arrayOfCitiesNames as $cityName) { - if ($cityName !== "Selangor") { - $cityName = trim($cityName); - - if ($countryName === "Korea") { - $countryName = "South Korea"; - } - $services = []; - - if ($hasBikes) { - $services[] = "bike"; - } - - if ($hasEscooters) { - $services[] = "escooter"; - } - $services = ["escooter", "bike"]; - $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } - } - } - } - } - } + if ($provider) { + $existingCityProviders[] = $provider; } } } diff --git a/app/Importers/BerylDataImporter.php b/app/Importers/BerylDataImporter.php index 1bf5208b..8156506d 100644 --- a/app/Importers/BerylDataImporter.php +++ b/app/Importers/BerylDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use GuzzleHttp\Exception\GuzzleException; use Symfony\Component\DomCrawler\Crawler; @@ -27,7 +28,7 @@ public function extract(): static } $crawler = new Crawler($html); - $this->sections = $crawler->filter("div > section > div .inner"); + $this->sections = $crawler->filter('div[class="view-content col-xs-12 col-sm-12"]')->filter("div[class='views-row']"); if (count($this->sections) === 0) { $this->createImportInfoDetails("204", self::getProviderName()); @@ -46,58 +47,69 @@ public function transform(): void $existingCityProviders = []; foreach ($this->sections as $section) { - foreach ($section->childNodes as $node) { - if ($node->nodeName === "div") { - foreach ($node->childNodes as $city) { - if ($city->nodeName === "h3") { - $cityName = $city->nodeValue; - } - - $eScootersFound = false; - - if ($city->nodeValue === "e-Scooters") { - $eScootersFound = true; - } - - if ($eScootersFound) { - $cityName = str_replace(["E-scooters", "and ", ","], "", $cityName); - $cityName = trim($cityName); - - if ($cityName === "Bournemouth Christchurch Poole") { - $arrayOfCityNames = explode(" ", $cityName); - - foreach ($arrayOfCityNames as $cityName) { - $provider = $this->load($cityName, self::COUNTRY_NAME); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } - } - } elseif ($cityName === "West Midlands") { - $cityName = "Birmingham"; - } - - if ($cityName === "Isle of Wight") { - $arrayOfCityNames = ["Cowes", "East Cowes", "Newport", "Ryde", "Sandown", "Shanklin"]; - - foreach ($arrayOfCityNames as $cityName) { - $provider = $this->load($cityName, self::COUNTRY_NAME); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } - } - } - $provider = $this->load($cityName, self::COUNTRY_NAME); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } - } + $crawler = new Crawler($section); + $cityName = $crawler->filter("h3")->text(); + $this->data[] = $cityName; + + $servicesNodes = $crawler->filter('div[class="field--label"]'); + $availableServices = []; + + foreach ($servicesNodes as $serviceNode) { + $serviceCrawler = new Crawler($serviceNode); + $service = $serviceCrawler->text(); + + if ($service === "e-Bikes") { + $availableServices[] = ServicesEnum::Bike; + } + + if ($service === "e-Scooters") { + $availableServices[] = ServicesEnum::Escooter; + } + + if ($service === "Cargo") { + $availableServices[] = ServicesEnum::Cargo; + } + } + + if ($cityName === "Bournemouth Christchurch Poole") { + $arrayOfCityNames = explode(" ", $cityName); + + foreach ($arrayOfCityNames as $cityName) { + $provider = $this->load($cityName, self::COUNTRY_NAME, $lat = "", $long = "", $availableServices); + + if ($provider !== "") { + $existingCityProviders[] = $provider; } } + } elseif ($cityName === "West Midlands") { + $cityName = "Birmingham"; + } elseif ($cityName === "Isle of Wight") { + $arrayOfCityNames = ["Cowes", "East Cowes", "Newport", "Ryde", "Sandown", "Shanklin"]; + + foreach ($arrayOfCityNames as $cityName) { + $provider = $this->load($cityName, self::COUNTRY_NAME, $lat = "", $long = "", $availableServices); + + if ($provider !== "") { + $existingCityProviders[] = $provider; + } + } + } + $provider = $this->load($cityName, self::COUNTRY_NAME, $lat = "", $long = "", $availableServices); + + if ($provider !== "") { + $existingCityProviders[] = $provider; } } + $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } + + public function test() + { + $this->extract()->transform(); + $sectionCount = count($this->sections); + $this->data[] = ["sections" => "$sectionCount"]; + + return $this->data; + } } diff --git a/app/Importers/BirdDataImporter.php b/app/Importers/BirdDataImporter.php index 21ff8bf3..b8f7d5d1 100644 --- a/app/Importers/BirdDataImporter.php +++ b/app/Importers/BirdDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Exceptions\MapboxGeocodingServiceException; use App\Models\City; use App\Models\CityAlternativeName; @@ -75,7 +76,7 @@ protected function parseData(string $html): array return array_map("trim", $coordinates); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index 91342338..a71c1e6e 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\CityProvider; @@ -77,8 +78,8 @@ protected function countryNotFound(string $cityName, string $countryName): void protected function createProvider(int $cityId, string $providerName, array $services): void { - foreach ($services as $service) { - $service = Service::query()->where("type", $service)->first(); + foreach ($services as $serviceEnum) { + $service = Service::query()->where("type", $serviceEnum->value)->first(); CityProvider::query()->updateOrCreate([ "provider_name" => $providerName, @@ -116,7 +117,7 @@ protected function createImportInfoDetails(string $code, string $providerName): ); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $country = Country::query()->where("name", $countryName)->orWhere("alternative_name", $countryName)->first(); diff --git a/app/Importers/LimeDataImporter.php b/app/Importers/LimeDataImporter.php index 9a3eaead..64bb4986 100644 --- a/app/Importers/LimeDataImporter.php +++ b/app/Importers/LimeDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -59,7 +60,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/SpinDataImporter.php b/app/Importers/SpinDataImporter.php index 563e0770..c3bec40a 100644 --- a/app/Importers/SpinDataImporter.php +++ b/app/Importers/SpinDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -76,7 +77,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); From e4df7ac54c3852a114c07a189363b419be0497d4 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 11 May 2024 12:29:14 +0200 Subject: [PATCH 11/14] fix example importer(beam) and rewrite loading to use enum --- app/Importers/BeamDataImporter.php | 88 ++++++++++++++++++++---------- app/Importers/BirdDataImporter.php | 3 +- app/Importers/DataImporter.php | 7 +-- app/Importers/LimeDataImporter.php | 3 +- app/Importers/SpinDataImporter.php | 3 +- 5 files changed, 66 insertions(+), 38 deletions(-) diff --git a/app/Importers/BeamDataImporter.php b/app/Importers/BeamDataImporter.php index 1c6eb46c..2735e05b 100644 --- a/app/Importers/BeamDataImporter.php +++ b/app/Importers/BeamDataImporter.php @@ -4,7 +4,6 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use GuzzleHttp\Exception\GuzzleException; use Symfony\Component\DomCrawler\Crawler; @@ -48,35 +47,68 @@ public function transform(): void $existingCityProviders = []; foreach ($this->sections as $section) { - $crawler = new Crawler($section); - $countryName = $crawler->filter("h4")->text(); - - $columns = $crawler->filter('div[class="beam-col"]'); - - foreach ($columns as $column) { - $columnCrawler = new Crawler($column); - $services = []; - $images = $columnCrawler->filter("img"); - - foreach ($images as $image) { - if ($image->getAttribute("src") === $escooterImageUrl) { - $services[] = ServicesEnum::Escooter; - } else if ($image->getAttribute("src") === $bikeImageUrl) { - $services[] = ServicesEnum::Bike; - } + foreach ($section->childNodes as $node) { + if ($node->nodeName === "h4") { + $countryName = $node->nodeValue; } - $cityNamesHtml = $columnCrawler->filter("p")->html(); - $cityNamesArray = explode("
", $cityNamesHtml); - - foreach ($cityNamesArray as $cityName) { - if ($countryName === "Korea") { - $countryName = "South Korea"; - } - - $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); - if ($provider) { - $existingCityProviders[] = $provider; + if ($node->nodeName === "div") { + foreach ($node->childNodes as $div) { + if ($div->nodeName === "div") { + foreach ($div->childNodes as $city) { + if (!$city->nodeName) { + continue; + } + + $hasEscooters = false; + $hasBikes = false; + + if ($city->getAttribute("src") === $escooterImageUrl) { + $hasEscooters = true; + } + + if ($city->getAttribute("src") === $bikeImageUrl) { + $hasBikes = true; + } + + if ($city->nodeName === "p") { + $search = ["\u{00A0}", "\u{200D}", "Prefecture"]; + $valueToDelete = "Selangor"; + $cityName = preg_replace('/[\p{Hiragana}\p{Katakana}\p{Han}]+/u', "", $city->nodeValue); + $cityName = str_replace($search, "", $cityName); + $cityName = preg_replace('/(?<=[^\s_\-])(?=[A-Z])/', " ", $cityName); + $arrayOfCitiesNames = explode(" ", $cityName); + $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($value) => $value !== $valueToDelete); + $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => strlen($record) > 1); + $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => !str_contains($record, "•")); + + foreach ($arrayOfCitiesNames as $cityName) { + if ($cityName !== "Selangor") { + $cityName = trim($cityName); + + if ($countryName === "Korea") { + $countryName = "South Korea"; + } + $services = []; + + if ($hasBikes) { + $services[] = "bike"; + } + + if ($hasEscooters) { + $services[] = "escooter"; + } + $services = ["escooter", "bike"]; + $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); + + if ($provider !== "") { + $existingCityProviders[] = $provider; + } + } + } + } + } + } } } } diff --git a/app/Importers/BirdDataImporter.php b/app/Importers/BirdDataImporter.php index b8f7d5d1..21ff8bf3 100644 --- a/app/Importers/BirdDataImporter.php +++ b/app/Importers/BirdDataImporter.php @@ -4,7 +4,6 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use App\Exceptions\MapboxGeocodingServiceException; use App\Models\City; use App\Models\CityAlternativeName; @@ -76,7 +75,7 @@ protected function parseData(string $html): array return array_map("trim", $coordinates); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index a71c1e6e..91342338 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -4,7 +4,6 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\CityProvider; @@ -78,8 +77,8 @@ protected function countryNotFound(string $cityName, string $countryName): void protected function createProvider(int $cityId, string $providerName, array $services): void { - foreach ($services as $serviceEnum) { - $service = Service::query()->where("type", $serviceEnum->value)->first(); + foreach ($services as $service) { + $service = Service::query()->where("type", $service)->first(); CityProvider::query()->updateOrCreate([ "provider_name" => $providerName, @@ -117,7 +116,7 @@ protected function createImportInfoDetails(string $code, string $providerName): ); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $country = Country::query()->where("name", $countryName)->orWhere("alternative_name", $countryName)->first(); diff --git a/app/Importers/LimeDataImporter.php b/app/Importers/LimeDataImporter.php index 64bb4986..9a3eaead 100644 --- a/app/Importers/LimeDataImporter.php +++ b/app/Importers/LimeDataImporter.php @@ -4,7 +4,6 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -60,7 +59,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/SpinDataImporter.php b/app/Importers/SpinDataImporter.php index c3bec40a..563e0770 100644 --- a/app/Importers/SpinDataImporter.php +++ b/app/Importers/SpinDataImporter.php @@ -4,7 +4,6 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -77,7 +76,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); From 06a2a2e0f320c87a94145dd2f1891342402ef349 Mon Sep 17 00:00:00 2001 From: JakubKermes Date: Sat, 11 May 2024 16:38:37 +0200 Subject: [PATCH 12/14] fix example importer(beam) and fix messed commit --- app/Importers/BeamDataImporter.php | 97 +++++++++++------------------- app/Importers/BirdDataImporter.php | 3 +- app/Importers/DataImporter.php | 3 +- app/Importers/LimeDataImporter.php | 3 +- app/Importers/SpinDataImporter.php | 3 +- 5 files changed, 42 insertions(+), 67 deletions(-) diff --git a/app/Importers/BeamDataImporter.php b/app/Importers/BeamDataImporter.php index 2735e05b..dd113a08 100644 --- a/app/Importers/BeamDataImporter.php +++ b/app/Importers/BeamDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use GuzzleHttp\Exception\GuzzleException; use Symfony\Component\DomCrawler\Crawler; @@ -18,12 +19,10 @@ public function extract(): static $html = $response->getBody()->getContents(); } catch (GuzzleException) { $this->createImportInfoDetails("400", self::getProviderName()); - $this->stopExecution = true; return $this; } - $crawler = new Crawler($html); $this->sections = $crawler->filter("div.find-beam-box"); @@ -43,72 +42,44 @@ public function transform(): void if ($this->stopExecution) { return; } - $existingCityProviders = []; foreach ($this->sections as $section) { - foreach ($section->childNodes as $node) { - if ($node->nodeName === "h4") { - $countryName = $node->nodeValue; + $crawler = new Crawler($section); + $countryName = $crawler->filter('h4[class="find-beam-title-map"]')->text(); + + $columns = $crawler->filter('div[class="beam-col"], div[class="beam-col-main-box"]'); + + foreach ($columns as $column) { + $columnCrawler = new Crawler($column); + $services = []; + $images = $columnCrawler->filter("img"); + + foreach ($images as $image) { + if ($image->getAttribute("src") === $escooterImageUrl) { + $services[] = ServicesEnum::Escooter; + } else if ($image->getAttribute("src") === $bikeImageUrl) { + $services[] = ServicesEnum::Bike; + } } - if ($node->nodeName === "div") { - foreach ($node->childNodes as $div) { - if ($div->nodeName === "div") { - foreach ($div->childNodes as $city) { - if (!$city->nodeName) { - continue; - } - - $hasEscooters = false; - $hasBikes = false; - - if ($city->getAttribute("src") === $escooterImageUrl) { - $hasEscooters = true; - } - - if ($city->getAttribute("src") === $bikeImageUrl) { - $hasBikes = true; - } - - if ($city->nodeName === "p") { - $search = ["\u{00A0}", "\u{200D}", "Prefecture"]; - $valueToDelete = "Selangor"; - $cityName = preg_replace('/[\p{Hiragana}\p{Katakana}\p{Han}]+/u', "", $city->nodeValue); - $cityName = str_replace($search, "", $cityName); - $cityName = preg_replace('/(?<=[^\s_\-])(?=[A-Z])/', " ", $cityName); - $arrayOfCitiesNames = explode(" ", $cityName); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($value) => $value !== $valueToDelete); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => strlen($record) > 1); - $arrayOfCitiesNames = array_filter($arrayOfCitiesNames, fn($record) => !str_contains($record, "•")); - - foreach ($arrayOfCitiesNames as $cityName) { - if ($cityName !== "Selangor") { - $cityName = trim($cityName); - - if ($countryName === "Korea") { - $countryName = "South Korea"; - } - $services = []; - - if ($hasBikes) { - $services[] = "bike"; - } - - if ($hasEscooters) { - $services[] = "escooter"; - } - $services = ["escooter", "bike"]; - $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); - - if ($provider !== "") { - $existingCityProviders[] = $provider; - } - } - } - } - } - } + $cityNamesHtml = $columnCrawler->filter("p")->html(); + $cityNamesArray = explode("
", $cityNamesHtml); + + foreach ($cityNamesArray as $cityName) { + $cityName = strip_tags($cityName); + $search = [" •", " "]; + $cityName = str_replace($search, "", $cityName); + $cityName = trim($cityName); + + if ($countryName === "Korea") { + $countryName = "South Korea"; + } + + $provider = $this->load($cityName, $countryName, $lat = "", $long = "", $services); + + if ($provider) { + $existingCityProviders[] = $provider; } } } diff --git a/app/Importers/BirdDataImporter.php b/app/Importers/BirdDataImporter.php index 21ff8bf3..b8f7d5d1 100644 --- a/app/Importers/BirdDataImporter.php +++ b/app/Importers/BirdDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Exceptions\MapboxGeocodingServiceException; use App\Models\City; use App\Models\CityAlternativeName; @@ -75,7 +76,7 @@ protected function parseData(string $html): array return array_map("trim", $coordinates); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index 91342338..ce23510e 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\CityProvider; @@ -116,7 +117,7 @@ protected function createImportInfoDetails(string $code, string $providerName): ); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $country = Country::query()->where("name", $countryName)->orWhere("alternative_name", $countryName)->first(); diff --git a/app/Importers/LimeDataImporter.php b/app/Importers/LimeDataImporter.php index 9a3eaead..64bb4986 100644 --- a/app/Importers/LimeDataImporter.php +++ b/app/Importers/LimeDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -59,7 +60,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); diff --git a/app/Importers/SpinDataImporter.php b/app/Importers/SpinDataImporter.php index 563e0770..c3bec40a 100644 --- a/app/Importers/SpinDataImporter.php +++ b/app/Importers/SpinDataImporter.php @@ -4,6 +4,7 @@ namespace App\Importers; +use App\Enums\ServicesEnum; use App\Models\City; use App\Models\CityAlternativeName; use App\Models\Country; @@ -76,7 +77,7 @@ public function transform(): void $this->deleteMissingProviders(self::getProviderName(), $existingCityProviders); } - protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = ["escooter"]): string + protected function load(string $cityName, string $countryName, string $lat = "", string $long = "", array $services = [ServicesEnum::Escooter]): string { $city = City::query()->where("name", $cityName)->first(); $alternativeCityName = CityAlternativeName::query()->where("name", $cityName)->first(); From bce9dcdc283fbaeb4fe7e74a1e77adb4d1d0b8f7 Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal Date: Mon, 13 May 2024 09:36:19 +0200 Subject: [PATCH 13/14] lint --- app/Importers/DataImporter.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Importers/DataImporter.php b/app/Importers/DataImporter.php index a653bc97..f1e8d68f 100644 --- a/app/Importers/DataImporter.php +++ b/app/Importers/DataImporter.php @@ -4,8 +4,8 @@ namespace App\Importers; -use App\Enums\ServicesEnum; use App\Enums\ChangeInFavoriteCityEnum; +use App\Enums\ServicesEnum; use App\Events\ChangeInFavoriteCityEvent; use App\Models\City; use App\Models\CityAlternativeName; @@ -83,9 +83,9 @@ protected function createProvider(int $cityId, string $providerName, array $serv foreach ($services as $service) { $service = Service::query()->where("type", $service)->first(); - if (!CityProvider::query()->where("city_id", $cityId)->where("provider_name", $providerName)->exists()) { - event(new ChangeInFavoriteCityEvent($cityId, $providerName, ChangeInFavoriteCityEnum::Added)); - } + if (!CityProvider::query()->where("city_id", $cityId)->where("provider_name", $providerName)->exists()) { + event(new ChangeInFavoriteCityEvent($cityId, $providerName, ChangeInFavoriteCityEnum::Added)); + } CityProvider::query()->updateOrCreate([ "provider_name" => $providerName, "city_id" => $cityId, From 32dca9b082c5cf01c18526dcb92e3764796fabec Mon Sep 17 00:00:00 2001 From: Aleksandra Kozubal <104600942+AleksandraKozubal@users.noreply.github.com> Date: Mon, 13 May 2024 09:42:46 +0200 Subject: [PATCH 14/14] Update database/seeders/ServiceSeeder.php Co-authored-by: Ewelina Skrzypacz <56546832+EwelinaSkrzypacz@users.noreply.github.com> --- database/seeders/ServiceSeeder.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/database/seeders/ServiceSeeder.php b/database/seeders/ServiceSeeder.php index 54b8230f..571b0410 100644 --- a/database/seeders/ServiceSeeder.php +++ b/database/seeders/ServiceSeeder.php @@ -10,9 +10,6 @@ class ServiceSeeder extends Seeder { - /** - * Run the database seeds. - */ public function run(): void { foreach (ServicesEnum::cases() as $service) {