!238 - Updated to latest v2.2. Changed the way we store ip2location a…

…s we need to index it for future use. Also moved the public repository to github and added the local repo as submodule.
sp-framework · Jun 16, 2024 · dae9815 · dae9815
1 parent 5d36fec
commit dae9815
Show file tree

Hide file tree

Showing 12 changed files with 165 additions and 99 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,7 +30,6 @@ apps/*
 !apps/Core/
 .ff/*
 .sql/*
-system/Base/Providers/BasepackagesServiceProvider/Packages/Geo/Data/*
 .api/
 phpstan.neon
 psalm.xml

diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "phpunit"]
 	path = phpunit
 	url = https://dev.bazaari.com.au/sp-core/phpunit.git
+[submodule "apps/Core/Packages/Devtools/GeoExtractData/Data"]
+	path = apps/Core/Packages/Devtools/GeoExtractData/Data
+	url = https://dev.bazaari.com.au/sp-core/geodata.git
diff --git a/apps/Core/Components/Devtools/Extractgeodata/ExtractgeodataComponent.php b/apps/Core/Components/Devtools/Extractgeodata/ExtractgeodataComponent.php
@@ -25,21 +25,19 @@ public function viewAction()
     //Ip2Location data from ip2location.com (lite version)
     public function processAction()
     {
-        if ($this->request->isPost()) {
-            if (!$this->checkCSRF()) {
-                return;
-            }
+        $this->requestIsPost();
 
-            if ($this->basepackages->progress->checkProgressFile()) {
-                $this->basepackages->progress->deleteProgressFile();
-            }
+        if ($this->basepackages->progress->checkProgressFile()) {
+            $this->basepackages->progress->deleteProgressFile();
+        }
 
-            if (!$this->registerProgressMethods()) {
-                $this->addResponse('No Methods Selected', 1);
+        if (!$this->registerProgressMethods()) {
+            $this->addResponse('No Methods Selected', 1);
 
-                return;
-            }
+            return;
+        }
 
+        try {
             if (isset($this->postData()['geo']) && $this->postData()['geo'] == 'true') {
                 $this->geoExtractDataPackage->downloadGeoData();
                 $this->geoExtractDataPackage->processGeoData();
@@ -66,8 +64,12 @@ public function processAction()
                 $this->geoExtractDataPackage->packagesData->responseMessage,
                 $this->geoExtractDataPackage->packagesData->responseCode
             );
-        } else {
-            $this->addResponse('Method Not Allowed', 1);
+        } catch (\throwable $e) {
+            $this->basepackages->progress->preCheckComplete(false);
+
+            $this->basepackages->progress->resetProgress();
+
+            $this->addResponse($e->getMessage(), 1);
         }
     }
 
@@ -110,28 +112,28 @@ protected function registerProgressMethods()
         if (isset($this->postData()['ip']) && $this->postData()['ip'] == 'true') {
             $methods = array_merge($methods,
                 [
-                    [
-                        'method'    => 'downloadGeoIpv4Data',
-                        'text'      => 'Download Geo Location IPv4 Data...',
-                        'remoteWeb' => true
-                    ],
-                    [
-                        'method'    => 'unzipGeoIpv4Data',
-                        'text'      => 'Unzip Geo Location IPv4 Data...',
-                    ],
+                    // [
+                    //     'method'    => 'downloadGeoIpv4Data',
+                    //     'text'      => 'Download Geo Location IPv4 Data...',
+                    //     'remoteWeb' => true
+                    // ],
+                    // [
+                    //     'method'    => 'unzipGeoIpv4Data',
+                    //     'text'      => 'Unzip Geo Location IPv4 Data...',
+                    // ],
                     [
                         'method'    => 'processGeoIpv4Data',
                         'text'      => 'Process Geo Location IPv4 Data. This will take a while...',
                     ],
-                    [
-                        'method'    => 'downloadGeoIpv6Data',
-                        'text'      => 'Download Geo Location IPv6 Data...',
-                        'remoteWeb' => true
-                    ],
-                    [
-                        'method'    => 'unzipGeoIpv6Data',
-                        'text'      => 'Unzip Geo Location IPv6 Data...',
-                    ],
+                    // [
+                    //     'method'    => 'downloadGeoIpv6Data',
+                    //     'text'      => 'Download Geo Location IPv6 Data...',
+                    //     'remoteWeb' => true
+                    // ],
+                    // [
+                    //     'method'    => 'unzipGeoIpv6Data',
+                    //     'text'      => 'Unzip Geo Location IPv6 Data...',
+                    // ],
                     [
                         'method'    => 'processGeoIpv6Data',
                         'text'      => 'Process Geo Location IPv6 Data. This will take a while...',

diff --git a/apps/Core/Packages/Devtools/GeoExtractData/Data b/apps/Core/Packages/Devtools/GeoExtractData/Data
diff --git a/apps/Core/Packages/Devtools/GeoExtractData/DevtoolsGeoExtractData.php b/apps/Core/Packages/Devtools/GeoExtractData/DevtoolsGeoExtractData.php
@@ -19,6 +19,8 @@ class DevtoolsGeoExtractData extends BasePackage
 
     public $method;
 
+    protected $zip;
+
     public function onConstruct()
     {
         if (!is_dir(base_path($this->sourceDir))) {
@@ -27,6 +29,21 @@ public function onConstruct()
             }
         }
 
+        $this->zip = new \ZipArchive;
+
+        // /etc/apache2.conf - Change the timeout to 3600 else you will get Gateway Timeout, revert back when done to 300 (5 mins)
+        // Timeout 3600
+
+        //Increase Exectimeout to 20 mins as this process takes time to extract and merge data.
+        if ((int) ini_get('max_execution_time') < 3600) {
+            set_time_limit(3600);
+        }
+
+        //Increase memory_limit to 2G as the process takes a bit of memory to process the array.
+        if ((int) ini_get('memory_limit') < 2048) {
+            ini_set('memory_limit', '2048M');
+        }
+
         parent::onConstruct();
     }
 
@@ -61,19 +78,6 @@ protected function downloadGeoData()
 
     protected function processGeoData()
     {
-        // /etc/apache2.conf - Change the timeout to 1800 else you will get Gateway Timeout, revert back when done to 300 (5 mins)
-        // Timeout 1800
-
-        //Increase Exectimeout to 20 mins as this process takes time to extract and merge data.
-        if ((int) ini_get('max_execution_time') < 1800) {
-            set_time_limit(1800);
-        }
-
-        //Increase memory_limit to 2G as the process takes a bit of memory to process the array.
-        if ((int) ini_get('memory_limit') < 2048) {
-            ini_set('memory_limit', '2048M');
-        }
-
         $countries = [];
 
         try {
@@ -146,26 +150,26 @@ protected function extractZip($sourceFile)
     {
         set_time_limit(120);
 
-        $zip = new \ZipArchive;
+        if ($this->zip->open(base_path($sourceFile)) === true) {
+            $this->zip->extractTo(base_path($this->sourceDir));
 
-        if ($zip->open(base_path($sourceFile))) {
-            $zip->extractTo(base_path($this->sourceDir));
+            $this->zip->close();
+
+            return true;
         }
 
-        $zip->close();
+        $this->addResponse('Not able to extract file: ' . $sourceFile . ' Check the file.', 1);
 
-        return true;
+        return false;
     }
 
     protected function compressZip($sourceFile)
     {
-        $zip = new \ZipArchive;
-
-        $zip->open(base_path($this->sourceDir . $sourceFile . '.zip'), $zip::CREATE);
+        $this->zip->open(base_path($this->sourceDir . $sourceFile . '.zip'), $this->zip::CREATE);
 
-        $zip->addFile(base_path($this->sourceDir . $sourceFile . '.json'), $sourceFile . '.json');
+        $this->zip->addFile(base_path($this->sourceDir . $sourceFile . '.json'), $sourceFile . '.json');
 
-        $zip->close();
+        $this->zip->close();
     }
 
     protected function downloadTimezoneData()
@@ -274,19 +278,6 @@ protected function processGeoIpv6Data()
 
     protected function processCSV($ipType)
     {
-        // /etc/apache2.conf - Change the timeout to 1800 else you will get Gateway Timeout, revert back when done to 300 (5 mins)
-        // Timeout 1800
-
-        //Increase Exectimeout to 20 mins as this process takes time to extract and merge data.
-        if ((int) ini_get('max_execution_time') < 1800) {
-            set_time_limit(1800);
-        }
-
-        //Increase memory_limit to 2G as the process takes a bit of memory to process the array.
-        if ((int) ini_get('memory_limit') < 2048) {
-            ini_set('memory_limit', '2048M');
-        }
-
         $countries = [];
         $countryKeys = [];
 
@@ -336,19 +327,6 @@ protected function processCSV($ipType)
 
     protected function mergeGeoIpData()
     {
-        // /etc/apache2.conf - Change the timeout to 1800 else you will get Gateway Timeout, revert back when done to 300 (5 mins)
-        // Timeout 1800
-
-        //Increase Exectimeout to 20 mins as this process takes time to extract and merge data.
-        if ((int) ini_get('max_execution_time') < 1800) {
-            set_time_limit(1800);
-        }
-
-        //Increase memory_limit to 2G as the process takes a bit of memory to process the array.
-        if ((int) ini_get('memory_limit') < 2048) {
-            ini_set('memory_limit', '2048M');
-        }
-
         try {
             $allCountries = $this->helper->decode($this->localContent->read($this->sourceDir . 'ip2location/AllCountries.json'), true);
 

diff --git a/system/Base/Installer/Packages/Setup/Register/Basepackages/Geo/Countries.php b/system/Base/Installer/Packages/Setup/Register/Basepackages/Geo/Countries.php
@@ -76,7 +76,7 @@ public function downloadSelectedCountryStatesAndCities($ff, $localContent, $remo
             $downloadCountry =
                 $remoteWebContent->request(
                     'GET',
-                    'https://dev.bazaari.com.au/sp-public/geodata/raw/branch/master/' . $country['iso2'] . '.zip',
+                    'https://github.com/oyeaussie/sp-geodata/raw/main/' . $country['iso2'] . '.zip',
                     [
                         'progress' => function(
                             $downloadTotal,
@@ -157,10 +157,9 @@ public function registerSelectedCountryStatesAndCities($ff, $localContent, $remo
         $ipv6Store = $ff->store('basepackages_geo_cities_ip2locationv6');
 
         try {
-            set_time_limit(300);//5Mins
 
-            if ($ip2location) {
-                set_time_limit(900);//15Mins
+            if (!$ip2location) {
+                set_time_limit(300);//5Mins
             }
 
             $countryData = $helper->decode($localContent->read($this->sourceDir . $country['iso2'] . '.json'), true);
@@ -187,15 +186,33 @@ public function registerSelectedCountryStatesAndCities($ff, $localContent, $remo
                         if ($ip2location == 'true' && isset($city['ip2locationv4'])) {
                             $ip2locationv4['id'] = $city['id'];
                             $ip2locationv4['city_id'] = $city['id'];
-                            $ip2locationv4['ip2locationv4'] = $city['ip2locationv4'];
-                            $ipv4Store->updateOrInsert($ip2locationv4, false);
+                            $ip2v4chunks = $helper->chunk($city['ip2locationv4'], 2);
+                            foreach ($ip2v4chunks as $chunk) {
+                                if (count($chunk) !== 2) {
+                                    continue;
+                                }
+
+                                $ip2locationv4['range_start'] = $chunk[0];
+                                $ip2locationv4['range_end'] = $chunk[1];
+
+                                $ipv4Store->updateOrInsert($ip2locationv4, false);
+                            }
                         }
 
                         if ($ip2location == 'true' && isset($city['ip2locationv6'])) {
                             $ip2locationv6['id'] = $city['id'];
                             $ip2locationv6['city_id'] = $city['id'];
-                            $ip2locationv6['ip2locationv6'] = $city['ip2locationv6'];
-                            $ipv6Store->updateOrInsert($ip2locationv6, false);
+                            $ip2v6chunks = $helper->chunk($city['ip2locationv6'], 2);
+                            foreach ($ip2v6chunks as $chunk) {
+                                if (count($chunk) !== 2) {
+                                    continue;
+                                }
+
+                                $ip2locationv6['range_start'] = $chunk[0];
+                                $ip2locationv6['range_end'] = $chunk[1];
+
+                                $ipv6Store->updateOrInsert($ip2locationv6, false);
+                            }
                         }
 
                         unset($city['ip2locationv4']);

diff --git a/system/Base/Installer/Packages/Setup/Schema/Basepackages/Geo/CitiesIp2LocationV4.php b/system/Base/Installer/Packages/Setup/Schema/Basepackages/Geo/CitiesIp2LocationV4.php
@@ -3,6 +3,7 @@
 namespace System\Base\Installer\Packages\Setup\Schema\Basepackages\Geo;
 
 use Phalcon\Db\Column;
+use Phalcon\Db\Index;
 
 class CitiesIp2LocationV4
 {
@@ -28,10 +29,19 @@ public function columns()
                     ]
                 ),
                 new Column(
-                    'ip2locationv4',
+                    'range_start',
                     [
-                        'type'          => Column::TYPE_JSON,
-                        'notNull'       => false,
+                        'type'          => Column::TYPE_VARCHAR,
+                        'size'          => 100,
+                        'notNull'       => true,
+                    ]
+                ),
+                new Column(
+                    'range_end',
+                    [
+                        'type'          => Column::TYPE_VARCHAR,
+                        'size'          => 100,
+                        'notNull'       => true,
                     ]
                 ),
             ],
@@ -40,4 +50,25 @@ public function columns()
             ]
         ];
     }
+
+    public function indexes()
+    {
+        return
+        [
+            new Index(
+                'column_range_start_index',
+                [
+                    'range_start'
+                ],
+                'INDEX'
+            ),
+            new Index(
+                'column_range_end_index',
+                [
+                    'range_end'
+                ],
+                'INDEX'
+            )
+        ];
+    }
 }