v7.0-b2: * **Crawler** Dropped Threads setting. Added PHP const `LI…

…TESPEED_CRAWLER_THREADS` support. * **Crawler** Dropped `Interval Between Runs` setting. Added PHP const `LITESPEED_CRAWLER_RUN_INTERVAL` support. * **Crawler** Dropped `Sitemap Timeout` setting. Added PHP const `LITESPEED_CRAWLER_MAP_TIMEOUT` support. * **Crawler** Dropped `Drop Domain from Sitemap` setting. Added PHP const `LITESPEED_CRAWLER_DROP_DOMAIN` support.
litespeedtech · Oct 31, 2024 · 778fb59 · 778fb59
1 parent 82eda98
commit 778fb59
Show file tree

Hide file tree

Showing 15 changed files with 103 additions and 209 deletions.
diff --git a/cli/crawler.cls.php b/cli/crawler.cls.php
@@ -60,18 +60,18 @@ public function list()
 		}
 		$is_running = time() - $summary['is_running'] <= 900;
 
-		$seconds = $this->conf(Base::O_CRAWLER_RUN_INTERVAL);
-		if ($seconds > 0) {
+		$CRAWLER_RUN_INTERVAL = defined('LITESPEED_CRAWLER_RUN_INTERVAL') ? LITESPEED_CRAWLER_RUN_INTERVAL : 600; // Specify time in seconds for the time between each run interval
+		if ($CRAWLER_RUN_INTERVAL > 0) {
 			$recurrence = '';
-			$hours = (int) floor($seconds / 3600);
+			$hours = (int) floor($CRAWLER_RUN_INTERVAL / 3600);
 			if ($hours) {
 				if ($hours > 1) {
 					$recurrence .= sprintf(__('%d hours', 'litespeed-cache'), $hours);
 				} else {
 					$recurrence .= sprintf(__('%d hour', 'litespeed-cache'), $hours);
 				}
 			}
-			$minutes = (int) floor(($seconds % 3600) / 60);
+			$minutes = (int) floor(($CRAWLER_RUN_INTERVAL % 3600) / 60);
 			if ($minutes) {
 				$recurrence .= ' ';
 				if ($minutes > 1) {

diff --git a/data/const.default.ini b/data/const.default.ini
@@ -582,19 +582,11 @@ crawler-run_interval = 600
 
 crawler-crawl_interval = 302400
 
-crawler-threads = 3
-
 crawler-load_limit = 1
 
 ; O_CRAWLER_SITEMAP
 crawler-sitemap = ''
 
-; O_CRAWLER_DROP_DOMAIN
-crawler-drop_domain = true
-
-; O_CRAWLER_MAP_TIMEOUT
-crawler-map_timeout = 120
-
 crawler-roles = ''
 
 crawler-cookies = ''

diff --git a/litespeed-cache.php b/litespeed-cache.php
@@ -4,7 +4,7 @@
  * Plugin Name:       LiteSpeed Cache
  * Plugin URI:        https://www.litespeedtech.com/products/cache-plugins/wordpress-acceleration
  * Description:       High-performance page caching and site optimization from LiteSpeed
- * Version:           7.0-b1
+ * Version:           7.0-b2
  * Author:            LiteSpeed Technologies
  * Author URI:        https://www.litespeedtech.com
  * License:           GPLv3
@@ -34,7 +34,7 @@
 	return;
 }
 
-!defined('LSCWP_V') && define('LSCWP_V', '7.0-b1');
+!defined('LSCWP_V') && define('LSCWP_V', '7.0-b2');
 
 !defined('LSCWP_CONTENT_DIR') && define('LSCWP_CONTENT_DIR', WP_CONTENT_DIR);
 !defined('LSCWP_DIR') && define('LSCWP_DIR', __DIR__ . '/'); // Full absolute path '/var/www/html/***/wp-content/plugins/litespeed-cache/' or MU

diff --git a/readme.txt b/readme.txt
@@ -273,6 +273,10 @@ You can report security bugs through the Patchstack Vulnerability Disclosure Pro
 * **Crawler** Used `127.0.0.1` instead of server IP setting for DNS resolve when crawling.
 * **Crawler** Dropped `Delay` setting. Added PHP const `LITESPEED_CRAWLER_USLEEP` support.
 * **Crawler** Dropped `Timeout` setting. Added PHP const `LITESPEED_CRAWLER_TIMEOUT` support.
+* **Crawler** Dropped `Threads` setting. Added PHP const `LITESPEED_CRAWLER_THREADS` support.
+* **Crawler** Dropped `Interval Between Runs` setting. Added PHP const `LITESPEED_CRAWLER_RUN_INTERVAL` support.
+* **Crawler** Dropped `Sitemap Timeout` setting. Added PHP const `LITESPEED_CRAWLER_MAP_TIMEOUT` support.
+* **Crawler** Dropped `Drop Domain from Sitemap` setting. Added PHP const `LITESPEED_CRAWLER_DROP_DOMAIN` support.
 * **Page Optimize** Updated request link parser to follow the site permalink. (Mijnheer Eetpraat #766)
 * **GUI** New QUIC.cloud CDN tab.
 * **GUI** Switch buttons rtl compatibility. (Eliza/Mehrshad Darzi #603)

diff --git a/src/base.cls.php b/src/base.cls.php
@@ -237,14 +237,14 @@ class Base extends Root
 	const O_CRAWLER = 'crawler';
 	const O_CRAWLER_USLEEP = 'crawler-usleep';  // @Deprecated since v7.0 TODO: remove after v7.5
 	const O_CRAWLER_RUN_DURATION = 'crawler-run_duration';  // @Deprecated since v7.0 TODO: remove after v7.5
-	const O_CRAWLER_RUN_INTERVAL = 'crawler-run_interval';
+	const O_CRAWLER_RUN_INTERVAL = 'crawler-run_interval';  // @Deprecated since v7.0 TODO: remove after v7.5
 	const O_CRAWLER_CRAWL_INTERVAL = 'crawler-crawl_interval';
-	const O_CRAWLER_THREADS = 'crawler-threads';
+	const O_CRAWLER_THREADS = 'crawler-threads'; // @Deprecated since v7.0 TODO: remove after v7.5
 	const O_CRAWLER_TIMEOUT = 'crawler-timeout'; // @Deprecated since v7.0 TODO: remove after v7.5
 	const O_CRAWLER_LOAD_LIMIT = 'crawler-load_limit';
 	const O_CRAWLER_SITEMAP = 'crawler-sitemap';
-	const O_CRAWLER_DROP_DOMAIN = 'crawler-drop_domain';
-	const O_CRAWLER_MAP_TIMEOUT = 'crawler-map_timeout';
+	const O_CRAWLER_DROP_DOMAIN = 'crawler-drop_domain'; // @Deprecated since v7.0 TODO: remove after v7.5
+	const O_CRAWLER_MAP_TIMEOUT = 'crawler-map_timeout'; // @Deprecated since v7.0 TODO: remove after v7.5
 	const O_CRAWLER_ROLES = 'crawler-roles';
 	const O_CRAWLER_COOKIES = 'crawler-cookies';
 
@@ -309,7 +309,6 @@ class Base extends Root
 	protected static $SINGLE_SITE_OPTIONS = array(
 		self::O_CRAWLER,
 		self::O_CRAWLER_SITEMAP,
-		self::O_CRAWLER_DROP_DOMAIN,
 		self::O_CDN,
 		self::O_CDN_ORI,
 		self::O_CDN_ORI_DIR,
@@ -511,13 +510,9 @@ class Base extends Root
 
 		// Crawler
 		self::O_CRAWLER => false,
-		self::O_CRAWLER_RUN_INTERVAL => 0,
 		self::O_CRAWLER_CRAWL_INTERVAL => 0,
-		self::O_CRAWLER_THREADS => 0,
 		self::O_CRAWLER_LOAD_LIMIT => 0,
 		self::O_CRAWLER_SITEMAP => '',
-		self::O_CRAWLER_DROP_DOMAIN => false,
-		self::O_CRAWLER_MAP_TIMEOUT => 0,
 		self::O_CRAWLER_ROLES => array(),
 		self::O_CRAWLER_COOKIES => array(),
 

diff --git a/src/conf.cls.php b/src/conf.cls.php
@@ -444,11 +444,6 @@ public function update_confs($the_matrix = false)
 
 		if ($this->_updated_ids) {
 			foreach ($this->_updated_ids as $id) {
-				// Special handler for crawler: reset sitemap when drop_domain setting changed
-				if ($id == self::O_CRAWLER_DROP_DOMAIN) {
-					$this->cls('Crawler_Map')->empty_map();
-				}
-
 				// Check if need to do a purge all or not
 				if ($this->_conf_purge_all($id)) {
 					Purge::purge_all('conf changed [id] ' . $id);

diff --git a/src/crawler-map.cls.php b/src/crawler-map.cls.php
@@ -36,7 +36,7 @@ public function __construct()
 		$this->__data = Data::cls();
 		$this->_tb = $this->__data->tb('crawler');
 		$this->_tb_blacklist = $this->__data->tb('crawler_blacklist');
-		$this->_conf_map_timeout = $this->conf(Base::O_CRAWLER_MAP_TIMEOUT);
+		$this->_conf_map_timeout = defined('LITESPEED_CRAWLER_MAP_TIMEOUT') ? LITESPEED_CRAWLER_MAP_TIMEOUT : 180; // Specify the timeout while parsing the sitemap
 	}
 
 	/**
@@ -415,7 +415,7 @@ private function _gen()
 		}
 
 		if (is_array($this->_urls) && !empty($this->_urls)) {
-			if ($this->conf(Base::O_CRAWLER_DROP_DOMAIN)) {
+			if (defined('LITESPEED_CRAWLER_DROP_DOMAIN') && LITESPEED_CRAWLER_DROP_DOMAIN) {
 				foreach ($this->_urls as $k => $v) {
 					if (stripos($v, $this->_home_url) !== 0) {
 						unset($this->_urls[$k]);

diff --git a/src/crawler.cls.php b/src/crawler.cls.php
@@ -518,6 +518,7 @@ private function _adjust_current_threads()
 
 		$curload /= $this->_ncpu;
 		// $curload = 1;
+		$CRAWLER_THREADS = defined('LITESPEED_CRAWLER_THREADS') ? LITESPEED_CRAWLER_THREADS : 3;
 
 		if ($this->_cur_threads == -1) {
 			// init
@@ -527,8 +528,8 @@ private function _adjust_current_threads()
 				$curthreads = 1;
 			} else {
 				$curthreads = intval($this->_crawler_conf['load_limit'] - $curload);
-				if ($curthreads > $this->conf(Base::O_CRAWLER_THREADS)) {
-					$curthreads = $this->conf(Base::O_CRAWLER_THREADS);
+				if ($curthreads > $CRAWLER_THREADS) {
+					$curthreads = $CRAWLER_THREADS;
 				}
 			}
 		} else {
@@ -544,14 +545,14 @@ private function _adjust_current_threads()
 				$curthreads--;
 				// }
 			} elseif ($curload + 1 < $this->_crawler_conf['load_limit']) {
-				if ($curthreads < $this->conf(Base::O_CRAWLER_THREADS)) {
+				if ($curthreads < $CRAWLER_THREADS) {
 					$curthreads++;
 				}
 			}
 		}
 
 		// $log = 'set current threads = ' . $curthreads . ' previous=' . $this->_cur_threads
-		// 	. ' max_allowed=' . $this->conf( Base::O_CRAWLER_THREADS ) . ' load_limit=' . $this->_crawler_conf[ 'load_limit' ] . ' current_load=' . $curload;
+		// 	. ' max_allowed=' . $CRAWLER_THREADS . ' load_limit=' . $this->_crawler_conf[ 'load_limit' ] . ' current_load=' . $curload;
 
 		$this->_cur_threads = $curthreads;
 		$this->_cur_thread_time = time();
@@ -781,6 +782,7 @@ private function _multi_request($rows, $options)
 			exit('curl_multi_init disabled');
 		}
 		$mh = curl_multi_init();
+		$CRAWLER_DROP_DOMAIN = defined('LITESPEED_CRAWLER_DROP_DOMAIN') ? LITESPEED_CRAWLER_DROP_DOMAIN : false;
 		$curls = array();
 		foreach ($rows as $row) {
 			if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'B') {
@@ -798,7 +800,7 @@ private function _multi_request($rows, $options)
 
 			// Append URL
 			$url = $row['url'];
-			if ($this->conf(Base::O_CRAWLER_DROP_DOMAIN)) {
+			if ($CRAWLER_DROP_DOMAIN) {
 				$url = $this->_crawler_conf['base'] . $row['url'];
 			}
 			curl_setopt($curls[$row['id']], CURLOPT_URL, $url);
@@ -958,7 +960,8 @@ private function _get_curl_options($crawler_only = false)
 		// IP resolve
 		// if ($this->conf(Base::O_SERVER_IP)) {
 		Utility::compatibility();
-		if (($this->conf(Base::O_CRAWLER_DROP_DOMAIN) || !$crawler_only) && $this->_crawler_conf['base']) {
+		$CRAWLER_DROP_DOMAIN = defined('LITESPEED_CRAWLER_DROP_DOMAIN') ? LITESPEED_CRAWLER_DROP_DOMAIN : false;
+		if (($CRAWLER_DROP_DOMAIN || !$crawler_only) && $this->_crawler_conf['base']) {
 			// Resolve URL to IP
 			$parsed_url = parse_url($this->_crawler_conf['base']);
 

diff --git a/src/lang.cls.php b/src/lang.cls.php
@@ -249,15 +249,11 @@ public static function title($id)
 			self::O_CDN_CLOUDFLARE => __('Cloudflare API', 'litespeed-cache'),
 
 			self::O_CRAWLER => __('Crawler', 'litespeed-cache'),
-			self::O_CRAWLER_RUN_INTERVAL => __('Interval Between Runs', 'litespeed-cache'),
 			self::O_CRAWLER_CRAWL_INTERVAL => __('Crawl Interval', 'litespeed-cache'),
-			self::O_CRAWLER_THREADS => __('Threads', 'litespeed-cache'),
 			self::O_CRAWLER_LOAD_LIMIT => __('Server Load Limit', 'litespeed-cache'),
 			self::O_CRAWLER_ROLES => __('Role Simulation', 'litespeed-cache'),
 			self::O_CRAWLER_COOKIES => __('Cookie Simulation', 'litespeed-cache'),
 			self::O_CRAWLER_SITEMAP => __('Custom Sitemap', 'litespeed-cache'),
-			self::O_CRAWLER_DROP_DOMAIN => __('Drop Domain from Sitemap', 'litespeed-cache'),
-			self::O_CRAWLER_MAP_TIMEOUT => __('Sitemap Timeout', 'litespeed-cache'),
 
 			self::O_DEBUG_DISABLE_ALL => __('Disable All Features', 'litespeed-cache'),
 			self::O_DEBUG => __('Debug Log', 'litespeed-cache'),

diff --git a/src/task.cls.php b/src/task.cls.php
@@ -200,12 +200,12 @@ public function lscache_cron_filter($schedules)
 	 */
 	public function lscache_cron_filter_crawler($schedules)
 	{
-		$interval = $this->conf(Base::O_CRAWLER_RUN_INTERVAL);
+		$CRAWLER_RUN_INTERVAL = defined('LITESPEED_CRAWLER_RUN_INTERVAL') ? LITESPEED_CRAWLER_RUN_INTERVAL : 600;
 		// $wp_schedules = wp_get_schedules();
 		if (!array_key_exists(self::FILTER_CRAWLER, $schedules)) {
 			// 	self::debug('Crawler cron log: cron filter '.$interval.' added');
 			$schedules[self::FILTER_CRAWLER] = array(
-				'interval' => $interval,
+				'interval' => $CRAWLER_RUN_INTERVAL,
 				'display' => __('LiteSpeed Crawler Cron', 'litespeed-cache'),
 			);
 		}

diff --git a/tpl/crawler/entry.tpl.php b/tpl/crawler/entry.tpl.php
@@ -1,53 +1,53 @@
 <?php
-namespace LiteSpeed ;
-defined( 'WPINC' ) || exit ;
+
+namespace LiteSpeed;
+
+defined('WPINC') || exit;
 
 $menu_list = array(
-	'summary'				=> __( 'Summary', 'litespeed-cache' ),
-	'map'					=> __( 'Map', 'litespeed-cache' ),
-	'blacklist'				=> __( 'Blocklist', 'litespeed-cache' ),
-	'settings-general'		=> __( 'General Settings', 'litespeed-cache' ),
-	'settings-simulation'	=> __( 'Simulation Settings', 'litespeed-cache' ),
-	'settings-sitemap'		=> __( 'Sitemap Settings', 'litespeed-cache' ),
-) ;
+	'summary'		=> __('Summary', 'litespeed-cache'),
+	'map'			=> __('Map', 'litespeed-cache'),
+	'blacklist'		=> __('Blocklist', 'litespeed-cache'),
+	'settings'		=> __('Settings', 'litespeed-cache'),
+);
 
 ?>
 
 <div class="wrap">
 	<h1 class="litespeed-h1">
-		<?php echo __( 'LiteSpeed Cache Crawler', 'litespeed-cache' ) ; ?>
+		<?php echo __('LiteSpeed Cache Crawler', 'litespeed-cache'); ?>
 	</h1>
 	<span class="litespeed-desc">
-		v<?php echo Core::VER ; ?>
+		v<?php echo Core::VER; ?>
 	</span>
 	<hr class="wp-header-end">
 </div>
 
 <div class="litespeed-wrap">
 	<h2 class="litespeed-header nav-tab-wrapper">
-	<?php
-		$i = 1 ;
-		foreach ($menu_list as $tab => $val){
-			$accesskey = $i <= 9 ? "litespeed-accesskey='$i'" : '' ;
-			echo "<a class='litespeed-tab nav-tab' href='#$tab' data-litespeed-tab='$tab' $accesskey>$val</a>" ;
-			$i ++ ;
+		<?php
+		$i = 1;
+		foreach ($menu_list as $tab => $val) {
+			$accesskey = $i <= 9 ? "litespeed-accesskey='$i'" : '';
+			echo "<a class='litespeed-tab nav-tab' href='#$tab' data-litespeed-tab='$tab' $accesskey>$val</a>";
+			$i++;
 		}
-	?>
+		?>
 	</h2>
 
 	<div class="litespeed-body">
-	<?php
+		<?php
 
 		// include all tpl for faster UE
 		foreach ($menu_list as $tab => $val) {
-			echo "<div data-litespeed-layout='$tab'>" ;
-			require LSCWP_DIR . "tpl/crawler/$tab.tpl.php" ;
-			echo "</div>" ;
+			echo "<div data-litespeed-layout='$tab'>";
+			require LSCWP_DIR . "tpl/crawler/$tab.tpl.php";
+			echo "</div>";
 		}
 
-	?>
+		?>
 	</div>
 
 </div>
 
-<iframe name="litespeedHiddenIframe" src="" width="0" height="0" frameborder="0"></iframe>
+<iframe name="litespeedHiddenIframe" src="" width="0" height="0" frameborder="0"></iframe>
diff --git a/tpl/crawler/settings-simulation.tpl.php b/tpl/crawler/settings-simulation.tpl.php