From 778fb59b6d6d20b6d0e8528ac4a1eb05d218ad7d Mon Sep 17 00:00:00 2001 From: Hai Zheng Date: Thu, 31 Oct 2024 11:33:04 -0400 Subject: [PATCH] v7.0-b2: * **Crawler** Dropped `Threads` setting. Added PHP const `LITESPEED_CRAWLER_THREADS` support. * **Crawler** Dropped `Interval Between Runs` setting. Added PHP const `LITESPEED_CRAWLER_RUN_INTERVAL` support. * **Crawler** Dropped `Sitemap Timeout` setting. Added PHP const `LITESPEED_CRAWLER_MAP_TIMEOUT` support. * **Crawler** Dropped `Drop Domain from Sitemap` setting. Added PHP const `LITESPEED_CRAWLER_DROP_DOMAIN` support. --- cli/crawler.cls.php | 8 +-- data/const.default.ini | 8 --- litespeed-cache.php | 4 +- readme.txt | 4 ++ src/base.cls.php | 13 ++-- src/conf.cls.php | 5 -- src/crawler-map.cls.php | 4 +- src/crawler.cls.php | 15 ++-- src/lang.cls.php | 4 -- src/task.cls.php | 4 +- tpl/crawler/entry.tpl.php | 48 ++++++------- tpl/crawler/settings-simulation.tpl.php | 60 ---------------- tpl/crawler/settings-sitemap.tpl.php | 59 ---------------- ...tings-general.tpl.php => settings.tpl.php} | 68 +++++++++++++------ tpl/crawler/summary.tpl.php | 8 +-- 15 files changed, 103 insertions(+), 209 deletions(-) delete mode 100644 tpl/crawler/settings-simulation.tpl.php delete mode 100644 tpl/crawler/settings-sitemap.tpl.php rename tpl/crawler/{settings-general.tpl.php => settings.tpl.php} (63%) diff --git a/cli/crawler.cls.php b/cli/crawler.cls.php index 2f678f09d..8505875d7 100644 --- a/cli/crawler.cls.php +++ b/cli/crawler.cls.php @@ -60,10 +60,10 @@ public function list() } $is_running = time() - $summary['is_running'] <= 900; - $seconds = $this->conf(Base::O_CRAWLER_RUN_INTERVAL); - if ($seconds > 0) { + $CRAWLER_RUN_INTERVAL = defined('LITESPEED_CRAWLER_RUN_INTERVAL') ? LITESPEED_CRAWLER_RUN_INTERVAL : 600; // Specify time in seconds for the time between each run interval + if ($CRAWLER_RUN_INTERVAL > 0) { $recurrence = ''; - $hours = (int) floor($seconds / 3600); + $hours = (int) floor($CRAWLER_RUN_INTERVAL / 3600); if ($hours) { if ($hours > 1) { $recurrence .= sprintf(__('%d hours', 'litespeed-cache'), $hours); @@ -71,7 +71,7 @@ public function list() $recurrence .= sprintf(__('%d hour', 'litespeed-cache'), $hours); } } - $minutes = (int) floor(($seconds % 3600) / 60); + $minutes = (int) floor(($CRAWLER_RUN_INTERVAL % 3600) / 60); if ($minutes) { $recurrence .= ' '; if ($minutes > 1) { diff --git a/data/const.default.ini b/data/const.default.ini index 8d8406fa8..8da5126cc 100644 --- a/data/const.default.ini +++ b/data/const.default.ini @@ -582,19 +582,11 @@ crawler-run_interval = 600 crawler-crawl_interval = 302400 -crawler-threads = 3 - crawler-load_limit = 1 ; O_CRAWLER_SITEMAP crawler-sitemap = '' -; O_CRAWLER_DROP_DOMAIN -crawler-drop_domain = true - -; O_CRAWLER_MAP_TIMEOUT -crawler-map_timeout = 120 - crawler-roles = '' crawler-cookies = '' diff --git a/litespeed-cache.php b/litespeed-cache.php index 3a81153d9..e7663a4b1 100644 --- a/litespeed-cache.php +++ b/litespeed-cache.php @@ -4,7 +4,7 @@ * Plugin Name: LiteSpeed Cache * Plugin URI: https://www.litespeedtech.com/products/cache-plugins/wordpress-acceleration * Description: High-performance page caching and site optimization from LiteSpeed - * Version: 7.0-b1 + * Version: 7.0-b2 * Author: LiteSpeed Technologies * Author URI: https://www.litespeedtech.com * License: GPLv3 @@ -34,7 +34,7 @@ return; } -!defined('LSCWP_V') && define('LSCWP_V', '7.0-b1'); +!defined('LSCWP_V') && define('LSCWP_V', '7.0-b2'); !defined('LSCWP_CONTENT_DIR') && define('LSCWP_CONTENT_DIR', WP_CONTENT_DIR); !defined('LSCWP_DIR') && define('LSCWP_DIR', __DIR__ . '/'); // Full absolute path '/var/www/html/***/wp-content/plugins/litespeed-cache/' or MU diff --git a/readme.txt b/readme.txt index 7f038b736..216cb46d0 100644 --- a/readme.txt +++ b/readme.txt @@ -273,6 +273,10 @@ You can report security bugs through the Patchstack Vulnerability Disclosure Pro * **Crawler** Used `127.0.0.1` instead of server IP setting for DNS resolve when crawling. * **Crawler** Dropped `Delay` setting. Added PHP const `LITESPEED_CRAWLER_USLEEP` support. * **Crawler** Dropped `Timeout` setting. Added PHP const `LITESPEED_CRAWLER_TIMEOUT` support. +* **Crawler** Dropped `Threads` setting. Added PHP const `LITESPEED_CRAWLER_THREADS` support. +* **Crawler** Dropped `Interval Between Runs` setting. Added PHP const `LITESPEED_CRAWLER_RUN_INTERVAL` support. +* **Crawler** Dropped `Sitemap Timeout` setting. Added PHP const `LITESPEED_CRAWLER_MAP_TIMEOUT` support. +* **Crawler** Dropped `Drop Domain from Sitemap` setting. Added PHP const `LITESPEED_CRAWLER_DROP_DOMAIN` support. * **Page Optimize** Updated request link parser to follow the site permalink. (Mijnheer Eetpraat #766) * **GUI** New QUIC.cloud CDN tab. * **GUI** Switch buttons rtl compatibility. (Eliza/Mehrshad Darzi #603) diff --git a/src/base.cls.php b/src/base.cls.php index abd12514f..02560302a 100644 --- a/src/base.cls.php +++ b/src/base.cls.php @@ -237,14 +237,14 @@ class Base extends Root const O_CRAWLER = 'crawler'; const O_CRAWLER_USLEEP = 'crawler-usleep'; // @Deprecated since v7.0 TODO: remove after v7.5 const O_CRAWLER_RUN_DURATION = 'crawler-run_duration'; // @Deprecated since v7.0 TODO: remove after v7.5 - const O_CRAWLER_RUN_INTERVAL = 'crawler-run_interval'; + const O_CRAWLER_RUN_INTERVAL = 'crawler-run_interval'; // @Deprecated since v7.0 TODO: remove after v7.5 const O_CRAWLER_CRAWL_INTERVAL = 'crawler-crawl_interval'; - const O_CRAWLER_THREADS = 'crawler-threads'; + const O_CRAWLER_THREADS = 'crawler-threads'; // @Deprecated since v7.0 TODO: remove after v7.5 const O_CRAWLER_TIMEOUT = 'crawler-timeout'; // @Deprecated since v7.0 TODO: remove after v7.5 const O_CRAWLER_LOAD_LIMIT = 'crawler-load_limit'; const O_CRAWLER_SITEMAP = 'crawler-sitemap'; - const O_CRAWLER_DROP_DOMAIN = 'crawler-drop_domain'; - const O_CRAWLER_MAP_TIMEOUT = 'crawler-map_timeout'; + const O_CRAWLER_DROP_DOMAIN = 'crawler-drop_domain'; // @Deprecated since v7.0 TODO: remove after v7.5 + const O_CRAWLER_MAP_TIMEOUT = 'crawler-map_timeout'; // @Deprecated since v7.0 TODO: remove after v7.5 const O_CRAWLER_ROLES = 'crawler-roles'; const O_CRAWLER_COOKIES = 'crawler-cookies'; @@ -309,7 +309,6 @@ class Base extends Root protected static $SINGLE_SITE_OPTIONS = array( self::O_CRAWLER, self::O_CRAWLER_SITEMAP, - self::O_CRAWLER_DROP_DOMAIN, self::O_CDN, self::O_CDN_ORI, self::O_CDN_ORI_DIR, @@ -511,13 +510,9 @@ class Base extends Root // Crawler self::O_CRAWLER => false, - self::O_CRAWLER_RUN_INTERVAL => 0, self::O_CRAWLER_CRAWL_INTERVAL => 0, - self::O_CRAWLER_THREADS => 0, self::O_CRAWLER_LOAD_LIMIT => 0, self::O_CRAWLER_SITEMAP => '', - self::O_CRAWLER_DROP_DOMAIN => false, - self::O_CRAWLER_MAP_TIMEOUT => 0, self::O_CRAWLER_ROLES => array(), self::O_CRAWLER_COOKIES => array(), diff --git a/src/conf.cls.php b/src/conf.cls.php index c48e092c7..34a1fa490 100644 --- a/src/conf.cls.php +++ b/src/conf.cls.php @@ -444,11 +444,6 @@ public function update_confs($the_matrix = false) if ($this->_updated_ids) { foreach ($this->_updated_ids as $id) { - // Special handler for crawler: reset sitemap when drop_domain setting changed - if ($id == self::O_CRAWLER_DROP_DOMAIN) { - $this->cls('Crawler_Map')->empty_map(); - } - // Check if need to do a purge all or not if ($this->_conf_purge_all($id)) { Purge::purge_all('conf changed [id] ' . $id); diff --git a/src/crawler-map.cls.php b/src/crawler-map.cls.php index d6c33f41e..de3762d3e 100644 --- a/src/crawler-map.cls.php +++ b/src/crawler-map.cls.php @@ -36,7 +36,7 @@ public function __construct() $this->__data = Data::cls(); $this->_tb = $this->__data->tb('crawler'); $this->_tb_blacklist = $this->__data->tb('crawler_blacklist'); - $this->_conf_map_timeout = $this->conf(Base::O_CRAWLER_MAP_TIMEOUT); + $this->_conf_map_timeout = defined('LITESPEED_CRAWLER_MAP_TIMEOUT') ? LITESPEED_CRAWLER_MAP_TIMEOUT : 180; // Specify the timeout while parsing the sitemap } /** @@ -415,7 +415,7 @@ private function _gen() } if (is_array($this->_urls) && !empty($this->_urls)) { - if ($this->conf(Base::O_CRAWLER_DROP_DOMAIN)) { + if (defined('LITESPEED_CRAWLER_DROP_DOMAIN') && LITESPEED_CRAWLER_DROP_DOMAIN) { foreach ($this->_urls as $k => $v) { if (stripos($v, $this->_home_url) !== 0) { unset($this->_urls[$k]); diff --git a/src/crawler.cls.php b/src/crawler.cls.php index 3e6d8c540..07b241c6e 100644 --- a/src/crawler.cls.php +++ b/src/crawler.cls.php @@ -518,6 +518,7 @@ private function _adjust_current_threads() $curload /= $this->_ncpu; // $curload = 1; + $CRAWLER_THREADS = defined('LITESPEED_CRAWLER_THREADS') ? LITESPEED_CRAWLER_THREADS : 3; if ($this->_cur_threads == -1) { // init @@ -527,8 +528,8 @@ private function _adjust_current_threads() $curthreads = 1; } else { $curthreads = intval($this->_crawler_conf['load_limit'] - $curload); - if ($curthreads > $this->conf(Base::O_CRAWLER_THREADS)) { - $curthreads = $this->conf(Base::O_CRAWLER_THREADS); + if ($curthreads > $CRAWLER_THREADS) { + $curthreads = $CRAWLER_THREADS; } } } else { @@ -544,14 +545,14 @@ private function _adjust_current_threads() $curthreads--; // } } elseif ($curload + 1 < $this->_crawler_conf['load_limit']) { - if ($curthreads < $this->conf(Base::O_CRAWLER_THREADS)) { + if ($curthreads < $CRAWLER_THREADS) { $curthreads++; } } } // $log = 'set current threads = ' . $curthreads . ' previous=' . $this->_cur_threads - // . ' max_allowed=' . $this->conf( Base::O_CRAWLER_THREADS ) . ' load_limit=' . $this->_crawler_conf[ 'load_limit' ] . ' current_load=' . $curload; + // . ' max_allowed=' . $CRAWLER_THREADS . ' load_limit=' . $this->_crawler_conf[ 'load_limit' ] . ' current_load=' . $curload; $this->_cur_threads = $curthreads; $this->_cur_thread_time = time(); @@ -781,6 +782,7 @@ private function _multi_request($rows, $options) exit('curl_multi_init disabled'); } $mh = curl_multi_init(); + $CRAWLER_DROP_DOMAIN = defined('LITESPEED_CRAWLER_DROP_DOMAIN') ? LITESPEED_CRAWLER_DROP_DOMAIN : false; $curls = array(); foreach ($rows as $row) { if (substr($row['res'], $this->_summary['curr_crawler'], 1) == 'B') { @@ -798,7 +800,7 @@ private function _multi_request($rows, $options) // Append URL $url = $row['url']; - if ($this->conf(Base::O_CRAWLER_DROP_DOMAIN)) { + if ($CRAWLER_DROP_DOMAIN) { $url = $this->_crawler_conf['base'] . $row['url']; } curl_setopt($curls[$row['id']], CURLOPT_URL, $url); @@ -958,7 +960,8 @@ private function _get_curl_options($crawler_only = false) // IP resolve // if ($this->conf(Base::O_SERVER_IP)) { Utility::compatibility(); - if (($this->conf(Base::O_CRAWLER_DROP_DOMAIN) || !$crawler_only) && $this->_crawler_conf['base']) { + $CRAWLER_DROP_DOMAIN = defined('LITESPEED_CRAWLER_DROP_DOMAIN') ? LITESPEED_CRAWLER_DROP_DOMAIN : false; + if (($CRAWLER_DROP_DOMAIN || !$crawler_only) && $this->_crawler_conf['base']) { // Resolve URL to IP $parsed_url = parse_url($this->_crawler_conf['base']); diff --git a/src/lang.cls.php b/src/lang.cls.php index f418d52c3..120d525b5 100644 --- a/src/lang.cls.php +++ b/src/lang.cls.php @@ -249,15 +249,11 @@ public static function title($id) self::O_CDN_CLOUDFLARE => __('Cloudflare API', 'litespeed-cache'), self::O_CRAWLER => __('Crawler', 'litespeed-cache'), - self::O_CRAWLER_RUN_INTERVAL => __('Interval Between Runs', 'litespeed-cache'), self::O_CRAWLER_CRAWL_INTERVAL => __('Crawl Interval', 'litespeed-cache'), - self::O_CRAWLER_THREADS => __('Threads', 'litespeed-cache'), self::O_CRAWLER_LOAD_LIMIT => __('Server Load Limit', 'litespeed-cache'), self::O_CRAWLER_ROLES => __('Role Simulation', 'litespeed-cache'), self::O_CRAWLER_COOKIES => __('Cookie Simulation', 'litespeed-cache'), self::O_CRAWLER_SITEMAP => __('Custom Sitemap', 'litespeed-cache'), - self::O_CRAWLER_DROP_DOMAIN => __('Drop Domain from Sitemap', 'litespeed-cache'), - self::O_CRAWLER_MAP_TIMEOUT => __('Sitemap Timeout', 'litespeed-cache'), self::O_DEBUG_DISABLE_ALL => __('Disable All Features', 'litespeed-cache'), self::O_DEBUG => __('Debug Log', 'litespeed-cache'), diff --git a/src/task.cls.php b/src/task.cls.php index adcf9fa0a..81bc44e8a 100644 --- a/src/task.cls.php +++ b/src/task.cls.php @@ -200,12 +200,12 @@ public function lscache_cron_filter($schedules) */ public function lscache_cron_filter_crawler($schedules) { - $interval = $this->conf(Base::O_CRAWLER_RUN_INTERVAL); + $CRAWLER_RUN_INTERVAL = defined('LITESPEED_CRAWLER_RUN_INTERVAL') ? LITESPEED_CRAWLER_RUN_INTERVAL : 600; // $wp_schedules = wp_get_schedules(); if (!array_key_exists(self::FILTER_CRAWLER, $schedules)) { // self::debug('Crawler cron log: cron filter '.$interval.' added'); $schedules[self::FILTER_CRAWLER] = array( - 'interval' => $interval, + 'interval' => $CRAWLER_RUN_INTERVAL, 'display' => __('LiteSpeed Crawler Cron', 'litespeed-cache'), ); } diff --git a/tpl/crawler/entry.tpl.php b/tpl/crawler/entry.tpl.php index 085238de7..ecbbcca48 100644 --- a/tpl/crawler/entry.tpl.php +++ b/tpl/crawler/entry.tpl.php @@ -1,53 +1,53 @@ __( 'Summary', 'litespeed-cache' ), - 'map' => __( 'Map', 'litespeed-cache' ), - 'blacklist' => __( 'Blocklist', 'litespeed-cache' ), - 'settings-general' => __( 'General Settings', 'litespeed-cache' ), - 'settings-simulation' => __( 'Simulation Settings', 'litespeed-cache' ), - 'settings-sitemap' => __( 'Sitemap Settings', 'litespeed-cache' ), -) ; + 'summary' => __('Summary', 'litespeed-cache'), + 'map' => __('Map', 'litespeed-cache'), + 'blacklist' => __('Blocklist', 'litespeed-cache'), + 'settings' => __('Settings', 'litespeed-cache'), +); ?>

- +

- v + v

- $val){ - $accesskey = $i <= 9 ? "litespeed-accesskey='$i'" : '' ; - echo "$val" ; - $i ++ ; + $val) { + $accesskey = $i <= 9 ? "litespeed-accesskey='$i'" : ''; + echo "$val"; + $i++; } - ?> + ?>

- $val) { - echo "
" ; - require LSCWP_DIR . "tpl/crawler/$tab.tpl.php" ; - echo "
" ; + echo "
"; + require LSCWP_DIR . "tpl/crawler/$tab.tpl.php"; + echo "
"; } - ?> + ?>
- + \ No newline at end of file diff --git a/tpl/crawler/settings-simulation.tpl.php b/tpl/crawler/settings-simulation.tpl.php deleted file mode 100644 index 252a51138..000000000 --- a/tpl/crawler/settings-simulation.tpl.php +++ /dev/null @@ -1,60 +0,0 @@ -form_action(); -?> - -

- - -

- - - - - - - - - - - - -
- - title( $id ); ?> - - build_textarea( $id, 20 ); ?> - -
- - -
- -
- - title( $id ); ?> - - enroll( $id . '[name][]' ); ?> - enroll( $id . '[vals][]' ); ?> - -
- - - -
- - -

_null', __( 'Cookie Values', 'litespeed-cache' ) ); ?>

-
- -
- -form_end(); diff --git a/tpl/crawler/settings-sitemap.tpl.php b/tpl/crawler/settings-sitemap.tpl.php deleted file mode 100644 index 05cdc2992..000000000 --- a/tpl/crawler/settings-sitemap.tpl.php +++ /dev/null @@ -1,59 +0,0 @@ -form_action(); -?> - -

- - -

- - - - - - - - - - - - - - - - -
- - title($id); ?> - - build_textarea($id); ?> -
- - -
-
- - title( $id ); ?> - - build_switch( $id ); ?> -
- - -
-
- - title( $id ); ?> - - build_input( $id, 'litespeed-input-short' ); ?> -
- - recommended( $id ); ?> - _validate_ttl( $id, 15, 1800 ); ?> -
-
- -form_end(); diff --git a/tpl/crawler/settings-general.tpl.php b/tpl/crawler/settings.tpl.php similarity index 63% rename from tpl/crawler/settings-general.tpl.php rename to tpl/crawler/settings.tpl.php index 3e9df09c9..b1a4f2190 100644 --- a/tpl/crawler/settings-general.tpl.php +++ b/tpl/crawler/settings.tpl.php @@ -28,21 +28,6 @@ - - - - title($id); ?> - - - build_input($id); ?> -
- - recommended($id); ?> - _validate_ttl($id, 60); ?> -
- - - @@ -59,15 +44,14 @@ - + title($id); ?> - build_input($id, 'litespeed-input-short'); ?> + build_textarea($id); ?>
- - recommended($id); ?> - _validate_ttl($id, 1, 16); ?> + +
@@ -101,6 +85,50 @@ + + + + title($id); ?> + + + build_textarea($id, 20); ?> + +
+ + +
+ + + + + + + + title($id); ?> + + + enroll($id . '[name][]'); ?> + enroll($id . '[vals][]'); ?> + +
+ + + +
+ + +

_null', __('Cookie Values', 'litespeed-cache')); ?>

+
+ + + + + diff --git a/tpl/crawler/summary.tpl.php b/tpl/crawler/summary.tpl.php index dd56b8a6b..400d1b7db 100644 --- a/tpl/crawler/summary.tpl.php +++ b/tpl/crawler/summary.tpl.php @@ -16,10 +16,10 @@ $disabled = Router::can_crawl() ? '' : 'disabled'; -$seconds = $this->conf(Base::O_CRAWLER_RUN_INTERVAL); -if ($seconds > 0) : +$CRAWLER_RUN_INTERVAL = defined('LITESPEED_CRAWLER_RUN_INTERVAL') ? LITESPEED_CRAWLER_RUN_INTERVAL : 600; +if ($CRAWLER_RUN_INTERVAL > 0) : $recurrence = ''; - $hours = (int)floor($seconds / 3600); + $hours = (int)floor($CRAWLER_RUN_INTERVAL / 3600); if ($hours) { if ($hours > 1) { $recurrence .= sprintf(__('%d hours', 'litespeed-cache'), $hours); @@ -27,7 +27,7 @@ $recurrence .= sprintf(__('%d hour', 'litespeed-cache'), $hours); } } - $minutes = (int)floor(($seconds % 3600) / 60); + $minutes = (int)floor(($CRAWLER_RUN_INTERVAL % 3600) / 60); if ($minutes) { $recurrence .= ' '; if ($minutes > 1) {