diff --git a/CHANGELOG.md b/CHANGELOG.md
index 21d7a2a..e78cbad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,24 @@
# Changelog
All notable changes to this project will be documented in this file.
+## [0.3.0] - 2022-11-06
+
+Hooks, log request method, improvements and bug fixes.
+
+### Added
+
+- Added Pre- and Post-Block-Hooks, which makes it possible, for example, to have your own whitelist rules or notifications.
+- Log request method
+- Validate ip addresses in cidr notation before adding to search crawler whitelist
+- Example of how to send notifications when a request is blocked
+- Bug fix: Remove unicode character "Zero Width Space" (200B) from bing ip adresses
+
+### Changed
+
+- Bug fix: replace quotation marks in logs (user agent and url)
+- Remove `googleusercontent.com` from trusted urls for fake search crawler detection
+- Remove `Not` and `Petalbot` from bad bot list
+
## [0.2.0] - 2022-10-23
The second beta release.
diff --git a/README.md b/README.md
index 084b2a4..226b446 100644
--- a/README.md
+++ b/README.md
@@ -74,6 +74,8 @@ app.use(easyWaf({
| ipWhitelist | array | [] | All requests by ips on the whitelist are never blocked. CIDR notation is supported. |
| modules[name].enabled | boolean | true, except "Block Tor Exit Nodes" | This option allows you to completely disable a specific module. |
| modules[name].excludePaths | boolean | undefined | Exclude paths from being checked by this module with a regex. |
+| postBlockHook | callback | undefined | Run your own code after a request is blocked. For example, you can send a notification. |
+| preBlockHook | callback | undefined | Run your own code before a request is blocked. Return false if the request should not be blocked. |
| trustProxy | string / array | [] | If a reverse proxy is used, this setting must be configured. See [npm/proxy-addr](https://www.npmjs.com/package/proxy-addr) for possible values. |
## What is checked?
diff --git a/examples/hooks.js b/examples/hooks.js
new file mode 100644
index 0000000..994d6b8
--- /dev/null
+++ b/examples/hooks.js
@@ -0,0 +1,22 @@
+const express = require('express');
+const easyWaf = require('easy-waf');
+const app = express();
+
+app.use(easyWaf({
+ preBlockHook: (req, moduleInfo, ip) => { // Executed before a request is blocked. If false is returned, the request will not be blocked.
+ if (moduleInfo.name === 'xss'
+ && ['::1', '127.0.0.1', '::ffff:127.0.0.1'].includes(ip)
+ && req.url.match('^[^?]*')[0] === '/test') { // Do not block xss from localhost at path /test
+ return false;
+ }
+ },
+ postBlockHook: (req, moduleInfo, ip) => { // Executed after a request is blocked. See send-notification.js for an example.
+ // ...
+ }
+}));
+
+app.get('/get', function(req, res){
+ res.status(200).send();
+});
+
+app.listen(3000);
\ No newline at end of file
diff --git a/examples/send-notification.js b/examples/send-notification.js
new file mode 100644
index 0000000..4aa72df
--- /dev/null
+++ b/examples/send-notification.js
@@ -0,0 +1,29 @@
+const express = require('express');
+const easyWaf = require('easy-waf');
+const nodemailer = require('nodemailer');
+const app = express();
+
+var mailTransporter = nodemailer.createTransport({
+ host: 'mail.example.com',
+ port: 587, secure: false,
+ auth: { user: 'user@example.com', pass: 'pass' }
+});
+
+app.use(easyWaf({
+ postBlockHook: (req, moduleInfo, ip) => { // Executed after a request is blocked.
+ mailTransporter.sendMail({
+ from: 'from@example.com',
+ to: 'to@example.com',
+ subject: `EasyWAF Blocked Request from ${ip}`,
+ html: `Module: ${moduleInfo.name}
Url: ${req.url}
...`
+ }).catch(function (e) {
+ console.log('Error on sendMail: ' + e.message);
+ });
+ }
+}));
+
+app.get('/get', function (req, res) {
+ res.status(200).send();
+});
+
+app.listen(3000);
\ No newline at end of file
diff --git a/index.d.ts b/index.d.ts
index a6d1fa7..81dce87 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -24,8 +24,9 @@ declare module "block" {
* @param {EasyWAFModuleInfo} moduleInfo
* @param {EasyWafConfig} config
* @param {String} ip
+ * @returns {Boolean}
*/
- function blocked(req: import('http').IncomingMessage, res: import('http').ServerResponse, moduleInfo: EasyWAFModuleInfo, config: EasyWafConfig, ip: string): void;
+ function blocked(req: import('http').IncomingMessage, res: import('http').ServerResponse, moduleInfo: EasyWAFModuleInfo, config: EasyWafConfig, ip: string): boolean;
}
declare module "modules/badBots" {
/**
@@ -343,6 +344,14 @@ type EasyWafConfig = {
* This option allows you to enable / disable modules or exclude paths with a regex
*/
modules?: EasyWafConfigModules;
+ /**
+ * Run your own code after a request is blocked. For example, you can send a notification.
+ */
+ postBlockHook?: EasyWAFPostBlockHook;
+ /**
+ * Run your own code before a request is blocked. Return false if the request should not be blocked.
+ */
+ preBlockHook?: EasyWAFPreBlockHook;
/**
* If a reverse proxy is used, this setting must be configured. See https://www.npmjs.com/package/proxy-addr for possible values.
*/
@@ -412,3 +421,5 @@ type EasyWAFRequestInfo = {
*/
url: string;
};
+type EasyWAFPreBlockHook = (req: import('http').IncomingMessage, moduleInfo: EasyWAFModuleInfo, ip: string) => boolean;
+type EasyWAFPostBlockHook = (req: import('http').IncomingMessage, moduleInfo: EasyWAFModuleInfo, ip: string) => any;
diff --git a/lib/block.js b/lib/block.js
index 0758a1e..d9233cf 100644
--- a/lib/block.js
+++ b/lib/block.js
@@ -8,6 +8,7 @@ const crypto = require('crypto');
* @param {EasyWAFModuleInfo} moduleInfo
* @param {EasyWafConfig} config
* @param {String} ip
+ * @returns {Boolean}
*/
function blocked(req, res, moduleInfo, config, ip){
@@ -16,6 +17,10 @@ function blocked(req, res, moduleInfo, config, ip){
/** @type {String} */
var referenceID = crypto.createHash('sha256').update(ip + date.getTime()).digest('hex');
+ if(typeof config.preBlockHook === 'function' && config.preBlockHook(req, moduleInfo, ip) === false){
+ return false;
+ }
+
if(!config.dryMode){
res.writeHead(403, {'Content-Type': 'text/html'});
if(!config.customBlockedPage){
@@ -57,6 +62,15 @@ function blocked(req, res, moduleInfo, config, ip){
}
logger.requestBlocked(moduleInfo, req, referenceID, config, ip);
+
+ if(typeof config.postBlockHook === 'function'){
+ config.postBlockHook(req, moduleInfo, ip);
+ }
+
+ if(config.dryMode){
+ return false;
+ }
+ return true;
}
module.exports = blocked;
\ No newline at end of file
diff --git a/lib/easy-waf.js b/lib/easy-waf.js
index f8d2612..ee0cb48 100644
--- a/lib/easy-waf.js
+++ b/lib/easy-waf.js
@@ -46,6 +46,8 @@ var config = {
enabled: true
}
},
+ postBlockHook: undefined,
+ preBlockHook: undefined,
trustProxy: []
};
@@ -123,22 +125,23 @@ function easyWaf(conf){
}
if(typeof ipBlacklist !== 'undefined' && ipBlacklist.contains(ip)){
- block(req, res, {name: 'IPBlacklist'}, config, ip);
- if(config.dryMode) next();
- return;
+ if(block(req, res, {name: 'IPBlacklist'}, config, ip)){
+ return;
+ }
}
if(Array.isArray(config.allowedHTTPMethods) && !config.allowedHTTPMethods.includes(req.method)){
- block(req, res, {name: 'HTTPMethod'}, config, ip);
- if(config.dryMode) next();
- return;
+ if(block(req, res, {name: 'HTTPMethod'}, config, ip)){
+ return;
+ }
}
try {
var url = decodeURIComponent(req.url);
} catch(e) {
- block(req, res, {name: 'uriMalformed'}, config, ip);
- if(config.dryMode) next();
+ if(!block(req, res, {name: 'uriMalformed'}, config, ip)){
+ next();
+ }
return;
}
@@ -189,16 +192,16 @@ function modulesLoop(modules, moduleKeys, i, req, res, reqInfo, next){
let key = moduleKeys[i];
if(typeof modules[key].check === 'function'){
if(!modules[key].check(reqInfo)){
- block(req, res, modules[key].info(), config, reqInfo.ip);
- if(config.dryMode) next();
- return;
+ if(block(req, res, modules[key].info(), config, reqInfo.ip)){
+ return;
+ }
}
} else if(typeof modules[key].checkCB === 'function'){
modules[key].checkCB(reqInfo, (ok) => {
if(!ok){
- block(req, res, modules[key].info(), config, reqInfo.ip);
- if(config.dryMode) next();
- return;
+ if(block(req, res, modules[key].info(), config, reqInfo.ip)){
+ return;
+ }
}
modulesLoop(modules, moduleKeys, ++i, req, res, reqInfo, next);
});
diff --git a/lib/logger.js b/lib/logger.js
index 0b55cdc..b740901 100644
--- a/lib/logger.js
+++ b/lib/logger.js
@@ -26,7 +26,7 @@ function requestBlocked(moduleInfo, req, referenceID, config, ip){
var url = req.url.replace(/(\n|\r|\v)/ig, '').replace(/"/g, '"');
var ua = (req.headers['user-agent'] || '').replace(/(\n|\r|\v)/ig, '').replace(/"/g, '"');
- console.warn((!config.dryMode ? 'EasyWAF - Blocked:' : 'EasyWAF DryMode - Blocked:') + ' ip=' + ip + ' module=' + moduleInfo.name + ' time=' + new Date().getTime() + ' url="' + url + '" ua="' + ua + '" rid=' + referenceID);
+ console.warn((!config.dryMode ? 'EasyWAF - Blocked:' : 'EasyWAF DryMode - Blocked:') + ' ip=' + ip + ' module=' + moduleInfo.name + ' time=' + new Date().getTime() + ' url="' + url + '" ua="' + ua + '" method=' + req.method + ' rid=' + referenceID);
}
module.exports = {
diff --git a/lib/modules/badBots.js b/lib/modules/badBots.js
index 147cf12..2b893a7 100644
--- a/lib/modules/badBots.js
+++ b/lib/modules/badBots.js
@@ -4,7 +4,7 @@
*/
//Performance tests showed that this regex is faster than checking for not exact match using arrays
-const regex = /(01h4x\.com|360Spider|404checker|404enemy|80legs|ADmantX|AIBOT|ALittle Client|ASPSeek|Abonti|Aboundex|Aboundexbot|Acunetix|AfD-Verbotsverfahren|AhrefsBot|AiHitBot|Aipbot|Alexibot|AllSubmitter|Alligator|AlphaBot|Anarchie|Anarchy|Anarchy99|Ankit|Anthill|Apexoo|Aspiegel|Asterias|Atomseobot|Attach|AwarioRssBot|AwarioSmartBot|BBBike|BDCbot|BDFetch|BLEXBot|BackDoorBot|BackStreet|BackWeb|Backlink-Ceck|BacklinkCrawler|Badass|Bandit|Barkrowler|BatchFTP|Battleztar Bazinga|BetaBot|Bigfoot|Bitacle|BlackWidow|Black Hole|Blackboard|Blow|BlowFish|Boardreader|Bolt|BotALot|Brandprotect|Brandwatch|Buck|Buddy|BuiltBotTough|BuiltWith|Bullseye|BunnySlippers|BuzzSumo|CATExplorador|CCBot|CODE87|CSHttp|Calculon|CazoodleBot|Cegbfeieh|CensysInspect|CheTeam|CheeseBot|CherryPicker|ChinaClaw|Chlooe|Citoid|Claritybot|Cliqzbot|Cloud mapping|Cocolyzebot|Cogentbot|Collector|Copier|CopyRightCheck|Copyscape|Cosmos|Craftbot|Crawling at Home Project|CrazyWebCrawler|Crescent|CrunchBot|Curious|Custo|CyotekWebCopy|DBLBot|DIIbot|DSearch|DTS Agent|DataCha0s|DatabaseDriverMysqli|Demon|Deusu|Devil|Digincore|DigitalPebble|Dirbuster|Disco|Discobot|Discoverybot|Dispatch|DittoSpyder|DnBCrawler-Analytics|DnyzBot|DomCopBot|DomainAppender|DomainCrawler|DomainSigmaCrawler|DomainStatsBot|Domains Project|Dotbot|Download Wonder|Dragonfly|Drip|ECCP\/1\.0|EMail Siphon|EMail Wolf|EasyDL|Ebingbong|Ecxi|EirGrabber|EroCrawler|Evil|Exabot|Express WebPictures|ExtLinksBot|Extractor|ExtractorPro|Extreme Picture Finder|EyeNetIE|Ezooms|FDM|FHscan|FemtosearchBot|Fimap|Firefox\/7\.0|FlashGet|Flunky|Foobot|Freeuploader|FrontPage|Fuzz|FyberSpider|Fyrebot|G-i-g-a-b-o-t|GT::WWW|GalaxyBot|Genieo|GermCrawler|GetRight|GetWeb|Getintent|Gigabot|Go!Zilla|Go-Ahead-Got-It|GoZilla|Gotit|GrabNet|Grabber|Grafula|GrapeFX|GrapeshotCrawler|GridBot|HEADMasterSEO|HMView|HTMLparser|HTTP::Lite|HTTrack|Haansoft|HaosouSpider|Harvest|Havij|Heritrix|Hloader|HonoluluBot|Humanlinks|HybridBot|IDBTE4M|IDBot|IRLbot|Iblog|Id-search|IlseBot|Image Fetch|Image Sucker|IndeedBot|Indy Library|InfoNaviRobot|InfoTekies|Intelliseek|InterGET|InternetSeer|Internet Ninja|Iria|Iskanie|IstellaBot|JOC Web Spider|JamesBOT|Jbrofuzz|JennyBot|JetCar|Jetty|JikeSpider|Joomla|Jorgee|JustView|Jyxobot|Kenjin Spider|Keybot Translation-Search-Machine|Keyword Density|Kinza|Kozmosbot|LNSpiderguy|LWP::Simple|Lanshanbot|Larbin|Leap|LeechFTP|LeechGet|LexiBot|Lftp|LibWeb|Libwhisker|LieBaoFast|Lightspeedsystems|Likse|LinkScan|LinkWalker|Linkbot|LinkextractorPro|LinkpadBot|LinksManager|LinqiaMetadataDownloaderBot|LinqiaRSSBot|LinqiaScrapeBot|Lipperhey|Lipperhey Spider|Litemage_walker|Lmspider|Ltx71|MFC_Tear_Sample|MIDown tool|MIIxpc|MJ12bot|MQQBrowser|MSFrontPage|MSIECrawler|MTRobot|Mag-Net|Magnet|Mail\.RU_Bot|Majestic-SEO|Majestic12|Majestic SEO|MarkMonitor|MarkWatch|Mass Downloader|Masscan|Mata Hari|MauiBot|Mb2345Browser|MeanPath Bot|Meanpathbot|Mediatoolkitbot|MegaIndex\.ru|Metauri|MicroMessenger|Microsoft Data Access|Microsoft URL Control|Minefield|Mister PiX|Moblie Safari|Mojeek|Mojolicious|MolokaiBot|Morfeus Fucking Scanner|Mozlila|Mr\.4x3|Msrabot|Musobot|NICErsPRO|NPbot|Name Intelligence|Nameprotect|Navroad|NearSite|Needle|Nessus|NetAnts|NetLyzer|NetMechanic|NetSpider|NetZIP|Net Vampire|Netcraft|Nettrack|Netvibes|NextGenSearchBot|Nibbler|Niki-bot|Nikto|NimbleCrawler|Nimbostratus|Ninja|Nmap|Nuclei|Nutch|Octopus|Offline Explorer|Offline Navigator|OnCrawl|OpenLinkProfiler|OpenVAS|Openfind|Openvas|OrangeBot|OrangeSpider|OutclicksBot|OutfoxBot|PECL::HTTP|PHPCrawl|POE-Component-Client-HTTP|PageAnalyzer|PageGrabber|PageScorer|PageThing\.com|Page Analyzer|Pandalytics|Panscient|Papa Foto|Pavuk|PeoplePal|Petalbot|Pi-Monster|Picscout|Picsearch|PictureFinder|Piepmatz|Pimonster|Pixray|PleaseCrawl|Pockey|ProPowerBot|ProWebWalker|Probethenet|Psbot|Pu_iN|Pump|PxBroker|PyCurl|QueryN Metasearch|Quick-Crawler|RSSingBot|RankActive|RankActiveLinkBot|RankFlex|RankingBot|RankingBot2|Rankivabot|RankurBot|Re-re|ReGet|RealDownload|Reaper|RebelMouse|Recorder|RedesScrapy|RepoMonkey|Ripper|RocketCrawler|Rogerbot|SBIder|SEOkicks|SEOkicks-Robot|SEOlyticsCrawler|SEOprofiler|SEOstats|SISTRIX|SMTBot|SalesIntelligent|ScanAlert|Scanbot|ScoutJet|Scrapy|Screaming|ScreenerBot|ScrepyBot|Searchestate|SearchmetricsBot|Seekport|SemanticJuice|Semrush|SemrushBot|SentiBot|SeoSiteCheckup|SeobilityBot|Seomoz|Shodan|Siphon|SiteCheckerBotCrawler|SiteExplorer|SiteLockSpider|SiteSnagger|SiteSucker|Site Sucker|Sitebeam|Siteimprove|Sitevigil|SlySearch|SmartDownload|Snake|Snapbot|Snoopy|SocialRankIOBot|Sociscraper|Sogou web spider|Sosospider|Sottopop|SpaceBison|Spammen|SpankBot|Spanner|Spbot|Spinn3r|SputnikBot|Sqlmap|Sqlworm|Sqworm|Steeler|Stripper|Sucker|Sucuri|SuperBot|SuperHTTP|Surfbot|SurveyBot|Suzuran|Swiftbot|Szukacz|T0PHackTeam|T8Abot|Teleport|TeleportPro|Telesoft|Telesphoreo|Telesphorep|TheNomad|The Intraformant|Thumbor|TightTwatBot|Titan|Toata|Toweyabot|Tracemyfile|Trendiction|Trendictionbot|True_Robot|Turingos|Turnitin|TurnitinBot|TwengaBot|Twice|Typhoeus|URLy\.Warning|URLy Warning|UnisterBot|Upflow|V-BOT|VB Project|VCI|Vacuum|Vagabondo|VelenPublicWebCrawler|VeriCiteCrawler|VidibleScraper|Virusdie|VoidEYE|Voil|Voltron|WASALive-Bot|WBSearchBot|WEBDAV|WISENutbot|WPScan|WWW-Collector-E|WWW-Mechanize|WWW::Mechanize|WWWOFFLE|Wallpapers|Wallpapers\/3\.0|WallpapersHD|WeSEE|WebAuto|WebBandit|WebCollage|WebCopier|WebEnhancer|WebFetch|WebFuck|WebGo IS|WebImageCollector|WebLeacher|WebPix|WebReaper|WebSauger|WebStripper|WebSucker|WebWhacker|WebZIP|Web Auto|Web Collage|Web Enhancer|Web Fetch|Web Fuck|Web Pix|Web Sauger|Web Sucker|Webalta|WebmasterWorldForumBot|Webshag|WebsiteExtractor|WebsiteQuester|Website Quester|Webster|Whack|Whacker|Whatweb|Who\.is Bot|Widow|WinHTTrack|WiseGuys Robot|Wonderbot|Woobot|Wotbox|Wprecon|Xaldon WebSpider|Xaldon_WebSpider|Xenu|YoudaoBot|Zade|Zauba|Zermelo|Zeus|Zitebot|ZmEu|ZoomBot|ZoominfoBot|ZumBot|ZyBorg|adscanner|arquivo-web-crawler|arquivo\.pt|autoemailspider|backlink-check|cah\.io\.community|check1\.exe|clark-crawler|coccocbot|cognitiveseo|com\.plumanalytics|crawl\.sogou\.com|crawler\.feedback|crawler4j|dataforseo\.com|demandbase-bot|domainsproject\.org|eCatch|evc-batch|facebookscraper|gopher|heritrix|instabid|internetVista monitor|ips-agent|isitwp\.com|iubenda-radar|linkdexbot|lwp-request|lwp-trivial|magpie-crawler|meanpathbot|mediawords|muhstik-scan|netEstate NE Crawler|page scorer|pcBrowser|plumanalytics|polaris version|probe-image-size|ripz|s1z\.ru|satoristudio\.net|scalaj-http|scan\.lol|seobility|seocompany\.store|seoscanners|seostar|serpstatbot|sexsearcher|sitechecker\.pro|siteripz|sogouspider|sp_auditbot|spyfu|sysscan|tAkeOut|trendiction\.com|trendiction\.de|ubermetrics-technologies\.com|voyagerx\.com|webgains-bot|webmeup-crawler|webpros\.com|webprosbot|x09Mozilla|x22Mozilla|xpymep1\.exe|zauba\.io|zgrab)/i;
+const regex = /(01h4x\.com|360Spider|404checker|404enemy|80legs|ADmantX|AIBOT|ALittle Client|ASPSeek|Abonti|Aboundex|Aboundexbot|Acunetix|AfD-Verbotsverfahren|AhrefsBot|AiHitBot|Aipbot|Alexibot|AllSubmitter|Alligator|AlphaBot|Anarchie|Anarchy|Anarchy99|Ankit|Anthill|Apexoo|Aspiegel|Asterias|Atomseobot|Attach|AwarioRssBot|AwarioSmartBot|BBBike|BDCbot|BDFetch|BLEXBot|BackDoorBot|BackStreet|BackWeb|Backlink-Ceck|BacklinkCrawler|Badass|Bandit|Barkrowler|BatchFTP|Battleztar Bazinga|BetaBot|Bigfoot|Bitacle|BlackWidow|Black Hole|Blackboard|Blow|BlowFish|Boardreader|Bolt|BotALot|Brandprotect|Brandwatch|Buck|Buddy|BuiltBotTough|BuiltWith|Bullseye|BunnySlippers|BuzzSumo|CATExplorador|CCBot|CODE87|CSHttp|Calculon|CazoodleBot|Cegbfeieh|CensysInspect|CheTeam|CheeseBot|CherryPicker|ChinaClaw|Chlooe|Citoid|Claritybot|Cliqzbot|Cloud mapping|Cocolyzebot|Cogentbot|Collector|Copier|CopyRightCheck|Copyscape|Cosmos|Craftbot|Crawling at Home Project|CrazyWebCrawler|Crescent|CrunchBot|Curious|Custo|CyotekWebCopy|DBLBot|DIIbot|DSearch|DTS Agent|DataCha0s|DatabaseDriverMysqli|Demon|Deusu|Devil|Digincore|DigitalPebble|Dirbuster|Disco|Discobot|Discoverybot|Dispatch|DittoSpyder|DnBCrawler-Analytics|DnyzBot|DomCopBot|DomainAppender|DomainCrawler|DomainSigmaCrawler|DomainStatsBot|Domains Project|Dotbot|Download Wonder|Dragonfly|Drip|ECCP\/1\.0|EMail Siphon|EMail Wolf|EasyDL|Ebingbong|Ecxi|EirGrabber|EroCrawler|Evil|Exabot|Express WebPictures|ExtLinksBot|Extractor|ExtractorPro|Extreme Picture Finder|EyeNetIE|Ezooms|FDM|FHscan|FemtosearchBot|Fimap|Firefox\/7\.0|FlashGet|Flunky|Foobot|Freeuploader|FrontPage|Fuzz|FyberSpider|Fyrebot|G-i-g-a-b-o-t|GT::WWW|GalaxyBot|Genieo|GermCrawler|GetRight|GetWeb|Getintent|Gigabot|Go!Zilla|Go-Ahead-Got-It|GoZilla|Gotit|GrabNet|Grabber|Grafula|GrapeFX|GrapeshotCrawler|GridBot|HEADMasterSEO|HMView|HTMLparser|HTTP::Lite|HTTrack|Haansoft|HaosouSpider|Harvest|Havij|Heritrix|Hloader|HonoluluBot|Humanlinks|HybridBot|IDBTE4M|IDBot|IRLbot|Iblog|Id-search|IlseBot|Image Fetch|Image Sucker|IndeedBot|Indy Library|InfoNaviRobot|InfoTekies|Intelliseek|InterGET|InternetSeer|Internet Ninja|Iria|Iskanie|IstellaBot|JOC Web Spider|JamesBOT|Jbrofuzz|JennyBot|JetCar|Jetty|JikeSpider|Joomla|Jorgee|JustView|Jyxobot|Kenjin Spider|Keybot Translation-Search-Machine|Keyword Density|Kinza|Kozmosbot|LNSpiderguy|LWP::Simple|Lanshanbot|Larbin|Leap|LeechFTP|LeechGet|LexiBot|Lftp|LibWeb|Libwhisker|LieBaoFast|Lightspeedsystems|Likse|LinkScan|LinkWalker|Linkbot|LinkextractorPro|LinkpadBot|LinksManager|LinqiaMetadataDownloaderBot|LinqiaRSSBot|LinqiaScrapeBot|Lipperhey|Lipperhey Spider|Litemage_walker|Lmspider|Ltx71|MFC_Tear_Sample|MIDown tool|MIIxpc|MJ12bot|MQQBrowser|MSFrontPage|MSIECrawler|MTRobot|Mag-Net|Magnet|Mail\.RU_Bot|Majestic-SEO|Majestic12|Majestic SEO|MarkMonitor|MarkWatch|Mass Downloader|Masscan|Mata Hari|MauiBot|Mb2345Browser|MeanPath Bot|Meanpathbot|Mediatoolkitbot|MegaIndex\.ru|Metauri|MicroMessenger|Microsoft Data Access|Microsoft URL Control|Minefield|Mister PiX|Moblie Safari|Mojeek|Mojolicious|MolokaiBot|Morfeus Fucking Scanner|Mozlila|Mr\.4x3|Msrabot|Musobot|NICErsPRO|NPbot|Name Intelligence|Nameprotect|Navroad|NearSite|Needle|Nessus|NetAnts|NetLyzer|NetMechanic|NetSpider|NetZIP|Net Vampire|Netcraft|Nettrack|Netvibes|NextGenSearchBot|Nibbler|Niki-bot|Nikto|NimbleCrawler|Nimbostratus|Ninja|Nmap|Nuclei|Nutch|Octopus|Offline Explorer|Offline Navigator|OnCrawl|OpenLinkProfiler|OpenVAS|Openfind|Openvas|OrangeBot|OrangeSpider|OutclicksBot|OutfoxBot|PECL::HTTP|PHPCrawl|POE-Component-Client-HTTP|PageAnalyzer|PageGrabber|PageScorer|PageThing\.com|Page Analyzer|Pandalytics|Panscient|Papa Foto|Pavuk|PeoplePal|Pi-Monster|Picscout|Picsearch|PictureFinder|Piepmatz|Pimonster|Pixray|PleaseCrawl|Pockey|ProPowerBot|ProWebWalker|Probethenet|Psbot|Pu_iN|Pump|PxBroker|PyCurl|QueryN Metasearch|Quick-Crawler|RSSingBot|RankActive|RankActiveLinkBot|RankFlex|RankingBot|RankingBot2|Rankivabot|RankurBot|Re-re|ReGet|RealDownload|Reaper|RebelMouse|Recorder|RedesScrapy|RepoMonkey|Ripper|RocketCrawler|Rogerbot|SBIder|SEOkicks|SEOkicks-Robot|SEOlyticsCrawler|SEOprofiler|SEOstats|SISTRIX|SMTBot|SalesIntelligent|ScanAlert|Scanbot|ScoutJet|Scrapy|Screaming|ScreenerBot|ScrepyBot|Searchestate|SearchmetricsBot|Seekport|SemanticJuice|Semrush|SemrushBot|SentiBot|SeoSiteCheckup|SeobilityBot|Seomoz|Shodan|Siphon|SiteCheckerBotCrawler|SiteExplorer|SiteLockSpider|SiteSnagger|SiteSucker|Site Sucker|Sitebeam|Siteimprove|Sitevigil|SlySearch|SmartDownload|Snake|Snapbot|Snoopy|SocialRankIOBot|Sociscraper|Sogou web spider|Sosospider|Sottopop|SpaceBison|Spammen|SpankBot|Spanner|Spbot|Spinn3r|SputnikBot|Sqlmap|Sqlworm|Sqworm|Steeler|Stripper|Sucker|Sucuri|SuperBot|SuperHTTP|Surfbot|SurveyBot|Suzuran|Swiftbot|Szukacz|T0PHackTeam|T8Abot|Teleport|TeleportPro|Telesoft|Telesphoreo|Telesphorep|TheNomad|The Intraformant|Thumbor|TightTwatBot|Titan|Toata|Toweyabot|Tracemyfile|Trendiction|Trendictionbot|True_Robot|Turingos|Turnitin|TurnitinBot|TwengaBot|Twice|Typhoeus|URLy\.Warning|URLy Warning|UnisterBot|Upflow|V-BOT|VB Project|VCI|Vacuum|Vagabondo|VelenPublicWebCrawler|VeriCiteCrawler|VidibleScraper|Virusdie|VoidEYE|Voil|Voltron|WASALive-Bot|WBSearchBot|WEBDAV|WISENutbot|WPScan|WWW-Collector-E|WWW-Mechanize|WWW::Mechanize|WWWOFFLE|Wallpapers|Wallpapers\/3\.0|WallpapersHD|WeSEE|WebAuto|WebBandit|WebCollage|WebCopier|WebEnhancer|WebFetch|WebFuck|WebGo IS|WebImageCollector|WebLeacher|WebPix|WebReaper|WebSauger|WebStripper|WebSucker|WebWhacker|WebZIP|Web Auto|Web Collage|Web Enhancer|Web Fetch|Web Fuck|Web Pix|Web Sauger|Web Sucker|Webalta|WebmasterWorldForumBot|Webshag|WebsiteExtractor|WebsiteQuester|Website Quester|Webster|Whack|Whacker|Whatweb|Who\.is Bot|Widow|WinHTTrack|WiseGuys Robot|Wonderbot|Woobot|Wotbox|Wprecon|Xaldon WebSpider|Xaldon_WebSpider|Xenu|YoudaoBot|Zade|Zauba|Zermelo|Zeus|Zitebot|ZmEu|ZoomBot|ZoominfoBot|ZumBot|ZyBorg|adscanner|arquivo-web-crawler|arquivo\.pt|autoemailspider|backlink-check|cah\.io\.community|check1\.exe|clark-crawler|coccocbot|cognitiveseo|com\.plumanalytics|crawl\.sogou\.com|crawler\.feedback|crawler4j|dataforseo\.com|demandbase-bot|domainsproject\.org|eCatch|evc-batch|facebookscraper|gopher|heritrix|instabid|internetVista monitor|ips-agent|isitwp\.com|iubenda-radar|linkdexbot|lwp-request|lwp-trivial|magpie-crawler|meanpathbot|mediawords|muhstik-scan|netEstate NE Crawler|page scorer|pcBrowser|plumanalytics|polaris version|probe-image-size|ripz|s1z\.ru|satoristudio\.net|scalaj-http|scan\.lol|seobility|seocompany\.store|seoscanners|seostar|serpstatbot|sexsearcher|sitechecker\.pro|siteripz|sogouspider|sp_auditbot|spyfu|sysscan|tAkeOut|trendiction\.com|trendiction\.de|ubermetrics-technologies\.com|voyagerx\.com|webgains-bot|webmeup-crawler|webpros\.com|webprosbot|x09Mozilla|x22Mozilla|xpymep1\.exe|zauba\.io|zgrab)/i;
/** @type {EasyWafConfig} */
var config;
diff --git a/lib/modules/fakeSearchCrawlers.js b/lib/modules/fakeSearchCrawlers.js
index 82bab2a..3675d59 100644
--- a/lib/modules/fakeSearchCrawlers.js
+++ b/lib/modules/fakeSearchCrawlers.js
@@ -2,6 +2,7 @@ const CIDRMatcher = require('cidr-matcher');
const utils = require('../utils');
const logger = require('../logger');
const dns = require('node:dns');
+const ip6addr = require('ip6addr');
/** @type {EasyWafConfig} */
var config;
@@ -73,6 +74,21 @@ function checkCB(reqInfo, cb){
cb(true);
}
+/**
+ *
+ * @param {String} ipCIDR
+ * @returns {Boolean}
+ */
+ function validateCIDR(ipCIDR){
+ try {
+ ip6addr.createCIDR(ipCIDR);
+ return true;
+ } catch (e) /* istanbul ignore next */ {
+ logger.log('Warn', `Invalid ip in CIDR format: ${ipCIDR} ${e.message}`);
+ return false;
+ }
+}
+
/**
* Changes the prefix list of google and bing into an array
* @param {Array.