Merge pull request #17 from gplumb/dev

Bumper update of bots (up to JayBizzle/Crawler-Detect#293)
gplumb · Oct 1, 2018 · 1cdffcc · 1cdffcc
2 parents f75f48b + 7dd7b66
commit 1cdffcc
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 245 deletions.
diff --git a/NetCrawlerDetect/NetCrawlerDetect.Tests/NetCrawlerDetect.Tests.csproj b/NetCrawlerDetect/NetCrawlerDetect.Tests/NetCrawlerDetect.Tests.csproj
@@ -4,7 +4,7 @@
     <TargetFramework>netcoreapp2.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
-    <ReleaseVersion>0.1.5</ReleaseVersion>
+    <ReleaseVersion>0.1.6</ReleaseVersion>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/NetCrawlerDetect/NetCrawlerDetect.Tests/crawlers.txt b/NetCrawlerDetect/NetCrawlerDetect.Tests/crawlers.txt
@@ -3284,4 +3284,18 @@ JolokiaPwn, https://matmannion.com/jolokiapwn/, mat.mannion@gmail.com
 CookieReports.com/0.1
 GoogleCloudMonitoring
 kube-probe/1.9+
-Mozilla/5.0 Chrome/10.0.613.0 Safari/534.15 Zombie.js/5.0.7
+Mozilla/5.0 Chrome/10.0.613.0 Safari/534.15 Zombie.js/5.0.7
+Microsoft Office/16.0 (Microsoft Outlook Mail 16.0.7167; Pro)Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0; InfoPath.3; Microsoft Outlook 14.0.6025; ms-office; MSOf
+Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Win64; x64; Trident/7.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3; .NET4.0E; Microsoft Outlook 14.0.7167; ms-office; MSOffice 14)
+Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Microsoft Outlook 15.0.4805; ms-office; MSOffice 15)
+Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; Tablet PC 2.0; MALNJS; Microsoft Outlook 16.0.6001; ms-office; MSOffice 16)
+Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; InfoPath.3; Microsoft Outlook 14.0.6025)
+Outlook-Express/7.0 (MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; McAfee; .NET4.0C; .NET4.0E; GWX:QUALIFIED; TmstmpExt)
+Mail/3445.4.7 CFNetwork/889.9 Darwin/17.2.0 (x86_64)
+SPDYCheck SPDY Protocol Tester, see http://spdycheck.org/about.html
+SpamExperts/128741 (Virus Scan)
+Unshorten.It! /1.0 (unshorten.it)
+Monit/5.25.2
+Mozilla/5.0 (compatible; PingAdmin.Ru/1.1; +http://pingadmin.ru/)
+Mozilla/5.0 (compatible; localsearch-web/2.0; +http://www.localsearch.ch/en/legal-notice)
+Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36
diff --git a/NetCrawlerDetect/NetCrawlerDetect.Tests/devices.txt b/NetCrawlerDetect/NetCrawlerDetect.Tests/devices.txt
diff --git a/NetCrawlerDetect/NetCrawlerDetect.sln b/NetCrawlerDetect/NetCrawlerDetect.sln
@@ -22,7 +22,7 @@ Global
 	EndGlobalSection
 	GlobalSection(MonoDevelopProperties) = preSolution
 		description = A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect).
-		version = 0.1.5
+		version = 0.1.6
 		Policies = $0
 		$0.VersionControlPolicy = $1
 	EndGlobalSection

diff --git a/NetCrawlerDetect/NetCrawlerDetect/Fixtures/Crawlers.cs b/NetCrawlerDetect/NetCrawlerDetect/Fixtures/Crawlers.cs
@@ -37,6 +37,7 @@ public Crawlers()
                 @"^Java\/",
                 @"^Jeode\/",
                 @"^Jetty\/",
+                @"^Mail\/",
                 @"^Mget",
                 @"^Microsoft URL Control",
                 @"^NG\/[0-9\.]",
@@ -228,6 +229,7 @@ public Crawlers()
                 @"DareBoost",
                 @"DatabaseDriverMysqli",
                 @"DataCha0s",
+                @"Datanyze",
                 @"DataparkSearch",
                 @"dataprovider",
                 @"DataXu",
@@ -572,6 +574,7 @@ public Crawlers()
                 @"Litemage_walker",
                 @"livedoor ScreenShot",
                 @"LoadImpactRload",
+                @"localsearch-web",
                 @"LongURL API",
                 @"looksystems\.net",
                 @"ltx71",
@@ -606,6 +609,7 @@ public Crawlers()
                 @"MFC_Tear_Sample",
                 @"Microsearch",
                 @"Microsoft Office ",
+                @"Microsoft Outlook",
                 @"Microsoft Windows Network Diagnostics",
                 @"Microsoft-WebDAV-MiniRedir",
                 @"Microsoft\ Data\ Access",
@@ -623,6 +627,7 @@ public Crawlers()
                 @"Mojeek",
                 @"Mojolicious \(Perl\)",
                 @"monitis",
+                @"Monit\/",
                 @"Monitority\/[0-9]",
                 @"montastic",
                 @"MonTools",
@@ -661,7 +666,7 @@ public Crawlers()
                 @"NetZIP",
                 @"Neustar WPM",
                 @"NeutrinoAPI",
-                @"NewRelicPinger\/1.0 \(\d+\)",
+                @"NewRelicPinger",
                 @"NewsBlur .*Finder",
                 @"NewsGator",
                 @"newsme",
@@ -706,6 +711,7 @@ public Crawlers()
                 @"Orbiter",
                 @"OrgProbe\/[0-9]",
                 @"orion-semantics",
+                @"Outlook-Express",
                 @"ow\.ly",
                 @"Owler",
                 @"ownCloud News",
@@ -747,6 +753,7 @@ public Crawlers()
                 @"Pimonster",
                 @"ping\.blo\.gs\/",
                 @"Pingability",
+                @"PingAdmin\.Ru",
                 @"Pingdom",
                 @"Pingoscope",
                 @"PingSpot",
@@ -918,6 +925,7 @@ public Crawlers()
                 @"Spammen",
                 @"Spanner",
                 @"spaziodati",
+                @"SPDYCheck",
                 @"Specificfeeds",
                 @"speedy",
                 @"SPEng",
@@ -1003,7 +1011,7 @@ public Crawlers()
                 @"uCrawlr\/",
                 @"UdmSearch",
                 @"UniversalFeedParser",
-                @"Unshorten\.It\!\/[0-9]",
+                @"Unshorten\.It",
                 @"Untiny",
                 @"UnwindFetchor",
                 @"updated",
@@ -1168,6 +1176,7 @@ public Crawlers()
                 @"ZnajdzFoto",
                 @"Zombie\.js",
                 @"ZyBorg",
+                @"SpamExperts",
                 @"[a-z0-9\-_]*(bot|crawler|archiver|transcoder|spider|uptime|validator|fetcher)",
             };
         }

diff --git a/NetCrawlerDetect/NetCrawlerDetect/Fixtures/Exclusions.cs b/NetCrawlerDetect/NetCrawlerDetect/Fixtures/Exclusions.cs
@@ -57,6 +57,11 @@ public Exclusions()
                 @"Opera",
                 @" \.NET[\d\.]*",
                 @"cubot",
+                @"; M bot",
+                @"; B bot",
+                @"; IDbot",
+                @"; ID bot",
+                @"; POWER BOT",
 
                 // Remove the following characters ;
                 @";",

diff --git a/NetCrawlerDetect/NetCrawlerDetect/NetCrawlerDetect.csproj b/NetCrawlerDetect/NetCrawlerDetect/NetCrawlerDetect.csproj
@@ -2,8 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <ReleaseVersion>0.1.5</ReleaseVersion>
-    <PackageVersion>0.1.5</PackageVersion>
+    <ReleaseVersion>0.1.6</ReleaseVersion>
+    <PackageVersion>0.1.6</PackageVersion>
     <Authors>Graham "Gee" Plumb</Authors>
     <PackageLicenseUrl>https://github.com/gplumb/NetCrawlerDetect/blob/master/LICENSE</PackageLicenseUrl>
     <Owners>Graham "Gee" Plumb</Owners>
@@ -12,12 +12,7 @@
     <Description>A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect).</Description>
     <PackOnBuild>true</PackOnBuild>
     <PackageId>NetCrawlerDetect</PackageId>
-    <PackageReleaseNotes>Bumper update of pulls from CrawlerDetect:
-
-#273: Arukereso
-#274: 10 new bots
-#275: Added 3 new crawlers
-#277: Zombie.js</PackageReleaseNotes>
+    <PackageReleaseNotes>Bumper update of bots (added 9 new bots and updated exclusions)</PackageReleaseNotes>
     <Summary>A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect).</Summary>
   </PropertyGroup>