Skip to content
This repository has been archived by the owner on Feb 17, 2023. It is now read-only.

Improved bot recognition #26

Merged
merged 2 commits into from
Jan 18, 2016
Merged

Improved bot recognition #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0319fcf
bot: check for in user agent
crackcomm Jan 17, 2016
e25e612
tests: more robots
crackcomm Jan 17, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions all_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,26 @@ var uastrings = []struct {
ua: "Facebot",
expected: "Browser:Facebot Bot:true Mobile:false",
},
{
title: "NutchCVS",
ua: "NutchCVS/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
expected: "Browser:NutchCVS Bot:true Mobile:false",
},
{
title: "MJ12bot",
ua: "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)",
expected: "Mozilla:5.0 Browser:MJ12bot-v1.2.4 Bot:true Mobile:false",
},
{
title: "MJ12bot",
ua: "MJ12bot/v1.0.8 (http://majestic12.co.uk/bot.php?+)",
expected: "Browser:MJ12bot Bot:true Mobile:false",
},
{
title: "AhrefsBot",
ua: "Mozilla/5.0 (compatible; AhrefsBot/4.0; +http://ahrefs.com/robot/)",
expected: "Mozilla:5.0 Browser:AhrefsBot-4.0 Bot:true Mobile:false",
},

// Internet Explorer
{
Expand Down
7 changes: 4 additions & 3 deletions bot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ func (p *UserAgent) fixOther(sections []section) {
}
}

var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")

// Check if we're dealing with a bot or with some weird browser. If that is the
// case, the receiver will be modified accordingly.
func (p *UserAgent) checkBot(sections []section) {
Expand All @@ -83,9 +85,8 @@ func (p *UserAgent) checkBot(sections []section) {
if len(sections) == 1 && sections[0].name != "Mozilla" {
p.mozilla = ""

// Check whether the name has some suspicious "bot" in his name.
reg, _ := regexp.Compile("(?i)bot")
if reg.Match([]byte(sections[0].name)) {
// Check whether the name has some suspicious "bot" or "crawler" in his name.
if botRegex.Match([]byte(sections[0].name)) {
p.setSimple(sections[0].name, "", true)
return
}
Expand Down