fix up

twiny · Jul 11, 2022 · 093db74 · 093db74
1 parent 99bd177
commit 093db74
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-## WBot - a web crawler
+## WBot
 
 A configurable, thread-safe web crawler, provides a minimal interface for crawling and downloading web pages.
 

diff --git a/filter.go b/filter.go
@@ -36,20 +36,22 @@ func newFilter(allowed, disallowed []string) *filter {
 
 // Allow
 func (f *filter) Allow(l *url.URL) bool {
+	raw := l.String()
+
 	if badExtensions.MatchString(l.Path) {
 		return false
 	}
 
 	// disallowed
-	for _, r := range f.disallowed {
-		if r.MatchString(l.String()) {
+	for _, d := range f.disallowed {
+		if d.MatchString(raw) {
 			return false
 		}
 	}
 
 	// allowed
-	for _, p := range f.allowed {
-		if !p.MatchString(l.String()) {
+	for _, a := range f.allowed {
+		if !a.MatchString(raw) {
 			return false
 		}
 	}

diff --git a/wbot.go b/wbot.go
@@ -73,11 +73,6 @@ func (wb *WBot) Crawl(link string) error {
 		return err
 	}
 
-	// check filter
-	if !wb.filter.Allow(req.URL) {
-		return fmt.Errorf("not allowed")
-	}
-
 	// rate limit
 	wb.limit.take(req.URL)