Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Filter out Cloudflare error pages + performance improvement #41

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Filter out Cloudflare error pages + performance improvement #41

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e69837
Add filter strings functionality
Jan 1, 2021
221d080
Skip unresolvable domains
Jan 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 34 additions & 5 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ func main() {
var preferHTTPS bool
flag.BoolVar(&preferHTTPS, "prefer-https", false, "only try plain HTTP if HTTPS fails")

// filter out cloudflare error pages
var filterCloudflareErrors bool
flag.BoolVar(&filterCloudflareErrors, "filter-cf-errors", false, "Filter out Cloudflare error pages")

// HTTP method to use
var method string
flag.StringVar(&method, "method", "GET", "HTTP method to use")
Expand All @@ -57,6 +61,13 @@ func main() {
// make an actual time.Duration out of the timeout
timeout := time.Duration(to * 1000000)

var filterStrings []string

// Add Cloudflare signatures to filterStrings if filterCloudflareErrors
if filterCloudflareErrors {
filterStrings = append(filterStrings, "<center>cloudflare</center>", "cf_styles-css")
}

var tr = &http.Transport{
MaxIdleConns: 30,
IdleConnTimeout: time.Second,
Expand Down Expand Up @@ -96,7 +107,7 @@ func main() {

// always try HTTPS first
withProto := "https://" + url
if isListening(client, withProto, method) {
if isListening(client, withProto, method, filterStrings) {
output <- withProto

// skip trying HTTP if --prefer-https is set
Expand All @@ -120,7 +131,7 @@ func main() {
go func() {
for url := range httpURLs {
withProto := "http://" + url
if isListening(client, withProto, method) {
if isListening(client, withProto, method, filterStrings) {
output <- withProto
continue
}
Expand Down Expand Up @@ -157,6 +168,11 @@ func main() {
for sc.Scan() {
domain := strings.ToLower(sc.Text())

// Skip unresolvable domains
if _, err := net.LookupIP(domain); err != nil {
continue
}

// submit standard port checks
if !skipDefault {
httpsURLs <- domain
Expand Down Expand Up @@ -210,7 +226,7 @@ func main() {
outputWG.Wait()
}

func isListening(client *http.Client, url, method string) bool {
func isListening(client *http.Client, url, method string, filterStrings []string) bool {

req, err := http.NewRequest(method, url, nil)
if err != nil {
Expand All @@ -222,8 +238,21 @@ func isListening(client *http.Client, url, method string) bool {

resp, err := client.Do(req)
if resp != nil {
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
defer resp.Body.Close()

if len(filterStrings) != 0 {
// Read the first 512 bytes of the response and check for presence of any filter strings
peek := make([]byte, 512)
resp.Body.Read(peek)
peekStr := string(peek)
for _, filterString := range filterStrings {
if strings.Contains(peekStr, filterString) {
return true
}
}
} else {
io.Copy(ioutil.Discard, resp.Body)
}
}

if err != nil {
Expand Down