Merge branch 'master' into lc/add-modules

README.md

@@ -275,4 +275,4 @@ http://mta-sts.managed.hackerone.com
 
 subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details.
 
-Read the disclaimer for usage at **[DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md)**
+Read the disclaimer for usage at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal.

config.yaml

@@ -27,6 +27,7 @@ sources:
   - hackertarget
   - ipv4info
   - passivetotal
+  - rapiddns
   - securitytrails
   - shodan
   - sitedossier
@@ -46,4 +47,4 @@ securitytrails: []
 shodan: []
 urlscan: []
 virustotal: []
-zoomeye: []
+zoomeye: []

pkg/passive/sources.go

@@ -18,6 +18,7 @@ import (
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal"
+	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier"
@@ -48,6 +49,7 @@ var DefaultSources = []string{
 	"hackertarget",
 	"ipv4info",
 	"passivetotal",
+	"rapiddns",
 	"securitytrails",
 	"shodan",
 	"sitedossier",
@@ -114,6 +116,8 @@ func (a *Agent) addSources(sources []string) {
 			a.sources[source] = &ipv4info.Source{}
 		case "passivetotal":
 			a.sources[source] = &passivetotal.Source{}
+		case "rapiddns":
+			a.sources[source] = &rapiddns.Source{}
 		case "securitytrails":
 			a.sources[source] = &securitytrails.Source{}
 		case "shodan":

pkg/subscraping/sources/rapiddns/rapiddns.go

@@ -0,0 +1,77 @@
+// Package rapiddns is a RapidDNS Scraping Engine in Golang
+package rapiddns
+
+import (
+	"context"
+	"io/ioutil"
+	"regexp"
+	"strings"
+
+	"github.com/projectdiscovery/subfinder/pkg/subscraping"
+)
+
+// RapidDNS is a struct for archiveurlsagent
+type RapidDNS struct {
+	Results chan subscraping.Result
+	Session *subscraping.Session
+}
+
+var reNext = regexp.MustCompile("<td><a href=\"(.*?)\" target=\"_blank\">.*?</a></td>")
+
+func (a *RapidDNS) enumerate(ctx context.Context, baseURL string) {
+	select {
+	case <-ctx.Done():
+		return
+	default:
+	}
+
+	resp, err := a.Session.NormalGetWithContext(ctx, baseURL)
+	if err != nil {
+		a.Results <- subscraping.Result{Source: "radpiddns", Type: subscraping.Error, Error: err}
+		return
+	}
+
+	// Get the response body
+	body, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		a.Results <- subscraping.Result{Source: "radpiddns", Type: subscraping.Error, Error: err}
+		return
+	}
+
+	src := string(body)
+
+	for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
+		a.Results <- subscraping.Result{Source: "radpiddns", Type: subscraping.Subdomain, Value: subdomain}
+	}
+
+	match1 := reNext.FindStringSubmatch(src)
+	if len(match1) > 0 {
+		a.enumerate(ctx, strings.TrimRight(match1[1], " "))
+	}
+}
+
+// Source is the passive scraping agent
+type Source struct{}
+
+// Run function returns all subdomains found with the service
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+
+	aInstance := RapidDNS{
+		Session: session,
+		Results: results,
+	}
+
+	go func() {
+		aInstance.enumerate(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1")
+		close(aInstance.Results)
+	}()
+
+	return aInstance.Results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "rapiddns"
+}