diff --git a/config.yaml b/config.yaml index 5ac88faaf..d0af439e5 100644 --- a/config.yaml +++ b/config.yaml @@ -10,6 +10,7 @@ resolvers: - 208.67.222.222 - 208.67.220.220 sources: + - alienvault - archiveis - binaryedge - bufferover @@ -20,6 +21,7 @@ sources: - crtsh - digicert - dnsdumpster + - dnsdb - entrust - googleter - hackertarget @@ -29,16 +31,20 @@ sources: - securitytrails - shodan - sitedossier + - sublist3r - threatcrowd - threatminer - urlscan - virustotal - waybackarchive + - zoomeye binaryedge: [] censys: [] certspotter: [] +dnsdb: [] passivetotal: [] securitytrails: [] shodan: [] urlscan: [] virustotal: [] +zoomeye: [] \ No newline at end of file diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index b33c6fff8..cb5d4a9d3 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -2,6 +2,7 @@ package passive import ( "github.com/projectdiscovery/subfinder/pkg/subscraping" + "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/alienvault" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/bufferover" @@ -11,6 +12,7 @@ import ( "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/digicert" + "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget" @@ -20,15 +22,18 @@ import ( "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/waybackarchive" + "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/zoomeye" ) // DefaultSources contains the list of sources used by default var DefaultSources = []string{ + "alienvault", "archiveis", "binaryedge", "bufferover", @@ -39,6 +44,7 @@ var DefaultSources = []string{ "crtsh", "digicert", "dnsdumpster", + "dnsdb", "entrust", "hackertarget", "ipv4info", @@ -47,11 +53,13 @@ var DefaultSources = []string{ "securitytrails", "shodan", "sitedossier", + "sublist3r", "threatcrowd", "threatminer", "urlscan", "virustotal", "waybackarchive", + "zoomeye", } // Agent is a struct for running passive subdomain enumeration @@ -76,6 +84,8 @@ func New(sources []string, exclusions []string) *Agent { func (a *Agent) addSources(sources []string) { for _, source := range sources { switch source { + case "alienvault": + a.sources[source] = &alienvault.Source{} case "archiveis": a.sources[source] = &archiveis.Source{} case "binaryedge": @@ -96,6 +106,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &digicert.Source{} case "dnsdumpster": a.sources[source] = &dnsdumpster.Source{} + case "dnsdb": + a.sources[source] = &dnsdb.Source{} case "entrust": a.sources[source] = &entrust.Source{} case "hackertarget": @@ -112,6 +124,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &shodan.Source{} case "sitedossier": a.sources[source] = &sitedossier.Source{} + case "sublist3r": + a.sources[source] = &sublist3r.Source{} case "threatcrowd": a.sources[source] = &threatcrowd.Source{} case "threatminer": @@ -122,6 +136,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &virustotal.Source{} case "waybackarchive": a.sources[source] = &waybackarchive.Source{} + case "zoomeye": + a.sources[source] = &zoomeye.Source{} } } } diff --git a/pkg/runner/config.go b/pkg/runner/config.go index 1d8795210..f4f6feb0d 100644 --- a/pkg/runner/config.go +++ b/pkg/runner/config.go @@ -22,11 +22,13 @@ type ConfigFile struct { Binaryedge []string `yaml:"binaryedge"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` + DNSDB []string `yaml:"dnsdb"` PassiveTotal []string `yaml:"passivetotal"` SecurityTrails []string `yaml:"securitytrails"` Shodan []string `yaml:"shodan"` URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` + ZoomEye []string `yaml:"zoomeye"` } // GetConfigDirectory gets the subfinder config directory for a user @@ -108,6 +110,10 @@ func (c ConfigFile) GetKeys() subscraping.Keys { keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] } + if (len(c.DNSDB)) > 0 { + keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] + } + if len(c.PassiveTotal) > 0 { passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))] parts := strings.Split(passiveTotalKeys, ":") @@ -129,5 +135,14 @@ func (c ConfigFile) GetKeys() subscraping.Keys { if len(c.Virustotal) > 0 { keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))] } + if len(c.ZoomEye) > 0 { + zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))] + parts := strings.Split(zoomEyeKeys, ":") + if len(parts) == 2 { + keys.ZoomEyeUsername = parts[0] + keys.ZoomEyePassword = parts[1] + } + } + return keys } diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go new file mode 100644 index 000000000..778dd5c1b --- /dev/null +++ b/pkg/subscraping/sources/alienvault/alienvault.go @@ -0,0 +1,62 @@ +package alienvault + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + "github.com/projectdiscovery/subfinder/pkg/subscraping" +) + +type alienvaultResponse struct { + PassiveDNS []struct { + Hostname string `json:"hostname"` + } `json:"passive_dns"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + close(results) + return + } + if resp.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("invalid status code received: %d", resp.StatusCode)} + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + close(results) + return + } + otxResp := &alienvaultResponse{} + // Get the response body and decode + err = json.NewDecoder(resp.Body).Decode(&otxResp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + close(results) + return + } + resp.Body.Close() + for _, record := range otxResp.PassiveDNS { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + } + close(results) + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "alienvault" +} diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go new file mode 100644 index 000000000..2b731ab8a --- /dev/null +++ b/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -0,0 +1,72 @@ +package dnsdb + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/projectdiscovery/subfinder/pkg/subscraping" +) + +type dnsdbResponse struct { + Name string `json:"rrname"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + headers := map[string]string{ + "X-API-KEY": session.Keys.DNSDB, + "Accept": "application/json", + "Content-Type": "application/json", + } + go func() { + resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + close(results) + return + } + // Check status code + if resp.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("invalid status code received: %d", resp.StatusCode)} + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + close(results) + return + } + defer resp.Body.Close() + // Get the response body + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + out := &dnsdbResponse{} + err := json.Unmarshal([]byte(line), out) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + close(results) + return + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(out.Name, ".")} + out = nil + } + close(results) + }() + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "DNSDB" +} diff --git a/pkg/subscraping/sources/sublist3r/subllist3r.go b/pkg/subscraping/sources/sublist3r/subllist3r.go new file mode 100644 index 000000000..cebbf107e --- /dev/null +++ b/pkg/subscraping/sources/sublist3r/subllist3r.go @@ -0,0 +1,48 @@ +package sublist3r + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/projectdiscovery/subfinder/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + close(results) + return + } + defer resp.Body.Close() + var subdomains []string + // Get the response body and unmarshal + err = json.NewDecoder(resp.Body).Decode(&subdomains) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + close(results) + return + } + + for _, subdomain := range subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + close(results) + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "sublist3r" +} diff --git a/pkg/subscraping/sources/zoomeye/zoomeye.go b/pkg/subscraping/sources/zoomeye/zoomeye.go new file mode 100644 index 000000000..135248aea --- /dev/null +++ b/pkg/subscraping/sources/zoomeye/zoomeye.go @@ -0,0 +1,148 @@ +package zoomeye + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/projectdiscovery/subfinder/pkg/subscraping" +) + +// zoomAuth holds the ZoomEye credentials +type zoomAuth struct { + User string `json:"username"` + Pass string `json:"password"` +} + +type loginResp struct { + JWT string `json:"access_token"` +} + +// search results +type zoomeyeResults struct { + Matches []struct { + Site string `json:"site"` + Domains []string `json:"domains"` + } `json:"matches"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" { + close(results) + return + } + jwt, err := doLogin(session) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + close(results) + return + } + // check if jwt is null + if jwt == "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} + close(results) + return + } + headers := map[string]string{ + "Authorization": fmt.Sprintf("JWT %s", jwt), + "Accept": "application/json", + "Content-Type": "application/json", + } + for currentPage := 0; currentPage <= 100; currentPage++ { + api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) + resp, err := session.Get(ctx, api, "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + close(results) + return + } + // if response is non-200 & current page is 0 return an error + if resp.StatusCode != 200 && currentPage == 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("invalid status code received: %d", resp.StatusCode)} + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + close(results) + return + } + // otherwise we hit 403 when there are no more results + if resp.StatusCode != 200 { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + close(results) + return + } + defer resp.Body.Close() + res := &zoomeyeResults{} + err = json.NewDecoder(resp.Body).Decode(res) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + close(results) + return + } + resp.Body.Close() + for _, r := range res.Matches { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} + for _, domain := range r.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} + } + } + currentPage++ + } + close(results) + }() + + return results +} + +// doLogin performs authentication on the ZoomEye API +func doLogin(session *subscraping.Session) (string, error) { + creds := &zoomAuth{ + User: session.Keys.ZoomEyeUsername, + Pass: session.Keys.ZoomEyePassword, + } + body, err := json.Marshal(&creds) + if err != nil { + return "", err + } + req, err := http.NewRequest("POST", "https://api.zoomeye.org/user/login", bytes.NewBuffer(body)) + if err != nil { + return "", err + } + req.Header.Add("Content-Type", "application/json") + resp, err := session.Client.Do(req) + if err != nil { + return "", err + } + // if not 200, bad credentials + if resp.StatusCode != 200 { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + return "", fmt.Errorf("login failed, non-200 response from zoomeye") + } + + defer resp.Body.Close() + login := &loginResp{} + err = json.NewDecoder(resp.Body).Decode(login) + if err != nil { + return "", err + } + return login.JWT, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "zoomeye" +} diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go index 1473d983c..d5d6b2ced 100755 --- a/pkg/subscraping/types.go +++ b/pkg/subscraping/types.go @@ -33,12 +33,15 @@ type Keys struct { CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` + DNSDB string `json:"dnsdb"` PassiveTotalUsername string `json:"passivetotal_username"` PassiveTotalPassword string `json:"passivetotal_password"` Securitytrails string `json:"securitytrails"` Shodan string `json:"shodan"` URLScan string `json:"urlscan"` Virustotal string `json:"virustotal"` + ZoomEyeUsername string `json:"zoomeye_username"` + ZoomEyePassword string `json:"zoomeye_password"` } // Result is a result structure returned by a source