From 6f9d857e5318cc688de32c9619013d2531eb2b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 16 Aug 2023 13:37:45 +0000 Subject: [PATCH] remove zoomeye --- .github/workflows/build-test.yml | 1 - .gitignore | 1 + README.md | 10 +- v2/pkg/passive/sources.go | 2 - v2/pkg/passive/sources_test.go | 1 - v2/pkg/runner/options.go | 2 +- v2/pkg/subscraping/sources/zoomeye/zoomeye.go | 179 ------------------ 7 files changed, 7 insertions(+), 189 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/zoomeye/zoomeye.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index aa01cc015..f25241cb9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -53,7 +53,6 @@ jobs: THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} - ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} uses: nick-invision/retry@v2 with: diff --git a/.gitignore b/.gitignore index 5e2c8c922..b0dd47ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ v2/subfinder vendor/ .idea .devcontainer +.vscode dist \ No newline at end of file diff --git a/README.md b/README.md index e3360cace..6b7677c8a 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ SOURCE: -s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources. -recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld) -all use all sources for enumeration (slow) - -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeye) + -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeyeapi) FILTER: -m, -match string[] subdomain or list of subdomain to match (file or comma separated) @@ -121,14 +121,14 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest `subfinder` can be used right after the installation, however the following services require configuring API keys to work: -[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com) +[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com) You can also use the `subfinder -ls` command to display all the available sources. These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. -Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix`, `360quake` and `ZoomEye`, need to be separated with a colon (`:`). +Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `360quake`, need to be separated with a colon (`:`). An example provider config file: @@ -147,8 +147,8 @@ shodan: github: - ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X - ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4 -zoomeye: - - ZOOMEYE_USERNAME:ZOOMEYE_PASSWORD +zoomeyeapi: + - 4f73021d-ff95-4f53-937f-83d6db719eec quake: - 0cb9030c-0a40-48a3-b8c4-fca28e466ba3 facebook: diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 700190464..500ebafe6 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -45,7 +45,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" ) @@ -86,7 +85,6 @@ var AllSources = [...]subscraping.Source{ &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, - &zoomeye.Source{}, &zoomeyeapi.Source{}, &facebook.Source{}, // &threatminer.Source{}, // failing api diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 34323c1af..c14b9c154 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -45,7 +45,6 @@ var ( "virustotal", "waybackarchive", "whoisxmlapi", - "zoomeye", "zoomeyeapi", "hunter", "leakix", diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 570b218b5..86c8254db 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -103,7 +103,7 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.Sources, "sources", "s", nil, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)"), flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"), - flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeyeapi)", goflags.NormalizedStringSliceOptions), ) flagSet.CreateGroup("filter", "Filter", diff --git a/v2/pkg/subscraping/sources/zoomeye/zoomeye.go b/v2/pkg/subscraping/sources/zoomeye/zoomeye.go deleted file mode 100644 index 10e42f57d..000000000 --- a/v2/pkg/subscraping/sources/zoomeye/zoomeye.go +++ /dev/null @@ -1,179 +0,0 @@ -// Package zoomeye logic -package zoomeye - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "time" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// zoomAuth holds the ZoomEye credentials -type zoomAuth struct { - User string `json:"username"` - Pass string `json:"password"` -} - -type loginResp struct { - JWT string `json:"access_token"` -} - -// search results -type zoomeyeResults struct { - Matches []struct { - Site string `json:"site"` - Domains []string `json:"domains"` - } `json:"matches"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []apiKey - timeTaken time.Duration - errors int - results int - skipped bool -} - -type apiKey struct { - username string - password string -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey.username == "" || randomApiKey.password == "" { - s.skipped = true - return - } - - jwt, err := doLogin(ctx, session, randomApiKey) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - // check if jwt is null - if jwt == "" { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye"), - } - s.errors++ - return - } - - headers := map[string]string{ - "Authorization": fmt.Sprintf("JWT %s", jwt), - "Accept": "application/json", - "Content-Type": "application/json", - } - for currentPage := 0; currentPage <= 100; currentPage++ { - api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) - resp, err := session.Get(ctx, api, "", headers) - isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden - if err != nil { - if !isForbidden && currentPage == 0 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - } - return - } - - var res zoomeyeResults - err = json.NewDecoder(resp.Body).Decode(&res) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - resp.Body.Close() - - for _, r := range res.Matches { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} - s.results++ - for _, domain := range r.Domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} - s.results++ - } - } - } - }() - - return results -} - -// doLogin performs authentication on the ZoomEye API -func doLogin(ctx context.Context, session *subscraping.Session, randomApiKey apiKey) (string, error) { - creds := &zoomAuth{ - User: randomApiKey.username, - Pass: randomApiKey.password, - } - body, err := json.Marshal(&creds) - if err != nil { - return "", err - } - resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body)) - if err != nil { - session.DiscardHTTPResponse(resp) - return "", err - } - - defer resp.Body.Close() - - var login loginResp - err = json.NewDecoder(resp.Body).Decode(&login) - if err != nil { - return "", err - } - return login.JWT, nil -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "zoomeye" -} - -func (s *Source) IsDefault() bool { - return false -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { - return apiKey{k, v} - }) -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -}