Skip to content

Commit

Permalink
优化后台查找,减少误报,
Browse files Browse the repository at this point in the history
修复一些潜在bug
  • Loading branch information
M0nster3 committed Jun 4, 2024
1 parent 322ccc2 commit 2ed47c7
Show file tree
Hide file tree
Showing 16 changed files with 174 additions and 127 deletions.
35 changes: 19 additions & 16 deletions Api/Company/tianyancha/bean.go
Original file line number Diff line number Diff line change
Expand Up @@ -318,27 +318,30 @@ func GetReqReturnPage(url string, options *Utils.LongOptions) *html.Node {
func UpCookie(res string, options *Utils.LongOptions) {
re := regexp.MustCompile(`arg1='([\w\s]+)';`)
rr := re.FindAllStringSubmatch(res, 1)
str := rr[0][1]
if str != "" {
if options.LongConfig.Cookies.Tianyancha != "" {
re = regexp.MustCompile(`acw_sc__v2=([\w\s]+)`)
rr = re.FindAllStringSubmatch(options.LongConfig.Cookies.Tianyancha, 1)
if len(rr) > 0 {
str2 := rr[0][1]
if str2 != "" {
gologger.Infof("【TYC】反爬计算签名成功!\n")
options.LongConfig.Cookies.Tianyancha = strings.ReplaceAll(options.LongConfig.Cookies.Tianyancha, str2, SingAwcSCV2(str))
} else {
gologger.Errorf("【TYC】反爬Cookie存在问题\n")
if len(rr) > 0 {
str := rr[0][1]
if str != "" {
if options.LongConfig.Cookies.Tianyancha != "" {
re = regexp.MustCompile(`acw_sc__v2=([\w\s]+)`)
rr = re.FindAllStringSubmatch(options.LongConfig.Cookies.Tianyancha, 1)
if len(rr) > 0 {
str2 := rr[0][1]
if str2 != "" {
gologger.Infof("【TYC】反爬计算签名成功!\n")
options.LongConfig.Cookies.Tianyancha = strings.ReplaceAll(options.LongConfig.Cookies.Tianyancha, str2, SingAwcSCV2(str))
} else {
gologger.Errorf("【TYC】反爬Cookie存在问题\n")
}
}
} else {
gologger.Infof("【TYC】未登录反爬计算签名成功!\n")
options.LongConfig.Cookies.Tianyancha = SingAwcSCV2(str)
}
} else {
gologger.Infof("【TYC】未登录反爬计算签名成功!\n")
options.LongConfig.Cookies.Tianyancha = SingAwcSCV2(str)
gologger.Errorf("【TYC】反爬存在问题\n")
}
} else {
gologger.Errorf("【TYC】反爬存在问题\n")
}

}

// SingAwcSCV2 acw_sc__v2
Expand Down
8 changes: 6 additions & 2 deletions Api/Domains/Google.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,9 @@ func Google(domain string, options *Utils.LongOptions, DomainsIP *outputfile.Dom
re := regexp.MustCompile(hostname)
for _, aa := range llink {
matches := re.FindAllStringSubmatch(strings.TrimSpace(aa.String()), -1)
buff = append(buff, matches[0][0])
if len(matches) > 0 {
buff = append(buff, matches[0][0])
}

}
// 查找匹配的内容
Expand Down Expand Up @@ -171,7 +173,9 @@ func Google(domain string, options *Utils.LongOptions, DomainsIP *outputfile.Dom
re := regexp.MustCompile(hostname)
for _, aa := range llink {
matches := re.FindAllStringSubmatch(strings.TrimSpace(aa.String()), -1)
buff = append(buff, matches[0][0])
if len(matches) > 0 {
buff = append(buff, matches[0][0])
}

}
// 查找匹配的内容
Expand Down
14 changes: 8 additions & 6 deletions Api/Domains/alienvault.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,15 @@ func GetEnInfoAlienvault(response string, DomainsIP *outputfile.DomainsIP) (*Uti
re := regexp.MustCompile(`(?:\d{1,3}\.){3}\d{1,3}`)
ip := gjson.Get(respons[aa].String(), "address").String()
matches := re.FindAllStringSubmatch(strings.TrimSpace(ip), -1)
for _, bu := range matches {
if !addedURLs[bu[0]] {
// 如果不存在重复则将 URL 添加到 Infos["Urls"] 中,并在 map 中标记为已添加
DomainsIP.IP = append(DomainsIP.IP, bu[0])
addedURLs[bu[0]] = true
if len(matches) > 0 {
for _, bu := range matches {
if !addedURLs[bu[0]] {
// 如果不存在重复则将 URL 添加到 Infos["Urls"] 中,并在 map 中标记为已添加
DomainsIP.IP = append(DomainsIP.IP, bu[0])
addedURLs[bu[0]] = true
}
break
}
break
}

}
Expand Down
20 changes: 12 additions & 8 deletions Api/Domains/commoncrawl.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func Commoncrawl(domain string, options *Utils.LongOptions, DomainsIP *outputfil
urls := "https://index.commoncrawl.org/collinfo.json"
client := resty.New()
client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
client.SetTimeout(time.Duration(options.TimeOut) * time.Minute)
client.SetTimeout(3 * time.Minute)
if options.Proxy != "" {
client.SetProxy(options.Proxy)
//client.SetProxy("192.168.203.111:1111")
Expand All @@ -90,7 +90,6 @@ func Commoncrawl(domain string, options *Utils.LongOptions, DomainsIP *outputfil
//加入随机延迟
time.Sleep(time.Duration(options.GetDelayRTime()) * time.Second)
clientR := client.R()

clientR.URL = urls
resp, err := clientR.Get(urls)
for add := 1; add < 4; add += 1 {
Expand Down Expand Up @@ -129,7 +128,7 @@ func Commoncrawl(domain string, options *Utils.LongOptions, DomainsIP *outputfil
clienta.URL = url
time.Sleep(3 * time.Second)
respa, err := clienta.Get(url)
for add := 1; add < 20; add += 1 {
for add := 1; add < 4; add += 1 {
if err != nil || respa.StatusCode() == 503 {
clients := resty.New()
clientaa := clients.R()
Expand All @@ -146,6 +145,9 @@ func Commoncrawl(domain string, options *Utils.LongOptions, DomainsIP *outputfil
break
}
}
if err != nil {
continue
}
if respa.StatusCode() == 503 {
fmt.Printf("503")
return ""
Expand All @@ -160,11 +162,13 @@ func Commoncrawl(domain string, options *Utils.LongOptions, DomainsIP *outputfil

// 查找匹配的内容
matches := re.FindAllStringSubmatch(strings.TrimSpace(string(respa.Body())), -1)
for _, bu := range matches {
if !addedURLs[bu[0]] {
// 如果不存在重复则将 URL 添加到 Infos["Urls"] 中,并在 map 中标记为已添加
result = append(result, bu[0])
addedURLs[bu[0]] = true
if len(matches) > 0 {
for _, bu := range matches {
if !addedURLs[bu[0]] {
// 如果不存在重复则将 URL 添加到 Infos["Urls"] 中,并在 map 中标记为已添加
result = append(result, bu[0])
addedURLs[bu[0]] = true
}
}
}

Expand Down
28 changes: 15 additions & 13 deletions Api/Domains/digitorus.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,20 +125,22 @@ func Digitorus(domain string, options *Utils.LongOptions, DomainsIP *outputfile.

// 查找匹配的内容
matches := re.FindAllStringSubmatch(string(resp.Body()), -1)
var ia5StringContent []string
// 遍历匹配项
ia5StringContent = append(ia5StringContent, "[")
for _, match := range matches {
zuo := strings.ReplaceAll(match[1], "IA5String &#39;", "\"")
you := strings.ReplaceAll(zuo, "&#39;", "\"")
// 第一个捕获组匹配的内容即为你需要的字符串
ia5StringContent = append(ia5StringContent, you+",")
}
ia5StringContent = append(ia5StringContent, "]")
respjoin := strings.Join(ia5StringContent, " ")
res, ensOutMap := GetEnInfoDigitorus(respjoin, DomainsIP)
if len(matches) > 0 {
var ia5StringContent []string
// 遍历匹配项
ia5StringContent = append(ia5StringContent, "[")
for _, match := range matches {
zuo := strings.ReplaceAll(match[1], "IA5String &#39;", "\"")
you := strings.ReplaceAll(zuo, "&#39;", "\"")
// 第一个捕获组匹配的内容即为你需要的字符串
ia5StringContent = append(ia5StringContent, you+",")
}
ia5StringContent = append(ia5StringContent, "]")
respjoin := strings.Join(ia5StringContent, " ")
res, ensOutMap := GetEnInfoDigitorus(respjoin, DomainsIP)

outputfile.MergeOutPut(res, ensOutMap, "Digitorus 证书查询", options)
outputfile.MergeOutPut(res, ensOutMap, "Digitorus 证书查询", options)
}

return "Success"
}
14 changes: 8 additions & 6 deletions Api/Domains/dnsdumpster.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,10 @@ func PostBuffer(domain string, options *Utils.LongOptions, token string, client

// 查找匹配的内容
matches := re.FindAllStringSubmatch(strings.TrimSpace(td3.Text()), -1)
for _, bu := range matches {

Hostname = append(Hostname, bu[0])

if len(matches) > 0 {
for _, bu := range matches {
Hostname = append(Hostname, bu[0])
}
}

})
Expand All @@ -125,9 +125,11 @@ func PostBuffer(domain string, options *Utils.LongOptions, token string, client

// 查找匹配的内容
matches := re.FindAllStringSubmatch(strings.TrimSpace(td4.Text()), -1)
for _, bu := range matches {
Address = append(Address, bu[0])
if len(matches) > 0 {
for _, bu := range matches {
Address = append(Address, bu[0])

}
}

})
Expand Down
7 changes: 5 additions & 2 deletions Api/Domains/dnsrepo.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,12 @@ func Dnsrepo(domain string, options *Utils.LongOptions, DomainsIP *outputfile.Do

// 查找匹配的内容
matches := re.FindAllStringSubmatch(href, -1)
for _, bu := range matches {
Address = append(Address, bu[0])
if len(matches) > 0 {
for _, bu := range matches {
Address = append(Address, bu[0])
}
}

})

})
Expand Down
9 changes: 6 additions & 3 deletions Api/Domains/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,14 @@ func Github(domain string, options *Utils.LongOptions, DomainsIP *outputfile.Dom
hostname := `(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.)+` + regexp.QuoteMeta(domain)
re := regexp.MustCompile(hostname)
matches := re.FindAllStringSubmatch(string(resp.Body()), -1)
for _, aa := range matches {
if strings.Contains(aa[0], domain) {
Hostname = append(Hostname, aa[0])
if len(matches) > 0 {
for _, aa := range matches {
if strings.Contains(aa[0], domain) {
Hostname = append(Hostname, aa[0])
}
}
}

}
Hostname = Utils.SetStr(Hostname)
result := "{\"passive_dns\":["
Expand Down
38 changes: 20 additions & 18 deletions Api/Domains/rapiddns.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,25 +103,27 @@ func Rapiddns(domain string, options *Utils.LongOptions, DomainsIP *outputfile.D
}
host := regexp.MustCompile(`<td>((?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,})</td>`)
hostrea := host.FindAllStringSubmatch(strings.TrimSpace(string(response.Body())), -1)
var Hostname []string
// 使用 HTML 解析器解析 HTML 内容
for _, bu := range hostrea {
Hostname = append(Hostname, bu[1])
}
Hostname = Utils.SetStr(Hostname)
// 查找具有特定 class 的元素并获取其内容
//var Hostname []string

var result string
result = "{\"passive_dns\":["
for i := 0; i < len(Hostname); i++ {
result += "{\"hostname\"" + ":" + "\"" + Hostname[i] + "\"" + "},"
DomainsIP.Domains = append(DomainsIP.Domains, Hostname[i])
}
result = result + "]}"
res, ensOutMap := GetEnInfoRapiddns(result, DomainsIP)
if len(hostrea) > 0 {
var Hostname []string
// 使用 HTML 解析器解析 HTML 内容
for _, bu := range hostrea {
Hostname = append(Hostname, bu[1])
}
Hostname = Utils.SetStr(Hostname)
// 查找具有特定 class 的元素并获取其内容
//var Hostname []string

var result string
result = "{\"passive_dns\":["
for i := 0; i < len(Hostname); i++ {
result += "{\"hostname\"" + ":" + "\"" + Hostname[i] + "\"" + "},"
DomainsIP.Domains = append(DomainsIP.Domains, Hostname[i])
}
result = result + "]}"
res, ensOutMap := GetEnInfoRapiddns(result, DomainsIP)

outputfile.MergeOutPut(res, ensOutMap, "Rapiddns", options)
outputfile.MergeOutPut(res, ensOutMap, "Rapiddns", options)
}

return "Success"
}
16 changes: 9 additions & 7 deletions Email/Intelx/IntelxEmail.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,16 +158,18 @@ func IntelxEmail(domain string, options *Utils.LongOptions, DomainsIP *outputfil
re := regexp.MustCompile(Email)

Emails := re.FindAllStringSubmatch(strings.TrimSpace(respnsehe), -1)
if len(Emails) > 0 {
result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","

result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","
}
result1 = result1 + "]}"

res, ensOutMap := GetEnInfo(result1, DomainsIP)
//
outputfile.MergeOutPut(res, ensOutMap, "Intelx", options)
}
result1 = result1 + "]}"

res, ensOutMap := GetEnInfo(result1, DomainsIP)
//
outputfile.MergeOutPut(res, ensOutMap, "Intelx", options)
return "Success"
}
16 changes: 9 additions & 7 deletions Email/Tomba/TombaEmail.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,16 +128,18 @@ func TombaEmail(domain string, options *Utils.LongOptions, DomainsIP *outputfile
re := regexp.MustCompile(Email)

Emails := re.FindAllStringSubmatch(strings.TrimSpace(respnsehe), -1)
if len(Emails) > 0 {
result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","

result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","
}
result1 = result1 + "]}"

res, ensOutMap := GetEnInfo(result1, DomainsIP)
//
outputfile.MergeOutPut(res, ensOutMap, "Tomba", options)
}
result1 = result1 + "]}"

res, ensOutMap := GetEnInfo(result1, DomainsIP)
//
outputfile.MergeOutPut(res, ensOutMap, "Tomba", options)
return "Success"
}
17 changes: 9 additions & 8 deletions Email/baidu/baidu.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,16 +195,17 @@ func Baidu(domain string, options *Utils.LongOptions, DomainsIP *outputfile.Doma
re := regexp.MustCompile(Email)

Emails := re.FindAllStringSubmatch(strings.TrimSpace(respnsehe), -1)
if len(Emails) > 0 {
result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","

result1 := "{\"Email\":["
for add := 0; add < len(Emails); add++ {
result1 += "{" + "\"Email\"" + ":" + "\"" + Emails[add][0] + "\"" + "}" + ","

}
result1 = result1 + "]}"
}
result1 = result1 + "]}"

res, ensOutMap := GetEnInfo(result1, DomainsIP)
res, ensOutMap := GetEnInfo(result1, DomainsIP)

outputfile.MergeOutPut(res, ensOutMap, "Baidu", options)
outputfile.MergeOutPut(res, ensOutMap, "Baidu", options)
}

}
Loading

0 comments on commit 2ed47c7

Please sign in to comment.