[feat] 泛解析过滤

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.0.5
+- 子域名爆破时泛解析过滤
+    -   参考 https://github.com/boy-hack/ksubdomain/issues/5
+
 ## v0.0.4
 - 修复修改二进制文件名可能读取不到配置文件的 bug
 

README.md

@@ -1,5 +1,6 @@
 # 🌟 Starmap
- 以 subfinder 为基础，融合 ksubdomain、 Amass 的一些优点进行二次开发的一款子域名收集工具，并增加了子域名接管检测功能。可以很方便作为 go 库集成进入项目中。
+![Starmap](https://socialify.git.ci/ZhuriLab/Starmap/image?description=1&font=Inter&forks=1&issues=1&logo=https%3A%2F%2Favatars.githubusercontent.com%2Fu%2F69614236%3Fs%3D200%26v%3D4&name=1&owner=1&pattern=Circuit%20Board&pulls=1&stargazers=1&theme=Light)
+以 subfinder 为基础，融合 ksubdomain、 Amass 的一些优点进行二次开发的一款子域名收集工具，并增加了子域名接管检测功能。可以很方便作为 go 库集成进入项目中。
 
 - [Amass](https://github.com/OWASP/Amass/) 虽然搜集的方法多，但太笨重，不方便集成，目标多了会内存爆炸
 - [subfinder](https://github.com/projectdiscovery/subfinder) 非常方便集成，但是只有被动的方式
@@ -105,7 +106,7 @@ func main() {
 	config.AllSources = passive.DefaultAllSources
 	config.Recursive = passive.DefaultRecursiveSources
 
-	runnerInstance, err := runner.NewRunner(&runner.Options{
+	options := &runner.Options{
 		Threads:            10, // Thread controls the number of threads to use for active enumerations
 		Timeout:            30, // Timeout is the seconds to wait for sources to respond
 		MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
@@ -118,17 +119,20 @@ func main() {
 		All: 				true,
 		Verbose: 			false,
 		Brute:				true,
-		Verify:             true,	// 验证找到的域名
+		Verify:             true,	// 验证找到的域名 
+		RemoveWildcard: 	true,	// 泛解析过滤 
 		Silent: 			false,	// 是否为静默模式，只输出找到的域名
 		DNS: 				"cn",	// dns 服务器区域选择，根据目标选择不同区域得到的结果不同，国内网站的话，选择 cn，dns 爆破结果比较多
 		BruteWordlist:      "",		// 爆破子域的域名字典，不填则使用内置的
 		Level: 				2,		// 枚举几级域名，默认为二级域名
 		LevelDic:           "",		// 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认
 		Takeover: 			false,	// 子域名接管检测
 		SAll: 				false,  // 子域名接管检测中请求全部 url，默认只对匹配的 cname 进行检测
+	}
 
-	})
+	options.ConfigureOutput()
 
+	runnerInstance, err := runner.NewRunner(options)
 
 	buf := bytes.Buffer{}
 	err, subdomains := runnerInstance.EnumerateSingleDomain(context.Background(), "baidu.com", []io.Writer{&buf})
@@ -168,7 +172,9 @@ func main() {
 
 # 💡 Tips
  - 指定不同的 dns ，获取到的结果会不同。比如：如果目标是国内的网站，选择国内的 dns 得到的子域名结果可能会比较多
-
+ - 提示 `pcap打开失败:vnic0: You don't have permission to capture on that device ((cannot open BPF device) /dev/bpf0: Permission denied)`等错误
+
+    执行`sudo chmod 777 /dev/bpf*` 或 `sudo ./Starmap`
 
 # 👀 参考
 - [subfinder](https://github.com/projectdiscovery/subfinder)

pkg/active/active.go

@@ -1,12 +1,13 @@
 package active
 
 import (
+	"context"
 	"github.com/ZhuriLab/Starmap/pkg/resolve"
 	"github.com/ZhuriLab/Starmap/pkg/util"
 	"github.com/projectdiscovery/gologger"
 )
 
-func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fileName string, level int, levelDict string, dns string) map[string]resolve.HostEntry {
+func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fileName string, level int, levelDict string, resolvers []string, wildcardIPs map[string]struct{}) map[string]resolve.HostEntry {
 	gologger.Info().Msgf("Start DNS blasting of %s", domain)
 	var levelDomains []string
 	if levelDict != "" {
@@ -19,53 +20,37 @@ func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fi
 		levelDomains = GetDefaultSubNextData()
 	}
 
-	var resolvers []string
-	switch dns {
-	case "cn":
-		resolvers = resolve.DefaultResolversCN
-	case "in":
-		resolvers = resolve.DefaultResolvers
-	case "all":
-		resolvers = append(resolve.DefaultResolvers, resolve.DefaultResolversCN...)
-	}
-
 	opt := &Options {
 		Rate:         Band2Rate("2m"),
 		Domain:       domain,
 		FileName:     fileName,
 		Resolvers:    resolvers,
 		Output:       "",
 		Silent:       silent,
-		SkipWildCard: false,
+		WildcardIPs:  wildcardIPs,
 		TimeOut:      5,
 		Retry:        6,
 		Level:        level,  				// 枚举几级域名，默认为2，二级域名,
 		LevelDomains: levelDomains,   		// 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认
 		Method:       "enum",
 	}
 
+	ctx := context.Background()
+
 	r, err := New(opt)
+
 	if err != nil {
 		gologger.Fatal().Msgf("%s", err)
 	}
 
-	enumMap := r.RunEnumeration(uniqueMap)
+	enumMap := r.RunEnumeration(uniqueMap, ctx)
 
 	r.Close()
 	return enumMap
 }
 
-func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, dns string) map[string]resolve.HostEntry {
+func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []string, wildcardIPs map[string]struct{}) map[string]resolve.HostEntry {
 	gologger.Info().Msgf("Start to verify the collected sub domain name results, a total of %d", len(uniqueMap))
-	var resolvers []string
-	switch dns {
-	case "cn":
-		resolvers = resolve.DefaultResolversCN
-	case "in":
-		resolvers = resolve.DefaultResolvers
-	case "all":
-		resolvers = append(resolve.DefaultResolvers, resolve.DefaultResolversCN...)
-	}
 
 	opt := &Options {
 		Rate:         Band2Rate("2m"),
@@ -74,18 +59,19 @@ func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, dns string) map
 		Resolvers:    resolvers,
 		Output:       "",
 		Silent:       silent,
-		SkipWildCard: false,
+		WildcardIPs:  wildcardIPs,
 		TimeOut:      5,
 		Retry:        6,
 		Method:       "verify",
 	}
+	ctx := context.Background()
 
 	r, err := New(opt)
 	if err != nil {
 		gologger.Fatal().Msgf("%s", err)
 	}
 
-	AuniqueMap := r.RunEnumerationVerify(uniqueMap)
+	AuniqueMap := r.RunEnumerationVerify(uniqueMap, ctx)
 
 	r.Close()
 

pkg/active/active_test.go

@@ -8,7 +8,10 @@ import (
 
 func TestEnum(t *testing.T) {
 	uniqueMap := make(map[string]resolve.HostEntry)
-	uniqueMap = Enum("baidu.com", uniqueMap, false,"", 2,"","cn")
+	resolvers := []string{
+		"114.114.114.114",
+	}
+	uniqueMap = Enum("baidu.com", uniqueMap, false,"", 2,"",resolvers, nil)
 	for k, v := range uniqueMap {
 		fmt.Println(k, v)
 	}
@@ -24,11 +27,14 @@ func TestVerify(t *testing.T) {
 
 	fmt.Println(uniqueMap)
 
-	uniqueMap = Verify(uniqueMap, true, "cn")
+	resolvers := []string{
+		"114.114.114.114",
+	}
+
+	uniqueMap = Verify(uniqueMap, true, resolvers, nil)
 	for k, v := range uniqueMap {
 		fmt.Println(k, v)
 	}
-
 }
 
 

pkg/active/device/device.go

@@ -12,7 +12,7 @@ import (
 )
 
 func AutoGetDevices() *EtherTable {
-	domain := util.RandomStr(4) + ".i.hacking8.com"
+	domain := util.RandomStr(6) + ".baidu.com"
 	signal := make(chan *EtherTable)
 	devices, err := pcap.FindAllDevs()
 	if err != nil {

pkg/active/options.go

@@ -13,7 +13,7 @@ type Options struct {
 	Resolvers    []string
 	Output       string // 输出文件名
 	Silent       bool
-	SkipWildCard bool
+	WildcardIPs  map[string]struct{}
 	TimeOut      int
 	Retry        int
 	Method       string // verify模式 enum模式 test模式

pkg/active/recv.go

@@ -82,18 +82,41 @@ func (r *runner) recvChanel(ctx context.Context) error {
 		r.hm.Del(domain)
 
 		if dns.ANCount > 0 {
-			atomic.AddUint64(&r.successIndex, 1)
-			result := RecvResult {
-				Subdomain: domain,
-				Answers:   dns.Answers,
-				ResponseCode: dns.ResponseCode,
+			var ips []string
+			for _, answers := range dns.Answers {
+				if answers.Class == layers.DNSClassIN {
+					if answers.IP != nil {
+						ips = append(ips, answers.IP.String())
+					}
+				}
 			}
 
-			select {
-			case <-ctx.Done():
-				gologger.Error().Msg("recvChanel ctx.Done()............")
-			default:
-				r.recver <- result
+			if len(ips) == 0 {
+				continue
+			}
+
+			var skip bool
+			for _, ip := range ips {
+				// Ignore the host if it exists in wildcard ips map
+				if _, ok := r.options.WildcardIPs[ip]; ok {
+					skip = true
+					break
+				}
+			}
+
+			// 不是泛解析出的 ip 的记录
+			if !skip {
+				select {
+				case <-ctx.Done():
+					gologger.Error().Msg("recvChanel ctx.Done()............")
+				default:
+					atomic.AddUint64(&r.successIndex, 1)
+					r.recver <- RecvResult {
+						Subdomain: domain,
+						Answers:   dns.Answers,
+						ResponseCode: dns.ResponseCode,
+					}
+				}
 			}
 		}
 	}