Skip to content

Commit

Permalink
feat bbscan 使用协程池
Browse files Browse the repository at this point in the history
  • Loading branch information
@yhy0
yhy0 committed May 19, 2024
1 parent 26b1e37 commit da4ac5f
Show file tree
Hide file tree
Showing 7 changed files with 702 additions and 44 deletions.
647 changes: 647 additions & 0 deletions 1.html
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions SCopilot/templates/SCopilot.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ <h4 class="card-title">端口信息</h4>
<pre class="card-text">{{ $info.Request }}</pre>
</div>
<div class="tab-pane fade" id="response{{$index}}" role="tabpanel" aria-labelledby="response-tab{{$index}}">
<pre class="card-text">{{ $info.Response }}</pre>
<pre class="card-text">{{ $info.Response | html }}</pre>
</div>
</div>
</li>
Expand Down Expand Up @@ -253,7 +253,7 @@ <h4 class="level mb-0">
{{ if $message.VulnData.Request}}<pre class="card-text">{{ $message.VulnData.Request}}</pre>{{ end }}
</div>
<div class="tab-pane fade" id="vuln-response{{$index}}" role="tabpanel" aria-labelledby="vuln-response-tab{{$index}}">
{{ if $message.VulnData.Response }}<pre class="card-text">{{ $message.VulnData.Response }}</pre>{{ end }}
{{ if $message.VulnData.Response }}<pre class="card-text">{{ $message.VulnData.Response | html }}</pre>{{ end }}
</div>
</div>
</div>
Expand Down
37 changes: 30 additions & 7 deletions cmd/other.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ package cmd
import (
"fmt"
"github.com/logrusorgru/aurora"
"github.com/panjf2000/ants/v2"
"github.com/spf13/cobra"
"github.com/yhy0/Jie/conf"
"github.com/yhy0/Jie/pkg/protocols/httpx"
"github.com/yhy0/Jie/scan/PerFolder/traversal"
"github.com/yhy0/Jie/scan/bbscan"
"github.com/yhy0/Jie/scan/gadget/brute"
"github.com/yhy0/Jie/scan/gadget/swagger"
"sync"
)

/**
Expand All @@ -23,21 +25,42 @@ var otherCmd = &cobra.Command{
Short: "other scan && exp bb:BasicBrute、swagger:Swagger、nat:NginxAliasTraversal、dir:dir)",
Run: func(cmd *cobra.Command, args []string) {
client := httpx.NewClient(nil)
pool, _ := ants.NewPool(20)
defer pool.Release() // 释放协程池

wg := sync.WaitGroup{}

for _, target := range conf.GlobalConfig.Options.Targets {
wg.Add(1)
switch conf.GlobalConfig.Options.Mode {
case "bb":
user, pwd, _ := brute.BasicBrute(target, client)
if user != "" {
fmt.Println(aurora.Red(fmt.Sprintf("[Success] %v %v", user, pwd)))
}
_ = pool.Submit(func() {
defer wg.Done()
user, pwd, _ := brute.BasicBrute(target, client)
if user != "" {
fmt.Println(aurora.Red(fmt.Sprintf("[Success] %v %v", user, pwd)))
}
})
case "nat":
traversal.NginxAlias(target, "", "")
_ = pool.Submit(func() {
defer wg.Done()
traversal.NginxAlias(target, "", "")
})

case "swagger":
swagger.Scan(target, client)
_ = pool.Submit(func() {
defer wg.Done()
swagger.Scan(target, client)
})
case "dir":
bbscan.BBscan(target, true, nil, nil, client)
_ = pool.Submit(func() {
defer wg.Done()
bbscan.BBscan(target, true, nil, nil, client)
})
}
}

wg.Wait()
},
}

Expand Down
16 changes: 2 additions & 14 deletions pkg/mode/active.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,13 +202,7 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s

// 分发扫描任务
t.WG.Add(1)
go func() {
err := t.Pool.Submit(t.Distribution(crawlResult))
if err != nil {
t.WG.Done()
logging.Logger.Errorf("add distribution err:%v, crawlResult:%v", err, crawlResult)
}
}()
_ = t.Pool.Submit(t.Distribution(crawlResult))
}

if conf.GlobalConfig.WebScan.Craw == "k" {
Expand Down Expand Up @@ -303,13 +297,7 @@ func Crawlergo(target string, waf []string, t *task.Task, fingerprint []string)

// 分发扫描任务
t.WG.Add(1)
go func() {
err := t.Pool.Submit(t.Distribution(crawlResult))
if err != nil {
t.WG.Done()
logging.Logger.Errorf("add distribution err:%v, crawlResult:%v", err, crawlResult)
}
}()
_ = t.Pool.Submit(t.Distribution(crawlResult))
}

// 开始爬虫任务
Expand Down
4 changes: 2 additions & 2 deletions pkg/output/vulnReport.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,10 @@ <h4 class="level mb-0">
<!-- Tab panes for Request and Response -->
<div class="tab-content">
<div class="tab-pane fade show active" id="vuln-request{{$index}}" role="tabpanel" aria-labelledby="vuln-request-tab{{$index}}">
{{ if $message.VulnData.Request}}<pre class="card-text">{{ $message.VulnData.Request}}</pre>{{ end }}
{{ if $message.VulnData.Request}}<pre class="card-text bg-light p-3 border">{{ $message.VulnData.Request}}</pre>{{ end }}
</div>
<div class="tab-pane fade" id="vuln-response{{$index}}" role="tabpanel" aria-labelledby="vuln-response-tab{{$index}}">
{{ if $message.VulnData.Response }}<pre class="card-text">{{ $message.VulnData.Response }}</pre>{{ end }}
{{ if $message.VulnData.Response }}<pre class="card-text bg-light p-3 border">{{ $message.VulnData.Response | html }}</pre>{{ end }}
</div>
</div>
</div>
Expand Down
2 changes: 1 addition & 1 deletion pkg/task/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ type Task struct {
Fingerprints []string // 这个只有主动会使用，被动只会新建一个 task，所以不会用到
Parallelism int // 同时扫描的最大 url 个数
Pool *ants.Pool // 协程池，目前来看只是用来优化被动扫描，减小被动扫描时的协程创建、销毁的开销
WG sync.WaitGroup // 等待协程池所有任务结束
WG *sync.WaitGroup // 等待协程池所有任务结束
ScanTask map[string]*ScanTask // 存储对目标扫描时的一些状态
Lock sync.Mutex // 对 Distribution函数中的一些 map 并发操作进行保护
WgLock sync.Mutex // ScanTask 是一个 map，运行插件时会并发操作，加锁保护
Expand Down
36 changes: 18 additions & 18 deletions scan/bbscan/bbscan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package bbscan

import (
"github.com/antlabs/strsim"
"github.com/panjf2000/ants/v2"
regexp "github.com/wasilibs/go-re2"
"github.com/yhy0/Jie/pkg/input"
"github.com/yhy0/Jie/pkg/output"
Expand Down Expand Up @@ -194,7 +195,6 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
if strings.HasSuffix(u, "/") {
u = u[:len(u)-1]
}

var (
technologies []string
resContents []string // 找到的页面返回集合，用来进行网页相似度比较，用来去除大量的返回一样的
Expand All @@ -208,8 +208,10 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
resContents = append(resContents, strings.ReplaceAll(url404res.Body, path404, ""))
}

pool, _ := ants.NewPool(20)
defer pool.Release() // 释放协程池

wg := sync.WaitGroup{}
ch := make(chan struct{}, 20)
var l sync.Mutex
count := 0

Expand Down Expand Up @@ -268,13 +270,11 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
}

wg.Add(1)
ch <- struct{}{}

go func(t string, _path string, r *Rule) {
_ = pool.Submit(func() {
defer wg.Done()
defer func() { <-ch }()
<-time.After(time.Duration(100) * time.Millisecond)
page, res, err := ReqPage(t, header, client)
page, res, err := ReqPage(target, header, client)

if err == nil && res != nil {
if res.StatusCode >= 500 {
Expand All @@ -285,7 +285,7 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
}

// 黑名单，跳过
if scan_util.IsBlackHtml(res.Body, res.Header["Content-Type"], _path) {
if scan_util.IsBlackHtml(res.Body, res.Header["Content-Type"], path) {
return
}

Expand All @@ -300,11 +300,11 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
return
}

if strings.HasSuffix(t, ".xml") {
if strings.HasSuffix(target, ".xml") {
if !util.Contains(contentType, "xml") {
return
}
} else if strings.HasSuffix(t, ".json") {
} else if strings.HasSuffix(target, ".json") {
if !util.Contains(contentType, "json") {
return
}
Expand All @@ -315,10 +315,10 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
if len(strings.TrimSpace(res.Body)) == 0 {
return
}
if (r.Type != "" && !util.Contains(contentType, r.Type)) || (r.TypeNo != "" && util.Contains(contentType, r.TypeNo)) {
if (rule.Type != "" && !util.Contains(contentType, rule.Type)) || (rule.TypeNo != "" && util.Contains(contentType, rule.TypeNo)) {
return
}
if r.Status != "" && strconv.Itoa(res.StatusCode) != r.Status {
if rule.Status != "" && strconv.Itoa(res.StatusCode) != rule.Status {
return
}
} else {
Expand All @@ -334,7 +334,7 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,

}

if r.Tag != "" && !util.Contains(res.Body, r.Tag) {
if rule.Tag != "" && !util.Contains(res.Body, rule.Tag) {
return
}

Expand All @@ -348,8 +348,8 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,

if !similar {
// 对扫到的 swagger 进行自动化测试
if strings.Contains(t, "swagger") {
swagger.Scan(t, client)
if strings.Contains(target, "swagger") {
swagger.Scan(target, client)
}
if res.StatusCode == 401 {
l.Lock()
Expand All @@ -358,8 +358,8 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
}

l.Lock()
technologies = append(addFingerprintsnormal(t, technologies, res, client)) // 基于200页面文件扫描指纹添加
resContents = append(resContents, strings.ReplaceAll(res.Body, t, ""))
technologies = append(addFingerprintsnormal(target, technologies, res, client)) // 基于200页面文件扫描指纹添加
resContents = append(resContents, strings.ReplaceAll(res.Body, target, ""))
l.Unlock()

output.OutChannel <- output.VulMessage{
Expand All @@ -368,7 +368,7 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
VulnData: output.VulnData{
CreateTime: time.Now().Format("2006-01-02 15:04:05"),
Target: u,
Payload: t,
Payload: target,
Method: "GET",
Request: res.RequestDump,
Response: res.ResponseDump,
Expand All @@ -377,7 +377,7 @@ func BBscan(u string, root bool, fingprints []string, header map[string]string,
}
}
}
}(target, path, rule)
})

}

Expand Down

0 comments on commit da4ac5f

Please sign in to comment.