From 045e826e6bc0825f62cf8dec0694e68f6f0c0b9f Mon Sep 17 00:00:00 2001
From: yhy <31311038+yhy0@users.noreply.github.com>
Date: Mon, 18 Mar 2024 21:53:17 +0800
Subject: [PATCH] fix Katana UniqueId

---
 pkg/mode/active.go           | 12 +++++++++++-
 pkg/util/uniqueness.go       | 26 +++++++++++++-------------
 scan/gadget/sensitive/key.go |  2 +-
 test/xss_test.go             |  2 +-
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/pkg/mode/active.go b/pkg/mode/active.go
index 253f112..8c190f6 100644
--- a/pkg/mode/active.go
+++ b/pkg/mode/active.go
@@ -13,6 +13,7 @@ import (
     "github.com/yhy0/Jie/crawler/crawlergo/model"
     "github.com/yhy0/Jie/fingprints"
     "github.com/yhy0/Jie/pkg/input"
+    "github.com/yhy0/Jie/pkg/mitmproxy/go-mitmproxy/proxy"
     "github.com/yhy0/Jie/pkg/protocols/httpx"
     "github.com/yhy0/Jie/pkg/task"
     "github.com/yhy0/Jie/pkg/util"
@@ -186,7 +187,16 @@ func Katana(target string, waf []string, t *task.Task, fingerprint []string) []s
             Fingerprints: fingerprint,
             Waf:          waf,
             Resp:         resp,
-            UniqueId:     util.UUID(), // 这里爬虫中已经判断过了，所以生成一个 uuid 就行
+            // UniqueId:     util.UUID(), // 这里爬虫中已经判断过了，所以生成一个 uuid 就行
+            // 需要先自己实现，Katana 去重逻辑不太行
+            UniqueId: util.UniqueId(&proxy.Request{
+                Method: result.Request.Method,
+                URL:    parseUrl,
+                Header: headers,
+                Body:   []byte(result.Request.Body),
+            }),
+            RawRequest:  result.Request.Raw,
+            RawResponse: result.Response.Raw,
         }
         
         // 分发扫描任务
diff --git a/pkg/util/uniqueness.go b/pkg/util/uniqueness.go
index 1d18599..212411e 100644
--- a/pkg/util/uniqueness.go
+++ b/pkg/util/uniqueness.go
@@ -33,7 +33,7 @@ func UniqueId(req *proxy.Request) string {
         logging.Logger.Errorln(err)
         return ""
     }
-
+    
     return key
 }
 
@@ -49,17 +49,17 @@ func getRequestKey(req *proxy.Request) (string, error) {
     } else {
         host = req.URL.Host
     }
-
+    
     // 将请求方法和 URL（不包括查询参数）连接在一起
     data := req.Method + req.URL.Scheme + "://" + host + req.URL.Path
-
+    
     // 提取查询参数的名称  有的即使是 POST 请求，url请求路径中也会存在参数，所以这里全部都要提取
     var paramNames []string
     queryParams := req.URL.Query()
     for paramName := range queryParams {
         paramNames = append(paramNames, paramName)
     }
-
+    
     if req.Method == "POST" {
         contentType := req.Header.Get("Content-Type")
         if strings.Contains(contentType, "application/x-www-form-urlencoded") {
@@ -91,13 +91,13 @@ func getRequestKey(req *proxy.Request) (string, error) {
             }
         }
     }
-
+    
     // 对查询参数名称进行排序，以确保相同的参数集合具有相同的哈希值
     sort.Strings(paramNames)
-
+    
     // 将排序后的参数名称连接在一起并添加到数据字符串中
     data += strings.Join(paramNames, "")
-
+    
     // 计算 MD5 哈希值
     hash := md5.Sum([]byte(data))
     return hex.EncodeToString(hash[:]), nil
@@ -109,29 +109,29 @@ func SimpleUniqueId(u string) string {
     if err != nil {
         return ""
     }
-
+    
     if parseUrl.Scheme == "http" && strings.HasSuffix(parseUrl.Host, ":80") {
         parseUrl.Host = strings.TrimRight(parseUrl.Host, ":80")
     } else if parseUrl.Scheme == "https" && strings.HasSuffix(parseUrl.Host, ":443") {
         parseUrl.Host = strings.TrimRight(parseUrl.Host, ":443")
     }
-
+    
     // 将请求方法和 URL（不包括查询参数）连接在一起
     data := parseUrl.Scheme + "://" + parseUrl.Host + parseUrl.Path
-
+    
     // 提取查询参数的名称  有的即使是 POST 请求，url请求路径中也会存在参数，所以这里全部都要提取
     var paramNames []string
     queryParams := parseUrl.Query()
     for paramName := range queryParams {
         paramNames = append(paramNames, paramName)
     }
-
+    
     // 对查询参数名称进行排序，以确保相同的参数集合具有相同的哈希值
     sort.Strings(paramNames)
-
+    
     // 将排序后的参数名称连接在一起并添加到数据字符串中
     data += strings.Join(paramNames, "")
-
+    
     // 计算 MD5 哈希值
     hash := md5.Sum([]byte(data))
     return hex.EncodeToString(hash[:])
diff --git a/scan/gadget/sensitive/key.go b/scan/gadget/sensitive/key.go
index f203d62..a047411 100644
--- a/scan/gadget/sensitive/key.go
+++ b/scan/gadget/sensitive/key.go
@@ -117,7 +117,7 @@ func KeyDetection(url, body string) {
                     CreateTime: time.Now().Format("2006-01-02 15:04:05"),
                     Target:     url,
                     Payload:    strings.Join(matchedRegexes, ","),
-                    Response:   body,
+                    // Response:   body, // todo js 这种文本过大，不显示了
                 },
                 Level: output.Medium,
             }
diff --git a/test/xss_test.go b/test/xss_test.go
index 151afcc..985e25f 100644
--- a/test/xss_test.go
+++ b/test/xss_test.go
@@ -53,7 +53,7 @@ func TestDomXss(t *testing.T) {
         // Fingerprints: technologies,
     }
     
-    mode.Crawler("https://public-firing-range.appspot.com/dom/", nil, task, nil)
+    mode.Crawlergo("https://public-firing-range.appspot.com/dom/", nil, task, nil)
     fmt.Println(count)
 }