diff --git a/cmd/output.go b/cmd/output.go new file mode 100644 index 0000000..a2ceb29 --- /dev/null +++ b/cmd/output.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "fmt" + "github.com/mstxq17/MoreFind/core" + "github.com/mstxq17/MoreFind/errx" + "os" + "sync" +) + +// try for refactor in future +// 为以后统一输出做铺垫 +func syncOutput(wg *sync.WaitGroup, outputchan chan string) { + // 任务完成,增加计数 + defer wg.Done() + var f *os.File + if output != "" { + var err error + f, err = os.Create(output) + if err != nil { + logger.Fatal(errx.NewWithMsgf(err, "Could not create output file '%s':", file)) + } + defer f.Close() + } + for o := range outputchan { + if o == "" { + continue + } + // output to stdout & file stream + // 输出到 stdout & 文件流 + if len(myIPFormats) > 0 { + outputItems(f, core.AlterIP(o, myIPFormats)...) + } else { + outputItems(f, o) + } + } +} + +func outputItems(f *os.File, items ...string) { + for _, item := range items { + fmt.Println(item) + if f != nil { + _, _ = f.WriteString(item + "\n") + } + } +} diff --git a/cmd/root.go b/cmd/root.go index dfe59d9..bbe10ee 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,6 +17,7 @@ import ( "regexp" "strconv" "strings" + "sync" ) const ( @@ -124,7 +125,6 @@ func searchIp(line string) []IPAndPort { result = append(result, entry) } return result - //return ipRegex.FindAllString(line, -1) } func filterLen(lenRange string) (int, int) { @@ -201,7 +201,7 @@ func inc(ip net.IP) { } } -func genIP(cidr string) { +func genIP(cidr string, outputchan chan string) { // fix parse errx because of \n in window env // 修复 window 因为多了换行符导致的错误 cidr = strings.TrimSpace(cidr) @@ -211,7 +211,7 @@ func genIP(cidr string) { logger.Println("无法解析CIDR地址:", err) } else { for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { - fmt.Println(ip) + outputchan <- ip.String() } } } @@ -235,12 +235,12 @@ func genIP(cidr string) { } ipList := core.IPRange(startIPStr, endIPStr) for _, ip := range ipList { - fmt.Println(ip) + outputchan <- ip } } if !strings.Contains(cidr, "/") && !strings.Contains(cidr, "-") { cidr = cidr + "/24" - genIP(cidr) + genIP(cidr, outputchan) } } @@ -277,13 +277,23 @@ func updateCommand(cmd *cobra.Command, args []string) { } func preCommand(cmd *cobra.Command, args []string) bool { + // 输出 + outputchan := make(chan string) + var wg sync.WaitGroup + wg.Add(1) + go syncOutput(&wg, outputchan) // if cidr flag be selected，deal with it first // 如果选择 cidr 参数，首先处理它 if myCidr != "" && myCidr != "__pipe__" { - genIP(myCidr) + genIP(myCidr, outputchan) + close(outputchan) + wg.Wait() return true + } else { + close(outputchan) + wg.Wait() + return false } - return false } func runCommand(cmd *cobra.Command, args []string) { @@ -315,12 +325,23 @@ func runCommand(cmd *cobra.Command, args []string) { // support maximum 512MB buffer every line // 支持最大读取单行 512MB 大小 scanner.Buffer(buf, MaxTokenSize) + // 输出 + outputchan := make(chan string) + var wg sync.WaitGroup + wg.Add(1) + go syncOutput(&wg, outputchan) // todo: current structure may be chaotic, should abstract the handle process if myCidr == "__pipe__" { for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - genIP(line) + genIP(line, outputchan) } + // close the channel first + // 先关闭通道 + close(outputchan) + // wait for ending, exit + // 等待输出结束，然后结束程序 + wg.Wait() return } if myUrl == false && myDomain == false && myIp == false { @@ -369,7 +390,6 @@ func runCommand(cmd *cobra.Command, args []string) { // remove duplicated url // 去除重复的url found := make(map[string]bool) - foundWrite := make(map[string]bool) // define stream myself // 定义自己的输出流 var outputBuffer *core.MyBuffer @@ -397,15 +417,14 @@ func runCommand(cmd *cobra.Command, args []string) { if _, ok := found[_url]; !ok { if myUrlFilter != "" { if !filterExt(_url, myUrlFilter) { - outputBuffer.WriteString(_url, &customStringHandler, NewLine) + outputBuffer.WriteString(_url, &customStringHandler) found[_url] = true - foundWrite[outputBuffer.TempString] = true } } else { - outputBuffer.WriteString(_url, &customStringHandler, NewLine) + outputBuffer.WriteString(_url, &customStringHandler) found[_url] = true - foundWrite[outputBuffer.TempString] = true } + outputchan <- outputBuffer.TempString } } if myDomain == true { @@ -423,9 +442,9 @@ func runCommand(cmd *cobra.Command, args []string) { } // remove repeated string if _, ok := found[_domain]; !ok { - outputBuffer.WriteString(_domain, &customStringHandler, NewLine) + outputBuffer.WriteString(_domain, &customStringHandler) found[_domain] = true - foundWrite[outputBuffer.TempString] = true + outputchan <- outputBuffer.TempString } } } @@ -445,50 +464,25 @@ func runCommand(cmd *cobra.Command, args []string) { if _, ok := found[ipWithPort]; !ok { if myPrivateIp == true { if isPrivateIP(ipWithPort) == false { - outputBuffer.WriteString(ipWithPort, &customStringHandler, NewLine) - foundWrite[outputBuffer.TempString] = true + outputBuffer.WriteString(ipWithPort, &customStringHandler) found[ipWithPort] = true } } else { - outputBuffer.WriteString(ipWithPort, &customStringHandler, NewLine) - foundWrite[outputBuffer.TempString] = true + outputBuffer.WriteString(ipWithPort, &customStringHandler) found[ipWithPort] = true } + outputchan <- outputBuffer.TempString } } } - fmt.Print(outputBuffer.String()) outputBuffer.Reset() } // maybe exceed maxTokenSize length if err := scanner.Err(); err != nil { logger.Println(err) } - if output != "" { - _output, err := os.Create(output) - if err != nil { - logger.Fatal(err) - } - defer func(_output *os.File) { - err := _output.Close() - if err != nil { - logger.Fatal(err) - } - }(_output) - writer := bufio.NewWriter(_output) - //for key := range found { - for key := range foundWrite { - _, err := writer.WriteString(key) - if err != nil { - return - } - } - err = writer.Flush() - if err != nil { - logger.Fatal(err) - return - } - } + close(outputchan) + wg.Wait() } var ( @@ -508,6 +502,7 @@ var ( myFlag string myProgress bool myUpdate bool + myIPFormats []string rootCmd = &cobra.Command{ Use: "morefind", Short: "MoreFind is a very rapid script for extracting URL、Domain and Ip from data stream", @@ -559,8 +554,8 @@ func init() { // help me a lot, so log it in the code， google dork: "flag needs an argument: cobra" // 感谢 https://stackoverflow.com/questions/70182858/how-to-create-flag-with-or-without-argument-in-golang-using-cobra 提供了如何解决--filter 默认参数的问题 rootCmd.PersistentFlags().Lookup("filter").NoOptDefVal = "js,css,json,png,jpg,html,xml,zip,rar" - rootCmd.PersistentFlags().StringVarP(&myCidr, "cidr", "c", "", vars.CidrHelpEn) + rootCmd.PersistentFlags().StringSliceVarP(&myIPFormats, "alter", "a", nil, vars.AlterHelpEn) rootCmd.PersistentFlags().Lookup("cidr").NoOptDefVal = "__pipe__" rootCmd.PersistentFlags().StringVarP(&myLimitLen, "len", "l", "", vars.LimitLenHelpEn) rootCmd.PersistentFlags().BoolVarP(&myShow, "show", "s", false, vars.ShowHelpEn) diff --git a/core/ip.go b/core/ip.go new file mode 100644 index 0000000..909d963 --- /dev/null +++ b/core/ip.go @@ -0,0 +1,125 @@ +package core + +import ( + "encoding/hex" + "fmt" + "math/big" + "net" + "regexp" +) + +// AlterIP make some come true +// AlterIP 只选取部分实现 +func AlterIP(ip string, formats []string) []string { + var alteredIPs []string + for _, format := range formats { + standardIP := net.ParseIP(ip) + switch format { + case "1": + alteredIPs = append(alteredIPs, standardIP.String()) + case "2": + // 0-optimized dotted-decimal notation + // the 0 value segments of an IP address can be ommitted (eg. 127.0.0.1 => 127.1) + // regex for zeroes with dot 0000. + var reZeroesWithDot = regexp.MustCompile(`(?m)[0]+\.`) + // regex for .0000 + var reDotWithZeroes = regexp.MustCompile(`(?m)\.[0^]+$`) + // suppress 0000. + alteredIP := reZeroesWithDot.ReplaceAllString(standardIP.String(), "") + // suppress .0000 + alteredIP = reDotWithZeroes.ReplaceAllString(alteredIP, "") + alteredIPs = append(alteredIPs, alteredIP) + case "3": + // Octal notation (leading zeroes are required): + // eg: 127.0.0.1 => 0177.0.0.01 + alteredIP := fmt.Sprintf("%#04o.%#o.%#o.%#o", standardIP[12], standardIP[13], standardIP[14], standardIP[15]) + alteredIPs = append(alteredIPs, alteredIP) + case "4": + alteredIPWithDots := fmt.Sprintf("%#x.%#x.%#x.%#x", standardIP[12], standardIP[13], standardIP[14], standardIP[15]) + alteredIPWithZeroX := fmt.Sprintf("0x%s", hex.EncodeToString(standardIP[12:])) + alteredIPWithRandomPrefixHex, _ := RandomHex(5, standardIP[12:]) + alteredIPWithRandomPrefix := fmt.Sprintf("0x%s", alteredIPWithRandomPrefixHex) + alteredIPs = append(alteredIPs, alteredIPWithDots, alteredIPWithZeroX, alteredIPWithRandomPrefix) + case "5": + // Decimal notation a.k.a dword notation + // 127.0.0.1 => 2130706433 + bigIP, _, _ := IPToInteger(standardIP) + alteredIPs = append(alteredIPs, bigIP.String()) + case "6": + // Binary notation# + // 127.0.0.1 => 01111111000000000000000000000001 + // converts to int + bigIP, _, _ := IPToInteger(standardIP) + // then to binary + alteredIP := fmt.Sprintf("%b", bigIP) + alteredIPs = append(alteredIPs, alteredIP) + case "7": + // Mixed notation + // Ipv4 only + alteredIP := fmt.Sprintf("%#x.%d.%#o.%#x", standardIP[12], standardIP[13], standardIP[14], standardIP[15]) + alteredIPs = append(alteredIPs, alteredIP) + case "8": + // URL-encoded IP address + // 127.0.0.1 => %31%32%37%2E%30%2E%30%2E%31 + // ::1 => %3A%3A%31 + alteredIP := escape(ip) + alteredIPs = append(alteredIPs, alteredIP) + } + } + return alteredIPs +} + +func IPRange(startIPStr, endIPStr string) []string { + var ipList []string + startIPInt := ipToUInt32(startIPStr) + endIPInt := ipToUInt32(endIPStr) + for currIPInt := new(big.Int).Set(startIPInt); currIPInt.Cmp(endIPInt) <= 0; incIP(currIPInt) { + ip := intToIP(currIPInt) + ipList = append(ipList, ip) + } + + return ipList +} + +// IPToInteger converts an IP address to its integer representation. +// It supports both IPv4 as well as IPv6 addresses. +func IPToInteger(ip net.IP) (*big.Int, int, error) { + // Binary operation, learn something + // 二进制操作 + val := &big.Int{} + val.SetBytes([]byte(ip)) + + if len(ip) == net.IPv4len { + return val, 32, nil //nolint + } else if len(ip) == net.IPv6len { + return val, 128, nil //nolint + } else { + return nil, 0, fmt.Errorf("unsupported address length %d", len(ip)) + } +} + +func intToIP(ipInt *big.Int) string { + ipBytes := ipInt.Bytes() + if len(ipBytes) < 4 { + // 补齐 4 个字节 + padBytes := make([]byte, 4-len(ipBytes)) + ipBytes = append(padBytes, ipBytes...) + } + return net.IP(ipBytes).String() +} + +func ipToUInt32(ipStr string) *big.Int { + ip := net.ParseIP(ipStr) + if ip == nil { + return nil + } + + // 将 net.IP 转换为 4 字节的大整数 + ipInt := new(big.Int) + ipInt.SetBytes(ip.To4()) + return ipInt +} + +func incIP(ipInt *big.Int) { + ipInt.Add(ipInt, big.NewInt(1)) +} diff --git a/core/iprange.go b/core/iprange.go deleted file mode 100644 index 34b172f..0000000 --- a/core/iprange.go +++ /dev/null @@ -1,62 +0,0 @@ -package core - -import ( - "math/big" - "net" -) - -func IPRange(startIPStr, endIPStr string) []string { - var ipList []string - startIPInt := ipToUInt32(startIPStr) - endIPInt := ipToUInt32(endIPStr) - for currIPInt := new(big.Int).Set(startIPInt); currIPInt.Cmp(endIPInt) <= 0; incIP(currIPInt) { - ip := intToIP(currIPInt) - ipList = append(ipList, ip) - } - - return ipList -} - -func intToIP(ipInt *big.Int) string { - ipBytes := ipInt.Bytes() - if len(ipBytes) < 4 { - // 补齐 4 个字节 - padBytes := make([]byte, 4-len(ipBytes)) - ipBytes = append(padBytes, ipBytes...) - } - return net.IP(ipBytes).String() -} - -func ipToUInt32(ipStr string) *big.Int { - ip := net.ParseIP(ipStr) - if ip == nil { - return nil - } - - // 将 net.IP 转换为 4 字节的大整数 - ipInt := new(big.Int) - ipInt.SetBytes(ip.To4()) - return ipInt -} - -func incIP(ipInt *big.Int) { - ipInt.Add(ipInt, big.NewInt(1)) -} - -//func check -//func main() { -//startIPStr := "192.168.1.10" -//endIPStr := "192.168.10.15" -// -//startIPInt := ipToUInt32(startIPStr) -//endIPInt := ipToUInt32(endIPStr) -// -//if startIPInt != nil && endIPInt != nil { -// ipList := ipRange(startIPInt, endIPInt) -// for _, ip := range ipList { -// fmt.Println(ip) -// } -//} else { -// fmt.Println("无效的IP地址") -//} -//} diff --git a/core/myBuffer.go b/core/myBuffer.go index b67217a..3ff4518 100644 --- a/core/myBuffer.go +++ b/core/myBuffer.go @@ -36,15 +36,15 @@ func NewMyBuffer(isFilter bool) *MyBuffer { } } -func (_bytes *MyBuffer) WriteString(s string, handler stringHandler, newLine string) (n int, err error) { +func (_bytes *MyBuffer) WriteString(s string, handler stringHandler) (n int, err error) { // change the action of WriteString method // 修改 WriteString 方法的行为 if _bytes.IsFilter { - _bytes.TempString = handler.HandleString(s) + newLine + _bytes.TempString = handler.HandleString(s) return _bytes.buffer.WriteString(_bytes.TempString) } - _bytes.TempString = s + newLine - return _bytes.buffer.WriteString(s + newLine) + _bytes.TempString = s + return _bytes.buffer.WriteString(s) } func (_bytes *MyBuffer) String() string { diff --git a/core/utils.go b/core/utils.go index 46282b8..e3c98f9 100644 --- a/core/utils.go +++ b/core/utils.go @@ -1,10 +1,33 @@ package core import ( + "bytes" + "crypto/rand" + "encoding/hex" "fmt" "os" ) +const upperhex = "0123456789ABCDEF" + +func escape(s string) string { + var b bytes.Buffer + for i := 0; i < len(s); i++ { + b.WriteString("%") + b.WriteByte(upperhex[s[i]>>4]) + b.WriteByte(upperhex[s[i]&15]) + } + return b.String() +} + +func RandomHex(n int, suffix []byte) (string, error) { + bytes := make([]byte, n) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(append(bytes, suffix...)), nil +} + func NewLine() string { var PS = fmt.Sprintf("%v", os.PathSeparator) var LineBreak = "\n" diff --git a/vars/help.go b/vars/help.go index dd54c80..36db922 100644 --- a/vars/help.go +++ b/vars/help.go @@ -66,4 +66,7 @@ const ( ThresholdHelpEn = "Set threshold for smart strategy." ThresholdHelpZh = "设置智能策略的阈值。" + + AlterHelpEn = "IP Alters (0,1,2,3,4,5,6,7,8)" + AlterHelpZh = "IP 变换 (0,1,2,3,4,5,6,7,8)" )