Skip to content

Commit

Permalink
feat: support alter ip format
Browse files Browse the repository at this point in the history
  • Loading branch information
mstxq17 committed Nov 4, 2023
1 parent 5701714 commit 95591e5
Show file tree
Hide file tree
Showing 7 changed files with 242 additions and 112 deletions.
46 changes: 46 additions & 0 deletions cmd/output.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package cmd

import (
"fmt"
"github.com/mstxq17/MoreFind/core"
"github.com/mstxq17/MoreFind/errx"
"os"
"sync"
)

// try for refactor in future
// 为以后统一输出做铺垫
func syncOutput(wg *sync.WaitGroup, outputchan chan string) {
// 任务完成,增加计数
defer wg.Done()
var f *os.File
if output != "" {
var err error
f, err = os.Create(output)
if err != nil {
logger.Fatal(errx.NewWithMsgf(err, "Could not create output file '%s':", file))
}
defer f.Close()
}
for o := range outputchan {
if o == "" {
continue
}
// output to stdout & file stream
// 输出到 stdout & 文件流
if len(myIPFormats) > 0 {
outputItems(f, core.AlterIP(o, myIPFormats)...)
} else {
outputItems(f, o)
}
}
}

func outputItems(f *os.File, items ...string) {
for _, item := range items {
fmt.Println(item)
if f != nil {
_, _ = f.WriteString(item + "\n")
}
}
}
87 changes: 41 additions & 46 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
)

const (
Expand Down Expand Up @@ -124,7 +125,6 @@ func searchIp(line string) []IPAndPort {
result = append(result, entry)
}
return result
//return ipRegex.FindAllString(line, -1)
}

func filterLen(lenRange string) (int, int) {
Expand Down Expand Up @@ -201,7 +201,7 @@ func inc(ip net.IP) {
}
}

func genIP(cidr string) {
func genIP(cidr string, outputchan chan string) {
// fix parse errx because of \n in window env
// 修复 window 因为多了换行符导致的错误
cidr = strings.TrimSpace(cidr)
Expand All @@ -211,7 +211,7 @@ func genIP(cidr string) {
logger.Println("无法解析CIDR地址:", err)
} else {
for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
fmt.Println(ip)
outputchan <- ip.String()
}
}
}
Expand All @@ -235,12 +235,12 @@ func genIP(cidr string) {
}
ipList := core.IPRange(startIPStr, endIPStr)
for _, ip := range ipList {
fmt.Println(ip)
outputchan <- ip
}
}
if !strings.Contains(cidr, "/") && !strings.Contains(cidr, "-") {
cidr = cidr + "/24"
genIP(cidr)
genIP(cidr, outputchan)
}
}

Expand Down Expand Up @@ -277,13 +277,23 @@ func updateCommand(cmd *cobra.Command, args []string) {
}

func preCommand(cmd *cobra.Command, args []string) bool {
// 输出
outputchan := make(chan string)
var wg sync.WaitGroup
wg.Add(1)
go syncOutput(&wg, outputchan)
// if cidr flag be selected,deal with it first
// 如果选择 cidr 参数,首先处理它
if myCidr != "" && myCidr != "__pipe__" {
genIP(myCidr)
genIP(myCidr, outputchan)
close(outputchan)
wg.Wait()
return true
} else {
close(outputchan)
wg.Wait()
return false
}
return false
}

func runCommand(cmd *cobra.Command, args []string) {
Expand Down Expand Up @@ -315,12 +325,23 @@ func runCommand(cmd *cobra.Command, args []string) {
// support maximum 512MB buffer every line
// 支持最大读取单行 512MB 大小
scanner.Buffer(buf, MaxTokenSize)
// 输出
outputchan := make(chan string)
var wg sync.WaitGroup
wg.Add(1)
go syncOutput(&wg, outputchan)
// todo: current structure may be chaotic, should abstract the handle process
if myCidr == "__pipe__" {
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
genIP(line)
genIP(line, outputchan)
}
// close the channel first
// 先关闭通道
close(outputchan)
// wait for ending, exit
// 等待输出结束,然后结束程序
wg.Wait()
return
}
if myUrl == false && myDomain == false && myIp == false {
Expand Down Expand Up @@ -369,7 +390,6 @@ func runCommand(cmd *cobra.Command, args []string) {
// remove duplicated url
// 去除重复的url
found := make(map[string]bool)
foundWrite := make(map[string]bool)
// define stream myself
// 定义自己的输出流
var outputBuffer *core.MyBuffer
Expand Down Expand Up @@ -397,15 +417,14 @@ func runCommand(cmd *cobra.Command, args []string) {
if _, ok := found[_url]; !ok {
if myUrlFilter != "" {
if !filterExt(_url, myUrlFilter) {
outputBuffer.WriteString(_url, &customStringHandler, NewLine)
outputBuffer.WriteString(_url, &customStringHandler)
found[_url] = true
foundWrite[outputBuffer.TempString] = true
}
} else {
outputBuffer.WriteString(_url, &customStringHandler, NewLine)
outputBuffer.WriteString(_url, &customStringHandler)
found[_url] = true
foundWrite[outputBuffer.TempString] = true
}
outputchan <- outputBuffer.TempString
}
}
if myDomain == true {
Expand All @@ -423,9 +442,9 @@ func runCommand(cmd *cobra.Command, args []string) {
}
// remove repeated string
if _, ok := found[_domain]; !ok {
outputBuffer.WriteString(_domain, &customStringHandler, NewLine)
outputBuffer.WriteString(_domain, &customStringHandler)
found[_domain] = true
foundWrite[outputBuffer.TempString] = true
outputchan <- outputBuffer.TempString
}
}
}
Expand All @@ -445,50 +464,25 @@ func runCommand(cmd *cobra.Command, args []string) {
if _, ok := found[ipWithPort]; !ok {
if myPrivateIp == true {
if isPrivateIP(ipWithPort) == false {
outputBuffer.WriteString(ipWithPort, &customStringHandler, NewLine)
foundWrite[outputBuffer.TempString] = true
outputBuffer.WriteString(ipWithPort, &customStringHandler)
found[ipWithPort] = true
}
} else {
outputBuffer.WriteString(ipWithPort, &customStringHandler, NewLine)
foundWrite[outputBuffer.TempString] = true
outputBuffer.WriteString(ipWithPort, &customStringHandler)
found[ipWithPort] = true
}
outputchan <- outputBuffer.TempString
}
}
}
fmt.Print(outputBuffer.String())
outputBuffer.Reset()
}
// maybe exceed maxTokenSize length
if err := scanner.Err(); err != nil {
logger.Println(err)
}
if output != "" {
_output, err := os.Create(output)
if err != nil {
logger.Fatal(err)
}
defer func(_output *os.File) {
err := _output.Close()
if err != nil {
logger.Fatal(err)
}
}(_output)
writer := bufio.NewWriter(_output)
//for key := range found {
for key := range foundWrite {
_, err := writer.WriteString(key)
if err != nil {
return
}
}
err = writer.Flush()
if err != nil {
logger.Fatal(err)
return
}
}
close(outputchan)
wg.Wait()
}

var (
Expand All @@ -508,6 +502,7 @@ var (
myFlag string
myProgress bool
myUpdate bool
myIPFormats []string
rootCmd = &cobra.Command{
Use: "morefind",
Short: "MoreFind is a very rapid script for extracting URL、Domain and Ip from data stream",
Expand Down Expand Up @@ -559,8 +554,8 @@ func init() {
// help me a lot, so log it in the code, google dork: "flag needs an argument: cobra"
// 感谢 https://stackoverflow.com/questions/70182858/how-to-create-flag-with-or-without-argument-in-golang-using-cobra 提供了如何解决--filter 默认参数的问题
rootCmd.PersistentFlags().Lookup("filter").NoOptDefVal = "js,css,json,png,jpg,html,xml,zip,rar"

rootCmd.PersistentFlags().StringVarP(&myCidr, "cidr", "c", "", vars.CidrHelpEn)
rootCmd.PersistentFlags().StringSliceVarP(&myIPFormats, "alter", "a", nil, vars.AlterHelpEn)
rootCmd.PersistentFlags().Lookup("cidr").NoOptDefVal = "__pipe__"
rootCmd.PersistentFlags().StringVarP(&myLimitLen, "len", "l", "", vars.LimitLenHelpEn)
rootCmd.PersistentFlags().BoolVarP(&myShow, "show", "s", false, vars.ShowHelpEn)
Expand Down
125 changes: 125 additions & 0 deletions core/ip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package core

import (
"encoding/hex"
"fmt"
"math/big"
"net"
"regexp"
)

// AlterIP make some come true
// AlterIP 只选取部分实现
func AlterIP(ip string, formats []string) []string {
var alteredIPs []string
for _, format := range formats {
standardIP := net.ParseIP(ip)
switch format {
case "1":
alteredIPs = append(alteredIPs, standardIP.String())
case "2":
// 0-optimized dotted-decimal notation
// the 0 value segments of an IP address can be ommitted (eg. 127.0.0.1 => 127.1)
// regex for zeroes with dot 0000.
var reZeroesWithDot = regexp.MustCompile(`(?m)[0]+\.`)
// regex for .0000
var reDotWithZeroes = regexp.MustCompile(`(?m)\.[0^]+$`)
// suppress 0000.
alteredIP := reZeroesWithDot.ReplaceAllString(standardIP.String(), "")
// suppress .0000
alteredIP = reDotWithZeroes.ReplaceAllString(alteredIP, "")
alteredIPs = append(alteredIPs, alteredIP)
case "3":
// Octal notation (leading zeroes are required):
// eg: 127.0.0.1 => 0177.0.0.01
alteredIP := fmt.Sprintf("%#04o.%#o.%#o.%#o", standardIP[12], standardIP[13], standardIP[14], standardIP[15])
alteredIPs = append(alteredIPs, alteredIP)
case "4":
alteredIPWithDots := fmt.Sprintf("%#x.%#x.%#x.%#x", standardIP[12], standardIP[13], standardIP[14], standardIP[15])
alteredIPWithZeroX := fmt.Sprintf("0x%s", hex.EncodeToString(standardIP[12:]))
alteredIPWithRandomPrefixHex, _ := RandomHex(5, standardIP[12:])
alteredIPWithRandomPrefix := fmt.Sprintf("0x%s", alteredIPWithRandomPrefixHex)
alteredIPs = append(alteredIPs, alteredIPWithDots, alteredIPWithZeroX, alteredIPWithRandomPrefix)
case "5":
// Decimal notation a.k.a dword notation
// 127.0.0.1 => 2130706433
bigIP, _, _ := IPToInteger(standardIP)
alteredIPs = append(alteredIPs, bigIP.String())
case "6":
// Binary notation#
// 127.0.0.1 => 01111111000000000000000000000001
// converts to int
bigIP, _, _ := IPToInteger(standardIP)
// then to binary
alteredIP := fmt.Sprintf("%b", bigIP)
alteredIPs = append(alteredIPs, alteredIP)
case "7":
// Mixed notation
// Ipv4 only
alteredIP := fmt.Sprintf("%#x.%d.%#o.%#x", standardIP[12], standardIP[13], standardIP[14], standardIP[15])
alteredIPs = append(alteredIPs, alteredIP)
case "8":
// URL-encoded IP address
// 127.0.0.1 => %31%32%37%2E%30%2E%30%2E%31
// ::1 => %3A%3A%31
alteredIP := escape(ip)
alteredIPs = append(alteredIPs, alteredIP)
}
}
return alteredIPs
}

func IPRange(startIPStr, endIPStr string) []string {
var ipList []string
startIPInt := ipToUInt32(startIPStr)
endIPInt := ipToUInt32(endIPStr)
for currIPInt := new(big.Int).Set(startIPInt); currIPInt.Cmp(endIPInt) <= 0; incIP(currIPInt) {
ip := intToIP(currIPInt)
ipList = append(ipList, ip)
}

return ipList
}

// IPToInteger converts an IP address to its integer representation.
// It supports both IPv4 as well as IPv6 addresses.
func IPToInteger(ip net.IP) (*big.Int, int, error) {
// Binary operation, learn something
// 二进制操作
val := &big.Int{}
val.SetBytes([]byte(ip))

if len(ip) == net.IPv4len {
return val, 32, nil //nolint
} else if len(ip) == net.IPv6len {
return val, 128, nil //nolint
} else {
return nil, 0, fmt.Errorf("unsupported address length %d", len(ip))
}
}

func intToIP(ipInt *big.Int) string {
ipBytes := ipInt.Bytes()
if len(ipBytes) < 4 {
// 补齐 4 个字节
padBytes := make([]byte, 4-len(ipBytes))
ipBytes = append(padBytes, ipBytes...)
}
return net.IP(ipBytes).String()
}

func ipToUInt32(ipStr string) *big.Int {
ip := net.ParseIP(ipStr)
if ip == nil {
return nil
}

// 将 net.IP 转换为 4 字节的大整数
ipInt := new(big.Int)
ipInt.SetBytes(ip.To4())
return ipInt
}

func incIP(ipInt *big.Int) {
ipInt.Add(ipInt, big.NewInt(1))
}
Loading

0 comments on commit 95591e5

Please sign in to comment.