feat: Match schema://host format from input data

cmd/root.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"github.com/cheggaaa/pb/v3"
 	"github.com/mstxq17/MoreFind/core"
+	"github.com/mstxq17/MoreFind/core/extract"
+	"github.com/mstxq17/MoreFind/core/utils/uniqueutil"
 	"github.com/mstxq17/MoreFind/update"
 	"github.com/mstxq17/MoreFind/vars"
 	"github.com/spf13/cobra"
@@ -354,7 +356,7 @@ func runCommand(cmd *cobra.Command, args []string) {
 			}
 		}
 	}
-	if myUrl == false && myDomain == false && myIp == false {
+	if myUrl == false && myDomain == false && myIp == false && myLink == false {
 		myUrl = true
 	}
 	var urlList []string
@@ -378,16 +380,40 @@ func runCommand(cmd *cobra.Command, args []string) {
 	}
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
-		if myUrl == true || myDomain == true {
+		if myUrl == true || myDomain == true || myLink == true {
 			searchUrl := searchUrl(line)
 			for _, _url := range searchUrl {
-				_url = strings.TrimSpace(_url)
+				trimUrl := strings.TrimSpace(_url)
+				if myLink == true {
+					simpleUrl, hasSchema, err := extract.SimpleUrl(trimUrl)
+					if err != nil {
+						logger.Printf("handle:  %v  Err: %v \n", trimUrl, err)
+						continue
+					}
+					if mySchema != "" && !hasSchema {
+						// 写入输出
+						simpleUrl = fmt.Sprintf("%v://%v", mySchema, simpleUrl)
+						if uniqueutil.IsKeyUniq(simpleUrl, found) {
+							outputBuffer.WriteString(simpleUrl, &customStringHandler)
+							found[simpleUrl] = struct{}{}
+							outputchan <- outputBuffer.TempString
+						}
+					} else {
+						// 写入输出
+						if uniqueutil.IsKeyUniq(simpleUrl, found) {
+							outputBuffer.WriteString(simpleUrl, &customStringHandler)
+							found[simpleUrl] = struct{}{}
+							outputchan <- outputBuffer.TempString
+						}
+					}
+				}
+
 				if myUrl == true {
 					if output != "" {
-						urlList = append(urlList, _url)
+						urlList = append(urlList, trimUrl)
 					}
 					// remove repeated string
-					if _, ok := found[_url]; !ok {
+					if _, ok := found[trimUrl]; !ok {
 						if myUrlFilter != "" {
 							if !filterExt(_url, myUrlFilter) {
 								outputBuffer.WriteString(_url, &customStringHandler)
@@ -401,7 +427,7 @@ func runCommand(cmd *cobra.Command, args []string) {
 					}
 				}
 				if myDomain == true {
-					port, _domain := searchDomain(_url, myRootDomain)
+					port, _domain := searchDomain(trimUrl, myRootDomain)
 					if _domain == "" || isIPAddr(_domain) {
 						continue
 					}
@@ -476,6 +502,8 @@ var (
 	myProgress   bool
 	myUpdate     bool
 	myQuiet      bool
+	myLink       bool
+	mySchema     string
 	myXlsx       string
 	myIPFormats  []string
 	rootCmd      = &cobra.Command{
@@ -516,6 +544,9 @@ func init() {
 	NewLine = core.NewLine()
 	rootCmd.PersistentFlags().StringVarP(&file, "file", "f", "", vars.FileHelpEn)
 	rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", vars.OutputHelpEn)
+	rootCmd.PersistentFlags().BoolVarP(&myLink, "link", "k", false, vars.TargetHelpEn)
+	rootCmd.PersistentFlags().StringVarP(&mySchema, "schema", "", "", vars.SchemaHelpEn)
+	rootCmd.PersistentFlags().Lookup("schema").NoOptDefVal = ""
 	rootCmd.PersistentFlags().BoolVarP(&myIp, "ip", "i", false, vars.IPHelpEn)
 	rootCmd.PersistentFlags().BoolVarP(&myPrivateIp, "exclude", "", false, vars.ExcludeHelpEn)
 	rootCmd.PersistentFlags().BoolVarP(&myDomain, "domain", "d", false, vars.DomainHelpEn)

core/extract/simple_url.go

@@ -0,0 +1,27 @@
+package extract
+
+import (
+	"fmt"
+	"net/url"
+)
+
+func SimpleUrl(input string) (string, bool, error) {
+	parsed, err := url.Parse(input)
+	if err != nil {
+		return "", false, err
+	}
+	if parsed.Scheme != "" {
+		tSimpleUrl := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
+		return tSimpleUrl, true, nil
+	} else {
+		//  拼接协议进行解析
+		modifiedHost := fmt.Sprintf("http://%s", input)
+		parsed, err := url.Parse(modifiedHost)
+		if err != nil {
+			return "", false, err
+		} else {
+			tSimpleUrl := fmt.Sprintf("%s", parsed.Hostname())
+			return tSimpleUrl, false, nil
+		}
+	}
+}

core/extract/simple_url_test.go

@@ -0,0 +1,34 @@
+package extract
+
+import "testing"
+
+// Test cases for SimpleUrl
+func TestSimpleUrl(t *testing.T) {
+	testCases := []struct {
+		input     string
+		expected  string
+		hasScheme bool
+	}{
+		{"http://example.com:8080", "http://example.com:8080", true},
+		{"http://example.com", "http://example.com", true},
+		{"https://example.com/path", "https://example.com", true},
+		{"ftp://example.com", "ftp://example.com", true},
+		{"example.com/123.php?a=1", "example.com", false},
+		{"example.com", "example.com", false},
+		{"http://", "http://", true}, // Invalid URL
+		{"", "", false},              // Empty string
+	}
+
+	for _, tc := range testCases {
+		result, hasScheme, err := SimpleUrl(tc.input)
+		if err != nil {
+			t.Errorf("input: %s, unexpected error: %v", tc.input, err)
+		}
+		if result != tc.expected {
+			t.Errorf("input: %s, expected: %s, got: %s", tc.input, tc.expected, result)
+		}
+		if hasScheme != tc.hasScheme {
+			t.Errorf("input: %s, expected hasScheme: %v, got: %v", tc.input, tc.hasScheme, hasScheme)
+		}
+	}
+}

core/utils/uniqueutil/uniqueutil.go

@@ -0,0 +1,8 @@
+package uniqueutil
+
+func IsKeyUniq(key string, found map[string]struct{}) bool {
+	if _, ok := found[key]; !ok {
+		return true
+	}
+	return false
+}

vars/help.go

@@ -13,6 +13,12 @@ const (
 	IPHelpEn = "Matches IPs from the input pipe or file."
 	IPHelpZh = "从输入管道或文件中匹配 IP。"
 
+	TargetHelpEn = "Matches schema://host from the input pipe or file."
+	TargetHelpZh = "从输入管道或文件中匹配 schema://host。"
+
+	SchemaHelpEn = "When use with -t, the default is set to the specified protocol"
+	SchemaHelpZh = "与 -t 一起使用，默认设置为指定协议"
+
 	ExcludeHelpEn = "Excludes internal/private IP segments when using -i/--ip."
 	ExcludeHelpZh = "在使用 -i/--ip 时排除内部/私有 IP 段。"
 

vars/version.go

@@ -2,4 +2,4 @@ package vars
 
 // VERSION number
 // 版本号
-const VERSION string = "1.5.6"
+const VERSION string = "1.5.7"