diff --git a/cmd/root.go b/cmd/root.go index cdaaa55..03dedea 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/cheggaaa/pb/v3" "github.com/mstxq17/MoreFind/core" + "github.com/mstxq17/MoreFind/core/extract" + "github.com/mstxq17/MoreFind/core/utils/uniqueutil" "github.com/mstxq17/MoreFind/update" "github.com/mstxq17/MoreFind/vars" "github.com/spf13/cobra" @@ -354,7 +356,7 @@ func runCommand(cmd *cobra.Command, args []string) { } } } - if myUrl == false && myDomain == false && myIp == false { + if myUrl == false && myDomain == false && myIp == false && myLink == false { myUrl = true } var urlList []string @@ -378,16 +380,40 @@ func runCommand(cmd *cobra.Command, args []string) { } for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - if myUrl == true || myDomain == true { + if myUrl == true || myDomain == true || myLink == true { searchUrl := searchUrl(line) for _, _url := range searchUrl { - _url = strings.TrimSpace(_url) + trimUrl := strings.TrimSpace(_url) + if myLink == true { + simpleUrl, hasSchema, err := extract.SimpleUrl(trimUrl) + if err != nil { + logger.Printf("handle: %v Err: %v \n", trimUrl, err) + continue + } + if mySchema != "" && !hasSchema { + // 写入输出 + simpleUrl = fmt.Sprintf("%v://%v", mySchema, simpleUrl) + if uniqueutil.IsKeyUniq(simpleUrl, found) { + outputBuffer.WriteString(simpleUrl, &customStringHandler) + found[simpleUrl] = struct{}{} + outputchan <- outputBuffer.TempString + } + } else { + // 写入输出 + if uniqueutil.IsKeyUniq(simpleUrl, found) { + outputBuffer.WriteString(simpleUrl, &customStringHandler) + found[simpleUrl] = struct{}{} + outputchan <- outputBuffer.TempString + } + } + } + if myUrl == true { if output != "" { - urlList = append(urlList, _url) + urlList = append(urlList, trimUrl) } // remove repeated string - if _, ok := found[_url]; !ok { + if _, ok := found[trimUrl]; !ok { if myUrlFilter != "" { if !filterExt(_url, myUrlFilter) { outputBuffer.WriteString(_url, &customStringHandler) @@ -401,7 +427,7 @@ func runCommand(cmd *cobra.Command, args []string) { } } if myDomain == true { - port, _domain := searchDomain(_url, myRootDomain) + port, _domain := searchDomain(trimUrl, myRootDomain) if _domain == "" || isIPAddr(_domain) { continue } @@ -476,6 +502,8 @@ var ( myProgress bool myUpdate bool myQuiet bool + myLink bool + mySchema string myXlsx string myIPFormats []string rootCmd = &cobra.Command{ @@ -516,6 +544,9 @@ func init() { NewLine = core.NewLine() rootCmd.PersistentFlags().StringVarP(&file, "file", "f", "", vars.FileHelpEn) rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", vars.OutputHelpEn) + rootCmd.PersistentFlags().BoolVarP(&myLink, "link", "k", false, vars.TargetHelpEn) + rootCmd.PersistentFlags().StringVarP(&mySchema, "schema", "", "", vars.SchemaHelpEn) + rootCmd.PersistentFlags().Lookup("schema").NoOptDefVal = "" rootCmd.PersistentFlags().BoolVarP(&myIp, "ip", "i", false, vars.IPHelpEn) rootCmd.PersistentFlags().BoolVarP(&myPrivateIp, "exclude", "", false, vars.ExcludeHelpEn) rootCmd.PersistentFlags().BoolVarP(&myDomain, "domain", "d", false, vars.DomainHelpEn) diff --git a/core/extract/simple_url.go b/core/extract/simple_url.go new file mode 100644 index 0000000..70a27d1 --- /dev/null +++ b/core/extract/simple_url.go @@ -0,0 +1,27 @@ +package extract + +import ( + "fmt" + "net/url" +) + +func SimpleUrl(input string) (string, bool, error) { + parsed, err := url.Parse(input) + if err != nil { + return "", false, err + } + if parsed.Scheme != "" { + tSimpleUrl := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) + return tSimpleUrl, true, nil + } else { + // 拼接协议进行解析 + modifiedHost := fmt.Sprintf("http://%s", input) + parsed, err := url.Parse(modifiedHost) + if err != nil { + return "", false, err + } else { + tSimpleUrl := fmt.Sprintf("%s", parsed.Hostname()) + return tSimpleUrl, false, nil + } + } +} diff --git a/core/extract/simple_url_test.go b/core/extract/simple_url_test.go new file mode 100644 index 0000000..ce204a8 --- /dev/null +++ b/core/extract/simple_url_test.go @@ -0,0 +1,34 @@ +package extract + +import "testing" + +// Test cases for SimpleUrl +func TestSimpleUrl(t *testing.T) { + testCases := []struct { + input string + expected string + hasScheme bool + }{ + {"http://example.com:8080", "http://example.com:8080", true}, + {"http://example.com", "http://example.com", true}, + {"https://example.com/path", "https://example.com", true}, + {"ftp://example.com", "ftp://example.com", true}, + {"example.com/123.php?a=1", "example.com", false}, + {"example.com", "example.com", false}, + {"http://", "http://", true}, // Invalid URL + {"", "", false}, // Empty string + } + + for _, tc := range testCases { + result, hasScheme, err := SimpleUrl(tc.input) + if err != nil { + t.Errorf("input: %s, unexpected error: %v", tc.input, err) + } + if result != tc.expected { + t.Errorf("input: %s, expected: %s, got: %s", tc.input, tc.expected, result) + } + if hasScheme != tc.hasScheme { + t.Errorf("input: %s, expected hasScheme: %v, got: %v", tc.input, tc.hasScheme, hasScheme) + } + } +} diff --git a/core/utils/uniqueutil/uniqueutil.go b/core/utils/uniqueutil/uniqueutil.go new file mode 100644 index 0000000..1f0264a --- /dev/null +++ b/core/utils/uniqueutil/uniqueutil.go @@ -0,0 +1,8 @@ +package uniqueutil + +func IsKeyUniq(key string, found map[string]struct{}) bool { + if _, ok := found[key]; !ok { + return true + } + return false +} diff --git a/vars/help.go b/vars/help.go index c345fa0..902860c 100644 --- a/vars/help.go +++ b/vars/help.go @@ -13,6 +13,12 @@ const ( IPHelpEn = "Matches IPs from the input pipe or file." IPHelpZh = "从输入管道或文件中匹配 IP。" + TargetHelpEn = "Matches schema://host from the input pipe or file." + TargetHelpZh = "从输入管道或文件中匹配 schema://host。" + + SchemaHelpEn = "When use with -t, the default is set to the specified protocol" + SchemaHelpZh = "与 -t 一起使用,默认设置为指定协议" + ExcludeHelpEn = "Excludes internal/private IP segments when using -i/--ip." ExcludeHelpZh = "在使用 -i/--ip 时排除内部/私有 IP 段。" diff --git a/vars/version.go b/vars/version.go index d64d103..e676dec 100644 --- a/vars/version.go +++ b/vars/version.go @@ -2,4 +2,4 @@ package vars // VERSION number // 版本号 -const VERSION string = "1.5.6" +const VERSION string = "1.5.7"