From abcea6f14438e80b2d27b3dd68c236448037f80d Mon Sep 17 00:00:00 2001 From: beerpiss Date: Sat, 14 May 2022 13:16:12 +0700 Subject: [PATCH 1/5] search from other known buckets --- main.go | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index b3f0622..c1ba58b 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,12 @@ package main import ( "fmt" + "io/ioutil" "log" + "net/http" "os" "path/filepath" + "regexp" "sort" "strings" "sync" @@ -80,11 +83,113 @@ func main() { wg.Wait() // print results and exit with status code - if !printResults(matches.data) { + var hasResult bool + hasResult = printResults(matches.data, false) + if !hasResult && !githubRatelimitReached() { + hasResult = printResults(searchRemoteAll(args.query), true) + } + + if !hasResult { + fmt.Println("No matches found.") os.Exit(1) } } +func githubRatelimitReached() bool { + var parser fastjson.Parser + + response, err := http.Get("https://api.github.com/rate_limit") + check(err) + + raw, err := ioutil.ReadAll(response.Body) + check(err) + + parse, _ := parser.ParseBytes(raw) + json, _ := parse.Object() + + return json.Get("resources").Get("core").GetInt("limit") == 0 +} + +func searchRemoteAll(term string) matchMap { + var parser fastjson.Parser + + raw, err := os.ReadFile(scoopHome() + "\\apps\\scoop\\current\\buckets.json") + check(err) + + result, _ := parser.ParseBytes(raw) + object, _ := result.Object() + var buckets []string + object.Visit(func(k []byte, v *fastjson.Value) { + _, err := os.Stat(scoopHome() + "\\buckets\\" + string(k)) + if os.IsNotExist(err) { + buckets = append(buckets, string(k)) + } + }) + + matches := struct { + sync.Mutex + data matchMap + }{} + matches.data = make(matchMap) + var wg sync.WaitGroup + for _, bucket := range buckets { + wg.Add(1) + go func(b string) { + res := searchRemote(b, term) + matches.Lock() + matches.data[b] = res + matches.Unlock() + wg.Done() + }(bucket) + } + wg.Wait() + return matches.data +} + +func searchRemote(bucket string, term string) []match { + var parser fastjson.Parser + + raw, err := os.ReadFile(scoopHome() + "\\apps\\scoop\\current\\buckets.json") + check(err) + + result, _ := parser.ParseBytes(raw) + bucketURL := string(result.GetStringBytes(bucket)) + bucketSplit := strings.Split(bucketURL, "/") + apiLink := "https://api.github.com/repos/" + bucketSplit[len(bucketSplit)-2] + "/" + bucketSplit[len(bucketSplit)-1] + "/git/trees/HEAD?recursive=1" + + response, err := http.Get(apiLink) + check(err) + + raw, err = ioutil.ReadAll(response.Body) + check(err) + + json, _ := parser.ParseBytes(raw) + fileTree := json.GetArray("tree") + + regex := regexp.MustCompile("(?i)^(?:bucket/)?(.*" + term + ".*)\\.json$") + + matches := struct { + sync.Mutex + data []match + }{} + var wg sync.WaitGroup + + for _, file := range fileTree { + wg.Add(1) + go func(path []byte) { + matching := regex.FindSubmatch(path) + if len(matching) > 1 { + matches.Lock() + matches.data = append(matches.data, match{name: string(matching[1]), version: "", bin: ""}) + matches.Unlock() + } + wg.Done() + }(file.GetStringBytes("path")) + } + wg.Wait() + return matches.data +} + func matchingManifests(path string, term string) (res []match) { term = strings.ToLower(term) files, err := os.ReadDir(path) @@ -165,7 +270,7 @@ func matchingManifests(path string, term string) (res []match) { return } -func printResults(data matchMap) (anyMatches bool) { +func printResults(data matchMap, fromKnownBucket bool) (anyMatches bool) { // sort by bucket names entries := 0 sortedKeys := make([]string, 0, len(data)) @@ -184,15 +289,37 @@ func printResults(data matchMap) (anyMatches bool) { if len(v) > 0 { anyMatches = true + } + } + + if fromKnownBucket && anyMatches { + fmt.Println("Results from other known buckets...") + fmt.Println("(add them using 'scoop bucket add ')") + fmt.Println() + } + + for _, k := range sortedKeys { + v := data[k] + + if len(v) > 0 { display.WriteString("'") display.WriteString(k) - display.WriteString("' bucket:\n") + display.WriteString("' bucket") + if fromKnownBucket { + display.WriteString(" (install using 'scoop install ") + display.WriteString(k) + display.WriteString("/'):\n") + } else { + display.WriteString(":\n") + } for _, m := range v { display.WriteString(" ") display.WriteString(m.name) - display.WriteString(" (") - display.WriteString(m.version) - display.WriteString(")") + if !fromKnownBucket { + display.WriteString(" (") + display.WriteString(m.version) + display.WriteString(")") + } if m.bin != "" { display.WriteString(" --> includes '") display.WriteString(m.bin) @@ -204,10 +331,6 @@ func printResults(data matchMap) (anyMatches bool) { } } - if !anyMatches { - display.WriteString("No matches found.") - } - os.Stdout.WriteString(display.String()) return } From 04b45162d41160848a6fc4c74613b33a78faacc6 Mon Sep 17 00:00:00 2001 From: beerpiss Date: Sat, 14 May 2022 15:18:00 +0700 Subject: [PATCH 2/5] improved ratelimit handling online searching prematurely ends when the API limit hits --- main.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index c1ba58b..c0c662f 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "sync" @@ -120,8 +121,7 @@ func searchRemoteAll(term string) matchMap { object, _ := result.Object() var buckets []string object.Visit(func(k []byte, v *fastjson.Value) { - _, err := os.Stat(scoopHome() + "\\buckets\\" + string(k)) - if os.IsNotExist(err) { + if _, err := os.Stat(scoopHome() + "\\buckets\\" + string(k)); os.IsNotExist(err) { buckets = append(buckets, string(k)) } }) @@ -132,21 +132,25 @@ func searchRemoteAll(term string) matchMap { }{} matches.data = make(matchMap) var wg sync.WaitGroup + var hasReachedApiLimit bool = false for _, bucket := range buckets { - wg.Add(1) - go func(b string) { - res := searchRemote(b, term) - matches.Lock() - matches.data[b] = res - matches.Unlock() - wg.Done() - }(bucket) + if !hasReachedApiLimit { + wg.Add(1) + go func(b string) { + var res []match + res, hasReachedApiLimit = searchRemote(b, term) + matches.Lock() + matches.data[b] = res + matches.Unlock() + wg.Done() + }(bucket) + } } wg.Wait() return matches.data } -func searchRemote(bucket string, term string) []match { +func searchRemote(bucket string, term string) (res []match, hasReachedApiLimit bool) { var parser fastjson.Parser raw, err := os.ReadFile(scoopHome() + "\\apps\\scoop\\current\\buckets.json") @@ -160,6 +164,10 @@ func searchRemote(bucket string, term string) []match { response, err := http.Get(apiLink) check(err) + requestsLeft, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining")) + check(err) + hasReachedApiLimit = requestsLeft == 0 + raw, err = ioutil.ReadAll(response.Body) check(err) @@ -187,7 +195,8 @@ func searchRemote(bucket string, term string) []match { }(file.GetStringBytes("path")) } wg.Wait() - return matches.data + res = matches.data + return } func matchingManifests(path string, term string) (res []match) { From d40aab1b7fb5df3b4cc880e56a308b166021713d Mon Sep 17 00:00:00 2001 From: beerpiss Date: Sat, 14 May 2022 16:24:12 +0700 Subject: [PATCH 3/5] close response body --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index c0c662f..2fb8086 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,7 @@ func githubRatelimitReached() bool { response, err := http.Get("https://api.github.com/rate_limit") check(err) + defer response.Body.Close() raw, err := ioutil.ReadAll(response.Body) check(err) @@ -163,6 +164,7 @@ func searchRemote(bucket string, term string) (res []match, hasReachedApiLimit b response, err := http.Get(apiLink) check(err) + defer response.Body.Close() requestsLeft, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining")) check(err) From 240e6685015ec2d3383cd5bc897d38e36ff7f7de Mon Sep 17 00:00:00 2001 From: beerpiss Date: Sat, 14 May 2022 16:46:01 +0700 Subject: [PATCH 4/5] optimization: query while object.visiting --- main.go | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index 2fb8086..3343e53 100644 --- a/main.go +++ b/main.go @@ -120,13 +120,6 @@ func searchRemoteAll(term string) matchMap { result, _ := parser.ParseBytes(raw) object, _ := result.Object() - var buckets []string - object.Visit(func(k []byte, v *fastjson.Value) { - if _, err := os.Stat(scoopHome() + "\\buckets\\" + string(k)); os.IsNotExist(err) { - buckets = append(buckets, string(k)) - } - }) - matches := struct { sync.Mutex data matchMap @@ -134,31 +127,29 @@ func searchRemoteAll(term string) matchMap { matches.data = make(matchMap) var wg sync.WaitGroup var hasReachedApiLimit bool = false - for _, bucket := range buckets { - if !hasReachedApiLimit { - wg.Add(1) - go func(b string) { - var res []match - res, hasReachedApiLimit = searchRemote(b, term) - matches.Lock() - matches.data[b] = res - matches.Unlock() - wg.Done() - }(bucket) + object.Visit(func(k []byte, v *fastjson.Value) { + if _, err := os.Stat(scoopHome() + "\\buckets\\" + string(k)); os.IsNotExist(err) { + if !hasReachedApiLimit { + wg.Add(1) + go func(b string, u string) { + var res []match + res, hasReachedApiLimit = searchRemote(u, term) + matches.Lock() + matches.data[b] = res + matches.Unlock() + wg.Done() + }(string(k), string(v.GetStringBytes())) + } + } - } + }) wg.Wait() return matches.data } -func searchRemote(bucket string, term string) (res []match, hasReachedApiLimit bool) { +func searchRemote(bucketURL string, term string) (res []match, hasReachedApiLimit bool) { var parser fastjson.Parser - raw, err := os.ReadFile(scoopHome() + "\\apps\\scoop\\current\\buckets.json") - check(err) - - result, _ := parser.ParseBytes(raw) - bucketURL := string(result.GetStringBytes(bucket)) bucketSplit := strings.Split(bucketURL, "/") apiLink := "https://api.github.com/repos/" + bucketSplit[len(bucketSplit)-2] + "/" + bucketSplit[len(bucketSplit)-1] + "/git/trees/HEAD?recursive=1" @@ -170,7 +161,7 @@ func searchRemote(bucket string, term string) (res []match, hasReachedApiLimit b check(err) hasReachedApiLimit = requestsLeft == 0 - raw, err = ioutil.ReadAll(response.Body) + raw, err := ioutil.ReadAll(response.Body) check(err) json, _ := parser.ParseBytes(raw) From 87c4593f75e4ea3a0a764add38bc85a9351dd424 Mon Sep 17 00:00:00 2001 From: beerpiss Date: Sat, 14 May 2022 17:03:44 +0700 Subject: [PATCH 5/5] disable network search if no conection --- main.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 3343e53..4e05f3d 100644 --- a/main.go +++ b/main.go @@ -100,7 +100,14 @@ func githubRatelimitReached() bool { var parser fastjson.Parser response, err := http.Get("https://api.github.com/rate_limit") - check(err) + if err != nil { + if strings.Contains(err.Error(), "no such host") { + // no internet connection + return true + } else { + check(err) + } + } defer response.Body.Close() raw, err := ioutil.ReadAll(response.Body)