Merge pull request #57 from xushiwei/q

tool: gopkgimps, etc.

chore/gopkgimps/gopkgimps.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "log"
+ "os"
+ "sort"
+
+ "github.com/goplus/hdq/fetcher"
+ "github.com/goplus/hdq/fetcher/gopkg"
+ _ "github.com/goplus/hdq/stream/http/cached"
+)
+
+// Usage: gopkgimps [pkgPath ...]
+func main() {
+ if len(os.Args) < 2 {
+ fmt.Fprintln(os.Stderr, "Usage: gopkgimps [pkgPath ...]")
+ os.Exit(1)
+ }
+ names := os.Args[1:]
+ docs := make([]gopkg.Result, 0, len(names))
+ for _, name := range names {
+ log.Println("==> Fetch", name)
+ doc, err := fetcher.FromInput("gopkg", name)
+ if err == fetcher.ErrUnknownPageType {
+ break
+ }
+ docs = append(docs, doc.(gopkg.Result))
+ }
+ sort.Slice(docs, func(i, j int) bool {
+ return docs[i].ImportedBy > docs[j].ImportedBy
+ })
+ enc := json.NewEncoder(os.Stdout)
+ enc.SetIndent("", " ")
+ enc.Encode(docs)
+}

chore/hreflinks/links.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "strings"
+
+ "github.com/goplus/hdq/fetcher"
+ _ "github.com/goplus/hdq/fetcher/hrefs"
+ _ "github.com/goplus/hdq/stream/http/cached"
+)
+
+// Usage: hreflinks [url ...]
+func main() {
+ if len(os.Args) < 2 {
+ fmt.Fprintln(os.Stderr, "Usage: hreflinks [url ...]")
+ os.Exit(1)
+ }
+ urls := os.Args[1:]
+ if len(urls) == 1 && urls[0] == "-" {
+ b, _ := io.ReadAll(os.Stdin)
+ urls = strings.Split(strings.TrimSpace(string(b)), "\n")
+ }
+ docs := make([]any, 0, len(urls))
+ for _, url := range urls {
+ log.Println("==> Fetch", url)
+ doc, err := fetcher.FromInput("hrefs", url)
+ if err == fetcher.ErrUnknownPageType {
+ break
+ }
+ docs = append(docs, doc)
+ }
+ json.NewEncoder(os.Stdout).Encode(docs)
+}

chore/pysigfetch/pysigfetch.go

@@ -45,7 +45,7 @@ func main() {
  b, _ := io.ReadAll(os.Stdin)
  names = strings.Split(strings.TrimSpace(string(b)), " ")
  }
- var docs = make([]any, 0, len(names))
+ docs := make([]any, 0, len(names))
  for _, name := range names {
  log.Println("==> Fetch", name)
  doc, err := fetcher.FromInput(moduleName, name)

fetcher/gopkg/_testdata/encoding/out.json

@@ -0,0 +1,4 @@
+{
+ "name": "",
+ "importedBy": 14960
+}

fetcher/gopkg/gopkg_imps.gop

@@ -0,0 +1,48 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gopkg
+
+import (
+ "strings"
+
+ "github.com/goplus/hdq"
+ "github.com/goplus/hdq/fetcher"
+)
+
+type Result struct {
+ Name string `json:"name"`
+ ImportedBy int `json:"importedBy"`
+}
+
+// New creates a new Result from a html document.
+func New(input any, doc hdq.NodeSet) Result {
+ const importedByPrefix = "Imported By:"
+ name := input.(string)
+ a := doc.any.a.attribute("aria-label", v => strings.hasPrefix(v, importedByPrefix)).one
+ label := a.attr("aria-label")!
+ labelVal := strings.trimSpace(label[len(importedByPrefix):])
+ importedBy := strings.replaceAll(labelVal, ",", "").int!
+ return {name, importedBy}
+}
+
+// URL returns the input URL for the given name.
+func URL(name any) string {
+ return "https://pkg.go.dev/" + name.(string)
+}
+
+func init() {
+ fetcher.Register("gopkg", New, URL)
+}

fetcher/hrefs/hrefs.gop

@@ -0,0 +1,41 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package links
+
+import (
+ "github.com/goplus/hdq"
+ "github.com/goplus/hdq/fetcher"
+)
+
+type Result struct {
+ URL string `json:"url,omitempty"`
+ Hrefs []string `json:"hrefs,omitempty"`
+}
+
+// New collects all href links from a html document.
+func New(input any, doc hdq.NodeSet) Result {
+ hrefs := [link for a <- doc.any.a if link := a.href?:""; link != ""]
+ return {input.(string), hrefs}
+}
+
+// URL returns the input URL for the given name.
+func URL(name any) string {
+ return name.(string)
+}
+
+func init() {
+ fetcher.Register("hrefs", New, URL)
+}