hdq: TestText; refactor hdq/stream; TestGithub

tutorial/02-GithubRepos/repos_test.gop → _testdata/github/repos/out.json

@@ -1,13 +1,4 @@
-package repos
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/goplus/hdq"
-)
-
-const outTestNew = `{
+{
 	"Repos": [
 		{
 			"Repo": "/xushiwei/linguist",
@@ -251,17 +242,4 @@ const outTestNew = `{
 		}
 	],
 	"Next": "https://github.com/xushiwei?after=Y3Vyc29yOnYyOpK0MjAyMC0wNS0xMlQxMToxNTozN1rOD7hDgA%3D%3D\u0026tab=repositories"
-}`
-
-func TestNew(t *testing.T) {
-	doc := hdq.Source("zip://data.zip#index.htm")
-	ret := New(doc)
-	b, err := json.MarshalIndent(ret, "", "\t")
-	if err != nil {
-		t.Fatal("json.MarshalIndent:", err)
-	}
-	out := string(b)
-	if out != outTestNew {
-		t.Fatal("TestNew failed:", out)
-	}
-}
+}

_testdata/text/eyesig/in.html

@@ -0,0 +1,2 @@
+<dt class="sig sig-object py" id="torch.eye">
+<span class="sig-prename descclassname"><span class="pre">torch.</span></span><span class="sig-name descname"><span class="pre">eye</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">n</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">m</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">out</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dtype</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">layout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">torch.strided</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">device</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">requires_grad</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference internal" href="../tensors.html#torch.Tensor" title="torch.Tensor"><span class="pre">Tensor</span></a></span></span><a class="headerlink" href="#torch.eye" title="Permalink to this definition">¶</a></dt>

_testdata/text/eyesig/out.json

@@ -0,0 +1 @@
+"torch.eye(n, m=None, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) → Tensor¶"

hdq_test.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hdq_test
+
+import (
+	"testing"
+
+	"github.com/goplus/hdq"
+	"github.com/goplus/hdq/hdqtest"
+	"github.com/goplus/hdq/pysig/torch"
+
+	repos "github.com/goplus/hdq/tutorial/02-GithubRepos"
+)
+
+func textOf(doc hdq.NodeSet) (ret string) {
+	ret, _ = doc.Text__0()
+	return
+}
+
+func TestText(t *testing.T) {
+	hdqtest.FromDir(t, "", "./_testdata/text", textOf)
+}
+
+func TestGithub(t *testing.T) {
+	hdqtest.FromDir(t, "", "./_testdata/github", repos.New, "data.zip#index.htm", "zip")
+}
+
+func TestTorch(t *testing.T) {
+	hdqtest.FromDir(t, "", "./pysig/torch/_testdata", torch.New)
+}

hdqtest/hdqtest.go

@@ -31,7 +31,8 @@ import (
 type Converter = any
 
 // FromDir tests all html files in a directory.
-func FromDir(t *testing.T, sel, relDir string, conv Converter) {
+// optional params: [filename, scheme]
+func FromDir(t *testing.T, sel, relDir string, conv Converter, params ...string) {
 	dir, err := os.Getwd()
 	if err != nil {
 		t.Fatal("Getwd failed:", err)
@@ -42,23 +43,30 @@ func FromDir(t *testing.T, sel, relDir string, conv Converter) {
 		t.Fatal("ReadDir failed:", err)
 	}
 	vConv := reflect.ValueOf(conv)
+	scheme, fname := "", "/in.html"
+	if len(params) > 0 {
+		fname = "/" + params[0]
+		if len(params) > 1 {
+			scheme = params[1] + ":"
+		}
+	}
 	for _, fi := range fis {
 		name := fi.Name()
 		if !fi.IsDir() || strings.HasPrefix(name, "_") {
 			continue
 		}
 		t.Run(name, func(t *testing.T) {
-			testFrom(t, dir+"/"+name, sel, vConv)
+			testFrom(t, dir+"/"+name, sel, vConv, fname, scheme)
 		})
 	}
 }
 
-func testFrom(t *testing.T, pkgDir, sel string, conv reflect.Value) {
+func testFrom(t *testing.T, pkgDir, sel string, conv reflect.Value, fname, scheme string) {
 	if sel != "" && !strings.Contains(pkgDir, sel) {
 		return
 	}
 	log.Println("Parsing", pkgDir)
-	in := pkgDir + "/in.html"
+	in := scheme + pkgDir + fname
 	out := pkgDir + "/out.json"
 	b, err := os.ReadFile(out)
 	if err != nil {

html_utils.go

@@ -74,6 +74,10 @@
 
 // -----------------------------------------------------------------------------
 
+const (
+	spaces = " \t\r\n"
+)
+
 // childEqualText returns true if the type of node's child is TextNode and it's Data equals `text`.
 func childEqualText(node *html.Node, text string) bool {
 	p := node.FirstChild
@@ -104,7 +108,7 @@
 	if node.Type != html.TextNode {
 		return false
 	}
-	return strings.Contains(strings.TrimLeft(node.Data, " \t\r\n"), text)
+	return strings.Contains(strings.TrimLeft(node.Data, spaces), text)
 }
 
 // exactText returns text of node if the type of node is TextNode.
@@ -125,26 +129,28 @@
 type textPrinter struct {
 	data         []byte
 	notLineStart bool
+	hasSpace     bool
 }
 
-func (p *textPrinter) printText(v string) {
+func (p *textPrinter) printText(v string, hasRightSpace bool) {
 	if v == "" {
 		return
 	}
-	if p.notLineStart {
+	if p.notLineStart && p.hasSpace {
 		p.data = append(p.data, ' ')
 	} else {
 		p.notLineStart = true
 	}
 	p.data = append(p.data, v...)
+	p.hasSpace = hasRightSpace
 }
 
 func (p *textPrinter) printNode(node *html.Node) {
 	if node == nil {
 		return
 	}
 	if node.Type == html.TextNode {
-		p.printText(strings.Trim(node.Data, " \t\r\n¶"))
+		p.printText(textTrimRight(textTrimLeft(node.Data, &p.hasSpace)))
 		return
 	}
 	for child := node.FirstChild; child != nil; child = child.NextSibling {
@@ -157,4 +163,17 @@
 	}
 }
 
+func textTrimLeft(v string, hasSpace *bool) string {
+	ret := strings.TrimLeft(v, spaces)
+	if len(v) != len(ret) {
+		*hasSpace = true
+	}
+	return ret
+}
+
+func textTrimRight(v string) (string, bool) {
+	ret := strings.TrimRight(v, spaces)
+	return ret, len(v) != len(ret)
+}
+
 // -----------------------------------------------------------------------------

pysig/torch/_testdata/eye/out.json

@@ -1,5 +1,5 @@
 {
 	"name": "eye",
 	"doc": "",
-	"sig": "( n , m = None , * , out = None , dtype = None , layout = torch.strided , device = None , requires_grad = False ) → Tensor"
+	"sig": "(n, m=None, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) → Tensor"
 }

pysig/torch/_testdata/invalid/in.html

@@ -0,0 +1,3 @@
+<dl class="py function">
+    <dt class="sig sig-object py">foo</dt>
+</dl>

pysig/torch/_testdata/invalid/out.json

@@ -0,0 +1,5 @@
+{
+	"name": "",
+	"doc": "",
+	"sig": "\u003cNULL\u003e"
+}

pysig/torch/pysig_torch.gop

@@ -23,6 +23,10 @@ import (
 
 // -----------------------------------------------------------------------------
 
+const (
+	spaces = " \t\r\n¶"
+)
+
 type Result struct {
 	Name string `json:"name"`
 	Doc  string `json:"doc"`
@@ -36,7 +40,7 @@ func New(doc hdq.NodeSet) Result {
 	if pos > 0 {
 		name := strings.trimPrefix(decl[:pos], "torch.")
 		sig := decl[pos:]
-		return {strings.trimSpace(name), "", strings.trimSpace(sig)}
+		return {strings.trimSpace(name), "", strings.trimRight(sig, spaces)}
 	}
 	return {"", "", "<NULL>"}
 }

stream/http/httpstrm.go

@@ -30,16 +30,8 @@ var (
 
 // -------------------------------------------------------------------------------------
 
-// Open opens a zipped file object.
-func Open(file string) (io.ReadCloser, error) {
-	return httpOpen("http://" + file)
-}
-
-func Opens(file string) (io.ReadCloser, error) {
-	return httpOpen("https://" + file)
-}
-
-func httpOpen(url string) (io.ReadCloser, error) {
+// Open opens a http file object.
+func Open(url string) (io.ReadCloser, error) {
 	resp, err := httpGet(url)
 	if err != nil {
 		return nil, err
@@ -62,9 +54,8 @@ func httpGet(url string) (resp *http.Response, err error) {
 }
 
 func init() {
-	// http://path, https://path
-	stream.RegisterSchema("http", Open)
-	stream.RegisterSchema("https", Opens)
+	stream.Register("http", Open)
+	stream.Register("https", Open)
 }
 
 // -------------------------------------------------------------------------------------

stream/inline/inline.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inline
+
+import (
+	"io"
+	"strings"
+
+	"github.com/goplus/hdq/stream"
+)
+
+type nilCloser struct {
+	io.Reader
+}
+
+func (p *nilCloser) Close() error {
+	return nil
+}
+
+// Open opens a inline text object.
+func Open(url string) (io.ReadCloser, error) {
+	file := strings.TrimPrefix(url, "inline:")
+	r := strings.NewReader(file)
+	return &nilCloser{r}, nil
+}
+
+func init() {
+	stream.Register("inline", Open)
+}