refactor hdq/stream; TestGithub

tutorial/02-GithubRepos/repos_test.gop → _testdata/github/repos/out.json

@@ -1,13 +1,4 @@
-package repos
-
-import (
- "encoding/json"
- "testing"
-
- "github.com/goplus/hdq"
-)
-
-const outTestNew = `{
+{
  "Repos": [
  {
  "Repo": "/xushiwei/linguist",
@@ -251,17 +242,4 @@ const outTestNew = `{
  }
  ],
  "Next": "https://github.com/xushiwei?after=Y3Vyc29yOnYyOpK0MjAyMC0wNS0xMlQxMToxNTozN1rOD7hDgA%3D%3D\u0026tab=repositories"
-}`
-
-func TestNew(t *testing.T) {
- doc := hdq.Source("zip://data.zip#index.htm")
- ret := New(doc)
- b, err := json.MarshalIndent(ret, "", "\t")
- if err != nil {
- t.Fatal("json.MarshalIndent:", err)
- }
- out := string(b)
- if out != outTestNew {
- t.Fatal("TestNew failed:", out)
- }
-}
+}

hdq_test.go

@@ -21,6 +21,8 @@ import (
  "github.com/goplus/hdq"
  "github.com/goplus/hdq/hdqtest"
  "github.com/goplus/hdq/pysig/torch"
+
+ repos "github.com/goplus/hdq/tutorial/02-GithubRepos"
 )
 
 func textOf(doc hdq.NodeSet) (ret string) {
@@ -32,6 +34,10 @@ func TestText(t *testing.T) {
  hdqtest.FromDir(t, "", "./_testdata/text", textOf)
 }
 
-func TestTestdata(t *testing.T) {
+func TestGithub(t *testing.T) {
+ hdqtest.FromDir(t, "", "./_testdata/github", repos.New, "data.zip#index.htm", "zip")
+}
+
+func TestTorch(t *testing.T) {
  hdqtest.FromDir(t, "", "./pysig/torch/_testdata", torch.New)
 }

hdqtest/hdqtest.go

@@ -31,7 +31,8 @@ import (
 type Converter = any
 
 // FromDir tests all html files in a directory.
-func FromDir(t *testing.T, sel, relDir string, conv Converter) {
+// optional params: [filename, scheme]
+func FromDir(t *testing.T, sel, relDir string, conv Converter, params ...string) {
  dir, err := os.Getwd()
  if err != nil {
  t.Fatal("Getwd failed:", err)
@@ -42,23 +43,30 @@ func FromDir(t *testing.T, sel, relDir string, conv Converter) {
  t.Fatal("ReadDir failed:", err)
  }
  vConv := reflect.ValueOf(conv)
+ scheme, fname := "", "/in.html"
+ if len(params) > 0 {
+ fname = "/" + params[0]
+ if len(params) > 1 {
+ scheme = params[1] + ":"
+ }
+ }
  for _, fi := range fis {
  name := fi.Name()
  if !fi.IsDir() || strings.HasPrefix(name, "_") {
  continue
  }
  t.Run(name, func(t *testing.T) {
- testFrom(t, dir+"/"+name, sel, vConv)
+ testFrom(t, dir+"/"+name, sel, vConv, fname, scheme)
  })
  }
 }
 
-func testFrom(t *testing.T, pkgDir, sel string, conv reflect.Value) {
+func testFrom(t *testing.T, pkgDir, sel string, conv reflect.Value, fname, scheme string) {
  if sel != "" && !strings.Contains(pkgDir, sel) {
  return
  }
  log.Println("Parsing", pkgDir)
- in := pkgDir + "/in.html"
+ in := scheme + pkgDir + fname
  out := pkgDir + "/out.json"
  b, err := os.ReadFile(out)
  if err != nil {

stream/http/httpstrm.go

@@ -30,16 +30,8 @@ var (
 
 // -------------------------------------------------------------------------------------
 
-// Open opens a zipped file object.
-func Open(file string) (io.ReadCloser, error) {
- return httpOpen("http://" + file)
-}
-
-func Opens(file string) (io.ReadCloser, error) {
- return httpOpen("https://" + file)
-}
-
-func httpOpen(url string) (io.ReadCloser, error) {
+// Open opens a http file object.
+func Open(url string) (io.ReadCloser, error) {
  resp, err := httpGet(url)
  if err != nil {
  return nil, err
@@ -62,9 +54,8 @@ func httpGet(url string) (resp *http.Response, err error) {
 }
 
 func init() {
- // http://path, https://path
- stream.RegisterSchema("http", Open)
- stream.RegisterSchema("https", Opens)
+ stream.Register("http", Open)
+ stream.Register("https", Open)
 }
 
 // -------------------------------------------------------------------------------------

stream/inline/inline.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inline
+
+import (
+ "io"
+ "strings"
+
+ "github.com/goplus/hdq/stream"
+)
+
+type nilCloser struct {
+ io.Reader
+}
+
+func (p *nilCloser) Close() error {
+ return nil
+}
+
+// Open opens a inline text object.
+func Open(url string) (io.ReadCloser, error) {
+ file := strings.TrimPrefix(url, "inline:")
+ r := strings.NewReader(file)
+ return &nilCloser{r}, nil
+}
+
+func init() {
+ stream.Register("inline", Open)
+}

stream/stream.go

@@ -18,43 +18,47 @@ package stream
 import (
  "errors"
  "io"
+ "io/fs"
  "os"
  "strings"
 )
 
-// -------------------------------------------------------------------------------------
-
-func SplitSchema(path string) (schema, file string) {
- idx := strings.IndexAny(path, ":/\\ ")
- if idx < 0 || path[idx] != ':' {
- return "", path
- }
- schema, file = path[:idx], path[idx+1:]
- file = strings.TrimPrefix(file, "//")
- return
-}
+var (
+ ErrUnknownScheme = errors.New("unknown scheme")
+)
 
 // -------------------------------------------------------------------------------------
 
 type OpenFunc = func(file string) (io.ReadCloser, error)
 
 var (
- openSchemas = map[string]OpenFunc{}
+ openers = map[string]OpenFunc{}
 )
 
-func RegisterSchema(schema string, open OpenFunc) {
- openSchemas[schema] = open
+// Register registers a scheme with an open function.
+func Register(scheme string, open OpenFunc) {
+ openers[scheme] = open
 }
 
-func Open(path string) (io.ReadCloser, error) {
- schema, file := SplitSchema(path)
- if schema == "" {
- return os.Open(path)
+func Open(url string) (io.ReadCloser, error) {
+ scheme := schemeOf(url)
+ if scheme == "" {
+ return os.Open(url)
+ }
+ if open, ok := openers[scheme]; ok {
+ return open(url)
  }
- if open, ok := openSchemas[schema]; ok {
- return open(file)
+ return nil, &fs.PathError{Op: "hdq/stream.Open", Err: ErrUnknownScheme, Path: url}
+}
+
+func schemeOf(url string) (scheme string) {
+ pos := strings.IndexAny(url, ":/")
+ if pos > 0 {
+ if url[pos] == ':' {
+ return url[:pos]
+ }
  }
- return nil, errors.New("stream.Open: unsupported schema - " + schema)
+ return ""
 }
 
 // -------------------------------------------------------------------------------------

stream/stream_test.go

@@ -13,30 +13,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package stream
+package stream_test
 
 import (
  "io"
- "strings"
  "testing"
-)
-
-type nilCloser struct {
- io.Reader
-}
 
-func (p *nilCloser) Close() error {
- return nil
-}
-
-func inlOpen(file string) (io.ReadCloser, error) {
- r := strings.NewReader(file)
- return &nilCloser{r}, nil
-}
+ "github.com/goplus/hdq/stream"
+ _ "github.com/goplus/hdq/stream/inline"
+)
 
 func TestBasic(t *testing.T) {
- RegisterSchema("inl", inlOpen)
- f, err := Open("inl://hello")
+ f, err := stream.Open("inline:hello")
  if err != nil {
  t.Fatal("Open failed:", err)
  }
@@ -49,15 +37,15 @@ func TestBasic(t *testing.T) {
  }
 }
 
-func TestUnknownSchema(t *testing.T) {
- _, err := Open("bad://foo")
- if err == nil || err.Error() != "stream.Open: unsupported schema - bad" {
+func TestUnknownScheme(t *testing.T) {
+ _, err := stream.Open("bad://foo")
+ if err == nil || err.Error() != "hdq/stream.Open bad://foo: unknown scheme" {
  t.Fatal("Open failed:", err)
  }
 }
 
 func TestOpenFile(t *testing.T) {
- _, err := Open("/bin/not-exists/foo")
+ _, err := stream.Open("/bin/not-exists/foo")
  if err == nil {
  t.Fatal("Open local file success?")
  }

stream/zip/zipstrm.go

@@ -37,7 +37,8 @@ func (p *readCloser) Close() error {
 }
 
 // Open opens a zipped file object.
-func Open(file string) (io.ReadCloser, error) {
+func Open(url string) (io.ReadCloser, error) {
+ file := strings.TrimPrefix(url, "zip:")
  pos := strings.Index(file, "#")
  if pos <= 0 {
  return nil, syscall.EINVAL
@@ -60,8 +61,8 @@ func Open(file string) (io.ReadCloser, error) {
 }
 
 func init() {
- // zip://file#index.htm
- stream.RegisterSchema("zip", Open)
+ // zip:file#index.htm
+ stream.Register("zip", Open)
 }
 
 // -------------------------------------------------------------------------------------