gop/tpl

parser/interface.go

@@ -23,6 +23,7 @@ import (
 	"go/scanner"
 
 	"github.com/goplus/gop/ast"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/token"
 )
 
@@ -83,7 +84,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode)
 	}
 
 	// get source
-	text, err := readSourceLocal(filename, src)
+	text, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return
 	}
@@ -133,7 +134,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode)
 // are returned via a scanner.ErrorList which is sorted by source position.
 func ParseExprFrom(fset *token.FileSet, filename string, src any, mode Mode) (expr ast.Expr, err error) {
 	// get source
-	text, err := readSourceLocal(filename, src)
+	text, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return
 	}

parser/iox/io.go

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package iox
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"os"
+)
+
+// -----------------------------------------------------------------------------
+
+var (
+	ErrInvalidSource = errors.New("invalid source")
+)
+
+func ReadSource(src any) ([]byte, error) {
+	switch s := src.(type) {
+	case string:
+		return []byte(s), nil
+	case []byte:
+		return s, nil
+	case *bytes.Buffer:
+		// is io.Reader, but src is already available in []byte form
+		if s != nil {
+			return s.Bytes(), nil
+		}
+	case io.Reader:
+		return io.ReadAll(s)
+	}
+	return nil, ErrInvalidSource
+}
+
+// If src != nil, readSource converts src to a []byte if possible;
+// otherwise it returns an error. If src == nil, readSource returns
+// the result of reading the file specified by filename.
+func ReadSourceLocal(filename string, src any) ([]byte, error) {
+	if src != nil {
+		return ReadSource(src)
+	}
+	return os.ReadFile(filename)
+}
+
+// -----------------------------------------------------------------------------

parser/parser_gop.go

@@ -17,11 +17,8 @@
 package parser
 
 import (
-	"bytes"
 	"errors"
-	"io"
 	"io/fs"
-	"os"
 	"path"
 	"strings"
 
@@ -30,6 +27,7 @@ import (
 
 	"github.com/goplus/gop/ast"
 	"github.com/goplus/gop/parser/fsx"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/token"
 )
 
@@ -340,40 +338,9 @@ func ParseFSFile(fset *token.FileSet, fs FileSystem, filename string, src interf
 	return parseFile(fset, filename, code, mode)
 }
 
-var (
-	errInvalidSource = errors.New("invalid source")
-)
-
-func readSource(src interface{}) ([]byte, error) {
-	switch s := src.(type) {
-	case string:
-		return []byte(s), nil
-	case []byte:
-		return s, nil
-	case *bytes.Buffer:
-		// is io.Reader, but src is already available in []byte form
-		if s != nil {
-			return s.Bytes(), nil
-		}
-	case io.Reader:
-		return io.ReadAll(s)
-	}
-	return nil, errInvalidSource
-}
-
-// If src != nil, readSource converts src to a []byte if possible;
-// otherwise it returns an error. If src == nil, readSource returns
-// the result of reading the file specified by filename.
-func readSourceLocal(filename string, src interface{}) ([]byte, error) {
-	if src != nil {
-		return readSource(src)
-	}
-	return os.ReadFile(filename)
-}
-
 func readSourceFS(fs FileSystem, filename string, src interface{}) ([]byte, error) {
 	if src != nil {
-		return readSource(src)
+		return iox.ReadSource(src)
 	}
 	return fs.ReadFile(filename)
 }

parser/parserdir_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/goplus/gop/ast"
 	"github.com/goplus/gop/parser/fsx"
 	"github.com/goplus/gop/parser/fsx/memfs"
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/parser/parsertest"
 	"github.com/goplus/gop/scanner"
 	"github.com/goplus/gop/token"
@@ -319,17 +320,17 @@ func TestParseExprFrom(t *testing.T) {
 
 func TestReadSource(t *testing.T) {
 	buf := bytes.NewBuffer(nil)
-	if _, err := readSource(buf); err != nil {
+	if _, err := iox.ReadSource(buf); err != nil {
 		t.Fatal("readSource failed:", err)
 	}
 	sr := strings.NewReader("")
-	if _, err := readSource(sr); err != nil {
+	if _, err := iox.ReadSource(sr); err != nil {
 		t.Fatal("readSource strings.Reader failed:", err)
 	}
-	if _, err := readSource(0); err == nil {
+	if _, err := iox.ReadSource(0); err == nil {
 		t.Fatal("readSource int failed: no error?")
 	}
-	if _, err := readSourceLocal("/foo/bar/not-exists", nil); err == nil {
+	if _, err := iox.ReadSourceLocal("/foo/bar/not-exists", nil); err == nil {
 		t.Fatal("readSourceLocal int failed: no error?")
 	}
 }
@@ -343,7 +344,7 @@ func TestParseFiles(t *testing.T) {
 
 func TestIparseFileInvalidSrc(t *testing.T) {
 	fset := token.NewFileSet()
-	if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != errInvalidSource {
+	if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != iox.ErrInvalidSource {
 		t.Fatal("ParseFile failed: not errInvalidSource?")
 	}
 }

tpl/cl/compile.go

@@ -35,7 +35,7 @@ var (
 // Result represents the result of compiling a set of rules.
 type Result struct {
 	Doc   *matcher.Var
-	rules map[string]*matcher.Var
+	Rules map[string]*matcher.Var
 }
 
 type context struct {

tpl/parser/parser.go

@@ -17,11 +17,7 @@
 package parser
 
 import (
-	"bytes"
-	"errors"
-	"io"
-	"os"
-
+	"github.com/goplus/gop/parser/iox"
 	"github.com/goplus/gop/tpl/ast"
 	"github.com/goplus/gop/tpl/scanner"
 	"github.com/goplus/gop/tpl/token"
@@ -36,7 +32,7 @@ type Mode uint
 
 // ParseFile parses a file and returns the AST.
 func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.File, err error) {
-	b, err := readSourceLocal(filename, src)
+	b, err := iox.ReadSourceLocal(filename, src)
 	if err != nil {
 		return nil, err
 	}
@@ -56,39 +52,6 @@ func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.Fi
 
 // -----------------------------------------------------------------------------
 
-var (
-	errInvalidSource = errors.New("invalid source")
-)
-
-func readSource(src any) ([]byte, error) {
-	switch s := src.(type) {
-	case string:
-		return []byte(s), nil
-	case []byte:
-		return s, nil
-	case *bytes.Buffer:
-		// is io.Reader, but src is already available in []byte form
-		if s != nil {
-			return s.Bytes(), nil
-		}
-	case io.Reader:
-		return io.ReadAll(s)
-	}
-	return nil, errInvalidSource
-}
-
-// If src != nil, readSource converts src to a []byte if possible;
-// otherwise it returns an error. If src == nil, readSource returns
-// the result of reading the file specified by filename.
-func readSourceLocal(filename string, src any) ([]byte, error) {
-	if src != nil {
-		return readSource(src)
-	}
-	return os.ReadFile(filename)
-}
-
-// -----------------------------------------------------------------------------
-
 // parser represents a parser.
 type parser struct {
 	scanner scanner.Scanner
@@ -106,7 +69,7 @@ type parser struct {
 func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
 	p.file = fset.AddFile(filename, -1, len(src))
 	eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) }
-	p.scanner.Init(p.file, src, eh, scanner.InsertSemis)
+	p.scanner.Init(p.file, src, eh, 0)
 	p.next() // initialize first token
 }
 

tpl/scanner/scanner.go

@@ -100,8 +100,8 @@ const (
 	// ScanComments means returning comments as COMMENT tokens
 	ScanComments ScanMode = 1 << iota
 
-	// InsertSemis means automatically insert semicolons
-	InsertSemis
+	// NoInsertSemis means don't automatically insert semicolons
+	NoInsertSemis
 )
 
 // Init prepares the scanner s to tokenize the text src by setting the
@@ -788,7 +788,7 @@ scanAgain:
 			t.Lit = string(ch)
 		}
 	}
-	if s.mode&InsertSemis != 0 {
+	if s.mode&NoInsertSemis == 0 {
 		s.insertSemi = insertSemi
 	}
 

tpl/scanner/scanner_test.go

@@ -128,7 +128,7 @@ world !=
 	fset := token.NewFileSet()
 	file := fset.AddFile("", -1, len(grammar))
 
-	s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments|InsertSemis)
+	s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments)
 	i := 0
 	for {
 		c := s.Scan()
@@ -145,7 +145,7 @@ world !=
 		t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i)
 	}
 
-	s.Init(file, []byte(grammar), nil, 0)
+	s.Init(file, []byte(grammar), nil, NoInsertSemis)
 	i = 0
 	for {
 		c := s.Scan()
@@ -176,7 +176,7 @@ world !=
 		t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i)
 	}
 
-	s.Init(file, []byte(grammar), nil, InsertSemis)
+	s.Init(file, []byte(grammar), nil, 0)
 	i = 0
 	for {
 		c := s.Scan()