diff --git a/parser/interface.go b/parser/interface.go index 7fd616af6..831ac793c 100644 --- a/parser/interface.go +++ b/parser/interface.go @@ -23,6 +23,7 @@ import ( "go/scanner" "github.com/goplus/gop/ast" + "github.com/goplus/gop/parser/iox" "github.com/goplus/gop/token" ) @@ -83,7 +84,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) } // get source - text, err := readSourceLocal(filename, src) + text, err := iox.ReadSourceLocal(filename, src) if err != nil { return } @@ -133,7 +134,7 @@ func parseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) // are returned via a scanner.ErrorList which is sorted by source position. func ParseExprFrom(fset *token.FileSet, filename string, src any, mode Mode) (expr ast.Expr, err error) { // get source - text, err := readSourceLocal(filename, src) + text, err := iox.ReadSourceLocal(filename, src) if err != nil { return } diff --git a/parser/iox/io.go b/parser/iox/io.go new file mode 100644 index 000000000..3aa296539 --- /dev/null +++ b/parser/iox/io.go @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package iox + +import ( + "bytes" + "errors" + "io" + "os" +) + +// ----------------------------------------------------------------------------- + +var ( + ErrInvalidSource = errors.New("invalid source") +) + +func ReadSource(src any) ([]byte, error) { + switch s := src.(type) { + case string: + return []byte(s), nil + case []byte: + return s, nil + case *bytes.Buffer: + // is io.Reader, but src is already available in []byte form + if s != nil { + return s.Bytes(), nil + } + case io.Reader: + return io.ReadAll(s) + } + return nil, ErrInvalidSource +} + +// If src != nil, readSource converts src to a []byte if possible; +// otherwise it returns an error. If src == nil, readSource returns +// the result of reading the file specified by filename. +func ReadSourceLocal(filename string, src any) ([]byte, error) { + if src != nil { + return ReadSource(src) + } + return os.ReadFile(filename) +} + +// ----------------------------------------------------------------------------- diff --git a/parser/parser_gop.go b/parser/parser_gop.go index e7b3b02b3..6309871b0 100644 --- a/parser/parser_gop.go +++ b/parser/parser_gop.go @@ -17,11 +17,8 @@ package parser import ( - "bytes" "errors" - "io" "io/fs" - "os" "path" "strings" @@ -30,6 +27,7 @@ import ( "github.com/goplus/gop/ast" "github.com/goplus/gop/parser/fsx" + "github.com/goplus/gop/parser/iox" "github.com/goplus/gop/token" ) @@ -340,40 +338,9 @@ func ParseFSFile(fset *token.FileSet, fs FileSystem, filename string, src interf return parseFile(fset, filename, code, mode) } -var ( - errInvalidSource = errors.New("invalid source") -) - -func readSource(src interface{}) ([]byte, error) { - switch s := src.(type) { - case string: - return []byte(s), nil - case []byte: - return s, nil - case *bytes.Buffer: - // is io.Reader, but src is already available in []byte form - if s != nil { - return s.Bytes(), nil - } - case io.Reader: - return io.ReadAll(s) - } - return nil, errInvalidSource -} - -// If src != nil, readSource converts src to a []byte if possible; -// otherwise it returns an error. If src == nil, readSource returns -// the result of reading the file specified by filename. -func readSourceLocal(filename string, src interface{}) ([]byte, error) { - if src != nil { - return readSource(src) - } - return os.ReadFile(filename) -} - func readSourceFS(fs FileSystem, filename string, src interface{}) ([]byte, error) { if src != nil { - return readSource(src) + return iox.ReadSource(src) } return fs.ReadFile(filename) } diff --git a/parser/parserdir_test.go b/parser/parserdir_test.go index 086a27fc3..8c4d29ed5 100644 --- a/parser/parserdir_test.go +++ b/parser/parserdir_test.go @@ -28,6 +28,7 @@ import ( "github.com/goplus/gop/ast" "github.com/goplus/gop/parser/fsx" "github.com/goplus/gop/parser/fsx/memfs" + "github.com/goplus/gop/parser/iox" "github.com/goplus/gop/parser/parsertest" "github.com/goplus/gop/scanner" "github.com/goplus/gop/token" @@ -319,17 +320,17 @@ func TestParseExprFrom(t *testing.T) { func TestReadSource(t *testing.T) { buf := bytes.NewBuffer(nil) - if _, err := readSource(buf); err != nil { + if _, err := iox.ReadSource(buf); err != nil { t.Fatal("readSource failed:", err) } sr := strings.NewReader("") - if _, err := readSource(sr); err != nil { + if _, err := iox.ReadSource(sr); err != nil { t.Fatal("readSource strings.Reader failed:", err) } - if _, err := readSource(0); err == nil { + if _, err := iox.ReadSource(0); err == nil { t.Fatal("readSource int failed: no error?") } - if _, err := readSourceLocal("/foo/bar/not-exists", nil); err == nil { + if _, err := iox.ReadSourceLocal("/foo/bar/not-exists", nil); err == nil { t.Fatal("readSourceLocal int failed: no error?") } } @@ -343,7 +344,7 @@ func TestParseFiles(t *testing.T) { func TestIparseFileInvalidSrc(t *testing.T) { fset := token.NewFileSet() - if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != errInvalidSource { + if _, err := parseFile(fset, "/foo/bar/not-exists", 1, PackageClauseOnly); err != iox.ErrInvalidSource { t.Fatal("ParseFile failed: not errInvalidSource?") } } diff --git a/tpl/cl/compile.go b/tpl/cl/compile.go index 290853b3f..e1d9ed7b3 100644 --- a/tpl/cl/compile.go +++ b/tpl/cl/compile.go @@ -35,7 +35,7 @@ var ( // Result represents the result of compiling a set of rules. type Result struct { Doc *matcher.Var - rules map[string]*matcher.Var + Rules map[string]*matcher.Var } type context struct { diff --git a/tpl/parser/parser.go b/tpl/parser/parser.go index fce1cd703..23cf5ee09 100644 --- a/tpl/parser/parser.go +++ b/tpl/parser/parser.go @@ -17,11 +17,7 @@ package parser import ( - "bytes" - "errors" - "io" - "os" - + "github.com/goplus/gop/parser/iox" "github.com/goplus/gop/tpl/ast" "github.com/goplus/gop/tpl/scanner" "github.com/goplus/gop/tpl/token" @@ -36,7 +32,7 @@ type Mode uint // ParseFile parses a file and returns the AST. func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.File, err error) { - b, err := readSourceLocal(filename, src) + b, err := iox.ReadSourceLocal(filename, src) if err != nil { return nil, err } @@ -56,39 +52,6 @@ func ParseFile(fset *token.FileSet, filename string, src any, _ Mode) (f *ast.Fi // ----------------------------------------------------------------------------- -var ( - errInvalidSource = errors.New("invalid source") -) - -func readSource(src any) ([]byte, error) { - switch s := src.(type) { - case string: - return []byte(s), nil - case []byte: - return s, nil - case *bytes.Buffer: - // is io.Reader, but src is already available in []byte form - if s != nil { - return s.Bytes(), nil - } - case io.Reader: - return io.ReadAll(s) - } - return nil, errInvalidSource -} - -// If src != nil, readSource converts src to a []byte if possible; -// otherwise it returns an error. If src == nil, readSource returns -// the result of reading the file specified by filename. -func readSourceLocal(filename string, src any) ([]byte, error) { - if src != nil { - return readSource(src) - } - return os.ReadFile(filename) -} - -// ----------------------------------------------------------------------------- - // parser represents a parser. type parser struct { scanner scanner.Scanner @@ -106,7 +69,7 @@ type parser struct { func (p *parser) init(fset *token.FileSet, filename string, src []byte) { p.file = fset.AddFile(filename, -1, len(src)) eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } - p.scanner.Init(p.file, src, eh, scanner.InsertSemis) + p.scanner.Init(p.file, src, eh, 0) p.next() // initialize first token } diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go index c696875ac..de65b44cd 100644 --- a/tpl/scanner/scanner.go +++ b/tpl/scanner/scanner.go @@ -100,8 +100,8 @@ const ( // ScanComments means returning comments as COMMENT tokens ScanComments ScanMode = 1 << iota - // InsertSemis means automatically insert semicolons - InsertSemis + // NoInsertSemis means don't automatically insert semicolons + NoInsertSemis ) // Init prepares the scanner s to tokenize the text src by setting the @@ -788,7 +788,7 @@ scanAgain: t.Lit = string(ch) } } - if s.mode&InsertSemis != 0 { + if s.mode&NoInsertSemis == 0 { s.insertSemi = insertSemi } diff --git a/tpl/scanner/scanner_test.go b/tpl/scanner/scanner_test.go index 867374115..ed0bcc2b2 100644 --- a/tpl/scanner/scanner_test.go +++ b/tpl/scanner/scanner_test.go @@ -128,7 +128,7 @@ world != fset := token.NewFileSet() file := fset.AddFile("", -1, len(grammar)) - s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments|InsertSemis) + s.Init(file, []byte(grammar), nil /* no error handler */, ScanComments) i := 0 for { c := s.Scan() @@ -145,7 +145,7 @@ world != t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i) } - s.Init(file, []byte(grammar), nil, 0) + s.Init(file, []byte(grammar), nil, NoInsertSemis) i = 0 for { c := s.Scan() @@ -176,7 +176,7 @@ world != t.Fatalf("len(expected) != i: %d, %d\n", len(expected), i) } - s.Init(file, []byte(grammar), nil, InsertSemis) + s.Init(file, []byte(grammar), nil, 0) i = 0 for { c := s.Scan() diff --git a/tpl/tpl.go b/tpl/tpl.go new file mode 100644 index 000000000..50f1b1c5b --- /dev/null +++ b/tpl/tpl.go @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tpl + +import ( + "github.com/goplus/gop/parser/iox" + "github.com/goplus/gop/tpl/cl" + "github.com/goplus/gop/tpl/matcher" + "github.com/goplus/gop/tpl/parser" + "github.com/goplus/gop/tpl/scanner" + "github.com/goplus/gop/tpl/token" + "github.com/goplus/gop/tpl/types" +) + +// ----------------------------------------------------------------------------- + +// Compiler represents a TPL compiler. +type Compiler struct { + cl.Result +} + +// New creates a new TPL compiler. +func New(filename string, src any, fset *token.FileSet) (ret Compiler, err error) { + if fset == nil { + fset = token.NewFileSet() + } + f, err := parser.ParseFile(fset, filename, src, 0) + if err != nil { + return + } + ret.Result, err = cl.New(fset, f) + return +} + +// ----------------------------------------------------------------------------- + +type Scanner interface { + Scan() types.Token + Init(file *token.File, src []byte, err scanner.ScanErrorHandler, mode scanner.ScanMode) +} + +type Config struct { + Fset *token.FileSet + Scanner Scanner + ScanMode scanner.ScanMode + ScanErrorHandler scanner.ScanErrorHandler +} + +func (p *Compiler) Eval(filename string, src any, conf *Config) (result any, err error) { + b, err := iox.ReadSourceLocal(filename, src) + if err != nil { + return + } + if conf == nil { + conf = &Config{} + } + fset := conf.Fset + if fset == nil { + fset = token.NewFileSet() + } + s := conf.Scanner + if s == nil { + s = new(scanner.Scanner) + } + f := fset.AddFile(filename, fset.Base(), len(b)) + s.Init(f, b, conf.ScanErrorHandler, conf.ScanMode) + toks := make([]*types.Token, 0, len(b)>>3) + for { + t := s.Scan() + if t.Tok == token.EOF { + break + } + toks = append(toks, &t) + } + ctx := &matcher.Context{ + Fset: fset, + FileEnd: token.Pos(f.Base() + len(b)), + } + n, result, err := p.Doc.Match(toks, ctx) + if err != nil { + return + } + if n < len(toks) { + err = ctx.NewErrorf(toks[n].Pos, "unexpected token: %v", toks[n]) + } + return +} + +// -----------------------------------------------------------------------------