fix: add envs for encoder/decoder and aligned rfc

.github/workflows/generic_test-opt.yml

@@ -0,0 +1,27 @@
+name: Generic Test Go1.18-Linux-X64
+
+on: pull_request
+
+jobs:
+  build:
+    runs-on: [self-hosted, X64]
+    steps:
+      - name: Clear repository
+        run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Generic Test
+        run: GOMAXPROCS=4 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./generic_test

.github/workflows/unit_test-linux-opt-x64.yml

@@ -30,17 +30,17 @@ jobs:
 
       - name: Unit Test
         run: |
-          GOMAXPROCS=4 SONIC_USE_OPTDEC=1 go test -race -covermode=atomic -coverprofile=coverage.txt ./...
+          GOMAXPROCS=4 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -race -covermode=atomic -coverprofile=coverage.txt ./...
 
       - name: external
         run: |
           cd ./external_jsonlib_test
-          SONIC_USE_OPTDEC=1 go test -v -race ./...
+          SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./...
 
       - name: external
         run: |
           cd ./loader
-          SONIC_USE_OPTDEC=1 go test -v -race ./...
+          SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./...
       
       - name: Codecov
         run: bash <(curl -s https://codecov.io/bash)

fuzz/Makefile

@@ -12,7 +12,7 @@ run:
 	SONIC_FUZZ_MEM_LIMIT=2 GOMAXPROCS=2 go test -fuzz=${testname} -v -fuzztime 5m
 
 runopt:
-	SONIC_FUZZ_MEM_LIMIT=2 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 GOMAXPROCS=2 go test -fuzz=${testname} -v -fuzztime 5m
+	SONIC_FUZZ_MEM_LIMIT=2 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1  GOMAXPROCS=2 go test -fuzz=${testname} -v -fuzztime 5m
 
 clean:
 	rm -rf ./go-fuzz-corpus/

fuzz/other_fuzz_test.go

@@ -51,7 +51,7 @@ func fuzzHtmlEscape(t *testing.T, data []byte){
 func fuzzStream(t *testing.T, data []byte) {
 	r := bytes.NewBuffer(data)
 	dc := decoder.NewStreamDecoder(r)
-	decoderEnableValidateString(dc)
+	dc.ValidateString()
 	r1 := bytes.NewBuffer(data)
 	dc1 := decoder.NewStreamDecoder(r1)
 

internal/decoder/optdec/native.go

@@ -1,6 +1,7 @@
 package optdec
 
 import (
+	"reflect"
 	"unsafe"
 
 	"sync"
@@ -171,6 +172,8 @@ func (p *Parser) JsonBytes() []byte {
 	}
 }
 
+var nodeType = rt.UnpackType(reflect.TypeOf(node{}))
+
 func (p *Parser) parse() ErrorCode {
 	var offset uintptr
 	// when decode into struct, we should decode number as possible
@@ -182,21 +185,30 @@ func (p *Parser) parse() ErrorCode {
 	// fast path with limited node buffer
 	err := ErrorCode(native.ParseWithPadding(unsafe.Pointer(p)))
 	if err != SONIC_VISIT_FAILED {
-		goto ret;
+		p.options = old
+		return err
 	}
 
-	// fallback parse
-	// maybe node buf is not enough, continue
+	// node buf is not enough, continue parse
+	// the maxCap is always meet all valid JSON
+	maxCap := len(p.Json) / 2 + 2 
+	slice := rt.GoSlice{
+		Ptr: rt.Mallocgc(uintptr(maxCap) * nodeType.Size, nodeType, false),
+		Len: maxCap,
+		Cap: maxCap,
+	}
+	offset = p.nbuf.ncur - p.nbuf.nstart
+	rt.Memmove(unsafe.Pointer(slice.Ptr), unsafe.Pointer(&p.nodes[0]), offset)
 	p.backup = p.nodes
-	p.nodes = make([]node, len(p.Json) / 2 + 2, len(p.Json) / 2 + 2)
-	copy(p.nodes, p.backup)
-	offset = (p.nbuf.ncur - p.nbuf.nstart) / unsafe.Sizeof(node{})
+	p.nodes = *(*[]node)(unsafe.Pointer(&slice))
+
+	// update node cursor
 	p.nbuf.nstart = uintptr(unsafe.Pointer(&p.nodes[0]))
 	p.nbuf.nend = p.nbuf.nstart + uintptr(cap(p.nodes)) * unsafe.Sizeof(node{})
-	p.nbuf.ncur = uintptr(unsafe.Pointer(&p.nodes[offset]))
-	err = ErrorCode(native.ParseWithPadding(unsafe.Pointer(p)))
+	p.nbuf.ncur = p.nbuf.nstart + offset
 
-ret:
+	// continue parse json
+	err = ErrorCode(native.ParseWithPadding(unsafe.Pointer(p)))
 	p.options = old
 	return err
 }

internal/decoder/optdec/node.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"unsafe"
 
+	"github.com/bytedance/sonic/internal/envs"
 	"github.com/bytedance/sonic/internal/rt"
 )
 
@@ -67,7 +68,6 @@ type efacePool struct{
 	t64   		rt.T64Pool
 	tslice 		rt.TslicePool
 	tstring 	rt.TstringPool
-	efaceMap    rt.MapPool
 	efaceSlice  rt.SlicePool
 }
 
@@ -84,13 +84,13 @@ func newEfacePool(stat *jsonStat, useNumber bool) *efacePool {
 		t64: rt.NewT64Pool(nums),
 		tslice: rt.NewTslicePool(int(stat.array)),
 		tstring: rt.NewTstringPool(strs),
-		efaceMap: rt.NewMapPool(rt.MapEfaceMapType, int(stat.object), int(stat.object_keys)),
 		efaceSlice: rt.NewPool(rt.AnyType, int(stat.array_elems)),
 	}
 }
 
 func (self *efacePool) GetMap(hint int) unsafe.Pointer {
-	return unsafe.Pointer(self.efaceMap.GetMap(hint))
+	m := make(map[string]interface{}, hint)
+	return *(*unsafe.Pointer)(unsafe.Pointer(&m))
 }
 
 func (self *efacePool) GetSlice(hint int) unsafe.Pointer {
@@ -116,7 +116,7 @@ func (self *efacePool) ConvTnum(val json.Number, dst unsafe.Pointer) {
 /********************************************************/
 
 func canUseFastMap( opts uint64, root *rt.GoType) bool {
-	return rt.EnbaleFastMap && (opts & (1 << _F_copy_string)) == 0 &&  (opts & (1 << _F_use_int64)) == 0  && (root == rt.AnyType || root == rt.MapEfaceType || root == rt.SliceEfaceType) 
+	return envs.UseFastMap && (opts & (1 << _F_copy_string)) == 0 &&  (opts & (1 << _F_use_int64)) == 0  && (root == rt.AnyType || root == rt.MapEfaceType || root == rt.SliceEfaceType) 
 }
 
 func NewContext(json string, pos int, opts uint64, root *rt.GoType) (Context, error) {
@@ -193,8 +193,8 @@ func (val Node) Next() uintptr {
 		return PtrOffset(val.cptr, 1)
 	}
 	cobj := ptrCast(val.cptr)
-	offset := int(uint64(cobj.val) >> ConLenBits)
-	return PtrOffset(val.cptr, uintptr(offset))
+	offset := int64(uint64(cobj.val) >> ConLenBits)
+	return PtrOffset(val.cptr, offset)
 }
 
 func (val *Node) next() {
@@ -1299,6 +1299,6 @@ func (node *Node) AsEfaceFallback(ctx *Context) (interface{}, error) {
 }
 
 //go:nosplit
-func PtrOffset(ptr uintptr, off uintptr) uintptr {
-	return ptr + off * unsafe.Sizeof(node{})
+func PtrOffset(ptr uintptr, off int64) uintptr {
+	return uintptr(int64(ptr) + off * int64(unsafe.Sizeof(node{})))
 }