fix: parse with large json

bytedance · Jul 1, 2024 · bd7e026 · bd7e026
1 parent 16634bd
commit bd7e026
Show file tree

Hide file tree

Showing 13 changed files with 26,185 additions and 26,140 deletions.
diff --git a/external_jsonlib_test/go.mod b/external_jsonlib_test/go.mod
@@ -13,6 +13,7 @@ require (
 )
 
 require (
+	github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect

diff --git a/external_jsonlib_test/go.sum b/external_jsonlib_test/go.sum
@@ -1,5 +1,7 @@
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2 h1:e+WTWDw35RetW+Zuhcy4YPFmNPRh7yPG7lAaWJeUQl0=
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
@@ -45,6 +47,13 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/bytedance/sonic
 go 1.16
 
 require (
+	github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2
 	github.com/bytedance/sonic/loader v0.1.1
 	github.com/cloudwego/base64x v0.1.4
 	github.com/davecgh/go-spew v1.1.1

diff --git a/go.sum b/go.sum
@@ -1,3 +1,5 @@
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2 h1:e+WTWDw35RetW+Zuhcy4YPFmNPRh7yPG7lAaWJeUQl0=
+github.com/bytedance/gopkg v0.0.0-20240531030433-5df24c0168e2/go.mod h1:FtQG3YbQG9L/91pbKSw787yBQPutC+457AvDW77fgUQ=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
@@ -24,6 +26,13 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

diff --git a/internal/decoder/optdec/native.go b/internal/decoder/optdec/native.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bytedance/sonic/internal/native/types"
 	"github.com/bytedance/sonic/internal/rt"
 	"github.com/bytedance/sonic/utf8"
+	"github.com/bytedance/gopkg/lang/dirtmake"
 )
 
 
@@ -173,6 +174,7 @@ func (p *Parser) JsonBytes() []byte {
 
 func (p *Parser) parse() ErrorCode {
 	var offset uintptr
+	var slice []byte
 	// when decode into struct, we should decode number as possible
 	old := p.options
 	if !p.isEface {
@@ -188,12 +190,14 @@ func (p *Parser) parse() ErrorCode {
 	// fallback parse
 	// maybe node buf is not enough, continue
 	p.backup = p.nodes
-	p.nodes = make([]node, len(p.Json) / 2 + 2, len(p.Json) / 2 + 2)
-	copy(p.nodes, p.backup)
-	offset = (p.nbuf.ncur - p.nbuf.nstart) / unsafe.Sizeof(node{})
+	slice = dirtmake.Bytes(0, (len(p.Json) / 2 + 2) * int(unsafe.Sizeof(node{})))
+	(*rt.GoSlice)(unsafe.Pointer(&slice)).Cap = len(p.Json) / 2 + 2;
+	p.nodes = *(*[]node)(unsafe.Pointer(&slice))
+	p.nodes = append(p.nodes, p.backup...)
+	offset = p.nbuf.ncur - p.nbuf.nstart
 	p.nbuf.nstart = uintptr(unsafe.Pointer(&p.nodes[0]))
 	p.nbuf.nend = p.nbuf.nstart + uintptr(cap(p.nodes)) * unsafe.Sizeof(node{})
-	p.nbuf.ncur = uintptr(unsafe.Pointer(&p.nodes[offset]))
+	p.nbuf.ncur = p.nbuf.nstart + offset
 	err = ErrorCode(native.ParseWithPadding(unsafe.Pointer(p)))
 
 ret:

diff --git a/internal/decoder/optdec/node.go b/internal/decoder/optdec/node.go
@@ -193,8 +193,8 @@ func (val Node) Next() uintptr {
 		return PtrOffset(val.cptr, 1)
 	}
 	cobj := ptrCast(val.cptr)
-	offset := int(uint64(cobj.val) >> ConLenBits)
-	return PtrOffset(val.cptr, uintptr(offset))
+	offset := int64(uint64(cobj.val) >> ConLenBits)
+	return PtrOffset(val.cptr, offset)
 }
 
 func (val *Node) next() {
@@ -1299,6 +1299,6 @@ func (node *Node) AsEfaceFallback(ctx *Context) (interface{}, error) {
 }
 
 //go:nosplit
-func PtrOffset(ptr uintptr, off uintptr) uintptr {
-	return ptr + off * unsafe.Sizeof(node{})
+func PtrOffset(ptr uintptr, off int64) uintptr {
+	return uintptr(int64(ptr) + off * int64(unsafe.Sizeof(node{})))
 }
diff --git a/internal/native/avx2/parse_with_padding_subr.go b/internal/native/avx2/parse_with_padding_subr.go