diff --git a/decoder/assembler_amd64_go116.go b/decoder/assembler_amd64_go116.go index 8069b3785..35e2d009c 100644 --- a/decoder/assembler_amd64_go116.go +++ b/decoder/assembler_amd64_go116.go @@ -215,6 +215,7 @@ var ( type _Assembler struct { jit.BaseAssembler p _Program + name string } func newAssembler(p _Program) *_Assembler { @@ -224,7 +225,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Decoder { - return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow)) + return ptodec(self.BaseAssembler.LoadWithFaker("decode_"+self.name, _FP_size, _FP_args, _Decoder_Shadow)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/decoder/assembler_amd64_go117.go b/decoder/assembler_amd64_go117.go index f791b124d..2b1b0e3fb 100644 --- a/decoder/assembler_amd64_go117.go +++ b/decoder/assembler_amd64_go117.go @@ -225,7 +225,7 @@ var ( ) func (self *_Assembler) Load() _Decoder { - return ptodec(self.BaseAssembler.Load(self.name+".Decoder", _FP_size, _FP_args, argPtrs, localPtrs)) + return ptodec(self.BaseAssembler.Load("decode_"+self.name, _FP_size, _FP_args, argPtrs, localPtrs)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/decoder/generic_amd64_go117.go b/decoder/generic_amd64_go117.go index 426ec3389..2d79290c1 100644 --- a/decoder/generic_amd64_go117.go +++ b/decoder/generic_amd64_go117.go @@ -90,7 +90,7 @@ var ( func (self *_ValueDecoder) build() uintptr { self.Init(self.compile) - return *(*uintptr)(self.Load("generic_decoder", _VD_size, _VD_args, argPtrs1, localPtrs1)) + return *(*uintptr)(self.Load("decode_value", _VD_size, _VD_args, argPtrs1, localPtrs1)) } /** Function Calling Helpers **/ diff --git a/decoder/stubs_go115.go b/decoder/stubs_go115.go new file mode 100644 index 000000000..b33dd5cb9 --- /dev/null +++ b/decoder/stubs_go115.go @@ -0,0 +1,112 @@ +//go:build go1.15,!go1.20 +// +build go1.15,!go1.20 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package decoder + +import ( + `unsafe` + `reflect` + + _ `github.com/chenzhuoyu/base64x` + + `github.com/bytedance/sonic/internal/rt` +) + +//go:linkname _subr__b64decode github.com/chenzhuoyu/base64x._subr__b64decode +var _subr__b64decode uintptr + +// runtime.maxElementSize +const _max_map_element_size uintptr = 128 + +func mapfast(vt reflect.Type) bool { + return vt.Elem().Size() <= _max_map_element_size +} + +//go:nosplit +//go:linkname throw runtime.throw +//goland:noinspection GoUnusedParameter +func throw(s string) + +//go:linkname convT64 runtime.convT64 +//goland:noinspection GoUnusedParameter +func convT64(v uint64) unsafe.Pointer + +//go:linkname convTslice runtime.convTslice +//goland:noinspection GoUnusedParameter +func convTslice(v []byte) unsafe.Pointer + +//go:linkname convTstring runtime.convTstring +//goland:noinspection GoUnusedParameter +func convTstring(v string) unsafe.Pointer + +//go:noescape +//go:linkname memequal runtime.memequal +//goland:noinspection GoUnusedParameter +func memequal(a unsafe.Pointer, b unsafe.Pointer, size uintptr) bool + +//go:noescape +//go:linkname memmove runtime.memmove +//goland:noinspection GoUnusedParameter +func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr) + +//go:linkname mallocgc runtime.mallocgc +//goland:noinspection GoUnusedParameter +func mallocgc(size uintptr, typ *rt.GoType, needzero bool) unsafe.Pointer + +//go:linkname makeslice runtime.makeslice +//goland:noinspection GoUnusedParameter +func makeslice(et *rt.GoType, len int, cap int) unsafe.Pointer + +//go:noescape +//go:linkname growslice runtime.growslice +//goland:noinspection GoUnusedParameter +func growslice(et *rt.GoType, old rt.GoSlice, cap int) rt.GoSlice + +//go:linkname makemap_small runtime.makemap_small +func makemap_small() unsafe.Pointer + +//go:linkname mapassign runtime.mapassign +//goland:noinspection GoUnusedParameter +func mapassign(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer + +//go:linkname mapassign_fast32 runtime.mapassign_fast32 +//goland:noinspection GoUnusedParameter +func mapassign_fast32(t *rt.GoType, h unsafe.Pointer, k uint32) unsafe.Pointer + +//go:linkname mapassign_fast64 runtime.mapassign_fast64 +//goland:noinspection GoUnusedParameter +func mapassign_fast64(t *rt.GoType, h unsafe.Pointer, k uint64) unsafe.Pointer + +//go:linkname mapassign_fast64ptr runtime.mapassign_fast64ptr +//goland:noinspection GoUnusedParameter +func mapassign_fast64ptr(t *rt.GoType, h unsafe.Pointer, k unsafe.Pointer) unsafe.Pointer + +//go:linkname mapassign_faststr runtime.mapassign_faststr +//goland:noinspection GoUnusedParameter +func mapassign_faststr(t *rt.GoType, h unsafe.Pointer, s string) unsafe.Pointer + +//go:nosplit +//go:linkname memclrHasPointers runtime.memclrHasPointers +//goland:noinspection GoUnusedParameter +func memclrHasPointers(ptr unsafe.Pointer, n uintptr) + +//go:noescape +//go:linkname memclrNoHeapPointers runtime.memclrNoHeapPointers +//goland:noinspection GoUnusedParameter +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) \ No newline at end of file diff --git a/encoder/assembler_amd64_go116.go b/encoder/assembler_amd64_go116.go index 56a898f0a..0c546b1b7 100644 --- a/encoder/assembler_amd64_go116.go +++ b/encoder/assembler_amd64_go116.go @@ -187,7 +187,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Encoder { - return ptoenc(self.BaseAssembler.Load(self.name+".Encoder", _FP_size, _FP_args, argStackmap, localStackmap)) + return ptoenc(self.BaseAssembler.LoadWithFaker("encode_"+self.name, _FP_size, _FP_args, _Encoder_Shadow)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/encoder/assembler_amd64_go117.go b/encoder/assembler_amd64_go117.go index d01cf8293..16f3c28e3 100644 --- a/encoder/assembler_amd64_go117.go +++ b/encoder/assembler_amd64_go117.go @@ -192,7 +192,7 @@ func newAssembler(p _Program) *_Assembler { /** Assembler Interface **/ func (self *_Assembler) Load() _Encoder { - return ptoenc(self.BaseAssembler.Load(self.name+".Encoder", _FP_size, _FP_args, argStackmap, localStackmap)) + return ptoenc(self.BaseAssembler.Load("encode_"+self.name, _FP_size, _FP_args, argStackmap, localStackmap)) } func (self *_Assembler) Init(p _Program) *_Assembler { diff --git a/internal/jit/assembler_amd64.go b/internal/jit/assembler_amd64.go index 5d337df90..dafd744f1 100644 --- a/internal/jit/assembler_amd64.go +++ b/internal/jit/assembler_amd64.go @@ -209,7 +209,7 @@ func (self *BaseAssembler) Init(f func()) { func (self *BaseAssembler) Load(name string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) loader.Function { self.build() - l := loader.Loader{ + l := loader.ModuleLoader{ Name: "sonic.jit", File: "github.com/bytedance/sonic/jit.go", } @@ -217,6 +217,11 @@ func (self *BaseAssembler) Load(name string, frameSize int, argSize int, argStac return l.LoadFunc(self.c, name, frameSize, argSize, argStackmap, localStackmap) } +func (self *BaseAssembler) LoadWithFaker(name string, frameSize int, argSize int, faker interface{}) loader.Function { + self.build() + return loader.Loader(self.c).LoadWithFaker(name, frameSize, argSize, faker) +} + /** Assembler Stages **/ func (self *BaseAssembler) init() { diff --git a/internal/loader/funcdata_go116.go b/internal/loader/funcdata_go116.go index 20d7df1a2..4570cbd06 100644 --- a/internal/loader/funcdata_go116.go +++ b/internal/loader/funcdata_go116.go @@ -22,7 +22,7 @@ import ( `unsafe` ) -type _Func struct { +type _func struct { entry uintptr // start pc nameoff int32 // function name args int32 // in/out args size @@ -39,7 +39,7 @@ type _Func struct { localptrs uintptr } -type _FuncTab struct { +type _funcTab struct { entry uintptr funcoff uintptr } @@ -74,14 +74,14 @@ type _TextSection struct { baseaddr uintptr // relocated section address } -type _ModuleData struct { +type moduledata struct { pcHeader *_PCHeader funcnametab []byte cutab []uint32 filetab []byte pctab []byte - pclntable []_Func - ftab []_FuncTab + pclntable []_func + ftab []_funcTab findfunctab *_FindFuncBucket minpc, maxpc uintptr text, etext uintptr @@ -103,7 +103,7 @@ type _ModuleData struct { gcdatamask, gcbssmask _BitVector typemap map[int32]unsafe.Pointer bad bool - next *_ModuleData + next *moduledata } type _FindFuncBucket struct { @@ -131,7 +131,7 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in maxpc := pc + size /* function entry */ - lnt := []_Func {{ + lnt := []_func {{ entry : pc, nameoff : 1, args : int32(args), @@ -142,14 +142,14 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in }} /* function table */ - tab := []_FuncTab { + tab := []_funcTab { {entry: pc}, {entry: pc}, {entry: maxpc}, } /* module data */ - mod := &_ModuleData { + mod := &moduledata { pcHeader : modHeader, funcnametab : append(append([]byte{0}, name...), 0), pctab : append(makePCtab(fp), encodeVariant(int(size))...), @@ -166,4 +166,6 @@ func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args in /* verify and register the new module */ moduledataverify1(mod) registerModule(mod) -} \ No newline at end of file +} + +func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) \ No newline at end of file diff --git a/internal/loader/loader.go b/internal/loader/loader.go index ad0ed6011..594d5a876 100644 --- a/internal/loader/loader.go +++ b/internal/loader/loader.go @@ -20,8 +20,12 @@ package loader import ( + "fmt" + "os" "syscall" "unsafe" + + "github.com/bytedance/sonic/internal/rt" ) const ( @@ -30,8 +34,28 @@ const ( _RW = syscall.PROT_READ | syscall.PROT_WRITE ) +type Loader []byte + +func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) { + p := os.Getpagesize() + n := int(rnd(int64(len(self)), int64(p))) + + /* register the function */ + m := mmap(n) + v := fmt.Sprintf("runtime.__%s_%x", fn, m) + argsptr, localsptr := GetStackMap(faker) + registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr) + + /* reference as a slice */ + s := rt.BytesFrom(unsafe.Pointer(m), len(self), n) + + /* copy the machine code, and make it executable */ + copy(s, self) + mprotect(m, n) + return Function(&m) +} -type Loader struct { +type ModuleLoader struct { Name string File string Options @@ -43,51 +67,8 @@ type Options struct { type Function unsafe.Pointer -// func (self Loader) LoadWithFaker(fn string, fp int, args int, faker interface{}) (f Function) { -// p := os.Getpagesize() -// n := int(rnd(int64(len(self)), int64(p))) - -// /* register the function */ -// m := mmap(n) -// v := fmt.Sprintf("runtime.__%s_%x", fn, m) -// argsptr, localsptr := GetStackMap(faker) -// registerFunction(v, m, uintptr(n), fp, args, uintptr(len(self)), argsptr, localsptr) - -// /* reference as a slice */ -// s := *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader { -// Data : m, -// Cap : n, -// Len : len(self), -// })) - -// /* copy the machine code, and make it executable */ -// copy(s, self) -// mprotect(m, n) -// return Function(&m) -// } - -// func (self Loader) Load(fn string, fp int, args int) (f Function) { -// return self.LoadWithFaker(fn, fp, args, func(){}) -// } - -func Load(modulename string, filenames []string, funcs []Func, text []byte) (out []Function) { - // generate module data and allocate memory address - mod := makeModuledata(modulename, filenames, funcs, text) - - // verify and register the new module - moduledataverify1(mod) - registerModule(mod) - - // encapsulate function address - out = make([]Function, len(funcs)) - for i, f := range funcs { - m := uintptr(mod.text + uintptr(f.EntryOff)) - out[i] = Function(&m) - } - return -} -func (self Loader) LoadFunc(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function { +func (self ModuleLoader) LoadFunc(text []byte, funcName string, frameSize int, argSize int, argStackmap []bool, localStackmap []bool) Function { size := uint32(len(text)) fn := Func{ @@ -132,4 +113,21 @@ func (self Loader) LoadFunc(text []byte, funcName string, frameSize int, argSize out := Load(self.Name + funcName, []string{self.File}, []Func{fn}, text) return out[0] +} + +func Load(modulename string, filenames []string, funcs []Func, text []byte) (out []Function) { + // generate module data and allocate memory address + mod := makeModuledata(modulename, filenames, funcs, text) + + // verify and register the new module + moduledataverify1(mod) + registerModule(mod) + + // encapsulate function address + out = make([]Function, len(funcs)) + for i, f := range funcs { + m := uintptr(mod.text + uintptr(f.EntryOff)) + out[i] = Function(&m) + } + return } \ No newline at end of file diff --git a/internal/loader/loader_test.go b/internal/loader/loader_test.go index c6015b42b..9edad4f02 100644 --- a/internal/loader/loader_test.go +++ b/internal/loader/loader_test.go @@ -32,7 +32,7 @@ func TestLoader_LoadFunc(t *testing.T) { 0xc3, // RET } v0 := 0 - l := Loader{ + l := ModuleLoader{ Name: "test", File: "test.go", }