diff --git a/.github/workflows/push-check-go115.yml b/.github/workflows/push-check-go115.yml new file mode 100644 index 000000000..3101e56e0 --- /dev/null +++ b/.github/workflows/push-check-go115.yml @@ -0,0 +1,24 @@ +name: Push Check Go1.15 + +on: push + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Unit Test + run: go test -v -race -covermode=atomic -coverprofile=coverage.out ./... diff --git a/.github/workflows/push-check.yml b/.github/workflows/push-check-go116.yml similarity index 96% rename from .github/workflows/push-check.yml rename to .github/workflows/push-check-go116.yml index 5dce5bd9c..e9831fb85 100644 --- a/.github/workflows/push-check.yml +++ b/.github/workflows/push-check-go116.yml @@ -1,4 +1,4 @@ -name: Push Check +name: Push Check Go1.16 on: push diff --git a/decoder/assembler_amd64_go115.go b/decoder/assembler_amd64_go115.go new file mode 100644 index 000000000..37341c68b --- /dev/null +++ b/decoder/assembler_amd64_go115.go @@ -0,0 +1,1563 @@ +// +build go1.15,!go1.16 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package decoder + + import ( + `encoding/json` + `fmt` + `math` + `reflect` + `unsafe` + + `github.com/bytedance/sonic/internal/caching` + `github.com/bytedance/sonic/internal/cpu` + `github.com/bytedance/sonic/internal/jit` + `github.com/bytedance/sonic/internal/native` + `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` + `github.com/twitchyliquid64/golang-asm/obj` + ) + + /** Register Allocations + * + * State Registers: + * + * %rbx : stack base + * %r12 : input pointer + * %r13 : input length + * %r14 : input cursor + * %r15 : value pointer + * + * Error Registers: + * + * %r10 : error type register + * %r11 : error pointer register + */ + + /** Function Prototype & Stack Map + * + * func (s string, ic int, vp unsafe.Pointer, sb *_Stack, fv uint64) (rc int, err error) + * + * s.buf : (FP) + * s.len : 8(FP) + * ic : 16(FP) + * vp : 24(FP) + * sb : 32(FP) + * fv : 40(FP) + * rc : 48(FP) + * err.vt : 56(FP) + * err.vp : 64(FP) + */ + + const ( + _FP_args = 72 // 72 bytes to pass arguments and return values for this function + _FP_fargs = 80 // 80 bytes for passing arguments to other Go functions + _FP_saves = 40 // 40 bytes for saving the registers before CALL instructions + _FP_locals = 96 // 96 bytes for local variables + ) + + const ( + _FP_offs = _FP_fargs + _FP_saves + _FP_locals + _FP_size = _FP_offs + 8 // 8 bytes for the parent frame pointer + _FP_base = _FP_size + 8 // 8 bytes for the return address + ) + + const ( + _IM_null = 0x6c6c756e // 'null' + _IM_true = 0x65757274 // 'true' + _IM_alse = 0x65736c61 // 'alse' ('false' without the 'f') + ) + + const ( + _BM_space = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n') + ) + + const ( + _LB_error = "_error" + _LB_im_error = "_im_error" + _LB_eof_error = "_eof_error" + _LB_type_error = "_type_error" + _LB_field_error = "_field_error" + _LB_range_error = "_range_error" + _LB_stack_error = "_stack_error" + _LB_base64_error = "_base64_error" + _LB_unquote_error = "_unquote_error" + _LB_parsing_error = "_parsing_error" + _LB_parsing_error_v = "_parsing_error_v" + ) + + const ( + _LB_char_0_error = "_char_0_error" + _LB_char_1_error = "_char_1_error" + _LB_char_2_error = "_char_2_error" + _LB_char_3_error = "_char_3_error" + _LB_char_4_error = "_char_4_error" + _LB_char_m2_error = "_char_m2_error" + _LB_char_m3_error = "_char_m3_error" + ) + + var ( + _AX = jit.Reg("AX") + _CX = jit.Reg("CX") + _DX = jit.Reg("DX") + _DI = jit.Reg("DI") + _SI = jit.Reg("SI") + _BP = jit.Reg("BP") + _SP = jit.Reg("SP") + _R8 = jit.Reg("R8") + _R9 = jit.Reg("R9") + _X0 = jit.Reg("X0") + _X1 = jit.Reg("X1") + ) + + var ( + _ST = jit.Reg("BX") + _IP = jit.Reg("R12") + _IL = jit.Reg("R13") + _IC = jit.Reg("R14") + _VP = jit.Reg("R15") + ) + + var ( + _DF = jit.Reg("R10") // reuse R10 in generic decoder for flags + _ET = jit.Reg("R10") + _EP = jit.Reg("R11") + ) + + var ( + _ARG_s = _ARG_sp + _ARG_sp = jit.Ptr(_SP, _FP_base) + _ARG_sl = jit.Ptr(_SP, _FP_base + 8) + _ARG_ic = jit.Ptr(_SP, _FP_base + 16) + _ARG_vp = jit.Ptr(_SP, _FP_base + 24) + _ARG_sb = jit.Ptr(_SP, _FP_base + 32) + _ARG_fv = jit.Ptr(_SP, _FP_base + 40) + ) + + var ( + _RET_rc = jit.Ptr(_SP, _FP_base + 48) + _RET_et = jit.Ptr(_SP, _FP_base + 56) + _RET_ep = jit.Ptr(_SP, _FP_base + 64) + ) + + var ( + _VAR_sv = _VAR_sv_p + _VAR_st = _VAR_st_Vt + _VAR_sr = jit.Ptr(_SP, _FP_fargs + _FP_saves) + ) + + var ( + _VAR_sv_p = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) + _VAR_sv_n = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + ) + + var ( + _VAR_st_Vt = jit.Ptr(_SP, _FP_fargs + _FP_saves + 24) + _VAR_st_Dv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 32) + _VAR_st_Iv = jit.Ptr(_SP, _FP_fargs + _FP_saves + 40) + _VAR_st_Ep = jit.Ptr(_SP, _FP_fargs + _FP_saves + 48) + ) + + var ( + _VAR_ss_AX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 56) + _VAR_ss_CX = jit.Ptr(_SP, _FP_fargs + _FP_saves + 64) + _VAR_ss_SI = jit.Ptr(_SP, _FP_fargs + _FP_saves + 72) + _VAR_ss_R8 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 80) + _VAR_ss_R9 = jit.Ptr(_SP, _FP_fargs + _FP_saves + 88) + ) + + type _Assembler struct { + jit.BaseAssembler + p _Program + } + + func newAssembler(p _Program) *_Assembler { + return new(_Assembler).Init(p) + } + + /** Assembler Interface **/ + + func (self *_Assembler) Load() _Decoder { + return ptodec(self.BaseAssembler.LoadWithFaker("json_decoder", _FP_size, _FP_args, _Decoder_Shadow)) + } + + func (self *_Assembler) Init(p _Program) *_Assembler { + self.p = p + self.BaseAssembler.Init(self.compile) + return self + } + + func (self *_Assembler) compile() { + self.prologue() + self.instrs() + self.epilogue() + self.type_error() + self.field_error() + self.range_error() + self.stack_error() + self.base64_error() + self.parsing_error() + } + + /** Assembler Stages **/ + + var _OpFuncTab = [256]func(*_Assembler, *_Instr) { + _OP_any : (*_Assembler)._asm_OP_any, + _OP_dyn : (*_Assembler)._asm_OP_dyn, + _OP_str : (*_Assembler)._asm_OP_str, + _OP_bin : (*_Assembler)._asm_OP_bin, + _OP_bool : (*_Assembler)._asm_OP_bool, + _OP_num : (*_Assembler)._asm_OP_num, + _OP_i8 : (*_Assembler)._asm_OP_i8, + _OP_i16 : (*_Assembler)._asm_OP_i16, + _OP_i32 : (*_Assembler)._asm_OP_i32, + _OP_i64 : (*_Assembler)._asm_OP_i64, + _OP_u8 : (*_Assembler)._asm_OP_u8, + _OP_u16 : (*_Assembler)._asm_OP_u16, + _OP_u32 : (*_Assembler)._asm_OP_u32, + _OP_u64 : (*_Assembler)._asm_OP_u64, + _OP_f32 : (*_Assembler)._asm_OP_f32, + _OP_f64 : (*_Assembler)._asm_OP_f64, + _OP_unquote : (*_Assembler)._asm_OP_unquote, + _OP_nil_1 : (*_Assembler)._asm_OP_nil_1, + _OP_nil_2 : (*_Assembler)._asm_OP_nil_2, + _OP_nil_3 : (*_Assembler)._asm_OP_nil_3, + _OP_deref : (*_Assembler)._asm_OP_deref, + _OP_index : (*_Assembler)._asm_OP_index, + _OP_is_null : (*_Assembler)._asm_OP_is_null, + _OP_map_init : (*_Assembler)._asm_OP_map_init, + _OP_map_key_i8 : (*_Assembler)._asm_OP_map_key_i8, + _OP_map_key_i16 : (*_Assembler)._asm_OP_map_key_i16, + _OP_map_key_i32 : (*_Assembler)._asm_OP_map_key_i32, + _OP_map_key_i64 : (*_Assembler)._asm_OP_map_key_i64, + _OP_map_key_u8 : (*_Assembler)._asm_OP_map_key_u8, + _OP_map_key_u16 : (*_Assembler)._asm_OP_map_key_u16, + _OP_map_key_u32 : (*_Assembler)._asm_OP_map_key_u32, + _OP_map_key_u64 : (*_Assembler)._asm_OP_map_key_u64, + _OP_map_key_f32 : (*_Assembler)._asm_OP_map_key_f32, + _OP_map_key_f64 : (*_Assembler)._asm_OP_map_key_f64, + _OP_map_key_str : (*_Assembler)._asm_OP_map_key_str, + _OP_map_key_utext : (*_Assembler)._asm_OP_map_key_utext, + _OP_map_key_utext_p : (*_Assembler)._asm_OP_map_key_utext_p, + _OP_array_skip : (*_Assembler)._asm_OP_array_skip, + _OP_array_clear : (*_Assembler)._asm_OP_array_clear, + _OP_array_clear_p : (*_Assembler)._asm_OP_array_clear_p, + _OP_slice_init : (*_Assembler)._asm_OP_slice_init, + _OP_slice_append : (*_Assembler)._asm_OP_slice_append, + _OP_object_skip : (*_Assembler)._asm_OP_object_skip, + _OP_object_next : (*_Assembler)._asm_OP_object_next, + _OP_struct_field : (*_Assembler)._asm_OP_struct_field, + _OP_unmarshal : (*_Assembler)._asm_OP_unmarshal, + _OP_unmarshal_p : (*_Assembler)._asm_OP_unmarshal_p, + _OP_unmarshal_text : (*_Assembler)._asm_OP_unmarshal_text, + _OP_unmarshal_text_p : (*_Assembler)._asm_OP_unmarshal_text_p, + _OP_lspace : (*_Assembler)._asm_OP_lspace, + _OP_match_char : (*_Assembler)._asm_OP_match_char, + _OP_check_char : (*_Assembler)._asm_OP_check_char, + _OP_load : (*_Assembler)._asm_OP_load, + _OP_save : (*_Assembler)._asm_OP_save, + _OP_drop : (*_Assembler)._asm_OP_drop, + _OP_drop_2 : (*_Assembler)._asm_OP_drop_2, + _OP_recurse : (*_Assembler)._asm_OP_recurse, + _OP_goto : (*_Assembler)._asm_OP_goto, + _OP_switch : (*_Assembler)._asm_OP_switch, + } + + func (self *_Assembler) instr(v *_Instr) { + if fn := _OpFuncTab[v.op()]; fn != nil { + fn(self, v) + } else { + panic(fmt.Sprintf("invalid opcode: %d", v.op())) + } + } + + func (self *_Assembler) instrs() { + for i, v := range self.p { + self.Mark(i) + self.instr(&v) + self.debug_instr(i, &v) + } + } + + func (self *_Assembler) epilogue() { + self.Mark(len(self.p)) + self.Emit("XORL", _ET, _ET) // XORL ET, ET + self.Emit("XORL", _EP, _EP) // XORL EP, EP + self.Link(_LB_error) // _error: + self.Emit("MOVQ", _IC, _RET_rc) // MOVQ IC, rc<>+40(FP) + self.Emit("MOVQ", _ET, _RET_et) // MOVQ ET, et<>+48(FP) + self.Emit("MOVQ", _EP, _RET_ep) // MOVQ EP, ep<>+56(FP) + self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP + self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP + self.Emit("RET") // RET + } + + func (self *_Assembler) prologue() { + self.Emit("SUBQ", jit.Imm(_FP_size), _SP) // SUBQ $_FP_size, SP + self.Emit("MOVQ", _BP, jit.Ptr(_SP, _FP_offs)) // MOVQ BP, _FP_offs(SP) + self.Emit("LEAQ", jit.Ptr(_SP, _FP_offs), _BP) // LEAQ _FP_offs(SP), BP + self.Emit("MOVQ", _ARG_sp, _IP) // MOVQ s.p<>+0(FP), IP + self.Emit("MOVQ", _ARG_sl, _IL) // MOVQ s.l<>+8(FP), IL + self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC + self.Emit("MOVQ", _ARG_vp, _VP) // MOVQ vp<>+24(FP), VP + self.Emit("MOVQ", _ARG_sb, _ST) // MOVQ vp<>+32(FP), ST + } + + /** Function Calling Helpers **/ + + var _REG_go = []obj.Addr { + _ST, + _VP, + _IP, + _IL, + _IC, + } + + func (self *_Assembler) save(r ...obj.Addr) { + for i, v := range r { + if i > _FP_saves / 8 - 1 { + panic("too many registers to save") + } else { + self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + int64(i) * 8)) + } + } + } + + func (self *_Assembler) load(r ...obj.Addr) { + for i, v := range r { + if i > _FP_saves / 8 - 1 { + panic("too many registers to load") + } else { + self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + int64(i) * 8), v) + } + } + } + + func (self *_Assembler) call(fn obj.Addr) { + self.Emit("MOVQ", fn, _AX) // MOVQ ${fn}, AX + self.Rjmp("CALL", _AX) // CALL AX + } + + func (self *_Assembler) call_go(fn obj.Addr) { + self.save(_REG_go...) // SAVE $REG_go + self.call(fn) // CALL ${fn} + self.load(_REG_go...) // LOAD $REG_go + } + + func (self *_Assembler) call_sf(fn obj.Addr) { + self.Emit("LEAQ", _ARG_s, _DI) // LEAQ s<>+0(FP), DI + self.Emit("MOVQ", _IC, _ARG_ic) // MOVQ IC, ic<>+16(FP) + self.Emit("LEAQ", _ARG_ic, _SI) // LEAQ ic<>+16(FP), SI + self.Emit("LEAQ", jit.Ptr(_ST, _FsmOffset), _DX) // LEAQ _FsmOffset(ST), DX + self.call(fn) // CALL ${fn} + self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC + } + + func (self *_Assembler) call_vf(fn obj.Addr) { + self.Emit("LEAQ", _ARG_s, _DI) // LEAQ s<>+0(FP), DI + self.Emit("MOVQ", _IC, _ARG_ic) // MOVQ IC, ic<>+16(FP) + self.Emit("LEAQ", _ARG_ic, _SI) // LEAQ ic<>+16(FP), SI + self.Emit("LEAQ", _VAR_st, _DX) // LEAQ st, DX + self.call(fn) // CALL ${fn} + self.Emit("MOVQ", _ARG_ic, _IC) // MOVQ ic<>+16(FP), IC + } + + /** Assembler Error Handlers **/ + + var ( + _F_convT64 = jit.Func(convT64) + _F_error_wrap = jit.Func(error_wrap) + _F_error_type = jit.Func(error_type) + _F_error_field = jit.Func(error_field) + _F_error_value = jit.Func(error_value) + ) + + var ( + _I_int8 , _T_int8 = rtype(reflect.TypeOf(int8(0))) + _I_int16 , _T_int16 = rtype(reflect.TypeOf(int16(0))) + _I_int32 , _T_int32 = rtype(reflect.TypeOf(int32(0))) + _I_uint8 , _T_uint8 = rtype(reflect.TypeOf(uint8(0))) + _I_uint16 , _T_uint16 = rtype(reflect.TypeOf(uint16(0))) + _I_uint32 , _T_uint32 = rtype(reflect.TypeOf(uint32(0))) + _I_float32 , _T_float32 = rtype(reflect.TypeOf(float32(0))) + ) + + var ( + _T_error = rt.UnpackType(errorType) + _I_base64_CorruptInputError = jit.Itab(_T_error, base64CorruptInputError) + ) + + var ( + _V_stackOverflow = jit.Imm(int64(uintptr(unsafe.Pointer(&stackOverflow)))) + _I_json_UnsupportedValueError = jit.Itab(_T_error, reflect.TypeOf(new(json.UnsupportedValueError))) + ) + + func (self *_Assembler) type_error() { + self.Link(_LB_type_error) // _type_error: + self.Emit("MOVQ", _ET, jit.Ptr(_SP, 0)) // MOVQ ET, (SP) + self.call_go(_F_error_type) // CALL_GO error_type + self.Emit("MOVQ", jit.Ptr(_SP, 8), _ET) // MOVQ 8(SP), ET + self.Emit("MOVQ", jit.Ptr(_SP, 16), _EP) // MOVQ 16(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) field_error() { + self.Link(_LB_field_error) // _field_error: + self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 0)) // MOVOU X0, (SP) + self.call_go(_F_error_field) // CALL_GO error_field + self.Emit("MOVQ" , jit.Ptr(_SP, 16), _ET) // MOVQ 16(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _EP) // MOVQ 24(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) range_error() { + self.Link(_LB_range_error) // _range_error: + self.slice_from(_VAR_st_Ep, 0) // SLICE st.Ep, $0 + self.Emit("MOVQ", _DI, jit.Ptr(_SP, 0)) // MOVQ DI, (SP) + self.Emit("MOVQ", _SI, jit.Ptr(_SP, 8)) // MOVQ SI, 8(SP) + self.Emit("MOVQ", _ET, jit.Ptr(_SP, 16)) // MOVQ ET, 16(SP) + self.Emit("MOVQ", _EP, jit.Ptr(_SP, 24)) // MOVQ EP, 24(SP) + self.call_go(_F_error_value) // CALL_GO error_value + self.Emit("MOVQ", jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ", jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) stack_error() { + self.Link(_LB_stack_error) // _stack_error: + self.Emit("MOVQ", _V_stackOverflow, _EP) // MOVQ ${_V_stackOverflow}, EP + self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET) // MOVQ ${_I_json_UnsupportedValueError}, ET + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) base64_error() { + self.Link(_LB_base64_error) + self.Emit("NEGQ", _AX) // NEGQ AX + self.Emit("SUBQ", jit.Imm(1), _AX) // SUBQ $1, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.call_go(_F_convT64) // CALL_GO convT64 + self.Emit("MOVQ", jit.Ptr(_SP, 8), _EP) // MOVQ 8(SP), EP + self.Emit("MOVQ", _I_base64_CorruptInputError, _ET) // MOVQ ${itab(base64.CorruptInputError)}, ET + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) parsing_error() { + self.Link(_LB_eof_error) // _eof_error: + self.Emit("MOVQ" , _IL, _IC) // MOVQ IL, IC + self.Emit("MOVL" , jit.Imm(int64(types.ERR_EOF)), _EP) // MOVL ${types.ERR_EOF}, EP + self.Sjmp("JMP" , _LB_parsing_error) // JMP _parsing_error + self.Link(_LB_unquote_error) // _unquote_error: + self.Emit("SUBQ" , _VAR_sr, _SI) // SUBQ sr, SI + self.Emit("SUBQ" , _SI, _IC) // SUBQ IL, IC + self.Link(_LB_parsing_error_v) // _parsing_error_v: + self.Emit("MOVQ" , _AX, _EP) // MOVQ AX, EP + self.Emit("NEGQ" , _EP) // NEGQ EP + self.Sjmp("JMP" , _LB_parsing_error) // JMP _parsing_error + self.Link(_LB_char_m3_error) // _char_m3_error: + self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC + self.Link(_LB_char_m2_error) // _char_m2_error: + self.Emit("SUBQ" , jit.Imm(2), _IC) // SUBQ $2, IC + self.Sjmp("JMP" , _LB_char_0_error) // JMP _char_0_error + self.Link(_LB_im_error) // _im_error: + self.Emit("CMPB" , _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPB CX, (IP)(IC) + self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error + self.Emit("SHRL" , jit.Imm(8), _CX) // SHRL $8, CX + self.Emit("CMPB" , _CX, jit.Sib(_IP, _IC, 1, 1)) // CMPB CX, 1(IP)(IC) + self.Sjmp("JNE" , _LB_char_1_error) // JNE _char_1_error + self.Emit("SHRL" , jit.Imm(8), _CX) // SHRL $8, CX + self.Emit("CMPB" , _CX, jit.Sib(_IP, _IC, 1, 2)) // CMPB CX, 2(IP)(IC) + self.Sjmp("JNE" , _LB_char_2_error) // JNE _char_2_error + self.Sjmp("JMP" , _LB_char_3_error) // JNE _char_3_error + self.Link(_LB_char_4_error) // _char_4_error: + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Link(_LB_char_3_error) // _char_3_error: + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Link(_LB_char_2_error) // _char_2_error: + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Link(_LB_char_1_error) // _char_1_error: + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Link(_LB_char_0_error) // _char_0_error: + self.Emit("MOVL" , jit.Imm(int64(types.ERR_INVALID_CHAR)), _EP) // MOVL ${types.ERR_INVALID_CHAR}, EP + self.Link(_LB_parsing_error) // _parsing_error: + self.Emit("MOVOU", _ARG_s, _X0) // MOVOU s, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 0)) // MOVOU X0, (SP) + self.Emit("MOVQ" , _IC, jit.Ptr(_SP, 16)) // MOVQ IC, 16(SP) + self.Emit("MOVQ" , _EP, jit.Ptr(_SP, 24)) // MOVQ EP, 24(SP) + self.call_go(_F_error_wrap) // CALL_GO error_wrap + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + /** Memory Management Routines **/ + + var ( + _T_byte = jit.Type(byteType) + _F_mallocgc = jit.Func(mallocgc) + ) + + func (self *_Assembler) malloc(nb obj.Addr, ret obj.Addr) { + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("MOVQ", _T_byte, _CX) // MOVQ ${type(byte)}, CX + self.Emit("MOVQ", nb, jit.Ptr(_SP, 0)) // MOVQ ${nb}, (SP) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.call_go(_F_mallocgc) // CALL_GO mallocgc + self.Emit("MOVQ", jit.Ptr(_SP, 24), ret) // MOVQ 24(SP), ${ret} + } + + func (self *_Assembler) valloc(vt reflect.Type, ret obj.Addr) { + self.Emit("MOVQ", jit.Imm(int64(vt.Size())), _AX) // MOVQ ${vt.Size()}, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ", jit.Type(vt), _AX) // MOVQ ${vt}, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.Emit("MOVB", jit.Imm(1), jit.Ptr(_SP, 16)) // MOVB $1, 16(SP) + self.call_go(_F_mallocgc) // CALL_GO mallocgc + self.Emit("MOVQ", jit.Ptr(_SP, 24), ret) // MOVQ 24(SP), ${ret} + } + + func (self *_Assembler) vfollow(vt reflect.Type) { + self.Emit("MOVQ" , jit.Ptr(_VP, 0), _AX) // MOVQ (VP), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNZ" , "_end_{n}") // JNZ _end_{n} + self.valloc(vt, _AX) // VALLOC ${vt}, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + self.Link("_end_{n}") // _end_{n}: + self.Emit("MOVQ" , _AX, _VP) // MOVQ AX, VP + } + + /** Value Parsing Routines **/ + + var ( + _F_vstring = jit.Imm(int64(native.S_vstring)) + _F_vnumber = jit.Imm(int64(native.S_vnumber)) + _F_vsigned = jit.Imm(int64(native.S_vsigned)) + _F_vunsigned = jit.Imm(int64(native.S_vunsigned)) + ) + + func (self *_Assembler) check_err() { + self.Emit("MOVQ" , _VAR_st_Vt, _AX) // MOVQ st.Vt, AX + self.Emit("TESTQ", _AX, _AX) // CMPQ AX, ${native.V_STRING} + self.Sjmp("JS" , _LB_parsing_error_v) // JNE _parsing_error_v + } + + func (self *_Assembler) check_eof(d int64) { + if d == 1 { + self.Emit("CMPQ", _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error + } else { + self.Emit("LEAQ", jit.Ptr(_IC, d), _AX) // LEAQ ${d}(IC), AX + self.Emit("CMPQ", _AX, _IL) // CMPQ AX, IL + self.Sjmp("JA" , _LB_eof_error) // JA _eof_error + } + } + + func (self *_Assembler) parse_string() { + self.call_vf(_F_vstring) + self.check_err() + } + + func (self *_Assembler) parse_number() { + self.call_vf(_F_vnumber) + self.check_err() + } + + func (self *_Assembler) parse_signed() { + self.call_vf(_F_vsigned) + self.check_err() + } + + func (self *_Assembler) parse_unsigned() { + self.call_vf(_F_vunsigned) + self.check_err() + } + + /** Range Checking Routines **/ + + var ( + _V_max_f32 = jit.Imm(int64(uintptr(unsafe.Pointer(_Vp_max_f32)))) + _V_min_f32 = jit.Imm(int64(uintptr(unsafe.Pointer(_Vp_min_f32)))) + ) + + var ( + _Vp_max_f32 = new(float64) + _Vp_min_f32 = new(float64) + ) + + func init() { + *_Vp_max_f32 = math.MaxFloat32 + *_Vp_min_f32 = -math.MaxFloat32 + } + + func (self *_Assembler) range_single() { + self.Emit("MOVSD" , _VAR_st_Dv, _X0) // MOVSD st.Dv, X0 + self.Emit("MOVQ" , _V_max_f32, _AX) // MOVQ _max_f32, AX + self.Emit("MOVQ" , jit.Gitab(_I_float32), _ET) // MOVQ ${itab(float32)}, ET + self.Emit("MOVQ" , jit.Gtype(_T_float32), _EP) // MOVQ ${type(float32)}, EP + self.Emit("UCOMISD" , jit.Ptr(_AX, 0), _X0) // UCOMISD (AX), X0 + self.Sjmp("JA" , _LB_range_error) // JA _range_error + self.Emit("MOVQ" , _V_min_f32, _AX) // MOVQ _min_f32, AX + self.Emit("MOVSD" , jit.Ptr(_AX, 0), _X1) // MOVSD (AX), X1 + self.Emit("UCOMISD" , _X0, _X1) // UCOMISD X0, X1 + self.Sjmp("JA" , _LB_range_error) // JA _range_error + self.Emit("CVTSD2SS", _X0, _X0) // CVTSD2SS X0, X0 + } + + func (self *_Assembler) range_signed(i *rt.GoItab, t *rt.GoType, a int64, b int64) { + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.Emit("MOVQ", jit.Gitab(i), _ET) // MOVQ ${i}, ET + self.Emit("MOVQ", jit.Gtype(t), _EP) // MOVQ ${t}, EP + self.Emit("CMPQ", _AX, jit.Imm(a)) // CMPQ AX, ${a} + self.Sjmp("JL" , _LB_range_error) // JL _range_error + self.Emit("CMPQ", _AX, jit.Imm(b)) // CMPQ AX, ${B} + self.Sjmp("JG" , _LB_range_error) // JG _range_error + } + + func (self *_Assembler) range_unsigned(i *rt.GoItab, t *rt.GoType, v uint64) { + self.Emit("MOVQ" , _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.Emit("MOVQ" , jit.Gitab(i), _ET) // MOVQ ${i}, ET + self.Emit("MOVQ" , jit.Gtype(t), _EP) // MOVQ ${t}, EP + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_range_error) // JS _range_error + self.Emit("CMPQ" , _AX, jit.Imm(int64(v))) // CMPQ AX, ${a} + self.Sjmp("JA" , _LB_range_error) // JA _range_error + } + + /** String Manipulating Routines **/ + + var ( + _F_unquote = jit.Imm(int64(native.S_unquote)) + ) + + func (self *_Assembler) slice_from(p obj.Addr, d int64) { + self.Emit("MOVQ", p, _SI) // MOVQ ${p}, SI + self.slice_from_r(_SI, d) // SLICE_R SI, ${d} + } + + func (self *_Assembler) slice_from_r(p obj.Addr, d int64) { + self.Emit("LEAQ", jit.Sib(_IP, p, 1, 0), _DI) // LEAQ (IP)(${p}), DI + self.Emit("NEGQ", p) // NEGQ ${p} + self.Emit("LEAQ", jit.Sib(_IC, p, 1, d), _SI) // LEAQ d(IC)(${p}), SI + } + + func (self *_Assembler) unquote_once(p obj.Addr, n obj.Addr) { + self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1 + self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p} + self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n} + self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1 + self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n} + self.malloc(_SI, _DX) // MALLOC SI, DX + self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI + self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI + self.Emit("MOVQ" , _DX, p) // MOVQ DX, ${p} + self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX + self.Emit("XORL" , _R8, _R8) // XORL R8, R8 + self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, fv + self.Emit("SETCC", _R8) // SETCC R8 + self.Emit("SHLQ" , jit.Imm(types.B_UNICODE_REPLACE), _R8) // SHLQ ${types.B_UNICODE_REPLACE}, R8 + self.call(_F_unquote) // CALL unquote + self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI + self.Emit("ADDQ" , jit.Imm(1), _SI) // ADDQ $1, SI + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error + self.Emit("MOVQ" , _AX, n) // MOVQ AX, ${n} + self.Link("_noescape_{n}") // _noescape_{n}: + } + + func (self *_Assembler) unquote_twice(p obj.Addr, n obj.Addr) { + self.Emit("CMPQ" , _VAR_st_Ep, jit.Imm(-1)) // CMPQ st.Ep, $-1 + self.Sjmp("JE" , _LB_eof_error) // JE _eof_error + self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -3), jit.Imm('\\')) // CMPB -3(IP)(IC), $'\\' + self.Sjmp("JNE" , _LB_char_m3_error) // JNE _char_m3_error + self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, -2), jit.Imm('"')) // CMPB -2(IP)(IC), $'"' + self.Sjmp("JNE" , _LB_char_m2_error) // JNE _char_m2_error + self.slice_from(_VAR_st_Iv, -3) // SLICE st.Iv, $-3 + self.Emit("MOVQ" , _DI, p) // MOVQ DI, ${p} + self.Emit("MOVQ" , _SI, n) // MOVQ SI, ${n} + self.Emit("MOVQ" , _SI, _AX) // MOVQ SI, AX + self.Emit("ADDQ" , _VAR_st_Iv, _AX) // ADDQ st.Iv, AX + self.Emit("CMPQ" , _VAR_st_Ep, _AX) // CMPQ st.Ep, AX + self.Sjmp("JE" , "_noescape_{n}") // JE _noescape_{n} + self.malloc(_SI, _DX) // MALLOC SI, DX + self.Emit("MOVQ" , p, _DI) // MOVQ ${p}, DI + self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI + self.Emit("MOVQ" , _DX, p) // MOVQ DX, ${p} + self.Emit("LEAQ" , _VAR_sr, _CX) // LEAQ sr, CX + self.Emit("MOVL" , jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8 + self.Emit("BTQ" , jit.Imm(_F_disable_urc), _ARG_fv) // BTQ ${_F_disable_urc}, AX + self.Emit("XORL" , _AX, _AX) // XORL AX, AX + self.Emit("SETCC", _AX) // SETCC AX + self.Emit("SHLQ" , jit.Imm(types.B_UNICODE_REPLACE), _AX) // SHLQ ${types.B_UNICODE_REPLACE}, AX + self.Emit("ORQ" , _AX, _R8) // ORQ AX, R8 + self.call(_F_unquote) // CALL unquote + self.Emit("MOVQ" , n, _SI) // MOVQ ${n}, SI + self.Emit("ADDQ" , jit.Imm(3), _SI) // ADDQ $3, SI + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_unquote_error) // JS _unquote_error + self.Emit("MOVQ" , _AX, n) // MOVQ AX, ${n} + self.Link("_noescape_{n}") // _noescape_{n}: + } + + /** Memory Clearing Routines **/ + + var ( + _F_memclrHasPointers = jit.Func(memclrHasPointers) + _F_memclrNoHeapPointers = jit.Func(memclrNoHeapPointers) + ) + + func (self *_Assembler) mem_clear_fn(ptrfree bool) { + if !ptrfree { + self.call_go(_F_memclrHasPointers) + } else { + self.call_go(_F_memclrNoHeapPointers) + } + } + + func (self *_Assembler) mem_clear_rem(size int64, ptrfree bool) { + self.Emit("MOVQ", jit.Imm(size), _CX) // MOVQ ${size}, CX + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _AX) // MOVQ (ST)(AX), AX + self.Emit("SUBQ", _VP, _AX) // SUBQ VP, AX + self.Emit("ADDQ", _AX, _CX) // ADDQ AX, CX + self.Emit("MOVQ", _VP, jit.Ptr(_SP, 0)) // MOVQ VP, (SP) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.mem_clear_fn(ptrfree) // CALL_GO memclr{Has,NoHeap}Pointers + } + + /** Map Assigning Routines **/ + + var ( + _F_mapassign = jit.Func(mapassign) + _F_mapassign_fast32 = jit.Func(mapassign_fast32) + _F_mapassign_faststr = jit.Func(mapassign_faststr) + _F_mapassign_fast64ptr = jit.Func(mapassign_fast64ptr) + ) + + var ( + _F_decodeJsonUnmarshaler obj.Addr + _F_decodeTextUnmarshaler obj.Addr + ) + + func init() { + _F_decodeJsonUnmarshaler = jit.Func(decodeJsonUnmarshaler) + _F_decodeTextUnmarshaler = jit.Func(decodeTextUnmarshaler) + } + + func (self *_Assembler) mapaccess_ptr(t reflect.Type) { + if rt.MapType(rt.UnpackType(t)).IndirectElem() { + self.vfollow(t.Elem()) + } + } + + func (self *_Assembler) mapassign_std(t reflect.Type, v obj.Addr) { + self.Emit("LEAQ", v, _AX) // LEAQ ${v}, AX + self.mapassign_call(t, _F_mapassign) // MAPASSIGN ${t}, mapassign + } + + func (self *_Assembler) mapassign_str_fast(t reflect.Type, p obj.Addr, n obj.Addr) { + self.Emit("MOVQ", jit.Type(t), _AX) // MOVQ ${t}, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ", _VP, jit.Ptr(_SP, 8)) // MOVQ VP, 8(SP) + self.Emit("MOVQ", p, jit.Ptr(_SP, 16)) // MOVQ ${p}, 16(SP) + self.Emit("MOVQ", n, jit.Ptr(_SP, 24)) // MOVQ ${n}, 24(SP) + self.call_go(_F_mapassign_faststr) // CALL_GO ${fn} + self.Emit("MOVQ", jit.Ptr(_SP, 32), _VP) // MOVQ 32(SP), VP + self.mapaccess_ptr(t) + } + + func (self *_Assembler) mapassign_call(t reflect.Type, fn obj.Addr) { + self.Emit("MOVQ", jit.Type(t), _SI) // MOVQ ${t}, SI + self.Emit("MOVQ", _SI, jit.Ptr(_SP, 0)) // MOVQ SI, (SP) + self.Emit("MOVQ", _VP, jit.Ptr(_SP, 8)) // MOVQ VP, 8(SP) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.call_go(fn) // CALL_GO ${fn} + self.Emit("MOVQ", jit.Ptr(_SP, 24), _VP) // MOVQ 24(SP), VP + } + + func (self *_Assembler) mapassign_fastx(t reflect.Type, fn obj.Addr) { + self.mapassign_call(t, fn) + self.mapaccess_ptr(t) + } + + func (self *_Assembler) mapassign_utext(t reflect.Type, addressable bool) { + pv := false + vk := t.Key() + tk := t.Key() + + /* deref pointer if needed */ + if vk.Kind() == reflect.Ptr { + pv = true + vk = vk.Elem() + } + + /* addressable value with pointer receiver */ + if addressable { + pv = false + tk = reflect.PtrTo(tk) + } + + /* allocate the key, and call the unmarshaler */ + self.valloc(vk, _DI) // VALLOC ${vk}, DI + self.Emit("MOVQ" , jit.Type(tk), _AX) // MOVQ ${tk}, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , _DI, jit.Ptr(_SP, 8)) // MOVQ DI, 8(SP) + self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 16)) // MOVOU X0, 16(SP) + self.call_go(_F_decodeTextUnmarshaler) // CALL_GO decodeTextUnmarshaler + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + self.Emit("MOVQ" , jit.Ptr(_SP, 8), _AX) // MOVQ 8(SP), AX + + /* select the correct assignment function */ + if !pv { + self.mapassign_call(t, _F_mapassign) + } else { + self.mapassign_fastx(t, _F_mapassign_fast64ptr) + } + } + + /** External Unmarshaler Routines **/ + + var ( + _F_skip_one = jit.Imm(int64(native.S_skip_one)) + ) + + func (self *_Assembler) unmarshal_json(t reflect.Type, deref bool) { + self.call_sf(_F_skip_one) // CALL_SF skip_one + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + self.slice_from_r(_AX, 0) // SLICE_R AX, $0 + self.Emit("MOVQ" , _DI, _VAR_sv_p) // MOVQ DI, sv.p + self.Emit("MOVQ" , _SI, _VAR_sv_n) // MOVQ SI, sv.n + self.unmarshal_func(t, _F_decodeJsonUnmarshaler, deref) // UNMARSHAL json, ${t}, ${deref} + } + + func (self *_Assembler) unmarshal_text(t reflect.Type, deref bool) { + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.unmarshal_func(t, _F_decodeTextUnmarshaler, deref) // UNMARSHAL text, ${t}, ${deref} + } + + func (self *_Assembler) unmarshal_func(t reflect.Type, fn obj.Addr, deref bool) { + pt := t + vk := t.Kind() + + /* allocate the field if needed */ + if deref && vk == reflect.Ptr { + self.Emit("MOVQ" , _VP, _AX) // MOVQ VP, AX + self.Emit("MOVQ" , jit.Ptr(_AX, 0), _AX) // MOVQ (AX), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNZ" , "_deref_{n}") // JNZ _deref_{n} + self.valloc(t.Elem(), _AX) // VALLOC ${t.Elem()}, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + self.Link("_deref_{n}") // _deref_{n}: + } + + /* set value type */ + self.Emit("MOVQ", jit.Type(pt), _CX) // MOVQ ${pt}, CX + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 0)) // MOVQ CX, (SP) + + /* set value pointer */ + if deref && vk == reflect.Ptr { + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + } else { + self.Emit("MOVQ", _VP, jit.Ptr(_SP, 8)) // MOVQ VP, 8(SP) + } + + /* set the source string and call the unmarshaler */ + self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 16)) // MOVOU X0, 16(SP) + self.call_go(fn) // CALL_GO ${fn} + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + /** Dynamic Decoding Routine **/ + + var ( + _F_decodeTypedPointer obj.Addr + ) + + func init() { + _F_decodeTypedPointer = jit.Func(decodeTypedPointer) + } + + func (self *_Assembler) decode_dynamic(vt obj.Addr, vp obj.Addr) { + self.Emit("MOVQ" , _ARG_fv, _CX) // MOVQ fv, CX + self.Emit("MOVOU", _ARG_sp, _X0) // MOVOU sp, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 0)) // MOVOU X0, (SP) + self.Emit("MOVQ" , _IC, jit.Ptr(_SP, 16)) // MOVQ IC, 16(SP) + self.Emit("MOVQ" , vt, jit.Ptr(_SP, 24)) // MOVQ ${vt}, 24(SP) + self.Emit("MOVQ" , vp, jit.Ptr(_SP, 32)) // MOVQ ${vp}, 32(SP) + self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 40)) // MOVQ ST, 40(SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 48)) // MOVQ CX, 48(SP) + self.call_go(_F_decodeTypedPointer) // CALL_GO decodeTypedPointer + self.Emit("MOVQ" , jit.Ptr(_SP, 64), _ET) // MOVQ 64(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 72), _EP) // MOVQ 72(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + self.Emit("MOVQ" , jit.Ptr(_SP, 56), _IC) // MOVQ 56(SP), IC + } + + /** OpCode Assembler Functions **/ + + var ( + _F_memequal = jit.Func(memequal) + _F_growslice = jit.Func(growslice) + _F_makeslice = jit.Func(makeslice) + _F_makemap_small = jit.Func(makemap_small) + _F_mapassign_fast64 = jit.Func(mapassign_fast64) + ) + + var ( + _F_lspace = jit.Imm(int64(native.S_lspace)) + _F_strhash = jit.Imm(int64(caching.S_strhash)) + ) + + var ( + _F_b64decode = jit.Imm(int64(_subr__b64decode)) + _F_decodeValue = jit.Imm(int64(_subr_decode_value)) + ) + + var ( + _F_skip_array = jit.Imm(int64(native.S_skip_array)) + _F_skip_object = jit.Imm(int64(native.S_skip_object)) + ) + + var ( + _F_FieldMap_GetCaseInsensitive obj.Addr + ) + + const ( + _MODE_AVX2 = 1 << 2 + ) + + const ( + _Fe_ID = int64(unsafe.Offsetof(caching.FieldEntry{}.ID)) + _Fe_Name = int64(unsafe.Offsetof(caching.FieldEntry{}.Name)) + _Fe_Hash = int64(unsafe.Offsetof(caching.FieldEntry{}.Hash)) + ) + + const ( + _Vk_Ptr = int64(reflect.Ptr) + _Gt_KindFlags = int64(unsafe.Offsetof(rt.GoType{}.KindFlags)) + ) + + func init() { + _F_FieldMap_GetCaseInsensitive = jit.Func((*caching.FieldMap).GetCaseInsensitive) + } + + func (self *_Assembler) _asm_OP_any(_ *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_VP, 8), _CX) // MOVQ 8(VP), CX + self.Emit("TESTQ" , _CX, _CX) // TESTQ CX, CX + self.Sjmp("JZ" , "_decode_{n}") // JZ _decode_{n} + self.Emit("CMPQ" , _CX, _VP) // CMPQ CX, VP + self.Sjmp("JE" , "_decode_{n}") // JE _decode_{n} + self.Emit("MOVQ" , jit.Ptr(_VP, 0), _AX) // MOVQ (VP), AX + self.Emit("MOVBLZX", jit.Ptr(_AX, _Gt_KindFlags), _DX) // MOVBLZX _Gt_KindFlags(AX), DX + self.Emit("ANDL" , jit.Imm(rt.F_kind_mask), _DX) // ANDL ${F_kind_mask}, DX + self.Emit("CMPL" , _DX, jit.Imm(_Vk_Ptr)) // CMPL DX, ${reflect.Ptr} + self.Sjmp("JNE" , "_decode_{n}") // JNE _decode_{n} + self.Emit("LEAQ" , jit.Ptr(_VP, 8), _DI) // LEAQ 8(VP), DI + self.decode_dynamic(_AX, _DI) // DECODE AX, DI + self.Sjmp("JMP" , "_decode_end_{n}") // JMP _decode_end_{n} + self.Link("_decode_{n}") // _decode_{n}: + self.Emit("MOVQ" , _ARG_fv, _DF) // MOVQ fv, DF + self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 0)) // MOVQ _ST, (SP) + self.call(_F_decodeValue) // CALL decodeValue + self.Emit("TESTQ" , _EP, _EP) // TESTQ EP, EP + self.Sjmp("JNZ" , _LB_parsing_error) // JNZ _parsing_error + self.Link("_decode_end_{n}") // _decode_end_{n}: + } + + func (self *_Assembler) _asm_OP_dyn(p *_Instr) { + self.Emit("MOVQ" , jit.Type(p.vt()), _ET) // MOVQ ${p.vt()}, ET + self.Emit("CMPQ" , jit.Ptr(_VP, 8), jit.Imm(0)) // CMPQ 8(VP), $0 + self.Sjmp("JE" , _LB_type_error) // JE _type_error + self.Emit("MOVQ" , jit.Ptr(_VP, 0), _AX) // MOVQ (VP), AX + self.Emit("MOVQ" , jit.Ptr(_AX, 8), _AX) // MOVQ 8(AX), AX + self.Emit("MOVBLZX", jit.Ptr(_AX, _Gt_KindFlags), _DX) // MOVBLZX _Gt_KindFlags(AX), DX + self.Emit("ANDL" , jit.Imm(rt.F_kind_mask), _DX) // ANDL ${F_kind_mask}, DX + self.Emit("CMPL" , _DX, jit.Imm(_Vk_Ptr)) // CMPL DX, ${reflect.Ptr} + self.Sjmp("JNE" , _LB_type_error) // JNE _type_error + self.Emit("LEAQ" , jit.Ptr(_VP, 8), _DI) // LEAQ 8(VP), DI + self.decode_dynamic(_AX, _DI) // DECODE AX, DI + self.Link("_decode_end_{n}") // _decode_end_{n}: + } + + func (self *_Assembler) _asm_OP_str(_ *_Instr) { + self.parse_string() // PARSE STRING + self.unquote_once(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE once, (VP), 8(VP) + } + + func (self *_Assembler) _asm_OP_bin(_ *_Instr) { + self.parse_string() // PARSE STRING + self.slice_from(_VAR_st_Iv, -1) // SLICE st.Iv, $-1 + self.Emit("MOVQ" , _DI, jit.Ptr(_VP, 0)) // MOVQ DI, (VP) + self.Emit("MOVQ" , _SI, jit.Ptr(_VP, 8)) // MOVQ SI, 8(VP) + self.Emit("SHRQ" , jit.Imm(2), _SI) // SHRQ $2, SI + self.Emit("LEAQ" , jit.Sib(_SI, _SI, 2, 0), _SI) // LEAQ (SI)(SI*2), SI + self.Emit("MOVQ" , _SI, jit.Ptr(_VP, 16)) // MOVQ SI, 16(VP) + self.malloc(_SI, _SI) // MALLOC SI, SI + + /* check for AVX2 support */ + if !cpu.HasAVX2 { + self.Emit("XORL", _CX, _CX) // XORL CX, CX + } else { + self.Emit("MOVL", jit.Imm(_MODE_AVX2), _CX) // MOVL $_MODE_AVX2, CX + } + + /* call the decoder */ + self.Emit("XORL" , _DX, _DX) // XORL DX, DX + self.Emit("MOVQ" , _VP, _DI) // MOVQ VP, DI + self.Emit("XCHGQ", _SI, jit.Ptr(_VP, 0)) // XCHGQ SI, (VP) + self.Emit("XCHGQ", _DX, jit.Ptr(_VP, 8)) // XCHGQ DX, 8(VP) + self.call(_F_b64decode) // CALL b64decode + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_base64_error) // JS _base64_error + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 8)) // MOVQ AX, 8(VP) + } + + func (self *_Assembler) _asm_OP_bool(_ *_Instr) { + self.Emit("LEAQ", jit.Ptr(_IC, 4), _AX) // LEAQ 4(IC), AX + self.Emit("CMPQ", _AX, _IL) // CMPQ AX, IL + self.Sjmp("JA" , _LB_eof_error) // JA _eof_error + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('f')) // CMPB (IP)(IC), $'f' + self.Sjmp("JE" , "_false_{n}") // JE _false_{n} + self.Emit("MOVL", jit.Imm(_IM_true), _CX) // MOVL $"true", CX + self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) + self.Sjmp("JNE" , _LB_im_error) // JNE _im_error + self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC + self.Emit("MOVB", jit.Imm(1), jit.Ptr(_VP, 0)) // MOVB $1, (VP) + self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} + self.Link("_false_{n}") // _false_{n}: + self.Emit("ADDQ", jit.Imm(1), _AX) // ADDQ $1, AX + self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC + self.Emit("CMPQ", _AX, _IL) // CMPQ AX, IL + self.Sjmp("JA" , _LB_eof_error) // JA _eof_error + self.Emit("MOVL", jit.Imm(_IM_alse), _CX) // MOVL $"alse", CX + self.Emit("CMPL", _CX, jit.Sib(_IP, _IC, 1, 0)) // CMPL CX, (IP)(IC) + self.Sjmp("JNE" , _LB_im_error) // JNE _im_error + self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("MOVB", _AX, jit.Ptr(_VP, 0)) // MOVB AX, (VP) + self.Link("_end_{n}") // _end_{n}: + } + + func (self *_Assembler) _asm_OP_num(_ *_Instr) { + self.parse_number() // PARSE NUMBER + self.slice_from(_VAR_st_Ep, 0) // SLICE st.Ep, $0 + self.Emit("MOVQ", _DI, jit.Ptr(_VP, 0)) // MOVQ DI, (VP) + self.Emit("MOVQ", _SI, jit.Ptr(_VP, 8)) // MOVQ SI, 8(VP) + } + + func (self *_Assembler) _asm_OP_i8(_ *_Instr) { + self.parse_signed() // PARSE int8 + self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 + self.Emit("MOVB", _AX, jit.Ptr(_VP, 0)) // MOVB AX, (VP) + } + + func (self *_Assembler) _asm_OP_i16(_ *_Instr) { + self.parse_signed() // PARSE int16 + self.range_signed(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16 + self.Emit("MOVW", _AX, jit.Ptr(_VP, 0)) // MOVW AX, (VP) + } + + func (self *_Assembler) _asm_OP_i32(_ *_Instr) { + self.parse_signed() // PARSE int32 + self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32 + self.Emit("MOVL", _AX, jit.Ptr(_VP, 0)) // MOVL AX, (VP) + } + + func (self *_Assembler) _asm_OP_i64(_ *_Instr) { + self.parse_signed() // PARSE int64 + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.Emit("MOVQ", _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + } + + func (self *_Assembler) _asm_OP_u8(_ *_Instr) { + self.parse_unsigned() // PARSE uint8 + self.range_unsigned(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8 + self.Emit("MOVB", _AX, jit.Ptr(_VP, 0)) // MOVB AX, (VP) + } + + func (self *_Assembler) _asm_OP_u16(_ *_Instr) { + self.parse_unsigned() // PARSE uint16 + self.range_unsigned(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16 + self.Emit("MOVW", _AX, jit.Ptr(_VP, 0)) // MOVW AX, (VP) + } + + func (self *_Assembler) _asm_OP_u32(_ *_Instr) { + self.parse_unsigned() // PARSE uint32 + self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32 + self.Emit("MOVL", _AX, jit.Ptr(_VP, 0)) // MOVL AX, (VP) + } + + func (self *_Assembler) _asm_OP_u64(_ *_Instr) { + self.parse_unsigned() // PARSE uint64 + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.Emit("MOVQ", _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + } + + func (self *_Assembler) _asm_OP_f32(_ *_Instr) { + self.parse_number() // PARSE NUMBER + self.range_single() // RANGE float32 + self.Emit("MOVSS", _X0, jit.Ptr(_VP, 0)) // MOVSS X0, (VP) + } + + func (self *_Assembler) _asm_OP_f64(_ *_Instr) { + self.parse_number() // PARSE NUMBER + self.Emit("MOVSD", _VAR_st_Dv, _X0) // MOVSD st.Dv, X0 + self.Emit("MOVSD", _X0, jit.Ptr(_VP, 0)) // MOVSD X0, (VP) + } + + func (self *_Assembler) _asm_OP_unquote(_ *_Instr) { + self.check_eof(2) + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm('\\')) // CMPB (IP)(IC), $'\\' + self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 1), jit.Imm('"')) // CMPB 1(IP)(IC), $'"' + self.Sjmp("JNE" , _LB_char_1_error) // JNE _char_1_error + self.Emit("ADDQ", jit.Imm(2), _IC) // ADDQ $2, IC + self.parse_string() // PARSE STRING + self.unquote_twice(jit.Ptr(_VP, 0), jit.Ptr(_VP, 8)) // UNQUOTE twice, (VP), 8(VP) + } + + func (self *_Assembler) _asm_OP_nil_1(_ *_Instr) { + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("MOVQ", _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + } + + func (self *_Assembler) _asm_OP_nil_2(_ *_Instr) { + self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_VP, 0)) // MOVOU X0, (VP) + } + + func (self *_Assembler) _asm_OP_nil_3(_ *_Instr) { + self.Emit("XORL" , _AX, _AX) // XORL AX, AX + self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_VP, 0)) // MOVOU X0, (VP) + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 16)) // MOVOU X0, 16(VP) + } + + func (self *_Assembler) _asm_OP_deref(p *_Instr) { + self.vfollow(p.vt()) + } + + func (self *_Assembler) _asm_OP_index(p *_Instr) { + self.Emit("MOVQ", jit.Imm(p.i64()), _AX) // MOVQ ${p.vi()}, AX + self.Emit("ADDQ", _AX, _VP) // ADDQ _AX, _VP + } + + func (self *_Assembler) _asm_OP_is_null(p *_Instr) { + self.Emit("LEAQ" , jit.Ptr(_IC, 4), _AX) // LEAQ 4(IC), AX + self.Emit("CMPQ" , _AX, _IL) // CMPQ AX, IL + self.Sjmp("JA" , "_not_null_{n}") // JA _not_null_{n} + self.Emit("CMPL" , jit.Sib(_IP, _IC, 1, 0), jit.Imm(_IM_null)) // CMPL (IP)(IC), $"null" + self.Emit("CMOVQEQ", _AX, _IC) // CMOVQEQ AX, IC + self.Xjmp("JE" , p.vi()) // JE {p.vi()} + self.Link("_not_null_{n}") // _not_null_{n}: + } + + func (self *_Assembler) _asm_OP_map_init(_ *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_VP, 0), _AX) // MOVQ (VP), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNZ" , "_end_{n}") // JNZ _end_{n} + self.call_go(_F_makemap_small) // CALL_GO makemap_small + self.Emit("MOVQ" , jit.Ptr(_SP, 0), _AX) // MOVQ (SP), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + self.Link("_end_{n}") // _end_{n}: + self.Emit("MOVQ" , _AX, _VP) // MOVQ AX, VP + } + + func (self *_Assembler) _asm_OP_map_key_i8(p *_Instr) { + self.parse_signed() // PARSE int8 + self.range_signed(_I_int8, _T_int8, math.MinInt8, math.MaxInt8) // RANGE int8 + self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int8, mapassign, st.Iv + } + + func (self *_Assembler) _asm_OP_map_key_i16(p *_Instr) { + self.parse_signed() // PARSE int16 + self.range_signed(_I_int16, _T_int16, math.MinInt16, math.MaxInt16) // RANGE int16 + self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN int16, mapassign, st.Iv + } + + func (self *_Assembler) _asm_OP_map_key_i32(p *_Instr) { + self.parse_signed() // PARSE int32 + self.range_signed(_I_int32, _T_int32, math.MinInt32, math.MaxInt32) // RANGE int32 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int32, mapassign, st.Iv + } else { + self.mapassign_fastx(vt, _F_mapassign_fast32) // MAPASSIGN int32, mapassign_fast32 + } + } + + func (self *_Assembler) _asm_OP_map_key_i64(p *_Instr) { + self.parse_signed() // PARSE int64 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN int64, mapassign, st.Iv + } else { + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.mapassign_fastx(vt, _F_mapassign_fast64) // MAPASSIGN int64, mapassign_fast64 + } + } + + func (self *_Assembler) _asm_OP_map_key_u8(p *_Instr) { + self.parse_unsigned() // PARSE uint8 + self.range_unsigned(_I_uint8, _T_uint8, math.MaxUint8) // RANGE uint8 + self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint8, vt.Iv + } + + func (self *_Assembler) _asm_OP_map_key_u16(p *_Instr) { + self.parse_unsigned() // PARSE uint16 + self.range_unsigned(_I_uint16, _T_uint16, math.MaxUint16) // RANGE uint16 + self.mapassign_std(p.vt(), _VAR_st_Iv) // MAPASSIGN uint16, vt.Iv + } + + func (self *_Assembler) _asm_OP_map_key_u32(p *_Instr) { + self.parse_unsigned() // PARSE uint32 + self.range_unsigned(_I_uint32, _T_uint32, math.MaxUint32) // RANGE uint32 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint32, vt.Iv + } else { + self.mapassign_fastx(vt, _F_mapassign_fast32) // MAPASSIGN uint32, mapassign_fast32 + } + } + + func (self *_Assembler) _asm_OP_map_key_u64(p *_Instr) { + self.parse_unsigned() // PARSE uint64 + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_st_Iv) // MAPASSIGN uint64, vt.Iv + } else { + self.Emit("MOVQ", _VAR_st_Iv, _AX) // MOVQ st.Iv, AX + self.mapassign_fastx(vt, _F_mapassign_fast64) // MAPASSIGN uint64, mapassign_fast64 + } + } + + func (self *_Assembler) _asm_OP_map_key_f32(p *_Instr) { + self.parse_number() // PARSE NUMBER + self.range_single() // RANGE float32 + self.Emit("MOVSS", _X0, _VAR_st_Dv) // MOVSS X0, st.Dv + self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv + } + + func (self *_Assembler) _asm_OP_map_key_f64(p *_Instr) { + self.parse_number() // PARSE NUMBER + self.mapassign_std(p.vt(), _VAR_st_Dv) // MAPASSIGN ${p.vt()}, mapassign, st.Dv + } + + func (self *_Assembler) _asm_OP_map_key_str(p *_Instr) { + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + if vt := p.vt(); !mapfast(vt) { + self.mapassign_std(vt, _VAR_sv_p) // MAPASSIGN string, DI, SI + } else { + self.Emit("MOVQ", _VAR_sv_p, _DI) // MOVQ sv.p, DI + self.Emit("MOVQ", _VAR_sv_n, _SI) // MOVQ sv.n, SI + self.mapassign_str_fast(vt, _DI, _SI) // MAPASSIGN string, DI, SI + } + } + + func (self *_Assembler) _asm_OP_map_key_utext(p *_Instr) { + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.mapassign_utext(p.vt(), false) // MAPASSIGN utext, ${p.vt()}, false + } + + func (self *_Assembler) _asm_OP_map_key_utext_p(p *_Instr) { + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.mapassign_utext(p.vt(), true) // MAPASSIGN utext, ${p.vt()}, true + } + + func (self *_Assembler) _asm_OP_array_skip(_ *_Instr) { + self.call_sf(_F_skip_array) // CALL_SF skip_array + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + } + + func (self *_Assembler) _asm_OP_array_clear(p *_Instr) { + self.mem_clear_rem(p.i64(), true) + } + + func (self *_Assembler) _asm_OP_array_clear_p(p *_Instr) { + self.mem_clear_rem(p.i64(), false) + } + + func (self *_Assembler) _asm_OP_slice_init(p *_Instr) { + self.Emit("XORL" , _AX, _AX) // XORL AX, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 8)) // MOVQ AX, 8(VP) + self.Emit("MOVQ" , jit.Ptr(_VP, 16), _AX) // MOVQ 16(VP), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNZ" , "_done_{n}") // JNZ _done_{n} + self.Emit("MOVQ" , jit.Imm(_MinSlice), _CX) // MOVQ ${_MinSlice}, CX + self.Emit("MOVQ" , _CX, jit.Ptr(_VP, 16)) // MOVQ CX, 16(VP) + self.Emit("MOVQ" , jit.Type(p.vt()), _DX) // MOVQ ${p.vt()}, DX + self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 0)) // MOVQ DX, (SP) + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 16)) // MOVQ CX, 16(SP) + self.call_go(_F_makeslice) // CALL_GO makeslice + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _AX) // MOVQ 24(SP), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 0)) // MOVQ AX, (VP) + self.Link("_done_{n}") // _done_{n}: + self.Emit("XORL" , _AX, _AX) // XORL AX, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 8)) // MOVQ AX, 8(VP) + } + + func (self *_Assembler) _asm_OP_slice_append(p *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_VP, 8), _AX) // MOVQ 8(VP), AX + self.Emit("CMPQ" , _AX, jit.Ptr(_VP, 16)) // CMPQ AX, 16(VP) + self.Sjmp("JB" , "_index_{n}") // JB _index_{n} + self.Emit("MOVQ" , jit.Type(p.vt()), _AX) // MOVQ ${p.vt()}, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVOU", jit.Ptr(_VP, 0), _X0) // MOVOU (VP), X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP) + self.Emit("MOVQ" , jit.Ptr(_VP, 16), _AX) // MOVQ 16(VP), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 24)) // MOVQ AX, 24(SP) + self.Emit("SHLQ" , jit.Imm(1), _AX) // SHLQ $1, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP) + self.call_go(_F_growslice) // CALL_GO growslice + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _DI) // MOVQ 40(SP), DI + self.Emit("MOVQ" , jit.Ptr(_SP, 48), _AX) // MOVQ 48(SP), AX + self.Emit("MOVQ" , jit.Ptr(_SP, 56), _SI) // MOVQ 56(SP), SI + self.Emit("MOVQ" , _DI, jit.Ptr(_VP, 0)) // MOVQ DI, (VP) + self.Emit("MOVQ" , _AX, jit.Ptr(_VP, 8)) // MOVQ AX, 8(VP) + self.Emit("MOVQ" , _SI, jit.Ptr(_VP, 16)) // MOVQ SI, 16(VP) + self.Link("_index_{n}") // _index_{n}: + self.Emit("ADDQ" , jit.Imm(1), jit.Ptr(_VP, 8)) // ADDQ $1, 8(VP) + self.Emit("MOVQ" , jit.Ptr(_VP, 0), _VP) // MOVQ (VP), VP + self.Emit("MOVQ" , jit.Imm(int64(p.vlen())), _CX) // MOVQ ${p.vlen()}, CX + self.From("MULQ" , _CX) // MULQ CX + self.Emit("ADDQ" , _AX, _VP) // ADDQ AX, VP + } + + func (self *_Assembler) _asm_OP_object_skip(_ *_Instr) { + self.call_sf(_F_skip_object) // CALL_SF skip_object + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + } + + func (self *_Assembler) _asm_OP_object_next(_ *_Instr) { + self.call_sf(_F_skip_one) // CALL_SF skip_one + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parse_error_v + } + + func (self *_Assembler) _asm_OP_struct_field(p *_Instr) { + assert_eq(caching.FieldEntrySize, 32, "invalid field entry size") + self.Emit("MOVQ" , jit.Imm(-1), _AX) // MOVQ $-1, AX + self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, sr + self.parse_string() // PARSE STRING + self.unquote_once(_VAR_sv_p, _VAR_sv_n) // UNQUOTE once, sv.p, sv.n + self.Emit("LEAQ" , _VAR_sv, _AX) // LEAQ sv, AX + self.Emit("XORL" , _CX, _CX) // XORL CX, CX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.call_go(_F_strhash) // CALL_GO strhash + self.Emit("MOVQ" , jit.Ptr(_SP, 16), _AX) // MOVQ 16(SP), AX + self.Emit("MOVQ" , _AX, _R9) // MOVQ AX, R9 + self.Emit("MOVQ" , jit.Imm(freezeFields(p.vf())), _CX) // MOVQ ${p.vf()}, CX + self.Emit("MOVQ" , jit.Ptr(_CX, caching.FieldMap_b), _SI) // MOVQ FieldMap.b(CX), SI + self.Emit("MOVQ" , jit.Ptr(_CX, caching.FieldMap_N), _CX) // MOVQ FieldMap.N(CX), CX + self.Emit("TESTQ", _CX, _CX) // TESTQ CX, CX + self.Sjmp("JZ" , "_try_lowercase_{n}") // JZ _try_lowercase_{n} + self.Link("_loop_{n}") // _loop_{n}: + self.Emit("XORL" , _DX, _DX) // XORL DX, DX + self.From("DIVQ" , _CX) // DIVQ CX + self.Emit("LEAQ" , jit.Ptr(_DX, 1), _AX) // LEAQ 1(DX), AX + self.Emit("SHLQ" , jit.Imm(5), _DX) // SHLQ $5, DX + self.Emit("LEAQ" , jit.Sib(_SI, _DX, 1, 0), _DI) // LEAQ (SI)(DX), DI + self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Hash), _R8) // MOVQ FieldEntry.Hash(DI), R8 + self.Emit("TESTQ", _R8, _R8) // TESTQ R8, R8 + self.Sjmp("JZ" , "_try_lowercase_{n}") // JZ _try_lowercase_{n} + self.Emit("CMPQ" , _R8, _R9) // CMPQ R8, R9 + self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n} + self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name + 8), _DX) // MOVQ FieldEntry.Name+8(DI), DX + self.Emit("CMPQ" , _DX, _VAR_sv_n) // CMPQ DX, sv.n + self.Sjmp("JNE" , "_loop_{n}") // JNE _loop_{n} + self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_ID), _R8) // MOVQ FieldEntry.ID(DI), R8 + self.Emit("MOVQ" , _AX, _VAR_ss_AX) // MOVQ AX, ss.AX + self.Emit("MOVQ" , _CX, _VAR_ss_CX) // MOVQ CX, ss.CX + self.Emit("MOVQ" , _SI, _VAR_ss_SI) // MOVQ SI, ss.SI + self.Emit("MOVQ" , _R8, _VAR_ss_R8) // MOVQ R8, ss.R8 + self.Emit("MOVQ" , _R9, _VAR_ss_R9) // MOVQ R9, ss.R9 + self.Emit("MOVQ" , _VAR_sv_p, _AX) // MOVQ _VAR_sv_p, AX + self.Emit("MOVQ" , jit.Ptr(_DI, _Fe_Name), _CX) // MOVQ FieldEntry.Name(DI), CX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 16)) // MOVQ DX, 16(SP) + self.call_go(_F_memequal) // CALL_GO memequal + self.Emit("MOVQ" , _VAR_ss_AX, _AX) // MOVQ ss.AX, AX + self.Emit("MOVQ" , _VAR_ss_CX, _CX) // MOVQ ss.CX, CX + self.Emit("MOVQ" , _VAR_ss_SI, _SI) // MOVQ ss.SI, SI + self.Emit("MOVQ" , _VAR_ss_R9, _R9) // MOVQ ss.R9, R9 + self.Emit("MOVB" , jit.Ptr(_SP, 24), _DX) // MOVB 24(SP), DX + self.Emit("TESTB", _DX, _DX) // TESTB DX, DX + self.Sjmp("JZ" , "_loop_{n}") // JZ _loop_{n} + self.Emit("MOVQ" , _VAR_ss_R8, _R8) // MOVQ ss.R8, R8 + self.Emit("MOVQ" , _R8, _VAR_sr) // MOVQ R8, sr + self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} + self.Link("_try_lowercase_{n}") // _try_lowercase_{n}: + self.Emit("MOVQ" , jit.Imm(referenceFields(p.vf())), _AX) // MOVQ ${p.vf()}, AX + self.Emit("MOVOU", _VAR_sv, _X0) // MOVOU sv, X0 + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP) + self.call_go(_F_FieldMap_GetCaseInsensitive) // CALL_GO FieldMap::GetCaseInsensitive + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _AX) // MOVQ 24(SP), AX + self.Emit("MOVQ" , _AX, _VAR_sr) // MOVQ AX, _VAR_sr + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JNS" , "_end_{n}") // JNS _end_{n} + self.Emit("BTQ" , jit.Imm(_F_disable_unknown), _ARG_fv) // BTQ ${_F_disable_unknown}, fv + self.Sjmp("JC" , _LB_field_error) // JC _field_error + self.Link("_end_{n}") // _end_{n}: + } + + func (self *_Assembler) _asm_OP_unmarshal(p *_Instr) { + self.unmarshal_json(p.vt(), true) + } + + func (self *_Assembler) _asm_OP_unmarshal_p(p *_Instr) { + self.unmarshal_json(p.vt(), false) + } + + func (self *_Assembler) _asm_OP_unmarshal_text(p *_Instr) { + self.unmarshal_text(p.vt(), true) + } + + func (self *_Assembler) _asm_OP_unmarshal_text_p(p *_Instr) { + self.unmarshal_text(p.vt(), false) + } + + func (self *_Assembler) _asm_OP_lspace(_ *_Instr) { + self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error + self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX + self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX + self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' ' + self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n} + self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX + self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n} + + /* test up to 4 characters */ + for i := 0; i < 3; i++ { + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error + self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX + self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' ' + self.Sjmp("JA" , "_nospace_{n}") // JA _nospace_{n} + self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX + self.Sjmp("JNC" , "_nospace_{n}") // JNC _nospace_{n} + } + + /* handle over to the native function */ + self.Emit("MOVQ" , _IP, _DI) // MOVQ IP, DI + self.Emit("MOVQ" , _IL, _SI) // MOVQ IL, SI + self.Emit("MOVQ" , _IC, _DX) // MOVQ IC, DX + self.call(_F_lspace) // CALL lspace + self.Emit("TESTQ" , _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , _LB_parsing_error_v) // JS _parsing_error_v + self.Emit("CMPQ" , _AX, _IL) // CMPQ AX, IL + self.Sjmp("JAE" , _LB_eof_error) // JAE _eof_error + self.Emit("MOVQ" , _AX, _IC) // MOVQ AX, IC + self.Link("_nospace_{n}") // _nospace_{n}: + } + + func (self *_Assembler) _asm_OP_match_char(p *_Instr) { + self.check_eof(1) + self.Emit("CMPB", jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} + self.Sjmp("JNE" , _LB_char_0_error) // JNE _char_0_error + self.Emit("ADDQ", jit.Imm(1), _IC) // ADDQ $1, IC + } + + func (self *_Assembler) _asm_OP_check_char(p *_Instr) { + self.check_eof(1) + self.Emit("LEAQ" , jit.Ptr(_IC, 1), _AX) // LEAQ 1(IC), AX + self.Emit("CMPB" , jit.Sib(_IP, _IC, 1, 0), jit.Imm(int64(p.vb()))) // CMPB (IP)(IC), ${p.vb()} + self.Emit("CMOVQEQ", _AX, _IC) // CMOVQEQ AX, IC + self.Xjmp("JE" , p.vi()) // JE {p.vi()} + } + + func (self *_Assembler) _asm_OP_load(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _VP) // MOVQ (ST)(AX), VP + } + + func (self *_Assembler) _asm_OP_save(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("CMPQ", _AX, jit.Imm(_MaxStack)) // CMPQ AX, ${_MaxStack} + self.Sjmp("JA" , _LB_stack_error) // JA _stack_error + self.Emit("MOVQ", _VP, jit.Sib(_ST, _AX, 1, 8)) // MOVQ VP, 8(ST)(AX) + self.Emit("ADDQ", jit.Imm(8), _AX) // ADDQ $8, AX + self.Emit("MOVQ", _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST) + } + + func (self *_Assembler) _asm_OP_drop(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("SUBQ", jit.Imm(8), _AX) // SUBQ $8, AX + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 8), _VP) // MOVQ 8(ST)(AX), VP + self.Emit("MOVQ", _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST) + self.Emit("XORL", _ET, _ET) // XORL ET, ET + self.Emit("MOVQ", _ET, jit.Sib(_ST, _AX, 1, 8)) // MOVQ ET, 8(ST)(AX) + } + + func (self *_Assembler) _asm_OP_drop_2(_ *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("SUBQ" , jit.Imm(16), _AX) // SUBQ $16, AX + self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 8), _VP) // MOVQ 8(ST)(AX), VP + self.Emit("MOVQ" , _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST) + self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 + self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 8)) // MOVOU X0, 8(ST)(AX) + } + + func (self *_Assembler) _asm_OP_recurse(p *_Instr) { + self.Emit("MOVQ", jit.Type(p.vt()), _AX) // MOVQ ${p.vt()}, AX + self.decode_dynamic(_AX, _VP) // DECODE AX, VP + } + + func (self *_Assembler) _asm_OP_goto(p *_Instr) { + self.Xjmp("JMP", p.vi()) + } + + func (self *_Assembler) _asm_OP_switch(p *_Instr) { + self.Emit("MOVQ", _VAR_sr, _AX) // MOVQ sr, AX + self.Emit("CMPQ", _AX, jit.Imm(p.i64())) // CMPQ AX, ${len(p.vs())} + self.Sjmp("JAE" , "_default_{n}") // JAE _default_{n} + + /* jump table selector */ + self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI + self.Sref("_switch_table_{n}", 4) // .... &_switch_table_{n} + self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, 0), _AX) // MOVLQSX (DI)(AX*4), AX + self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX + self.Rjmp("JMP" , _AX) // JMP AX + self.Link("_switch_table_{n}") // _switch_table_{n}: + + /* generate the jump table */ + for i, v := range p.vs() { + self.Xref(v, int64(-i) * 4) + } + + /* default case */ + self.Link("_default_{n}") + self.NOP() + } \ No newline at end of file diff --git a/decoder/assembler_amd64.go b/decoder/assembler_amd64_go116.go similarity index 99% rename from decoder/assembler_amd64.go rename to decoder/assembler_amd64_go116.go index 679e84b9e..e5ee03583 100644 --- a/decoder/assembler_amd64.go +++ b/decoder/assembler_amd64_go116.go @@ -1,3 +1,5 @@ +// +build go1.16,!go1.17 + /* * Copyright 2021 ByteDance Inc. * @@ -1567,6 +1569,21 @@ func (self *_Assembler) _asm_OP_switch(p *_Instr) { self.NOP() } + +type writeBarrier struct { + enabled bool // compiler emits a check of this before calling write barrier + pad [3]byte // compiler uses 32-bit load for "enabled" field + needed bool // whether we need a write barrier for current GC phase + cgo bool // whether we need a write barrier for a cgo check + alignme uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load +} + +//go:linkname _runtime_writeBarrier runtime.writeBarrier +var _runtime_writeBarrier writeBarrier + +//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier +func gcWriteBarrierAX() + var ( _V_writeBarrier = jit.Imm(int64(uintptr(unsafe.Pointer(&_runtime_writeBarrier)))) diff --git a/decoder/generic_amd64_go115.go b/decoder/generic_amd64_go115.go new file mode 100644 index 000000000..23c8fad2b --- /dev/null +++ b/decoder/generic_amd64_go115.go @@ -0,0 +1,680 @@ +// +build go1.15,!go1.16 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package decoder + + import ( + `encoding/json` + `fmt` + `reflect` + + `github.com/bytedance/sonic/internal/jit` + `github.com/bytedance/sonic/internal/native` + `github.com/bytedance/sonic/internal/native/types` + `github.com/twitchyliquid64/golang-asm/obj` + ) + + /** Crucial Registers: + * + * ST(BX) : ro, decoder stack + * DF(R10) : ro, decoder flags + * EP(R11) : wo, error pointer + * IP(R12) : ro, input pointer + * IL(R13) : ro, input length + * IC(R14) : rw, input cursor + * VP(R15) : ro, value pointer (to an interface{}) + */ + + const ( + _VD_args = 8 // 8 bytes for passing arguments to this functions + _VD_fargs = 64 // 64 bytes for passing arguments to other Go functions + _VD_saves = 40 // 40 bytes for saving the registers before CALL instructions + _VD_locals = 40 // 40 bytes for local variables + ) + + const ( + _VD_offs = _VD_fargs + _VD_saves + _VD_locals + _VD_size = _VD_offs + 8 // 8 bytes for the parent frame pointer + ) + + var ( + _VAR_ss = _VAR_ss_Vt + _VAR_df = jit.Ptr(_SP, _VD_fargs + _VD_saves) + ) + + var ( + _VAR_ss_Vt = jit.Ptr(_SP, _VD_fargs + _VD_saves + 8) + _VAR_ss_Dv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 16) + _VAR_ss_Iv = jit.Ptr(_SP, _VD_fargs + _VD_saves + 24) + _VAR_ss_Ep = jit.Ptr(_SP, _VD_fargs + _VD_saves + 32) + ) + + type _ValueDecoder struct { + jit.BaseAssembler + } + + func (self *_ValueDecoder) build() uintptr { + self.Init(self.compile) + return *(*uintptr)(self.LoadWithFaker("decode_value", _VD_size, _VD_fargs, _Decoder_Generic_Shadow)) + } + + /** Function Calling Helpers **/ + + func (self *_ValueDecoder) save(r ...obj.Addr) { + for i, v := range r { + if i > _VD_saves / 8 - 1 { + panic("too many registers to save") + } else { + self.Emit("MOVQ", v, jit.Ptr(_SP, _VD_fargs + int64(i) * 8)) + } + } + } + + func (self *_ValueDecoder) load(r ...obj.Addr) { + for i, v := range r { + if i > _VD_saves / 8 - 1 { + panic("too many registers to load") + } else { + self.Emit("MOVQ", jit.Ptr(_SP, _VD_fargs + int64(i) * 8), v) + } + } + } + + func (self *_ValueDecoder) call(fn obj.Addr) { + self.Emit("MOVQ", fn, _AX) // MOVQ ${fn}, AX + self.Rjmp("CALL", _AX) // CALL AX + } + + func (self *_ValueDecoder) call_go(fn obj.Addr) { + self.save(_REG_go...) // SAVE $REG_go + self.call(fn) // CALL ${fn} + self.load(_REG_go...) // LOAD $REG_go + } + + /** Decoder Assembler **/ + + const ( + _S_val = iota + 1 + _S_arr + _S_arr_0 + _S_obj + _S_obj_x + _S_obj_delim + ) + + const ( + _S_omask = (1 << _S_obj) | (1 << _S_obj_x) + _S_vmask = (1 << _S_val) | (1 << _S_arr_0) + ) + + const ( + _A_init_len = 1 + _A_init_cap = 16 + ) + + const ( + _ST_Sp = 0 + _ST_Vt = _PtrBytes + _ST_Vp = _PtrBytes * (types.MAX_RECURSE + 1) + ) + + var ( + _V_true = jit.Imm(int64(pbool(true))) + _V_false = jit.Imm(int64(pbool(false))) + _F_value = jit.Imm(int64(native.S_value)) + ) + + var ( + _V_max = jit.Imm(int64(types.V_MAX)) + _E_eof = jit.Imm(int64(types.ERR_EOF)) + _E_invalid = jit.Imm(int64(types.ERR_INVALID_CHAR)) + _E_recurse = jit.Imm(int64(types.ERR_RECURSE_EXCEED_MAX)) + ) + + var ( + _F_convTslice = jit.Func(convTslice) + _F_convTstring = jit.Func(convTstring) + _F_invalid_vtype = jit.Func(invalid_vtype) + ) + + var ( + _T_map = jit.Type(reflect.TypeOf((map[string]interface{})(nil))) + _T_bool = jit.Type(reflect.TypeOf(false)) + _T_int64 = jit.Type(reflect.TypeOf(int64(0))) + _T_eface = jit.Type(reflect.TypeOf((*interface{})(nil)).Elem()) + _T_slice = jit.Type(reflect.TypeOf(([]interface{})(nil))) + _T_string = jit.Type(reflect.TypeOf("")) + _T_number = jit.Type(reflect.TypeOf(json.Number(""))) + _T_float64 = jit.Type(reflect.TypeOf(float64(0))) + ) + + var _R_tab = map[int]string { + '[': "_decode_V_ARRAY", + '{': "_decode_V_OBJECT", + ':': "_decode_V_KEY_SEP", + ',': "_decode_V_ELEM_SEP", + ']': "_decode_V_ARRAY_END", + '}': "_decode_V_OBJECT_END", + } + + func (self *_ValueDecoder) compile() { + self.Emit("SUBQ", jit.Imm(_VD_size), _SP) // SUBQ $_VD_size, SP + self.Emit("MOVQ", _BP, jit.Ptr(_SP, _VD_offs)) // MOVQ BP, _VD_offs(SP) + self.Emit("LEAQ", jit.Ptr(_SP, _VD_offs), _BP) // LEAQ _VD_offs(SP), BP + + /* initialize the state machine */ + self.Emit("XORL", _CX, _CX) // XORL CX, CX + self.Emit("MOVQ", _DF, _VAR_df) // MOVQ DF, df + self.Emit("ADDQ", jit.Imm(_FsmOffset), _ST) // ADDQ _FsmOffset, _ST + self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp + self.Emit("MOVQ", _VP, jit.Ptr(_ST, _ST_Vp)) // MOVQ VP, ST.Vp[0] + self.Emit("MOVQ", jit.Imm(_S_val), jit.Ptr(_ST, _ST_Vt)) // MOVQ _S_val, ST.Vt[0] + self.Sjmp("JMP" , "_next") // JMP _next + + /* set the value from previous round */ + self.Link("_set_value") // _set_value: + self.Emit("MOVL" , jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX + self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ" , jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX + self.Sjmp("JNC" , "_vtype_error") // JNC _vtype_error + self.Emit("XORL" , _AX, _AX) // XORL AX, AX + self.Emit("SUBQ" , jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp + self.Emit("XCHGQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _AX) // XCHGQ ST.Vp[CX], AX + self.Emit("MOVQ" , _R8, jit.Ptr(_AX, 0)) // MOVQ R8, (AX) + self.Emit("MOVQ" , _R9, jit.Ptr(_AX, 8)) // MOVQ R9, 8(AX) + + /* check for value stack */ + self.Link("_next") // _next: + self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _AX) // MOVQ ST.Sp, AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , "_return") // JS _return + + /* fast path: test up to 4 characters manually */ + self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF + self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX + self.Emit("MOVQ" , jit.Imm(_BM_space), _DX) // MOVQ _BM_space, DX + self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' ' + self.Sjmp("JA" , "_decode_fast") // JA _decode_fast + self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX + self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + + /* at least 1 to 3 spaces */ + for i := 0; i < 3; i++ { + self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF + self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX + self.Emit("CMPQ" , _AX, jit.Imm(' ')) // CMPQ AX, $' ' + self.Sjmp("JA" , "_decode_fast") // JA _decode_fast + self.Emit("BTQ" , _AX, _DX) // BTQ _AX, _DX + self.Sjmp("JNC" , "_decode_fast") // JNC _decode_fast + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + } + + /* at least 4 spaces */ + self.Emit("CMPQ" , _IC, _IL) // CMPQ IC, IL + self.Sjmp("JAE" , "_decode_V_EOF") // JAE _decode_V_EOF + self.Emit("MOVBQZX", jit.Sib(_IP, _IC, 1, 0), _AX) // MOVBQZX (IP)(IC), AX + + /* fast path: use lookup table to select decoder */ + self.Link("_decode_fast") // _decode_fast: + self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI + self.Sref("_decode_tab", 4) // .... &_decode_tab + self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, 0), _AX) // MOVLQSX (DI)(AX*4), AX + self.Emit("TESTQ" , _AX, _AX) // TESTQ AX, AX + self.Sjmp("JZ" , "_decode_native") // JZ _decode_native + self.Emit("ADDQ" , jit.Imm(1), _IC) // ADDQ $1, IC + self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX + self.Rjmp("JMP" , _AX) // JMP AX + + /* decode with native decoder */ + self.Link("_decode_native") // _decode_native: + self.Emit("MOVQ", _IP, _DI) // MOVQ IP, DI + self.Emit("MOVQ", _IL, _SI) // MOVQ IL, SI + self.Emit("MOVQ", _IC, _DX) // MOVQ IC, DX + self.Emit("LEAQ", _VAR_ss, _CX) // LEAQ ss, CX + self.Emit("MOVL", jit.Imm(1), _R8) // MOVL $1, R8 + self.call(_F_value) // CALL value + self.Emit("MOVQ", _AX, _IC) // MOVQ AX, IC + + /* check for errors */ + self.Emit("MOVQ" , _VAR_ss_Vt, _AX) // MOVQ ss.Vt, AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , "_parsing_error") // JS _parsing_error + self.Sjmp("JZ" , "_invalid_vtype") // JZ _invalid_vtype + self.Emit("CMPQ" , _AX, _V_max) // CMPQ AX, _V_max + self.Sjmp("JA" , "_invalid_vtype") // JA _invalid_vtype + + /* jump table selector */ + self.Byte(0x48, 0x8d, 0x3d) // LEAQ ?(PC), DI + self.Sref("_switch_table", 4) // .... &_switch_table + self.Emit("MOVLQSX", jit.Sib(_DI, _AX, 4, -4), _AX) // MOVLQSX -4(DI)(AX*4), AX + self.Emit("ADDQ" , _DI, _AX) // ADDQ DI, AX + self.Rjmp("JMP" , _AX) // JMP AX + + /** V_EOF **/ + self.Link("_decode_V_EOF") // _decode_V_EOF: + self.Emit("MOVL", _E_eof, _EP) // MOVL _E_eof, EP + self.Sjmp("JMP" , "_error") // JMP _error + + /** V_NULL **/ + self.Link("_decode_V_NULL") // _decode_V_NULL: + self.Emit("XORL", _R8, _R8) // XORL R8, R8 + self.Emit("XORL", _R9, _R9) // XORL R9, R9 + self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /** V_TRUE **/ + self.Link("_decode_V_TRUE") // _decode_V_TRUE: + self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8 + self.Emit("MOVQ", _V_true, _R9) // MOVQ _V_true, R9 + self.Emit("LEAQ", jit.Ptr(_IC, -4), _DI) // LEAQ -4(IC), DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /** V_FALSE **/ + self.Link("_decode_V_FALSE") // _decode_V_FALSE: + self.Emit("MOVQ", _T_bool, _R8) // MOVQ _T_bool, R8 + self.Emit("MOVQ", _V_false, _R9) // MOVQ _V_false, R9 + self.Emit("LEAQ", jit.Ptr(_IC, -5), _DI) // LEAQ -5(IC), DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /** V_ARRAY **/ + self.Link("_decode_V_ARRAY") // _decode_V_ARRAY: + self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX + self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char + + /* create a new array */ + self.Emit("MOVQ", _T_eface, _AX) // MOVQ _T_eface, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ", jit.Imm(_A_init_len), jit.Ptr(_SP, 8)) // MOVQ _A_init_len, 8(SP) + self.Emit("MOVQ", jit.Imm(_A_init_cap), jit.Ptr(_SP, 16)) // MOVQ _A_init_cap, 16(SP) + self.call_go(_F_makeslice) // CALL_GO runtime.makeslice + self.Emit("MOVQ", jit.Ptr(_SP, 24), _DX) // MOVQ 24(SP), DX + + /* pack into an interface */ + self.Emit("MOVQ", _DX, jit.Ptr(_SP, 0)) // MOVQ DX, (SP) + self.Emit("MOVQ", jit.Imm(_A_init_len), jit.Ptr(_SP, 8)) // MOVQ _A_init_len, 8(SP) + self.Emit("MOVQ", jit.Imm(_A_init_cap), jit.Ptr(_SP, 16)) // MOVQ _A_init_cap, 16(SP) + self.call_go(_F_convTslice) // CALL_GO runtime.convTslice + self.Emit("MOVQ", jit.Ptr(_SP, 24), _R8) // MOVQ 24(SP), R8 + + /* replace current state with an array */ + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI + self.Emit("MOVQ", jit.Imm(_S_arr), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr, ST.Vt[CX] + self.Emit("MOVQ", _T_slice, _AX) // MOVQ _T_slice, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SI, 0)) // MOVQ AX, (SI) + self.Emit("MOVQ", _R8, jit.Ptr(_SI, 8)) // MOVQ R8, 8(SI) + + /* add a new slot for the first element */ + self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} + self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Emit("MOVQ", jit.Ptr(_R8, 0), _AX) // MOVQ (R8), AX + self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp + self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX] + self.Emit("MOVQ", jit.Imm(_S_arr_0), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_arr_0, ST.Vt[CX] + self.Sjmp("JMP" , "_next") // JMP _next + + /** V_OBJECT **/ + self.Link("_decode_V_OBJECT") // _decode_V_OBJECT: + self.Emit("MOVL", jit.Imm(_S_vmask), _DX) // MOVL _S_vmask, DX + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("BTQ" , _AX, _DX) // BTQ AX, DX + self.Sjmp("JNC" , "_invalid_char") // JNC _invalid_char + self.call_go(_F_makemap_small) // CALL_GO runtime.makemap_small + self.Emit("MOVQ", jit.Ptr(_SP, 0), _AX) // MOVQ (SP), AX + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI + self.Emit("MOVQ", _T_map, _DX) // MOVQ _T_map, DX + self.Emit("MOVQ", _DX, jit.Ptr(_SI, 0)) // MOVQ DX, (SI) + self.Emit("MOVQ", _AX, jit.Ptr(_SI, 8)) // MOVQ AX, 8(SI) + self.Emit("MOVQ", jit.Imm(_S_obj), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj, ST.Vt[CX] + self.Sjmp("JMP" , "_next") // JMP _next + + /** V_STRING **/ + self.Link("_decode_V_STRING") // _decode_V_STRING: + self.Emit("XORL", _DX, _DX) // XORL DX, DX + self.Emit("MOVQ", _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX + self.Emit("MOVQ", _IC, _AX) // MOVQ IC, AX + self.Emit("SUBQ", _CX, _AX) // SUBQ CX, AX + + /* check for escapes */ + self.Emit("CMPQ", _VAR_ss_Ep, jit.Imm(-1)) // CMPQ ss.Ep, $-1 + self.Sjmp("JNE" , "_unquote") // JNE _unquote + self.Emit("SUBQ", jit.Imm(1), _AX) // SUBQ $1, AX + self.Emit("LEAQ", jit.Sib(_IP, _CX, 1, 0), _R8) // LEAQ (IP)(CX), R8 + + /* strings with no escape sequences */ + self.Link("_noescape") // _noescape: + self.Emit("MOVL", jit.Imm(_S_omask), _DI) // MOVL _S_omask, DI + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _SI) // MOVQ ST.Vt[CX], SI + self.Emit("BTQ" , _SI, _DI) // BTQ SI, DI + self.Sjmp("JC" , "_object_key") // JC _object_key + + /* check for pre-packed strings, avoid 1 allocation */ + self.Emit("TESTQ", _DX, _DX) // TESTQ DX, DX + self.Sjmp("JNZ" , "_packed_str") // JNZ _packed_str + self.Emit("MOVQ" , _R8, jit.Ptr(_SP, 0)) // MOVQ R8, (SP) + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.call_go(_F_convTstring) // CALL_GO runtime.convTstring + self.Emit("MOVQ" , jit.Ptr(_SP, 16), _R9) // MOVQ 16(SP), R9 + + /* packed string already in R9 */ + self.Link("_packed_str") // _packed_str: + self.Emit("MOVQ", _T_string, _R8) // MOVQ _T_string, R8 + self.Emit("MOVQ", _VAR_ss_Iv, _DI) // MOVQ ss.Iv, DI + self.Emit("SUBQ", jit.Imm(1), _DI) // SUBQ $1, DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /* the string is an object key, get the map */ + self.Link("_object_key") + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI + self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI + + /* add a new delimiter */ + self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("CMPQ", _CX, jit.Imm(types.MAX_RECURSE)) // CMPQ CX, ${types.MAX_RECURSE} + self.Sjmp("JA" , "_stack_overflow") // JA _stack_overflow + self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp + self.Emit("MOVQ", jit.Imm(_S_obj_delim), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_obj_delim, ST.Vt[CX] + + /* add a new slot int the map */ + self.Emit("MOVQ", _T_map, _DX) // MOVQ _T_map, DX + self.Emit("MOVQ", _DX, jit.Ptr(_SP, 0)) // MOVQ DX, (SP) + self.Emit("MOVQ", _SI, jit.Ptr(_SP, 8)) // MOVQ SI, 8(SP) + self.Emit("MOVQ", _R8, jit.Ptr(_SP, 16)) // MOVQ R9, 16(SP) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 24)) // MOVQ AX, 24(SP) + self.call_go(_F_mapassign_faststr) // CALL_GO runtime.mapassign_faststr + self.Emit("MOVQ", jit.Ptr(_SP, 32), _AX) // MOVQ 32(SP), AX + + /* add to the pointer stack */ + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX] + self.Sjmp("JMP" , "_next") // JMP _next + + /* allocate memory to store the string header and unquoted result */ + self.Link("_unquote") // _unquote: + self.Emit("ADDQ", jit.Imm(15), _AX) // ADDQ $15, AX + self.Emit("MOVQ", _T_byte, _CX) // MOVQ _T_byte, CX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.Emit("MOVB", jit.Imm(0), jit.Ptr(_SP, 16)) // MOVB $0, 16(SP) + self.call_go(_F_mallocgc) // CALL_GO runtime.mallocgc + self.Emit("MOVQ", jit.Ptr(_SP, 24), _R9) // MOVQ 24(SP), R9 + + /* prepare the unquoting parameters */ + self.Emit("MOVQ" , _VAR_ss_Iv, _CX) // MOVQ ss.Iv, CX + self.Emit("LEAQ" , jit.Sib(_IP, _CX, 1, 0), _DI) // LEAQ (IP)(CX), DI + self.Emit("NEGQ" , _CX) // NEGQ CX + self.Emit("LEAQ" , jit.Sib(_IC, _CX, 1, -1), _SI) // LEAQ -1(IC)(CX), SI + self.Emit("LEAQ" , jit.Ptr(_R9, 16), _DX) // LEAQ 16(R8), DX + self.Emit("LEAQ" , _VAR_ss_Ep, _CX) // LEAQ ss.Ep, CX + self.Emit("XORL" , _R8, _R8) // XORL R8, R8 + self.Emit("BTQ" , jit.Imm(_F_disable_urc), _VAR_df) // BTQ ${_F_disable_urc}, fv + self.Emit("SETCC", _R8) // SETCC R8 + self.Emit("SHLQ" , jit.Imm(types.B_UNICODE_REPLACE), _R8) // SHLQ ${types.B_UNICODE_REPLACE}, R8 + + /* unquote the string, with R9 been preserved */ + self.save(_R9) // SAVE R9 + self.call(_F_unquote) // CALL unquote + self.load(_R9) // LOAD R9 + + /* check for errors */ + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , "_unquote_error") // JS _unquote_error + self.Emit("MOVL" , jit.Imm(1), _DX) // MOVL $1, DX + self.Emit("LEAQ" , jit.Ptr(_R9, 16), _R8) // ADDQ $16, R8 + self.Emit("MOVQ" , _R8, jit.Ptr(_R9, 0)) // MOVQ R8, (R9) + self.Emit("MOVQ" , _AX, jit.Ptr(_R9, 8)) // MOVQ AX, 8(R9) + self.Sjmp("JMP" , "_noescape") // JMP _noescape + + /** V_DOUBLE **/ + self.Link("_decode_V_DOUBLE") // _decode_V_DOUBLE: + self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df + self.Sjmp("JC" , "_use_number") // JC _use_number + self.Emit("MOVSD", _VAR_ss_Dv, _X0) // MOVSD ss.Dv, X0 + self.Sjmp("JMP" , "_use_float64") // JMP _use_float64 + + /** V_INTEGER **/ + self.Link("_decode_V_INTEGER") // _decode_V_INTEGER: + self.Emit("BTQ" , jit.Imm(_F_use_number), _VAR_df) // BTQ _F_use_number, df + self.Sjmp("JC" , "_use_number") // JC _use_number + self.Emit("BTQ" , jit.Imm(_F_use_int64), _VAR_df) // BTQ _F_use_int64, df + self.Sjmp("JC" , "_use_int64") // JC _use_int64 + self.Emit("MOVQ" , _VAR_ss_Iv, _AX) // MOVQ ss.Iv, AX + self.Emit("CVTSQ2SD", _AX, _X0) // CVTSQ2SD AX, X0 + + /* represent numbers as `float64` */ + self.Link("_use_float64") // _use_float64: + self.Emit("MOVSD", _X0, jit.Ptr(_SP, 0)) // MOVSD X0, (SP) + self.call_go(_F_convT64) // CALL_GO runtime.convT64 + self.Emit("MOVQ" , _T_float64, _R8) // MOVQ _T_float64, R8 + self.Emit("MOVQ" , jit.Ptr(_SP, 8), _R9) // MOVQ 8(SP), R9 + self.Emit("MOVQ" , _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /* represent numbers as `json.Number` */ + self.Link("_use_number") // _use_number + self.Emit("MOVQ", _VAR_ss_Ep, _AX) // MOVQ ss.Ep, AX + self.Emit("LEAQ", jit.Sib(_IP, _AX, 1, 0), _SI) // LEAQ (IP)(AX), SI + self.Emit("MOVQ", _IC, _CX) // MOVQ IC, CX + self.Emit("SUBQ", _AX, _CX) // SUBQ AX, CX + self.Emit("MOVQ", _SI, jit.Ptr(_SP, 0)) // MOVQ SI, (SP) + self.Emit("MOVQ", _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.call_go(_F_convTstring) // CALL_GO runtime.convTstring + self.Emit("MOVQ", _T_number, _R8) // MOVQ _T_number, R8 + self.Emit("MOVQ", jit.Ptr(_SP, 16), _R9) // MOVQ 16(SP), R9 + self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /* represent numbers as `int64` */ + self.Link("_use_int64") // _use_int64: + self.Emit("MOVQ", _VAR_ss_Iv, _AX) // MOVQ ss.Iv, AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.call_go(_F_convT64) // CALL_GO runtime.convT64 + self.Emit("MOVQ", _T_int64, _R8) // MOVQ _T_int64, R8 + self.Emit("MOVQ", jit.Ptr(_SP, 8), _R9) // MOVQ 8(SP), R9 + self.Emit("MOVQ", _VAR_ss_Ep, _DI) // MOVQ ss.Ep, DI + self.Sjmp("JMP" , "_set_value") // JMP _set_value + + /** V_KEY_SEP **/ + self.Link("_decode_V_KEY_SEP") // _decode_V_KEY_SEP: + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("CMPQ", _AX, jit.Imm(_S_obj_delim)) // CMPQ AX, _S_obj_delim + self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char + self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX] + self.Emit("MOVQ", jit.Imm(_S_obj), jit.Sib(_ST, _CX, 8, _ST_Vt - 8)) // MOVQ _S_obj, ST.Vt[CX - 1] + self.Sjmp("JMP" , "_next") // JMP _next + + /** V_ELEM_SEP **/ + self.Link("_decode_V_ELEM_SEP") // _decode_V_ELEM_SEP: + self.Emit("MOVQ" , jit.Imm(_S_obj), _AX) // MOVQ _S_obj, AX + self.Emit("MOVQ" , jit.Imm(_S_obj_x), _DX) // MOVQ _S_obj_x, DX + self.Emit("MOVQ" , jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("CMPXCHGQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vt)) // CMPXCHGQ DX, ST.Vt[CX] + self.Sjmp("JZ" , "_next") // JZ _next + self.Emit("CMPQ" , _AX, jit.Imm(_S_arr)) // CMPQ _AX, _S_arr + self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char + + /* arrays */ + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI + self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI + self.Emit("MOVQ", jit.Ptr(_SI, 8), _DX) // MOVQ 8(SI), DX + self.Emit("CMPQ", _DX, jit.Ptr(_SI, 16)) // CMPQ DX, 16(SI) + self.Sjmp("JAE" , "_array_more") // JAE _array_more + + /* add a slot for the new element */ + self.Link("_array_append") // _array_append: + self.Emit("ADDQ", jit.Imm(1), jit.Ptr(_SI, 8)) // ADDQ $1, 8(SI) + self.Emit("MOVQ", jit.Ptr(_SI, 0), _SI) // MOVQ (SI), SI + self.Emit("ADDQ", jit.Imm(1), _CX) // ADDQ $1, CX + self.Emit("SHLQ", jit.Imm(1), _DX) // SHLQ $1, DX + self.Emit("LEAQ", jit.Sib(_SI, _DX, 8, 0), _SI) // LEAQ (SI)(DX*8), SI + self.Emit("MOVQ", _CX, jit.Ptr(_ST, _ST_Sp)) // MOVQ CX, ST.Sp + self.Emit("MOVQ", _SI, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ SI, ST.Vp[CX] + self.Emit("MOVQ", jit.Imm(_S_val), jit.Sib(_ST, _CX, 8, _ST_Vt)) // MOVQ _S_val, ST.Vt[CX} + self.Sjmp("JMP" , "_next") // JMP _next + + /** V_ARRAY_END **/ + self.Link("_decode_V_ARRAY_END") // _decode_V_ARRAY_END: + self.Emit("XORL", _DX, _DX) // XORL DX, DX + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("CMPQ", _AX, jit.Imm(_S_arr_0)) // CMPQ AX, _S_arr_0 + self.Sjmp("JE" , "_first_item") // JE _first_item + self.Emit("CMPQ", _AX, jit.Imm(_S_arr)) // CMPQ AX, _S_arr + self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char + self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp + self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX] + self.Sjmp("JMP" , "_next") // JMP _next + + /* first element of an array */ + self.Link("_first_item") // _first_item: + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("SUBQ", jit.Imm(2), jit.Ptr(_ST, _ST_Sp)) // SUBQ $2, ST.Sp + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp - 8), _SI) // MOVQ ST.Vp[CX - 1], SI + self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI + self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp - 8)) // MOVQ DX, ST.Vp[CX - 1] + self.Emit("MOVQ", _DX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ DX, ST.Vp[CX] + self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI) + self.Sjmp("JMP" , "_next") // JMP _next + + /** V_OBJECT_END **/ + self.Link("_decode_V_OBJECT_END") // _decode_V_OBJECT_END: + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vt), _AX) // MOVQ ST.Vt[CX], AX + self.Emit("CMPQ", _AX, jit.Imm(_S_obj)) // CMPQ AX, _S_obj + self.Sjmp("JNE" , "_invalid_char") // JNE _invalid_char + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("SUBQ", jit.Imm(1), jit.Ptr(_ST, _ST_Sp)) // SUBQ $1, ST.Sp + self.Emit("MOVQ", _AX, jit.Sib(_ST, _CX, 8, _ST_Vp)) // MOVQ AX, ST.Vp[CX] + self.Sjmp("JMP" , "_next") // JMP _next + + /* return from decoder */ + self.Link("_return") // _return: + self.Emit("XORL", _EP, _EP) // XORL EP, EP + self.Emit("MOVQ", _EP, jit.Ptr(_ST, _ST_Vp)) // MOVQ EP, ST.Vp[0] + self.Link("_epilogue") // _epilogue: + self.Emit("SUBQ", jit.Imm(_FsmOffset), _ST) // SUBQ _FsmOffset, _ST + self.Emit("MOVQ", jit.Ptr(_SP, _VD_offs), _BP) // MOVQ _VD_offs(SP), BP + self.Emit("ADDQ", jit.Imm(_VD_size), _SP) // ADDQ $_VD_size, SP + self.Emit("RET") // RET + + /* array expand */ + self.Link("_array_more") // _array_more: + self.Emit("MOVQ" , _T_eface, _AX) // MOVQ _T_eface, AX + self.Emit("MOVOU", jit.Ptr(_SI, 0), _X0) // MOVOU (SI), X0 + self.Emit("MOVQ" , jit.Ptr(_SI, 16), _DX) // MOVQ 16(SI), DX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP) + self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 24)) // MOVQ DX, 24(SP) + self.Emit("SHLQ" , jit.Imm(1), _DX) // SHLQ $1, DX + self.Emit("MOVQ" , _DX, jit.Ptr(_SP, 32)) // MOVQ DX, 32(SP) + self.call_go(_F_growslice) // CALL_GO runtime.growslice + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _DI) // MOVOU 40(SP), DI + self.Emit("MOVQ" , jit.Ptr(_SP, 48), _DX) // MOVOU 48(SP), DX + self.Emit("MOVQ" , jit.Ptr(_SP, 56), _AX) // MOVQ 56(SP), AX + + /* update the slice */ + self.Emit("MOVQ", jit.Ptr(_ST, _ST_Sp), _CX) // MOVQ ST.Sp, CX + self.Emit("MOVQ", jit.Sib(_ST, _CX, 8, _ST_Vp), _SI) // MOVQ ST.Vp[CX], SI + self.Emit("MOVQ", jit.Ptr(_SI, 8), _SI) // MOVQ 8(SI), SI + self.Emit("MOVQ", _DI, jit.Ptr(_SI, 0)) // MOVQ DI, (SI) + self.Emit("MOVQ", _DX, jit.Ptr(_SI, 8)) // MOVQ DX, 8(SI) + self.Emit("MOVQ", _AX, jit.Ptr(_SI, 16)) // MOVQ AX, 16(AX) + self.Sjmp("JMP" , "_array_append") // JMP _array_append + + /* error handlers */ + self.Link("_stack_overflow") + self.Emit("MOVL" , _E_recurse, _EP) // MOVQ _E_recurse, EP + self.Sjmp("JMP" , "_error") // JMP _error + self.Link("_vtype_error") // _vtype_error: + self.Emit("MOVQ" , _DI, _IC) // MOVQ DI, IC + self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP + self.Sjmp("JMP" , "_error") // JMP _error + self.Link("_invalid_char") // _invalid_char: + self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC + self.Emit("MOVL" , _E_invalid, _EP) // MOVL _E_invalid, EP + self.Sjmp("JMP" , "_error") // JMP _error + self.Link("_unquote_error") // _unquote_error: + self.Emit("MOVQ" , _VAR_ss_Iv, _IC) // MOVQ ss.Iv, IC + self.Emit("SUBQ" , jit.Imm(1), _IC) // SUBQ $1, IC + self.Link("_parsing_error") // _parsing_error: + self.Emit("NEGQ" , _AX) // NEGQ AX + self.Emit("MOVQ" , _AX, _EP) // MOVQ AX, EP + self.Link("_error") // _error: + self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 + self.Emit("MOVOU", _X0, jit.Ptr(_VP, 0)) // MOVOU X0, (VP) + self.Sjmp("JMP" , "_epilogue") // JMP _epilogue + + /* invalid value type, never returns */ + self.Link("_invalid_vtype") + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.call(_F_invalid_vtype) // CALL invalid_type + self.Emit("UD2") // UD2 + + /* switch jump table */ + self.Link("_switch_table") // _switch_table: + self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0 + self.Sref("_decode_V_NULL", -4) // SREF &_decode_V_NULL, $-4 + self.Sref("_decode_V_TRUE", -8) // SREF &_decode_V_TRUE, $-8 + self.Sref("_decode_V_FALSE", -12) // SREF &_decode_V_FALSE, $-12 + self.Sref("_decode_V_ARRAY", -16) // SREF &_decode_V_ARRAY, $-16 + self.Sref("_decode_V_OBJECT", -20) // SREF &_decode_V_OBJECT, $-20 + self.Sref("_decode_V_STRING", -24) // SREF &_decode_V_STRING, $-24 + self.Sref("_decode_V_DOUBLE", -28) // SREF &_decode_V_DOUBLE, $-28 + self.Sref("_decode_V_INTEGER", -32) // SREF &_decode_V_INTEGER, $-32 + self.Sref("_decode_V_KEY_SEP", -36) // SREF &_decode_V_KEY_SEP, $-36 + self.Sref("_decode_V_ELEM_SEP", -40) // SREF &_decode_V_ELEM_SEP, $-40 + self.Sref("_decode_V_ARRAY_END", -44) // SREF &_decode_V_ARRAY_END, $-44 + self.Sref("_decode_V_OBJECT_END", -48) // SREF &_decode_V_OBJECT_END, $-48 + + /* fast character lookup table */ + self.Link("_decode_tab") // _decode_tab: + self.Sref("_decode_V_EOF", 0) // SREF &_decode_V_EOF, $0 + + /* generate rest of the tabs */ + for i := 1; i < 256; i++ { + if to, ok := _R_tab[i]; ok { + self.Sref(to, -int64(i) * 4) + } else { + self.Byte(0x00, 0x00, 0x00, 0x00) + } + } + } + + /** Generic Decoder **/ + + var ( + _subr_decode_value = new(_ValueDecoder).build() + ) + + //go:nosplit + func invalid_vtype(vt types.ValueType) { + throw(fmt.Sprintf("invalid value type: %d", vt)) + } \ No newline at end of file diff --git a/decoder/generic_amd64.go b/decoder/generic_amd64_go116.go similarity index 99% rename from decoder/generic_amd64.go rename to decoder/generic_amd64_go116.go index 134a5126e..2a1d29796 100644 --- a/decoder/generic_amd64.go +++ b/decoder/generic_amd64_go116.go @@ -1,3 +1,5 @@ +// +build go1.16,!go1.17 + /* * Copyright 2021 ByteDance Inc. * diff --git a/decoder/stubs.go b/decoder/stubs.go index f595e807c..efa160fa7 100644 --- a/decoder/stubs.go +++ b/decoder/stubs.go @@ -35,20 +35,6 @@ func mapfast(vt reflect.Type) bool { return vt.Elem().Size() <= _max_map_element_size } -type writeBarrier struct { - enabled bool // compiler emits a check of this before calling write barrier - pad [3]byte // compiler uses 32-bit load for "enabled" field - needed bool // whether we need a write barrier for current GC phase - cgo bool // whether we need a write barrier for a cgo check - alignme uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load -} - -//go:linkname _runtime_writeBarrier runtime.writeBarrier -var _runtime_writeBarrier writeBarrier - -//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier -func gcWriteBarrierAX() - //go:nosplit //go:linkname throw runtime.throw //goland:noinspection GoUnusedParameter diff --git a/encoder/assembler_amd64_go115.go b/encoder/assembler_amd64_go115.go new file mode 100644 index 000000000..86b831f7d --- /dev/null +++ b/encoder/assembler_amd64_go115.go @@ -0,0 +1,1114 @@ +// +build go1.15,!go1.16 + +/* + * Copyright 2021 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package encoder + + import ( + `fmt` + `reflect` + `strconv` + `unsafe` + + `github.com/bytedance/sonic/internal/cpu` + `github.com/bytedance/sonic/internal/jit` + `github.com/bytedance/sonic/internal/native/types` + `github.com/twitchyliquid64/golang-asm/obj` + `github.com/twitchyliquid64/golang-asm/obj/x86` + + `github.com/bytedance/sonic/internal/native` + `github.com/bytedance/sonic/internal/rt` + ) + + /** Register Allocations + * + * State Registers: + * + * %rbx : stack base + * %rdi : result pointer + * %rsi : result length + * %rdx : result capacity + * %r12 : sp->p + * %r13 : sp->q + * %r14 : sp->x + * %r15 : sp->f + * + * Error Registers: + * + * %r10 : error type register + * %r11 : error pointer register + */ + + /** Function Prototype & Stack Map + * + * func (buf *[]byte, p unsafe.Pointer, sb *_Stack, fv uint64) (err error) + * + * buf : (FP) + * p : 8(FP) + * sb : 16(FP) + * fv : 24(FP) + * err.vt : 32(FP) + * err.vp : 40(FP) + */ + + const ( + _S_cond = iota + _S_init + ) + + const ( + _FP_args = 48 // 48 bytes for passing arguments to this function + _FP_fargs = 64 // 64 bytes for passing arguments to other Go functions + _FP_saves = 64 // 64 bytes for saving the registers before CALL instructions + _FP_locals = 24 // 24 bytes for local variables + ) + + const ( + _FP_offs = _FP_fargs + _FP_saves + _FP_locals + _FP_size = _FP_offs + 8 // 8 bytes for the parent frame pointer + _FP_base = _FP_size + 8 // 8 bytes for the return address + ) + + const ( + _FM_exp32 = 0x7f800000 + _FM_exp64 = 0x7ff0000000000000 + ) + + const ( + _IM_null = 0x6c6c756e // 'null' + _IM_true = 0x65757274 // 'true' + _IM_fals = 0x736c6166 // 'fals' ('false' without the 'e') + _IM_open = 0x00225c22 // '"\"∅' + _IM_mulv = -0x5555555555555555 + ) + + const ( + _LB_more_space = "_more_space" + _LB_more_space_return = "_more_space_return_" + ) + + const ( + _LB_error = "_error" + _LB_error_too_deep = "_error_too_deep" + _LB_error_invalid_number = "_error_invalid_number" + _LB_error_nan_or_infinite = "_error_nan_or_infinite" + ) + + var ( + _AX = jit.Reg("AX") + _CX = jit.Reg("CX") + _DX = jit.Reg("DX") + _DI = jit.Reg("DI") + _SI = jit.Reg("SI") + _BP = jit.Reg("BP") + _SP = jit.Reg("SP") + _R8 = jit.Reg("R8") + ) + + var ( + _X0 = jit.Reg("X0") + _Y0 = jit.Reg("Y0") + ) + + var ( + _ST = jit.Reg("BX") + _RP = jit.Reg("DI") + _RL = jit.Reg("SI") + _RC = jit.Reg("DX") + ) + + var ( + _LR = jit.Reg("R9") + _ET = jit.Reg("R10") + _EP = jit.Reg("R11") + ) + + var ( + _SP_p = jit.Reg("R12") + _SP_q = jit.Reg("R13") + _SP_x = jit.Reg("R14") + _SP_f = jit.Reg("R15") + ) + + var ( + _ARG_rb = jit.Ptr(_SP, _FP_base) + _ARG_vp = jit.Ptr(_SP, _FP_base + 8) + _ARG_sb = jit.Ptr(_SP, _FP_base + 16) + _ARG_fv = jit.Ptr(_SP, _FP_base + 24) + ) + + var ( + _RET_et = jit.Ptr(_SP, _FP_base + 32) + _RET_ep = jit.Ptr(_SP, _FP_base + 40) + ) + + var ( + _VAR_sp = jit.Ptr(_SP, _FP_fargs + _FP_saves) + _VAR_dn = jit.Ptr(_SP, _FP_fargs + _FP_saves + 8) + _VAR_vp = jit.Ptr(_SP, _FP_fargs + _FP_saves + 16) + ) + + var ( + _REG_ffi = []obj.Addr{_RP, _RL, _RC} + _REG_enc = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q} + _REG_jsr = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _LR} + _REG_all = []obj.Addr{_ST, _SP_x, _SP_f, _SP_p, _SP_q, _RP, _RL, _RC} + ) + + type _Assembler struct { + jit.BaseAssembler + p _Program + x int + } + + func newAssembler(p _Program) *_Assembler { + return new(_Assembler).Init(p) + } + + /** Assembler Interface **/ + + func (self *_Assembler) Load() _Encoder { + return ptoenc(self.BaseAssembler.LoadWithFaker("json_encoder", _FP_size, _FP_args, _Encoder_Shadow)) + } + + func (self *_Assembler) Init(p _Program) *_Assembler { + self.p = p + self.BaseAssembler.Init(self.compile) + return self + } + + func (self *_Assembler) compile() { + self.prologue() + self.instrs() + self.epilogue() + self.builtins() + } + + /** Assembler Stages **/ + + var _OpFuncTab = [256]func(*_Assembler, *_Instr) { + _OP_null : (*_Assembler)._asm_OP_null, + _OP_bool : (*_Assembler)._asm_OP_bool, + _OP_i8 : (*_Assembler)._asm_OP_i8, + _OP_i16 : (*_Assembler)._asm_OP_i16, + _OP_i32 : (*_Assembler)._asm_OP_i32, + _OP_i64 : (*_Assembler)._asm_OP_i64, + _OP_u8 : (*_Assembler)._asm_OP_u8, + _OP_u16 : (*_Assembler)._asm_OP_u16, + _OP_u32 : (*_Assembler)._asm_OP_u32, + _OP_u64 : (*_Assembler)._asm_OP_u64, + _OP_f32 : (*_Assembler)._asm_OP_f32, + _OP_f64 : (*_Assembler)._asm_OP_f64, + _OP_str : (*_Assembler)._asm_OP_str, + _OP_bin : (*_Assembler)._asm_OP_bin, + _OP_quote : (*_Assembler)._asm_OP_quote, + _OP_number : (*_Assembler)._asm_OP_number, + _OP_eface : (*_Assembler)._asm_OP_eface, + _OP_iface : (*_Assembler)._asm_OP_iface, + _OP_byte : (*_Assembler)._asm_OP_byte, + _OP_text : (*_Assembler)._asm_OP_text, + _OP_deref : (*_Assembler)._asm_OP_deref, + _OP_index : (*_Assembler)._asm_OP_index, + _OP_load : (*_Assembler)._asm_OP_load, + _OP_save : (*_Assembler)._asm_OP_save, + _OP_drop : (*_Assembler)._asm_OP_drop, + _OP_drop_2 : (*_Assembler)._asm_OP_drop_2, + _OP_recurse : (*_Assembler)._asm_OP_recurse, + _OP_is_nil : (*_Assembler)._asm_OP_is_nil, + _OP_is_nil_p1 : (*_Assembler)._asm_OP_is_nil_p1, + _OP_is_zero_1 : (*_Assembler)._asm_OP_is_zero_1, + _OP_is_zero_2 : (*_Assembler)._asm_OP_is_zero_2, + _OP_is_zero_4 : (*_Assembler)._asm_OP_is_zero_4, + _OP_is_zero_8 : (*_Assembler)._asm_OP_is_zero_8, + _OP_is_zero_map : (*_Assembler)._asm_OP_is_zero_map, + _OP_goto : (*_Assembler)._asm_OP_goto, + _OP_map_iter : (*_Assembler)._asm_OP_map_iter, + _OP_map_stop : (*_Assembler)._asm_OP_map_stop, + _OP_map_check_key : (*_Assembler)._asm_OP_map_check_key, + _OP_map_write_key : (*_Assembler)._asm_OP_map_write_key, + _OP_map_value_next : (*_Assembler)._asm_OP_map_value_next, + _OP_slice_len : (*_Assembler)._asm_OP_slice_len, + _OP_slice_next : (*_Assembler)._asm_OP_slice_next, + _OP_marshal : (*_Assembler)._asm_OP_marshal, + _OP_marshal_p : (*_Assembler)._asm_OP_marshal_p, + _OP_marshal_text : (*_Assembler)._asm_OP_marshal_text, + _OP_marshal_text_p : (*_Assembler)._asm_OP_marshal_text_p, + _OP_cond_set : (*_Assembler)._asm_OP_cond_set, + _OP_cond_testc : (*_Assembler)._asm_OP_cond_testc, + } + + func (self *_Assembler) instr(v *_Instr) { + if fn := _OpFuncTab[v.op()]; fn != nil { + fn(self, v) + } else { + panic(fmt.Sprintf("invalid opcode: %d", v.op())) + } + } + + func (self *_Assembler) instrs() { + for i, v := range self.p { + self.Mark(i) + self.instr(&v) + self.debug_instr(i, &v) + } + } + + func (self *_Assembler) builtins() { + self.more_space() + self.error_too_deep() + self.error_invalid_number() + self.error_nan_or_infinite() + } + + func (self *_Assembler) epilogue() { + self.Mark(len(self.p)) + self.Emit("XORL", _ET, _ET) + self.Emit("XORL", _EP, _EP) + self.Link(_LB_error) + self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX + self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8)) // MOVQ RL, 8(AX) + self.Emit("MOVQ", _ET, _RET_et) // MOVQ ET, et<>+24(FP) + self.Emit("MOVQ", _EP, _RET_ep) // MOVQ EP, ep<>+32(FP) + self.Emit("MOVQ", jit.Ptr(_SP, _FP_offs), _BP) // MOVQ _FP_offs(SP), BP + self.Emit("ADDQ", jit.Imm(_FP_size), _SP) // ADDQ $_FP_size, SP + self.Emit("RET") // RET + } + + func (self *_Assembler) prologue() { + self.Emit("SUBQ", jit.Imm(_FP_size), _SP) // SUBQ $_FP_size, SP + self.Emit("MOVQ", _BP, jit.Ptr(_SP, _FP_offs)) // MOVQ BP, _FP_offs(SP) + self.Emit("LEAQ", jit.Ptr(_SP, _FP_offs), _BP) // LEAQ _FP_offs(SP), BP + self.load_buffer() // LOAD {buf} + self.Emit("MOVQ", _ARG_vp, _SP_p) // MOVQ vp<>+8(FP), SP.p + self.Emit("MOVQ", _ARG_sb, _ST) // MOVQ sb<>+16(FP), ST + self.Emit("XORL", _SP_x, _SP_x) // XORL SP.x, SP.x + self.Emit("XORL", _SP_f, _SP_f) // XORL SP.f, SP.f + self.Emit("XORL", _SP_q, _SP_q) // XORL SP.q, SP.q + } + + /** Assembler Inline Functions **/ + + func (self *_Assembler) xsave(reg ...obj.Addr) { + for i, v := range reg { + if i > _FP_saves / 8 - 1 { + panic("too many registers to save") + } else { + self.Emit("MOVQ", v, jit.Ptr(_SP, _FP_fargs + int64(i) * 8)) + } + } + } + + func (self *_Assembler) xload(reg ...obj.Addr) { + for i, v := range reg { + if i > _FP_saves / 8 - 1 { + panic("too many registers to load") + } else { + self.Emit("MOVQ", jit.Ptr(_SP, _FP_fargs + int64(i) * 8), v) + } + } + } + + func (self *_Assembler) rbuf_di() { + if _RP.Reg != x86.REG_DI { + panic("register allocation messed up: RP != DI") + } else { + self.Emit("ADDQ", _RL, _RP) + } + } + + func (self *_Assembler) store_int(nd int, fn obj.Addr, ins string) { + self.check_size(nd) + self.save_c() // SAVE $C_regs + self.rbuf_di() // MOVQ RP, DI + self.Emit(ins, jit.Ptr(_SP_p, 0), _SI) // $ins (SP.p), SI + self.call_c(fn) // CALL_C $fn + self.Emit("ADDQ", _AX, _RL) // ADDQ AX, RL + } + + func (self *_Assembler) store_str(s string) { + i := 0 + m := rt.Str2Mem(s) + + /* 8-byte stores */ + for i <= len(m) - 8 { + self.Emit("MOVQ", jit.Imm(rt.Get64(m[i:])), _AX) // MOVQ $s[i:], AX + self.Emit("MOVQ", _AX, jit.Sib(_RP, _RL, 1, int64(i))) // MOVQ AX, i(RP)(RL) + i += 8 + } + + /* 4-byte stores */ + if i <= len(m) - 4 { + self.Emit("MOVL", jit.Imm(int64(rt.Get32(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i))) // MOVL $s[i:], i(RP)(RL) + i += 4 + } + + /* 2-byte stores */ + if i <= len(m) - 2 { + self.Emit("MOVW", jit.Imm(int64(rt.Get16(m[i:]))), jit.Sib(_RP, _RL, 1, int64(i))) // MOVW $s[i:], i(RP)(RL) + i += 2 + } + + /* last byte */ + if i < len(m) { + self.Emit("MOVB", jit.Imm(int64(m[i])), jit.Sib(_RP, _RL, 1, int64(i))) // MOVB $s[i:], i(RP)(RL) + } + } + + func (self *_Assembler) check_size(n int) { + self.check_size_rl(jit.Ptr(_RL, int64(n))) + } + + func (self *_Assembler) check_size_r(r obj.Addr, d int) { + self.check_size_rl(jit.Sib(_RL, r, 1, int64(d))) + } + + func (self *_Assembler) check_size_rl(v obj.Addr) { + idx := self.x + key := _LB_more_space_return + strconv.Itoa(idx) + + /* the following code relies on LR == R9 to work */ + if _LR.Reg != x86.REG_R9 { + panic("register allocation messed up: LR != R9") + } + + /* check for buffer capacity */ + self.x++ + self.Emit("LEAQ", v, _AX) // LEAQ $v, AX + self.Emit("CMPQ", _AX, _RC) // CMPQ AX, RC + self.Sjmp("JBE" , key) // JBE _more_space_return_{n} + self.slice_grow_ax(key) // GROW $key + self.Link(key) // _more_space_return_{n}: + } + + func (self *_Assembler) slice_grow_ax(ret string) { + self.Byte(0x4c, 0x8d, 0x0d) // LEAQ ?(PC), R9 + self.Sref(ret, 4) // .... &ret + self.Sjmp("JMP" , _LB_more_space) // JMP _more_space + } + + /** State Stack Helpers **/ + + const ( + _StateSize = int64(unsafe.Sizeof(_State{})) + _StackLimit = _MaxStack * _StateSize + ) + + func (self *_Assembler) save_state() { + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("LEAQ", jit.Ptr(_AX, _StateSize), _R8) // LEAQ _StateSize(AX), R8 + self.Emit("CMPQ", _R8, jit.Imm(_StackLimit)) // CMPQ R8, $_StackLimit + self.Sjmp("JA" , _LB_error_too_deep) // JA _error_too_deep + self.Emit("MOVQ", _SP_x, jit.Sib(_ST, _AX, 1, 8)) // MOVQ SP.x, 8(ST)(AX) + self.Emit("MOVQ", _SP_f, jit.Sib(_ST, _AX, 1, 16)) // MOVQ SP.f, 16(ST)(AX) + self.Emit("MOVQ", _SP_p, jit.Sib(_ST, _AX, 1, 24)) // MOVQ SP.p, 24(ST)(AX) + self.Emit("MOVQ", _SP_q, jit.Sib(_ST, _AX, 1, 32)) // MOVQ SP.q, 32(ST)(AX) + self.Emit("MOVQ", _R8, jit.Ptr(_ST, 0)) // MOVQ R8, (ST) + } + + func (self *_Assembler) drop_state(decr int64) { + self.Emit("MOVQ" , jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("SUBQ" , jit.Imm(decr), _AX) // SUBQ $decr, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_ST, 0)) // MOVQ AX, (ST) + self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 8), _SP_x) // MOVQ 8(ST)(AX), SP.x + self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 16), _SP_f) // MOVQ 16(ST)(AX), SP.f + self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 24), _SP_p) // MOVQ 24(ST)(AX), SP.p + self.Emit("MOVQ" , jit.Sib(_ST, _AX, 1, 32), _SP_q) // MOVQ 32(ST)(AX), SP.q + self.Emit("PXOR" , _X0, _X0) // PXOR X0, X0 + self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 8)) // MOVOU X0, 8(ST)(AX) + self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 24)) // MOVOU X0, 24(ST)(AX) + } + + /** Buffer Helpers **/ + + func (self *_Assembler) add_char(ch byte) { + self.Emit("MOVB", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0)) // MOVB $ch, (RP)(RL) + self.Emit("ADDQ", jit.Imm(1), _RL) // ADDQ $1, RL + } + + func (self *_Assembler) add_long(ch uint32, n int64) { + self.Emit("MOVL", jit.Imm(int64(ch)), jit.Sib(_RP, _RL, 1, 0)) // MOVL $ch, (RP)(RL) + self.Emit("ADDQ", jit.Imm(n), _RL) // ADDQ $n, RL + } + + func (self *_Assembler) add_text(ss string) { + self.store_str(ss) // TEXT $ss + self.Emit("ADDQ", jit.Imm(int64(len(ss))), _RL) // ADDQ ${len(ss)}, RL + } + + func (self *_Assembler) prep_buffer() { + self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX + self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8)) // MOVQ RL, 8(AX) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + } + + func (self *_Assembler) prep_buffer_c() { + self.Emit("MOVQ", _ARG_rb, _DI) // MOVQ rb<>+0(FP), DI + self.Emit("MOVQ", _RL, jit.Ptr(_DI, 8)) // MOVQ RL, 8(DI) + } + + func (self *_Assembler) save_buffer() { + self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX + self.Emit("MOVQ", _RP, jit.Ptr(_AX, 0)) // MOVQ RP, (AX) + self.Emit("MOVQ", _RL, jit.Ptr(_AX, 8)) // MOVQ RL, 8(AX) + self.Emit("MOVQ", _RC, jit.Ptr(_AX, 16)) // MOVQ RC, 16(AX) + } + + func (self *_Assembler) load_buffer() { + self.Emit("MOVQ", _ARG_rb, _AX) // MOVQ rb<>+0(FP), AX + self.Emit("MOVQ", jit.Ptr(_AX, 0), _RP) // MOVQ (AX), RP + self.Emit("MOVQ", jit.Ptr(_AX, 8), _RL) // MOVQ 8(AX), RL + self.Emit("MOVQ", jit.Ptr(_AX, 16), _RC) // MOVQ 16(AX), RC + } + + /** Function Interface Helpers **/ + + var ( + _F_assertI2I = jit.Func(assertI2I) + ) + + func (self *_Assembler) call(pc obj.Addr) { + self.Emit("MOVQ", pc, _AX) // MOVQ $pc, AX + self.Rjmp("CALL", _AX) // CALL AX + } + + func (self *_Assembler) save_c() { + self.xsave(_REG_ffi...) // SAVE $REG_ffi + } + + func (self *_Assembler) call_c(pc obj.Addr) { + self.call(pc) // CALL $pc + self.xload(_REG_ffi...) // LOAD $REG_ffi + } + + func (self *_Assembler) call_go(pc obj.Addr) { + self.xsave(_REG_all...) // SAVE $REG_all + self.call(pc) // CALL $pc + self.xload(_REG_all...) // LOAD $REG_all + } + + func (self *_Assembler) call_encoder(pc obj.Addr) { + self.xsave(_REG_enc...) // SAVE $REG_enc + self.call(pc) // CALL $pc + self.xload(_REG_enc...) // LOAD $REG_enc + self.load_buffer() // LOAD {buf} + } + + func (self *_Assembler) call_marshaler(fn obj.Addr, it *rt.GoType, vt reflect.Type) { + switch vt.Kind() { + case reflect.Interface : self.call_marshaler_i(fn, it) + case reflect.Ptr : self.call_marshaler_v(fn, it, vt, true) + default : self.call_marshaler_v(fn, it, vt, false) + } + } + + func (self *_Assembler) call_marshaler_i(fn obj.Addr, it *rt.GoType) { + self.Emit("MOVQ" , jit.Gtype(it), _AX) // MOVQ $it, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _CX) // MOVQ 8(SP.p), CX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JZ" , "_null_{n}") // JZ _null_{n} + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 16)) // MOVQ CX, 16(SP) + self.call_go(_F_assertI2I) // CALL_GO assertI2I + self.prep_buffer() // MOVE {buf}, (SP) + self.Emit("MOVOU", jit.Ptr(_SP, 24), _X0) // MOVOU 24(SP), X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP) + self.call_encoder(fn) // CALL $fn + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _ET) // MOVQ 24(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _EP) // MOVQ 32(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + self.Sjmp("JMP" , "_done_{n}") // JMP _done_{n} + self.Link("_null_{n}") // _null_{n}: + self.check_size(4) // SIZE $4 + self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'null', (RP)(RL*1) + self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL + self.Link("_done_{n}") // _done_{n}: + } + + func (self *_Assembler) call_marshaler_v(fn obj.Addr, it *rt.GoType, vt reflect.Type, deref bool) { + self.prep_buffer() // MOVE {buf}, (SP) + self.Emit("MOVQ", jit.Itab(it, vt), _AX) // MOVQ $(itab(it, vt)), AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + + /* dereference the pointer if needed */ + if !deref { + self.Emit("MOVQ", _SP_p, jit.Ptr(_SP, 16)) // MOVQ SP.p, 16(SP) + } else { + self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + } + + /* call the encoder, and perform error checks */ + self.call_encoder(fn) // CALL $fn + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _ET) // MOVQ 24(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _EP) // MOVQ 32(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + /** Builtin: _more_space **/ + + var ( + _T_byte = jit.Type(byteType) + _F_growslice = jit.Func(growslice) + ) + + func (self *_Assembler) more_space() { + self.Link(_LB_more_space) + self.Emit("MOVQ", _T_byte, jit.Ptr(_SP, 0)) // MOVQ $_T_byte, (SP) + self.Emit("MOVQ", _RP, jit.Ptr(_SP, 8)) // MOVQ RP, 8(SP) + self.Emit("MOVQ", _RL, jit.Ptr(_SP, 16)) // MOVQ RL, 16(SP) + self.Emit("MOVQ", _RC, jit.Ptr(_SP, 24)) // MOVQ RC, 24(SP) + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP) + self.xsave(_REG_jsr...) // SAVE $REG_jsr + self.call(_F_growslice) // CALL $pc + self.xload(_REG_jsr...) // LOAD $REG_jsr + self.Emit("MOVQ", jit.Ptr(_SP, 40), _RP) // MOVQ 40(SP), RP + self.Emit("MOVQ", jit.Ptr(_SP, 48), _RL) // MOVQ 48(SP), RL + self.Emit("MOVQ", jit.Ptr(_SP, 56), _RC) // MOVQ 56(SP), RC + self.save_buffer() // SAVE {buf} + self.Rjmp("JMP" , _LR) // JMP LR + } + + /** Builtin Errors **/ + + var ( + _V_ERR_too_deep = jit.Imm(int64(uintptr(unsafe.Pointer(_ERR_too_deep)))) + _V_ERR_nan_or_infinite = jit.Imm(int64(uintptr(unsafe.Pointer(_ERR_nan_or_infinite)))) + _I_json_UnsupportedValueError = jit.Itab(rt.UnpackType(errorType), jsonUnsupportedValueType) + ) + + func (self *_Assembler) error_too_deep() { + self.Link(_LB_error_too_deep) + self.Emit("MOVQ", _V_ERR_too_deep, _EP) // MOVQ $_V_ERR_too_deep, EP + self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET) // MOVQ $_I_json_UnsupportedValuError, ET + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) error_invalid_number() { + self.Link(_LB_error_invalid_number) + self.call_go(_F_error_number) // CALL_GO error_number + self.Emit("MOVQ", jit.Ptr(_SP, 16), _ET) // MOVQ 16(SP), ET + self.Emit("MOVQ", jit.Ptr(_SP, 24), _EP) // MOVQ 24(SP), EP + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + func (self *_Assembler) error_nan_or_infinite() { + self.Link(_LB_error_nan_or_infinite) + self.Emit("MOVQ", _V_ERR_nan_or_infinite, _EP) // MOVQ $_V_ERR_nan_or_infinite, EP + self.Emit("MOVQ", _I_json_UnsupportedValueError, _ET) // MOVQ $_I_json_UnsupportedValuError, ET + self.Sjmp("JMP" , _LB_error) // JMP _error + } + + /** String Encoding Routine **/ + + var ( + _F_quote = jit.Imm(int64(native.S_quote)) + ) + + func (self *_Assembler) encode_string(doubleQuote bool) { + self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _AX) // MOVQ 8(SP.p), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JZ" , "_str_empty_{n}") // JZ _str_empty_{n} + + /* openning quote, check for double quote */ + if !doubleQuote { + self.check_size_r(_AX, 2) // SIZE $2 + self.add_char('"') // CHAR $'"' + } else { + self.check_size_r(_AX, 6) // SIZE $6 + self.add_long(_IM_open, 3) // TEXT $`"\"` + } + + /* quoting loop */ + self.Emit("XORL", _AX, _AX) // XORL AX, AX + self.Emit("MOVQ", _AX, _VAR_sp) // MOVQ AX, sp + self.Link("_str_loop_{n}") // _str_loop_{n}: + self.save_c() // SAVE $REG_ffi + + /* load the output buffer first, and then input buffer, + * because the parameter registers collide with RP / RL / RC */ + self.Emit("MOVQ", _RC, _CX) // MOVQ RC, CX + self.Emit("SUBQ", _RL, _CX) // SUBQ RL, CX + self.Emit("MOVQ", _CX, _VAR_dn) // MOVQ CX, dn + self.Emit("LEAQ", jit.Sib(_RP, _RL, 1, 0), _DX) // LEAQ (RP)(RL), DX + self.Emit("LEAQ", _VAR_dn, _CX) // LEAQ dn, CX + self.Emit("MOVQ", _VAR_sp, _AX) // MOVQ sp, AX + self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _DI) // MOVQ (SP.p), DI + self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _SI) // MOVQ 8(SP.p), SI + self.Emit("ADDQ", _AX, _DI) // ADDQ AX, DI + self.Emit("SUBQ", _AX, _SI) // SUBQ AX, SI + + /* set the flags based on `doubleQuote` */ + if !doubleQuote { + self.Emit("XORL", _R8, _R8) // XORL R8, R8 + } else { + self.Emit("MOVL", jit.Imm(types.F_DOUBLE_UNQUOTE), _R8) // MOVL ${types.F_DOUBLE_UNQUOTE}, R8 + } + + /* call the native quoter */ + self.call_c(_F_quote) // CALL quote + self.Emit("ADDQ" , _VAR_dn, _RL) // ADDQ dn, RL + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Sjmp("JS" , "_str_space_{n}") // JS _str_space_{n} + + /* close the string, check for double quote */ + if !doubleQuote { + self.check_size(1) // SIZE $1 + self.add_char('"') // CHAR $'"' + self.Sjmp("JMP", "_str_end_{n}") // JMP _str_end_{n} + } else { + self.check_size(3) // SIZE $3 + self.add_text("\\\"\"") // TEXT $'\""' + self.Sjmp("JMP", "_str_end_{n}") // JMP _str_end_{n} + } + + /* not enough space to contain the quoted string */ + self.Link("_str_space_{n}") // _str_space_{n}: + self.Emit("NOTQ", _AX) // NOTQ AX + self.Emit("ADDQ", _AX, _VAR_sp) // ADDQ AX, sp + self.Emit("LEAQ", jit.Sib(_RC, _RC, 1, 0), _AX) // LEAQ (RC)(RC), AX + self.slice_grow_ax("_str_loop_{n}") // GROW _str_loop_{n} + + /* empty string, check for double quote */ + if !doubleQuote { + self.Link("_str_empty_{n}") // _str_empty_{n}: + self.check_size(2) // SIZE $2 + self.add_text("\"\"") // TEXT $'""' + self.Link("_str_end_{n}") // _str_end_{n}: + } else { + self.Link("_str_empty_{n}") // _str_empty_{n}: + self.check_size(6) // SIZE $6 + self.add_text("\"\\\"\\\"\"") // TEXT $'"\"\""' + self.Link("_str_end_{n}") // _str_end_{n}: + } + } + + /** OpCode Assembler Functions **/ + + var ( + _T_json_Marshaler = rt.UnpackType(jsonMarshalerType) + _T_encoding_TextMarshaler = rt.UnpackType(encodingTextMarshalerType) + ) + + var ( + _F_f64toa = jit.Imm(int64(native.S_f64toa)) + _F_i64toa = jit.Imm(int64(native.S_i64toa)) + _F_u64toa = jit.Imm(int64(native.S_u64toa)) + _F_b64encode = jit.Imm(int64(_subr__b64encode)) + ) + + var ( + _F_memmove = jit.Func(memmove) + _F_error_number = jit.Func(error_number) + _F_isValidNumber = jit.Func(isValidNumber) + ) + + var ( + _F_iteratorStop = jit.Func(iteratorStop) + _F_iteratorNext = jit.Func(iteratorNext) + _F_iteratorStart = jit.Func(iteratorStart) + ) + + var ( + _F_encodeTypedPointer obj.Addr + _F_encodeJsonMarshaler obj.Addr + _F_encodeTextMarshaler obj.Addr + ) + + const ( + _MODE_AVX2 = 1 << 2 + ) + + func init() { + _F_encodeTypedPointer = jit.Func(encodeTypedPointer) + _F_encodeJsonMarshaler = jit.Func(encodeJsonMarshaler) + _F_encodeTextMarshaler = jit.Func(encodeTextMarshaler) + } + + func (self *_Assembler) _asm_OP_null(_ *_Instr) { + self.check_size(4) + self.Emit("MOVL", jit.Imm(_IM_null), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'null', (RP)(RL*1) + self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL + } + + func (self *_Assembler) _asm_OP_bool(_ *_Instr) { + self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0 + self.Sjmp("JE" , "_false_{n}") // JE _false_{n} + self.check_size(4) // SIZE $4 + self.Emit("MOVL", jit.Imm(_IM_true), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'true', (RP)(RL*1) + self.Emit("ADDQ", jit.Imm(4), _RL) // ADDQ $4, RL + self.Sjmp("JMP" , "_end_{n}") // JMP _end_{n} + self.Link("_false_{n}") // _false_{n}: + self.check_size(5) // SIZE $5 + self.Emit("MOVL", jit.Imm(_IM_fals), jit.Sib(_RP, _RL, 1, 0)) // MOVL $'fals', (RP)(RL*1) + self.Emit("MOVB", jit.Imm('e'), jit.Sib(_RP, _RL, 1, 4)) // MOVB $'e', 4(RP)(RL*1) + self.Emit("ADDQ", jit.Imm(5), _RL) // ADDQ $5, RL + self.Link("_end_{n}") // _end_{n}: + } + + func (self *_Assembler) _asm_OP_i8(_ *_Instr) { + self.store_int(4, _F_i64toa, "MOVBQSX") + } + + func (self *_Assembler) _asm_OP_i16(_ *_Instr) { + self.store_int(6, _F_i64toa, "MOVWQSX") + } + + func (self *_Assembler) _asm_OP_i32(_ *_Instr) { + self.store_int(11, _F_i64toa, "MOVLQSX") + } + + func (self *_Assembler) _asm_OP_i64(_ *_Instr) { + self.store_int(21, _F_i64toa, "MOVQ") + } + + func (self *_Assembler) _asm_OP_u8(_ *_Instr) { + self.store_int(3, _F_u64toa, "MOVBQZX") + } + + func (self *_Assembler) _asm_OP_u16(_ *_Instr) { + self.store_int(5, _F_u64toa, "MOVWQZX") + } + + func (self *_Assembler) _asm_OP_u32(_ *_Instr) { + self.store_int(10, _F_u64toa, "MOVLQZX") + } + + func (self *_Assembler) _asm_OP_u64(_ *_Instr) { + self.store_int(20, _F_u64toa, "MOVQ") + } + + func (self *_Assembler) _asm_OP_f32(_ *_Instr) { + self.check_size(32) + self.Emit("MOVL" , jit.Ptr(_SP_p, 0), _AX) // MOVL (SP.p), AX + self.Emit("ANDL" , jit.Imm(_FM_exp32), _AX) // ANDL $_FM_exp32, AX + self.Emit("XORL" , jit.Imm(_FM_exp32), _AX) // XORL $_FM_exp32, AX + self.Sjmp("JZ" , _LB_error_nan_or_infinite) // JZ _error_nan_or_infinite + self.save_c() // SAVE $C_regs + self.rbuf_di() // MOVQ RP, DI + self.Emit("MOVSS" , jit.Ptr(_SP_p, 0), _X0) // MOVSS (SP.p), X0 + self.Emit("CVTSS2SD", _X0, _X0) // CVTSS2SD X0, X0 + self.call_c(_F_f64toa) // CALL_C f64toa + self.Emit("ADDQ" , _AX, _RL) // ADDQ AX, RL + } + + func (self *_Assembler) _asm_OP_f64(_ *_Instr) { + self.check_size(32) + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ" , jit.Imm(_FM_exp64), _CX) // MOVQ $_FM_exp64, CX + self.Emit("ANDQ" , _CX, _AX) // ANDQ CX, AX + self.Emit("XORQ" , _CX, _AX) // XORQ CX, AX + self.Sjmp("JZ" , _LB_error_nan_or_infinite) // JZ _error_nan_or_infinite + self.save_c() // SAVE $C_regs + self.rbuf_di() // MOVQ RP, DI + self.Emit("MOVSD" , jit.Ptr(_SP_p, 0), _X0) // MOVSD (SP.p), X0 + self.call_c(_F_f64toa) // CALL_C f64toa + self.Emit("ADDQ" , _AX, _RL) // ADDQ AX, RL + } + + func (self *_Assembler) _asm_OP_str(_ *_Instr) { + self.encode_string(false) + } + + func (self *_Assembler) _asm_OP_bin(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_SP_p, 8), _AX) // MOVQ 8(SP.p), AX + self.Emit("ADDQ", jit.Imm(2), _AX) // ADDQ $2, AX + self.Emit("MOVQ", jit.Imm(_IM_mulv), _CX) // MOVQ $_MF_mulv, CX + self.Emit("MOVQ", _DX, _R8) // MOVQ DX, R8 + self.From("MULQ", _CX) // MULQ CX + self.Emit("LEAQ", jit.Sib(_DX, _DX, 1, 1), _AX) // LEAQ 1(DX)(DX), AX + self.Emit("ORQ" , jit.Imm(2), _AX) // ORQ $2, AX + self.Emit("MOVQ", _R8, _DX) // MOVQ R8, DX + self.check_size_r(_AX, 0) // SIZE AX + self.add_char('"') // CHAR $'"' + self.save_c() // SAVE $REG_ffi + self.prep_buffer_c() // MOVE {buf}, DI + self.Emit("MOVQ", _SP_p, _SI) // MOVQ SP.p, SI + + /* check for AVX2 support */ + if !cpu.HasAVX2 { + self.Emit("XORL", _DX, _DX) // XORL DX, DX + } else { + self.Emit("MOVL", jit.Imm(_MODE_AVX2), _DX) // MOVL $_MODE_AVX2, DX + } + + /* call the encoder */ + self.call_c(_F_b64encode) // CALL b64encode + self.load_buffer() // LOAD {buf} + self.add_char('"') // CHAR $'"' + } + + func (self *_Assembler) _asm_OP_quote(_ *_Instr) { + self.encode_string(true) + } + + func (self *_Assembler) _asm_OP_number(_ *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _CX) // MOVQ (SP.p), CX + self.Emit("TESTQ", _CX, _CX) // TESTQ CX, CX + self.Sjmp("JZ" , "_empty_{n}") // JZ _empty_{n} + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.call_go(_F_isValidNumber) // CALL_GO isValidNumber + self.Emit("CMPB" , jit.Ptr(_SP, 16), jit.Imm(0)) // CMPB 16(SP), $0 + self.Sjmp("JE" , _LB_error_invalid_number) // JE _error_invalid_number + self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _AX) // MOVQ 8(SP.p), AX + self.check_size_r(_AX, 0) // SIZE AX + self.Emit("LEAQ" , jit.Sib(_RP, _RL, 1, 0), _AX) // LEAQ (RP)(RL), AX + self.Emit("ADDQ" , jit.Ptr(_SP_p, 8), _RL) // ADDQ 8(SP.p), RL + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVOU", jit.Ptr(_SP_p, 0), _X0) // MOVOU (SP.p), X0 + self.Emit("MOVOU", _X0, jit.Ptr(_SP, 8)) // MOVOU X0, 8(SP) + self.call_go(_F_memmove) // CALL_GO memmove + self.Sjmp("JMP" , "_done_{n}") // JMP _done_{n} + self.Link("_empty_{n}") // _empty_{n}: + self.check_size(1) // SIZE $1 + self.add_char('0') // CHAR $'0' + self.Link("_done_{n}") // _done_{n}: + } + + func (self *_Assembler) _asm_OP_eface(_ *_Instr) { + self.prep_buffer() // MOVE {buf}, (SP)s + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.Emit("LEAQ" , jit.Ptr(_SP_p, 8), _AX) // LEAQ 8(SP.p), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24)) // MOVQ ST, 24(SP) + self.Emit("MOVQ" , _ARG_fv, _AX) // MOVQ fv, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP) + self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET) // MOVQ 40(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP) // MOVQ 48(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + func (self *_Assembler) _asm_OP_iface(_ *_Instr) { + self.prep_buffer() // MOVE {buf}, (SP) + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("MOVQ" , jit.Ptr(_AX, 8), _AX) // MOVQ 8(AX), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + self.Emit("LEAQ" , jit.Ptr(_SP_p, 8), _AX) // LEAQ 8(SP.p), AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24)) // MOVQ ST, 24(SP) + self.Emit("MOVQ" , _ARG_fv, _AX) // MOVQ fv, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP) + self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET) // MOVQ 40(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP) // MOVQ 48(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + func (self *_Assembler) _asm_OP_byte(p *_Instr) { + self.check_size(1) + self.Emit("MOVB", jit.Imm(p.i64()), jit.Sib(_RP, _RL, 1, 0)) // MOVL p.vi(), (RP)(RL*1) + self.Emit("ADDQ", jit.Imm(1), _RL) // ADDQ $1, RL + } + + func (self *_Assembler) _asm_OP_text(p *_Instr) { + self.check_size(len(p.vs())) // SIZE ${len(p.vs())} + self.add_text(p.vs()) // TEXT ${p.vs()} + } + + func (self *_Assembler) _asm_OP_deref(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_SP_p, 0), _SP_p) // MOVQ (SP.p), SP.p + } + + func (self *_Assembler) _asm_OP_index(p *_Instr) { + self.Emit("MOVQ", jit.Imm(p.i64()), _AX) // MOVQ $p.vi(), AX + self.Emit("ADDQ", _AX, _SP_p) // ADDQ AX, SP.p + } + + func (self *_Assembler) _asm_OP_load(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_ST, 0), _AX) // MOVQ (ST), AX + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -24), _SP_x) // MOVQ -24(ST)(AX), SP.x + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, -8), _SP_p) // MOVQ -8(ST)(AX), SP.p + self.Emit("MOVQ", jit.Sib(_ST, _AX, 1, 0), _SP_q) // MOVQ (ST)(AX), SP.q + } + + func (self *_Assembler) _asm_OP_save(_ *_Instr) { + self.save_state() + } + + func (self *_Assembler) _asm_OP_drop(_ *_Instr) { + self.drop_state(_StateSize) + } + + func (self *_Assembler) _asm_OP_drop_2(_ *_Instr) { + self.drop_state(_StateSize * 2) // DROP $(_StateSize * 2) + self.Emit("MOVOU", _X0, jit.Sib(_ST, _AX, 1, 56)) // MOVOU X0, 56(ST)(AX) + } + + func (self *_Assembler) _asm_OP_recurse(p *_Instr) { + self.prep_buffer() // MOVE {buf}, (SP) + self.Emit("MOVQ", jit.Type(p.vt()), _AX) // MOVQ $(type(p.vt())), AX + self.Emit("MOVQ", _AX, jit.Ptr(_SP, 8)) // MOVQ AX, 8(SP) + + /* check for indirection */ + if (p.vf() & rt.F_direct) != 0 { + self.Emit("MOVQ", _SP_p, _AX) // MOVQ SP.p, AX + } else { + self.Emit("MOVQ", _SP_p, _VAR_vp) // MOVQ SP.p, 48(SP) + self.Emit("LEAQ", _VAR_vp, _AX) // LEAQ 48(SP), AX + } + + /* call the encoder */ + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.Emit("MOVQ" , _ST, jit.Ptr(_SP, 24)) // MOVQ ST, 24(SP) + self.Emit("MOVQ" , _ARG_fv, _AX) // MOVQ fv, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 32)) // MOVQ AX, 32(SP) + self.call_encoder(_F_encodeTypedPointer) // CALL encodeTypedPointer + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _ET) // MOVQ 40(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 48), _EP) // MOVQ 48(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + func (self *_Assembler) _asm_OP_is_nil(p *_Instr) { + self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPQ (SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_nil_p1(p *_Instr) { + self.Emit("CMPQ", jit.Ptr(_SP_p, 8), jit.Imm(0)) // CMPQ 8(SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_zero_1(p *_Instr) { + self.Emit("CMPB", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPB (SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_zero_2(p *_Instr) { + self.Emit("CMPW", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPW (SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_zero_4(p *_Instr) { + self.Emit("CMPL", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPL (SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_zero_8(p *_Instr) { + self.Emit("CMPQ", jit.Ptr(_SP_p, 0), jit.Imm(0)) // CMPQ (SP.p), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_is_zero_map(p *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _AX) // MOVQ (SP.p), AX + self.Emit("TESTQ", _AX, _AX) // TESTQ AX, AX + self.Xjmp("JZ" , p.vi()) // JZ p.vi() + self.Emit("CMPQ" , jit.Ptr(_AX, 0), jit.Imm(0)) // CMPQ (AX), $0 + self.Xjmp("JE" , p.vi()) // JE p.vi() + } + + func (self *_Assembler) _asm_OP_goto(p *_Instr) { + self.Xjmp("JMP", p.vi()) + } + + func (self *_Assembler) _asm_OP_map_iter(p *_Instr) { + self.Emit("MOVQ" , jit.Type(p.vt()), _AX) // MOVQ $p.vt(), AX + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _CX) // MOVQ (SP.p), CX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 0)) // MOVQ AX, (SP) + self.Emit("MOVQ" , _CX, jit.Ptr(_SP, 8)) // MOVQ CX, 8(SP) + self.Emit("MOVQ" , _ARG_fv, _AX) // MOVQ fv, AX + self.Emit("MOVQ" , _AX, jit.Ptr(_SP, 16)) // MOVQ AX, 16(SP) + self.call_go(_F_iteratorStart) // CALL_GO iteratorStart + self.Emit("MOVQ" , jit.Ptr(_SP, 24), _SP_q) // MOVQ 24(SP), SP.q + self.Emit("MOVQ" , jit.Ptr(_SP, 32), _ET) // MOVQ 32(SP), ET + self.Emit("MOVQ" , jit.Ptr(_SP, 40), _EP) // MOVQ 40(SP), EP + self.Emit("TESTQ", _ET, _ET) // TESTQ ET, ET + self.Sjmp("JNZ" , _LB_error) // JNZ _error + } + + func (self *_Assembler) _asm_OP_map_stop(_ *_Instr) { + self.Emit("MOVQ", _SP_q, jit.Ptr(_SP, 0)) // MOVQ SP.q, 0(SP) + self.call_go(_F_iteratorStop) // CALL_GO iteratorStop + self.Emit("XORL", _SP_q, _SP_q) // XORL SP.q, SP.q + } + + func (self *_Assembler) _asm_OP_map_check_key(p *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_SP_q, 0), _SP_p) // MOVQ (SP.q), SP.p + self.Emit("TESTQ", _SP_p, _SP_p) // TESTQ SP.p, SP.p + self.Xjmp("JZ" , p.vi()) // JNZ p.vi() + } + + func (self *_Assembler) _asm_OP_map_write_key(p *_Instr) { + self.Emit("BTQ", jit.Imm(bitSortMapKeys), _ARG_fv) // BTQ ${SortMapKeys}, fv + self.Sjmp("JNC", "_unordered_key_{n}") // JNC _unordered_key_{n} + self.encode_string(false) // STR $false + self.Xjmp("JMP", p.vi()) // JMP ${p.vi()} + self.Link("_unordered_key_{n}") // _unordered_key_{n}: + } + + func (self *_Assembler) _asm_OP_map_value_next(_ *_Instr) { + self.Emit("MOVQ", jit.Ptr(_SP_q, 8), _SP_p) // MOVQ 8(SP.q), SP.p + self.Emit("MOVQ", _SP_q, jit.Ptr(_SP, 0)) // MOVQ SP.q, (SP) + self.call_go(_F_iteratorNext) // CALL_GO iteratorNext + } + + func (self *_Assembler) _asm_OP_slice_len(_ *_Instr) { + self.Emit("MOVQ" , jit.Ptr(_SP_p, 8), _SP_x) // MOVQ 8(SP.p), SP.x + self.Emit("MOVQ" , jit.Ptr(_SP_p, 0), _SP_p) // MOVQ (SP.p), SP.p + self.Emit("ORQ" , jit.Imm(1 << _S_init), _SP_f) // ORQ $(1<<_S_init), SP.f + } + + func (self *_Assembler) _asm_OP_slice_next(p *_Instr) { + self.Emit("TESTQ" , _SP_x, _SP_x) // TESTQ SP.x, SP.x + self.Xjmp("JZ" , p.vi()) // JZ p.vi() + self.Emit("SUBQ" , jit.Imm(1), _SP_x) // SUBQ $1, SP.x + self.Emit("BTRQ" , jit.Imm(_S_init), _SP_f) // BTRQ $_S_init, SP.f + self.Emit("LEAQ" , jit.Ptr(_SP_p, int64(p.vlen())), _AX) // LEAQ $(p.vlen())(SP.p), AX + self.Emit("CMOVQCC", _AX, _SP_p) // CMOVQNC AX, SP.p + } + + func (self *_Assembler) _asm_OP_marshal(p *_Instr) { + self.call_marshaler(_F_encodeJsonMarshaler, _T_json_Marshaler, p.vt()) + } + + func (self *_Assembler) _asm_OP_marshal_p(p *_Instr) { + if p.vk() != reflect.Ptr { + panic("marshal_p: invalid type") + } else { + self.call_marshaler_v(_F_encodeJsonMarshaler, _T_json_Marshaler, p.vt(), false) + } + } + + func (self *_Assembler) _asm_OP_marshal_text(p *_Instr) { + self.call_marshaler(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.vt()) + } + + func (self *_Assembler) _asm_OP_marshal_text_p(p *_Instr) { + if p.vk() != reflect.Ptr { + panic("marshal_text_p: invalid type") + } else { + self.call_marshaler_v(_F_encodeTextMarshaler, _T_encoding_TextMarshaler, p.vt(), false) + } + } + + func (self *_Assembler) _asm_OP_cond_set(_ *_Instr) { + self.Emit("ORQ", jit.Imm(1 << _S_cond), _SP_f) // ORQ $(1<<_S_cond), SP.f + } + + func (self *_Assembler) _asm_OP_cond_testc(p *_Instr) { + self.Emit("BTRQ", jit.Imm(_S_cond), _SP_f) // BTRQ $_S_cond, SP.f + self.Xjmp("JC" , p.vi()) + } \ No newline at end of file diff --git a/encoder/assembler_amd64.go b/encoder/assembler_amd64_go116.go similarity index 98% rename from encoder/assembler_amd64.go rename to encoder/assembler_amd64_go116.go index dd906e177..14196911d 100644 --- a/encoder/assembler_amd64.go +++ b/encoder/assembler_amd64_go116.go @@ -1,3 +1,5 @@ +// +build go1.16,!go1.17 + /* * Copyright 2021 ByteDance Inc. * @@ -1112,6 +1114,20 @@ func (self *_Assembler) _asm_OP_cond_testc(p *_Instr) { self.Xjmp("JC" , p.vi()) } +type writeBarrier struct { + enabled bool // compiler emits a check of this before calling write barrier + pad [3]byte // compiler uses 32-bit load for "enabled" field + needed bool // whether we need a write barrier for current GC phase + cgo bool // whether we need a write barrier for a cgo check + alignme uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load +} + +//go:linkname _runtime_writeBarrier runtime.writeBarrier +var _runtime_writeBarrier writeBarrier + +//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier +func gcWriteBarrierAX() + var ( _V_writeBarrier = jit.Imm(int64(uintptr(unsafe.Pointer(&_runtime_writeBarrier)))) diff --git a/encoder/stubs.go b/encoder/stubs.go index 96a374951..0c2850ccf 100644 --- a/encoder/stubs.go +++ b/encoder/stubs.go @@ -27,20 +27,6 @@ import ( //go:linkname _subr__b64encode github.com/chenzhuoyu/base64x._subr__b64encode var _subr__b64encode uintptr -type writeBarrier struct { - enabled bool // compiler emits a check of this before calling write barrier - pad [3]byte // compiler uses 32-bit load for "enabled" field - needed bool // whether we need a write barrier for current GC phase - cgo bool // whether we need a write barrier for a cgo check - alignme uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load -} - -//go:linkname _runtime_writeBarrier runtime.writeBarrier -var _runtime_writeBarrier writeBarrier - -//go:linkname gcWriteBarrierAX runtime.gcWriteBarrier -func gcWriteBarrierAX() - //go:noescape //go:linkname memmove runtime.memmove //goland:noinspection GoUnusedParameter