| %def header(): |
| /* |
| * Copyright (C) 2019 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /* |
| * This is a #include, not a %include, because we want the C pre-processor |
| * to expand the macros into assembler assignment statements. |
| */ |
| #include "asm_support.h" |
| #include "arch/x86_64/asm_support_x86_64.S" |
| |
| /** |
| * x86_64 ABI general notes: |
| * |
| * Caller save set: |
| * rax, rdx, rcx, rsi, rdi, r8-r11, st(0)-st(7) |
| * Callee save set: |
| * rbx, rbp, r12-r15 |
| * Return regs: |
| * 32-bit in eax |
| * 64-bit in rax |
| * fp on xmm0 |
| * |
| * First 8 fp parameters came in xmm0-xmm7. |
| * First 6 non-fp parameters came in rdi, rsi, rdx, rcx, r8, r9. |
| * Other parameters passed on stack, pushed right-to-left. On entry to target, first |
| * param is at 8(%esp). |
| * |
| * Stack must be 16-byte aligned to support SSE in native code. |
| */ |
| |
| #define IN_ARG3 %rcx |
| #define IN_ARG2 %rdx |
| #define IN_ARG1 %rsi |
| #define IN_ARG0 %rdi |
| /* Out Args */ |
| #define OUT_ARG3 %rcx |
| #define OUT_ARG2 %rdx |
| #define OUT_ARG1 %rsi |
| #define OUT_ARG0 %rdi |
| #define OUT_32_ARG3 %ecx |
| #define OUT_32_ARG2 %edx |
| #define OUT_32_ARG1 %esi |
| #define OUT_32_ARG0 %edi |
| #define OUT_FP_ARG1 %xmm1 |
| #define OUT_FP_ARG0 %xmm0 |
| |
| /* |
| * single-purpose registers, given names for clarity |
| */ |
| #define rSELF %gs |
| #define rPC %r12 |
| #define CFI_DEX 12 // DWARF register number of the register holding dex-pc (rPC). |
| #define CFI_TMP 5 // DWARF register number of the first argument register (rdi). |
| #define rFP %r13 |
| #define rINST %ebx |
| #define rINSTq %rbx |
| #define rINSTw %bx |
| #define rINSTbh %bh |
| #define rINSTbl %bl |
| #define rIBASE %r14 |
| #define rREFS %r15 |
| #define rREFS32 %r15d |
| #define CFI_REFS 15 // DWARF register number of the reference array (r15). |
| |
| // Temporary registers while setting up a frame. |
| #define rNEW_FP %r8 |
| #define rNEW_REFS %r9 |
| #define rNEW_REFS32 %r9d |
| #define CFI_NEW_REFS 9 |
| |
| /* |
| * Get/set the 32-bit value from a Dalvik register. |
| */ |
| #define VREG_ADDRESS(_vreg) (rFP,_vreg,4) |
| #define VREG_HIGH_ADDRESS(_vreg) 4(rFP,_vreg,4) |
| #define VREG_REF_ADDRESS(_vreg) (rREFS,_vreg,4) |
| #define VREG_REF_HIGH_ADDRESS(_vreg) 4(rREFS,_vreg,4) |
| |
| // Includes the return address implictly pushed on stack by 'call'. |
| #define CALLEE_SAVES_SIZE (6 * 8 + 4 * 8 + 1 * 8) |
| |
| // +8 for the ArtMethod of the caller. |
| #define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) |
| |
| /* |
| * Refresh rINST. |
| * At enter to handler rINST does not contain the opcode number. |
| * However some utilities require the full value, so this macro |
| * restores the opcode number. |
| */ |
| .macro REFRESH_INST _opnum |
| movb rINSTbl, rINSTbh |
| movb $$\_opnum, rINSTbl |
| .endm |
| |
| /* |
| * Fetch the next instruction from rPC into rINSTw. Does not advance rPC. |
| */ |
| .macro FETCH_INST |
| movzwq (rPC), rINSTq |
| .endm |
| |
| /* |
| * Remove opcode from rINST, compute the address of handler and jump to it. |
| */ |
| .macro GOTO_NEXT |
| movzx rINSTbl,%ecx |
| movzbl rINSTbh,rINST |
| shll MACRO_LITERAL(${handler_size_bits}), %ecx |
| addq rIBASE, %rcx |
| jmp *%rcx |
| .endm |
| |
| /* |
| * Advance rPC by instruction count. |
| */ |
| .macro ADVANCE_PC _count |
| leaq 2*\_count(rPC), rPC |
| .endm |
| |
| /* |
| * Advance rPC by instruction count, fetch instruction and jump to handler. |
| */ |
| .macro ADVANCE_PC_FETCH_AND_GOTO_NEXT _count |
| ADVANCE_PC \_count |
| FETCH_INST |
| GOTO_NEXT |
| .endm |
| |
| .macro GET_VREG _reg _vreg |
| movl VREG_ADDRESS(\_vreg), \_reg |
| .endm |
| |
| .macro GET_VREG_OBJECT _reg _vreg |
| movl VREG_REF_ADDRESS(\_vreg), \_reg |
| .endm |
| |
| /* Read wide value. */ |
| .macro GET_WIDE_VREG _reg _vreg |
| movq VREG_ADDRESS(\_vreg), \_reg |
| .endm |
| |
| .macro SET_VREG _reg _vreg |
| movl \_reg, VREG_ADDRESS(\_vreg) |
| movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) |
| .endm |
| |
| /* Write wide value. reg is clobbered. */ |
| .macro SET_WIDE_VREG _reg _vreg |
| movq \_reg, VREG_ADDRESS(\_vreg) |
| xorq \_reg, \_reg |
| movq \_reg, VREG_REF_ADDRESS(\_vreg) |
| .endm |
| |
| .macro SET_VREG_OBJECT _reg _vreg |
| movl \_reg, VREG_ADDRESS(\_vreg) |
| movl \_reg, VREG_REF_ADDRESS(\_vreg) |
| .endm |
| |
| .macro GET_VREG_HIGH _reg _vreg |
| movl VREG_HIGH_ADDRESS(\_vreg), \_reg |
| .endm |
| |
| .macro SET_VREG_HIGH _reg _vreg |
| movl \_reg, VREG_HIGH_ADDRESS(\_vreg) |
| movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) |
| .endm |
| |
| .macro CLEAR_REF _vreg |
| movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) |
| .endm |
| |
| .macro CLEAR_WIDE_REF _vreg |
| movl MACRO_LITERAL(0), VREG_REF_ADDRESS(\_vreg) |
| movl MACRO_LITERAL(0), VREG_REF_HIGH_ADDRESS(\_vreg) |
| .endm |
| |
| .macro GET_VREG_XMMs _xmmreg _vreg |
| movss VREG_ADDRESS(\_vreg), \_xmmreg |
| .endm |
| .macro GET_VREG_XMMd _xmmreg _vreg |
| movsd VREG_ADDRESS(\_vreg), \_xmmreg |
| .endm |
| .macro SET_VREG_XMMs _xmmreg _vreg |
| movss \_xmmreg, VREG_ADDRESS(\_vreg) |
| .endm |
| .macro SET_VREG_XMMd _xmmreg _vreg |
| movsd \_xmmreg, VREG_ADDRESS(\_vreg) |
| .endm |
| |
| // An assembly entry that has a OatQuickMethodHeader prefix. |
| .macro OAT_ENTRY name, end |
| FUNCTION_TYPE(\name) |
| ASM_HIDDEN SYMBOL(\name) |
| .global SYMBOL(\name) |
| .balign 16 |
| .long 0 |
| .long (SYMBOL(\end) - SYMBOL(\name)) |
| SYMBOL(\name): |
| .endm |
| |
| .macro ENTRY name |
| .text |
| ASM_HIDDEN SYMBOL(\name) |
| .global SYMBOL(\name) |
| FUNCTION_TYPE(\name) |
| SYMBOL(\name): |
| .endm |
| |
| .macro END name |
| SIZE(\name) |
| .endm |
| |
| // Macro for defining entrypoints into runtime. We don't need to save registers |
| // (we're not holding references there), but there is no |
| // kDontSave runtime method. So just use the kSaveRefsOnly runtime method. |
| .macro NTERP_TRAMPOLINE name, helper |
| DEFINE_FUNCTION \name |
| SETUP_SAVE_REFS_ONLY_FRAME |
| call \helper |
| RESTORE_SAVE_REFS_ONLY_FRAME |
| RETURN_OR_DELIVER_PENDING_EXCEPTION |
| END_FUNCTION \name |
| .endm |
| |
| .macro CLEAR_VOLATILE_MARKER reg |
| andq MACRO_LITERAL(-2), \reg |
| .endm |
| |
| .macro EXPORT_PC |
| movq rPC, -16(rREFS) |
| .endm |
| |
| |
| .macro BRANCH |
| // Update method counter and do a suspend check if the branch is negative. |
| testq rINSTq, rINSTq |
| js 3f |
| 2: |
| leaq (rPC, rINSTq, 2), rPC |
| FETCH_INST |
| GOTO_NEXT |
| 3: |
| movq (%rsp), %rdi |
| addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) |
| andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) |
| // If the counter overflows, handle this in the runtime. |
| jz NterpHandleHotnessOverflow |
| // Otherwise, do a suspend check. |
| testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET |
| jz 2b |
| EXPORT_PC |
| call SYMBOL(art_quick_test_suspend) |
| jmp 2b |
| .endm |
| |
| // Expects: |
| // - r10, and r11 to be available. |
| // Outputs: |
| // - \registers contains the dex registers size |
| // - \outs contains the outs size |
| // - if load_ins is 1, \ins contains the ins |
| // - \code_item is replace with a pointer to the instructions |
| .macro FETCH_CODE_ITEM_INFO code_item, registers, outs, ins, load_ins |
| testq MACRO_LITERAL(1), \code_item |
| je 5f |
| andq $$-2, \code_item // Remove the extra bit that marks it's a compact dex file. |
| movzwl COMPACT_CODE_ITEM_FIELDS_OFFSET(\code_item), %r10d |
| movl %r10d, \registers |
| sarl $$COMPACT_CODE_ITEM_REGISTERS_SIZE_SHIFT, \registers |
| andl $$0xf, \registers |
| movl %r10d, \outs |
| sarl $$COMPACT_CODE_ITEM_OUTS_SIZE_SHIFT, \outs |
| andl $$0xf, \outs |
| .if \load_ins |
| movl %r10d, \ins |
| sarl $$COMPACT_CODE_ITEM_INS_SIZE_SHIFT, \ins |
| andl $$0xf, \ins |
| .else |
| movl %r10d, %r11d |
| sarl $$COMPACT_CODE_ITEM_INS_SIZE_SHIFT, %r11d |
| andl $$0xf, %r11d |
| addl %r11d, \registers |
| .endif |
| testw $$COMPACT_CODE_ITEM_REGISTERS_INS_OUTS_FLAGS, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) |
| je 4f |
| movq \code_item, %r11 |
| testw $$COMPACT_CODE_ITEM_INSNS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) |
| je 1f |
| subq $$4, %r11 |
| 1: |
| testw $$COMPACT_CODE_ITEM_REGISTERS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) |
| je 2f |
| subq $$2, %r11 |
| movzwl (%r11), %r10d |
| addl %r10d, \registers |
| 2: |
| testw $$COMPACT_CODE_ITEM_INS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) |
| je 3f |
| subq $$2, %r11 |
| movzwl (%r11), %r10d |
| .if \load_ins |
| addl %r10d, \ins |
| .else |
| addl %r10d, \registers |
| .endif |
| 3: |
| testw $$COMPACT_CODE_ITEM_OUTS_FLAG, COMPACT_CODE_ITEM_FLAGS_OFFSET(\code_item) |
| je 4f |
| subq $$2, %r11 |
| movzwl (%r11), %r10d |
| addl %r10d, \outs |
| 4: |
| .if \load_ins |
| addl \ins, \registers |
| .endif |
| addq $$COMPACT_CODE_ITEM_INSNS_OFFSET, \code_item |
| jmp 6f |
| 5: |
| // Fetch dex register size. |
| movzwl CODE_ITEM_REGISTERS_SIZE_OFFSET(\code_item), \registers |
| // Fetch outs size. |
| movzwl CODE_ITEM_OUTS_SIZE_OFFSET(\code_item), \outs |
| .if \load_ins |
| movzwl CODE_ITEM_INS_SIZE_OFFSET(\code_item), \ins |
| .endif |
| addq $$CODE_ITEM_INSNS_OFFSET, \code_item |
| 6: |
| .endm |
| |
| // Setup the stack to start executing the method. Expects: |
| // - rdi to contain the ArtMethod |
| // - rbx, r10, r11 to be available. |
| // |
| // Outputs |
| // - rbx contains the dex registers size |
| // - r11 contains the old stack pointer. |
| // - \code_item is replace with a pointer to the instructions |
| // - if load_ins is 1, r14 contains the ins |
| .macro SETUP_STACK_FRAME code_item, refs, refs32, fp, cfi_refs, load_ins |
| FETCH_CODE_ITEM_INFO \code_item, %ebx, \refs32, %r14d, \load_ins |
| |
| // Compute required frame size for dex registers: ((2 * ebx) + refs) |
| leaq (\refs, %rbx, 2), %r11 |
| salq $$2, %r11 |
| |
| // Compute new stack pointer in r10: add 24 for saving the previous frame, |
| // pc, and method being executed. |
| leaq -24(%rsp), %r10 |
| subq %r11, %r10 |
| // Alignment |
| // Note: There may be two pieces of alignment but there is no need to align |
| // out args to `kPointerSize` separately before aligning to kStackAlignment. |
| andq $$-16, %r10 |
| |
| // Set reference and dex registers, align to pointer size for previous frame and dex pc. |
| leaq 24 + 4(%r10, \refs, 4), \refs |
| andq LITERAL(-__SIZEOF_POINTER__), \refs |
| leaq (\refs, %rbx, 4), \fp |
| |
| // Now setup the stack pointer. |
| movq %rsp, %r11 |
| CFI_DEF_CFA_REGISTER(r11) |
| movq %r10, %rsp |
| movq %r11, -8(\refs) |
| CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, ((6 + 4 + 1) * 8) |
| |
| // Put nulls in reference frame. |
| testl %ebx, %ebx |
| je 2f |
| movq \refs, %r10 |
| 1: |
| movl $$0, (%r10) |
| addq $$4, %r10 |
| cmpq %r10, \fp |
| jne 1b |
| 2: |
| // Save the ArtMethod. |
| movq %rdi, (%rsp) |
| .endm |
| |
| // Puts the next floating point argument into the expected register, |
| // fetching values based on a non-range invoke. |
| // Uses rax as temporary. |
| // |
| // TODO: We could simplify a lot of code by loading the G argument into |
| // the "inst" register. Given that we enter the handler with "1(rPC)" in |
| // the rINST, we can just add rINST<<16 to the args and we don't even |
| // need to pass "arg_index" around. |
| .macro LOOP_OVER_SHORTY_LOADING_XMMS xmm_reg, inst, shorty, arg_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| // Handle extra argument in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| subq MACRO_LITERAL(8), %rsp |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| GET_VREG %eax, %rax |
| movl %eax, (%rsp) |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| cmpq MACRO_LITERAL(4), REG_VAR(arg_index) |
| je 5f |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 6f |
| 5: |
| movzbl 1(rPC), %eax |
| andq MACRO_LITERAL(0xf), %rax |
| 6: |
| GET_VREG %eax, %rax |
| movl %eax, 4(%rsp) |
| movsd (%rsp), REG_VAR(xmm_reg) |
| addq MACRO_LITERAL(8), %rsp |
| jmp 4f |
| 3: // FOUND_FLOAT |
| cmpq MACRO_LITERAL(4), REG_VAR(arg_index) |
| je 7f |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 8f |
| 7: |
| movzbl 1(rPC), %eax |
| andq MACRO_LITERAL(0xf), %rax |
| 8: |
| GET_VREG_XMMs REG_VAR(xmm_reg), %rax |
| 4: |
| .endm |
| |
| // Puts the next int/long/object argument in the expected register, |
| // fetching values based on a non-range invoke. |
| // Uses rax as temporary. |
| .macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| cmpq MACRO_LITERAL(4), REG_VAR(arg_index) |
| je 7f |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 8f |
| 7: |
| movzbl 1(rPC), %eax |
| andq MACRO_LITERAL(0xf), %rax |
| 8: |
| GET_VREG REG_VAR(gpr_reg32), %rax |
| jmp 5f |
| 2: // FOUND_LONG |
| subq MACRO_LITERAL(8), %rsp |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| GET_VREG %eax, %rax |
| movl %eax, (%rsp) |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| cmpq MACRO_LITERAL(4), REG_VAR(arg_index) |
| je 9f |
| movq REG_VAR(inst), %rax |
| andq MACRO_LITERAL(0xf), %rax |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 10f |
| 9: |
| movzbl 1(rPC), %eax |
| andq MACRO_LITERAL(0xf), %rax |
| 10: |
| GET_VREG %eax, %rax |
| movl %eax, 4(%rsp) |
| movq (%rsp), REG_VAR(gpr_reg64) |
| addq MACRO_LITERAL(8), %rsp |
| jmp 5f |
| 3: // SKIP_FLOAT |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| cmpq MACRO_LITERAL(4), REG_VAR(arg_index) |
| je 1b |
| shrq MACRO_LITERAL(4), REG_VAR(inst) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 5: |
| .endm |
| |
| // Puts the next floating point argument into the expected register, |
| // fetching values based on a range invoke. |
| // Uses rax as temporary. |
| .macro LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm_reg, shorty, arg_index, stack_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| // Handle extra argument in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| GET_VREG_XMMd REG_VAR(xmm_reg), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 4f |
| 3: // FOUND_FLOAT |
| GET_VREG_XMMs REG_VAR(xmm_reg), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| 4: |
| .endm |
| |
| // Puts the next floating point argument into the expected stack slot, |
| // fetching values based on a range invoke. |
| // Uses rax as temporary. |
| // |
| // TODO: We could just copy all the vregs to the stack slots in a simple loop |
| // (or REP MOVSD) without looking at the shorty at all. (We could also drop |
| // the "stack_index" from the macros for loading registers.) We could also do |
| // that conditionally if argument word count > 6; otherwise we know that all |
| // args fit into registers. |
| .macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| // Handle extra argument in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| movq (rFP, REG_VAR(arg_index), 4), %rax |
| movq %rax, 8(%rsp, REG_VAR(stack_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 1b |
| 3: // FOUND_FLOAT |
| movl (rFP, REG_VAR(arg_index), 4), %eax |
| movl %eax, 8(%rsp, REG_VAR(stack_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b |
| .endm |
| |
| // Puts the next int/long/object argument in the expected register, |
| // fetching values based on a range invoke. |
| // Uses rax as temporary. |
| .macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, stack_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| movl (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg32) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 5f |
| 2: // FOUND_LONG |
| movq (rFP, REG_VAR(arg_index), 4), REG_VAR(gpr_reg64) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 5f |
| 3: // SKIP_FLOAT |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 1b |
| 5: |
| .endm |
| |
| // Puts the next int/long/object argument in the expected stack slot, |
| // fetching values based on a range invoke. |
| // Uses rax as temporary. |
| .macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // al := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| movl (rFP, REG_VAR(arg_index), 4), %eax |
| movl %eax, 8(%rsp, REG_VAR(stack_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b |
| 2: // FOUND_LONG |
| movq (rFP, REG_VAR(arg_index), 4), %rax |
| movq %rax, 8(%rsp, REG_VAR(stack_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 1b |
| 3: // SKIP_FLOAT |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| addq MACRO_LITERAL(1), REG_VAR(stack_index) |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| addq MACRO_LITERAL(2), REG_VAR(stack_index) |
| jmp 1b |
| .endm |
| |
| // Puts the next floating point parameter passed in physical register |
| // in the expected dex register array entry. |
| // Uses rax as temporary. |
| .macro LOOP_OVER_SHORTY_STORING_XMMS xmm_reg, shorty, arg_index, fp, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // al := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| // Handle extra argument in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| movsd REG_VAR(xmm_reg),(REG_VAR(fp), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 4f |
| 3: // FOUND_FLOAT |
| movss REG_VAR(xmm_reg), (REG_VAR(fp), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| 4: |
| .endm |
| |
| // Puts the next int/long/object parameter passed in physical register |
| // in the expected dex register array entry, and in case of object in the |
| // expected reference array entry. |
| // Uses rax as temporary. |
| .macro LOOP_OVER_SHORTY_STORING_GPRS gpr_reg64, gpr_reg32, shorty, arg_index, regs, refs, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| movl REG_VAR(gpr_reg32), (REG_VAR(regs), REG_VAR(arg_index), 4) |
| cmpb MACRO_LITERAL(76), %al // if (al != 'L') goto NOT_REFERENCE |
| jne 6f |
| movl REG_VAR(gpr_reg32), (REG_VAR(refs), REG_VAR(arg_index), 4) |
| 6: // NOT_REFERENCE |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 5f |
| 2: // FOUND_LONG |
| movq REG_VAR(gpr_reg64), (REG_VAR(regs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 5f |
| 3: // SKIP_FLOAT |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 1b |
| 5: |
| .endm |
| |
| // Puts the next floating point parameter passed in stack |
| // in the expected dex register array entry. |
| // Uses rax as temporary. |
| // |
| // TODO: Or we could just spill regs to the reserved slots in the caller's |
| // frame and copy all regs in a simple loop. This time, however, we would |
| // need to look at the shorty anyway to look for the references. |
| // (The trade-off is different for passing arguments and receiving them.) |
| .macro LOOP_OVER_FPs shorty, arg_index, regs, stack_ptr, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| // Handle extra argument in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax |
| movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 1b |
| 3: // FOUND_FLOAT |
| movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax |
| movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| .endm |
| |
| // Puts the next int/long/object parameter passed in stack |
| // in the expected dex register array entry, and in case of object in the |
| // expected reference array entry. |
| // Uses rax as temporary. |
| .macro LOOP_OVER_INTs shorty, arg_index, regs, refs, stack_ptr, finished |
| 1: // LOOP |
| movb (REG_VAR(shorty)), %al // bl := *shorty |
| addq MACRO_LITERAL(1), REG_VAR(shorty) // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(76), %al // if (al == 'L') goto FOUND_REFERENCE |
| je 6f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax |
| movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 6: // FOUND_REFERENCE |
| movl OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %eax |
| movl %eax, (REG_VAR(regs), REG_VAR(arg_index), 4) |
| movl %eax, (REG_VAR(refs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 2: // FOUND_LONG |
| movq OFFSET_TO_FIRST_ARGUMENT_IN_STACK(REG_VAR(stack_ptr), REG_VAR(arg_index), 4), %rax |
| movq %rax, (REG_VAR(regs), REG_VAR(arg_index), 4) |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 1b |
| 3: // SKIP_FLOAT |
| addq MACRO_LITERAL(1), REG_VAR(arg_index) |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| addq MACRO_LITERAL(2), REG_VAR(arg_index) |
| jmp 1b |
| .endm |
| |
| // Increase method hotness and do suspend check before starting executing the method. |
| .macro START_EXECUTING_INSTRUCTIONS |
| movq (%rsp), %rdi |
| addw $$1, ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) |
| andw $$(NTERP_HOTNESS_MASK), ART_METHOD_HOTNESS_COUNT_OFFSET(%rdi) |
| jz 2f |
| testl $$(THREAD_SUSPEND_OR_CHECKPOINT_REQUEST), rSELF:THREAD_FLAGS_OFFSET |
| jz 1f |
| EXPORT_PC |
| call SYMBOL(art_quick_test_suspend) |
| 1: |
| FETCH_INST |
| GOTO_NEXT |
| 2: |
| movq $$0, %rsi |
| movq rFP, %rdx |
| call nterp_hot_method |
| jmp 1b |
| .endm |
| |
| .macro SPILL_ALL_CALLEE_SAVES |
| PUSH r15 |
| PUSH r14 |
| PUSH r13 |
| PUSH r12 |
| PUSH rbp |
| PUSH rbx |
| SETUP_FP_CALLEE_SAVE_FRAME |
| .endm |
| |
| .macro RESTORE_ALL_CALLEE_SAVES |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| POP rbx |
| POP rbp |
| POP r12 |
| POP r13 |
| POP r14 |
| POP r15 |
| .endm |
| |
| // Helper to setup the stack after doing a nterp to nterp call. This will setup: |
| // - rNEW_FP: the new pointer to dex registers |
| // - rNEW_REFS: the new pointer to references |
| // - rPC: the new PC pointer to execute |
| // - edi: number of arguments |
| // - ecx: first dex register |
| // |
| // This helper expects: |
| // - rax to contain the code item |
| .macro SETUP_STACK_FOR_INVOKE |
| // We do the same stack overflow check as the compiler. See CanMethodUseNterp |
| // in how we limit the maximum nterp frame size. |
| testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) |
| |
| // Spill all callee saves to have a consistent stack frame whether we |
| // are called by compiled code or nterp. |
| SPILL_ALL_CALLEE_SAVES |
| |
| // Setup the frame. |
| SETUP_STACK_FRAME %rax, rNEW_REFS, rNEW_REFS32, rNEW_FP, CFI_NEW_REFS, load_ins=0 |
| // Make r11 point to the top of the dex register array. |
| leaq (rNEW_FP, %rbx, 4), %r11 |
| |
| // Fetch instruction information before replacing rPC. |
| movzbl 1(rPC), %edi |
| movzwl 4(rPC), %ecx |
| |
| // Set the dex pc pointer. |
| movq %rax, rPC |
| CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) |
| .endm |
| |
| // Setup arguments based on a non-range nterp to nterp call, and start executing |
| // the method. We expect: |
| // - rNEW_FP: the new pointer to dex registers |
| // - rNEW_REFS: the new pointer to references |
| // - rPC: the new PC pointer to execute |
| // - edi: number of arguments |
| // - ecx: first dex register |
| // - r11: top of dex register array |
| // - esi: receiver if non-static. |
| .macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 |
| // Now all temporary registers (except r11 containing top of registers array) |
| // are available, copy the parameters. |
| // /* op vA, vB, {vC...vG} */ |
| movl %edi, %eax |
| shrl $$4, %eax # Number of arguments |
| jz 6f # shl sets the Z flag |
| movq MACRO_LITERAL(-1), %r10 |
| cmpl MACRO_LITERAL(2), %eax |
| jl 1f |
| je 2f |
| cmpl MACRO_LITERAL(4), %eax |
| jl 3f |
| je 4f |
| |
| // We use a decrementing r10 to store references relative |
| // to rNEW_FP and dex registers relative to r11. |
| // |
| // TODO: We could set up r10 as the number of registers (this can be an additional output from |
| // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg to |
| // (rNEW_FP, r10, 4) and (rNEW_REFS, r10, 4). |
| // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. |
| 5: |
| andq MACRO_LITERAL(15), %rdi |
| GET_VREG_OBJECT %edx, %rdi |
| movl %edx, (rNEW_FP, %r10, 4) |
| GET_VREG %edx, %rdi |
| movl %edx, (%r11, %r10, 4) |
| subq MACRO_LITERAL(1), %r10 |
| 4: |
| movl %ecx, %eax |
| shrl MACRO_LITERAL(12), %eax |
| GET_VREG_OBJECT %edx, %rax |
| movl %edx, (rNEW_FP, %r10, 4) |
| GET_VREG %edx, %rax |
| movl %edx, (%r11, %r10, 4) |
| subq MACRO_LITERAL(1), %r10 |
| 3: |
| movl %ecx, %eax |
| shrl MACRO_LITERAL(8), %eax |
| andl MACRO_LITERAL(0xf), %eax |
| GET_VREG_OBJECT %edx, %rax |
| movl %edx, (rNEW_FP, %r10, 4) |
| GET_VREG %edx, %rax |
| movl %edx, (%r11, %r10, 4) |
| subq MACRO_LITERAL(1), %r10 |
| 2: |
| movl %ecx, %eax |
| shrl MACRO_LITERAL(4), %eax |
| andl MACRO_LITERAL(0xf), %eax |
| GET_VREG_OBJECT %edx, %rax |
| movl %edx, (rNEW_FP, %r10, 4) |
| GET_VREG %edx, %rax |
| movl %edx, (%r11, %r10, 4) |
| subq MACRO_LITERAL(1), %r10 |
| 1: |
| .if \is_string_init |
| // Ignore the first argument |
| .elseif \is_static |
| movl %ecx, %eax |
| andq MACRO_LITERAL(0x000f), %rax |
| GET_VREG_OBJECT %edx, %rax |
| movl %edx, (rNEW_FP, %r10, 4) |
| GET_VREG %edx, %rax |
| movl %edx, (%r11, %r10, 4) |
| .else |
| movl %esi, (rNEW_FP, %r10, 4) |
| movl %esi, (%r11, %r10, 4) |
| .endif |
| |
| 6: |
| // Start executing the method. |
| movq rNEW_FP, rFP |
| movq rNEW_REFS, rREFS |
| CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) |
| START_EXECUTING_INSTRUCTIONS |
| .endm |
| |
| // Setup arguments based on a range nterp to nterp call, and start executing |
| // the method. |
| .macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 |
| // edi is number of arguments |
| // ecx is first register |
| movq MACRO_LITERAL(-4), %r10 |
| .if \is_string_init |
| // Ignore the first argument |
| subl $$1, %edi |
| addl $$1, %ecx |
| .elseif !\is_static |
| subl $$1, %edi |
| addl $$1, %ecx |
| .endif |
| |
| testl %edi, %edi |
| je 2f |
| leaq (rREFS, %rcx, 4), %rax # pointer to first argument in reference array |
| leaq (%rax, %rdi, 4), %rax # pointer to last argument in reference array |
| leaq (rFP, %rcx, 4), %rcx # pointer to first argument in register array |
| leaq (%rcx, %rdi, 4), %rdi # pointer to last argument in register array |
| // TODO: Same comment for copying arguments as in SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE. |
| 1: |
| movl -4(%rax), %edx |
| movl %edx, (rNEW_FP, %r10, 1) |
| movl -4(%rdi), %edx |
| movl %edx, (%r11, %r10, 1) |
| subq MACRO_LITERAL(4), %r10 |
| subq MACRO_LITERAL(4), %rax |
| subq MACRO_LITERAL(4), %rdi |
| cmpq %rcx, %rdi |
| jne 1b |
| |
| 2: |
| .if \is_string_init |
| // Ignore first argument |
| .elseif !\is_static |
| movl %esi, (rNEW_FP, %r10, 1) |
| movl %esi, (%r11, %r10, 1) |
| .endif |
| movq rNEW_FP, rFP |
| movq rNEW_REFS, rREFS |
| CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, ((6 + 4 + 1) * 8) |
| START_EXECUTING_INSTRUCTIONS |
| .endm |
| |
| .macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom |
| push %rdi |
| push %rsi |
| .if \is_polymorphic |
| movq 16(%rsp), %rdi |
| movq rPC, %rsi |
| call SYMBOL(NterpGetShortyFromInvokePolymorphic) |
| .elseif \is_custom |
| movq 16(%rsp), %rdi |
| movq rPC, %rsi |
| call SYMBOL(NterpGetShortyFromInvokeCustom) |
| .elseif \is_interface |
| movq 16(%rsp), %rdi |
| movzwl 2(rPC), %esi |
| call SYMBOL(NterpGetShortyFromMethodId) |
| .else |
| call SYMBOL(NterpGetShorty) |
| .endif |
| pop %rsi |
| pop %rdi |
| movq %rax, \dest |
| .endm |
| |
| .macro DO_ENTRY_POINT_CHECK call_compiled_code |
| // On entry, the method is %rdi, the instance is %rsi |
| leaq ExecuteNterpImpl(%rip), %rax |
| cmpq %rax, ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) |
| jne VAR(call_compiled_code) |
| |
| movq ART_METHOD_DATA_OFFSET_64(%rdi), %rax |
| .endm |
| |
| // Uses r9 and r10 as temporary |
| .macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value |
| movq rREFS, %r9 |
| movq rFP, %r10 |
| 1: |
| cmpl (%r9), \old_value |
| jne 2f |
| movl \new_value, (%r9) |
| movl \new_value, (%r10) |
| 2: |
| addq $$4, %r9 |
| addq $$4, %r10 |
| cmpq %r9, rFP |
| jne 1b |
| .endm |
| |
| .macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 |
| .if \is_polymorphic |
| // We always go to compiled code for polymorphic calls. |
| .elseif \is_custom |
| // We always go to compiled code for custom calls. |
| .else |
| DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix |
| .if \is_string_init |
| call nterp_to_nterp_string_init_non_range |
| .elseif \is_static |
| call nterp_to_nterp_static_non_range |
| .else |
| call nterp_to_nterp_instance_non_range |
| .endif |
| jmp .Ldone_return_\suffix |
| .endif |
| |
| .Lcall_compiled_code_\suffix: |
| GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom |
| // From this point: |
| // - rISNTq contains shorty (in callee-save to switch over return value after call). |
| // - rdi contains method |
| // - rsi contains 'this' pointer for instance method. |
| leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character |
| movzwl 4(rPC), %r11d // arguments |
| .if \is_string_init |
| shrq MACRO_LITERAL(4), %r11 |
| movq $$1, %r10 // ignore first argument |
| .elseif \is_static |
| movq $$0, %r10 // arg_index |
| .else |
| shrq MACRO_LITERAL(4), %r11 |
| movq $$1, %r10 // arg_index |
| .endif |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm0, r11, r9, r10, .Lxmm_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm1, r11, r9, r10, .Lxmm_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm2, r11, r9, r10, .Lxmm_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm3, r11, r9, r10, .Lxmm_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm4, r11, r9, r10, .Lxmm_setup_finished_\suffix |
| .Lxmm_setup_finished_\suffix: |
| leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character |
| movzwl 4(rPC), %r11d // arguments |
| .if \is_string_init |
| movq $$1, %r10 // ignore first argument |
| shrq MACRO_LITERAL(4), %r11 |
| LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| .elseif \is_static |
| movq $$0, %r10 // arg_index |
| LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| .else |
| shrq MACRO_LITERAL(4), %r11 |
| movq $$1, %r10 // arg_index |
| .endif |
| LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r9, r10, .Lgpr_setup_finished_\suffix |
| .Lgpr_setup_finished_\suffix: |
| .if \is_polymorphic |
| call SYMBOL(art_quick_invoke_polymorphic) |
| .elseif \is_custom |
| call SYMBOL(art_quick_invoke_custom) |
| .else |
| .if \is_interface |
| // Setup hidden argument. As we don't have access to the interface method, |
| // just pass the method from the IMT. If the method is the conflict trampoline, |
| // this will make the stub go to runtime, otherwise the hidden argument is unused. |
| movq %rdi, %rax |
| .endif |
| call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. |
| .endif |
| cmpb LITERAL(68), (rINSTq) // Test if result type char == 'D'. |
| je .Lreturn_double_\suffix |
| cmpb LITERAL(70), (rINSTq) // Test if result type char == 'F'. |
| jne .Ldone_return_\suffix |
| .Lreturn_float_\suffix: |
| movd %xmm0, %eax |
| jmp .Ldone_return_\suffix |
| .Lreturn_double_\suffix: |
| movq %xmm0, %rax |
| .Ldone_return_\suffix: |
| /* resume execution of caller */ |
| .if \is_string_init |
| movzwl 4(rPC), %r11d // arguments |
| andq $$0xf, %r11 |
| GET_VREG %esi, %r11 |
| UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax |
| .endif |
| |
| .if \is_polymorphic |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 |
| .else |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 |
| .endif |
| .endm |
| |
| .macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 |
| .if \is_polymorphic |
| // We always go to compiled code for polymorphic calls. |
| .elseif \is_custom |
| // We always go to compiled code for custom calls. |
| .else |
| DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix |
| .if \is_string_init |
| call nterp_to_nterp_string_init_range |
| .elseif \is_static |
| call nterp_to_nterp_static_range |
| .else |
| call nterp_to_nterp_instance_range |
| .endif |
| jmp .Ldone_return_range_\suffix |
| .endif |
| |
| .Lcall_compiled_code_range_\suffix: |
| GET_SHORTY rINSTq, \is_interface, \is_polymorphic, \is_custom |
| // From this point: |
| // - rINSTq contains shorty (in callee-save to switch over return value after call). |
| // - rdi contains method |
| // - rsi contains 'this' pointer for instance method. |
| leaq 1(rINSTq), %r9 // shorty + 1 ; ie skip return arg character |
| movzwl 4(rPC), %r10d // arg start index |
| .if \is_string_init |
| addq $$1, %r10 // arg start index |
| movq $$1, %rbp // index in stack |
| .elseif \is_static |
| movq $$0, %rbp // index in stack |
| .else |
| addq $$1, %r10 // arg start index |
| movq $$1, %rbp // index in stack |
| .endif |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm0, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm1, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm2, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm3, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm4, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm5, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm6, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_XMMS xmm7, r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_FPs r9, r10, rbp, .Lxmm_setup_finished_range_\suffix |
| .Lxmm_setup_finished_range_\suffix: |
| leaq 1(%rbx), %r11 // shorty + 1 ; ie skip return arg character |
| movzwl 4(rPC), %r10d // arg start index |
| .if \is_string_init |
| addq $$1, %r10 // arg start index |
| movq $$1, %rbp // index in stack |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| .elseif \is_static |
| movq $$0, %rbp // index in stack |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rsi, esi, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| .else |
| addq $$1, %r10 // arg start index |
| movq $$1, %rbp // index in stack |
| .endif |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rdx, edx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS rcx, ecx, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r8, r8d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_SHORTY_LOADING_GPRS r9, r9d, r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| LOOP_RANGE_OVER_INTs r11, r10, rbp, .Lgpr_setup_finished_range_\suffix |
| |
| .Lgpr_setup_finished_range_\suffix: |
| .if \is_polymorphic |
| call SYMBOL(art_quick_invoke_polymorphic) |
| .elseif \is_custom |
| call SYMBOL(art_quick_invoke_custom) |
| .else |
| .if \is_interface |
| // Setup hidden argument. As we don't have access to the interface method, |
| // just pass the method from the IMT. If the method is the conflict trampoline, |
| // this will make the stub go to runtime, otherwise the hidden argument is unused. |
| movq %rdi, %rax |
| .endif |
| call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. |
| .endif |
| cmpb LITERAL(68), (%rbx) // Test if result type char == 'D'. |
| je .Lreturn_range_double_\suffix |
| cmpb LITERAL(70), (%rbx) // Test if result type char == 'F'. |
| je .Lreturn_range_float_\suffix |
| /* resume execution of caller */ |
| .Ldone_return_range_\suffix: |
| .if \is_string_init |
| movzwl 4(rPC), %r11d // arguments |
| GET_VREG %esi, %r11 |
| UPDATE_REGISTERS_FOR_STRING_INIT %esi, %eax |
| .endif |
| |
| .if \is_polymorphic |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 4 |
| .else |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 3 |
| .endif |
| .Lreturn_range_double_\suffix: |
| movq %xmm0, %rax |
| jmp .Ldone_return_range_\suffix |
| .Lreturn_range_float_\suffix: |
| movd %xmm0, %eax |
| jmp .Ldone_return_range_\suffix |
| .endm |
| |
| // Fetch some information from the thread cache. |
| // Uses rax, rdx, rcx as temporaries. |
| .macro FETCH_FROM_THREAD_CACHE dest_reg, slow_path |
| movq rSELF:THREAD_SELF_OFFSET, %rax |
| movq rPC, %rdx |
| salq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_SHIFT), %rdx |
| andq MACRO_LITERAL(THREAD_INTERPRETER_CACHE_SIZE_MASK), %rdx |
| cmpq THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), rPC |
| jne \slow_path |
| movq __SIZEOF_POINTER__+THREAD_INTERPRETER_CACHE_OFFSET(%rax, %rdx, 1), \dest_reg |
| .endm |
| |
| // Helper for static field get. |
| .macro OP_SGET load="movl", wide="0" |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 3f |
| 4: |
| .if \wide |
| movq (%eax,%edx,1), %rax |
| SET_WIDE_VREG %rax, rINSTq # fp[A] <- value |
| .else |
| \load (%eax, %edx, 1), %eax |
| SET_VREG %eax, rINSTq # fp[A] <- value |
| .endif |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_static_field |
| // Clear the marker that we put for volatile fields. The x86 memory |
| // model doesn't require a barrier. |
| andq $$-2, %rax |
| jmp 1b |
| 3: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 4b |
| .endm |
| |
| // Helper for static field put. |
| .macro OP_SPUT rINST_reg="rINST", store="movl", wide="0": |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 3f |
| 4: |
| .if \wide |
| GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] |
| .else |
| GET_VREG rINST, rINSTq # rINST <- v[A] |
| .endif |
| \store \rINST_reg, (%rax,%rdx,1) |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_static_field |
| testq MACRO_LITERAL(1), %rax |
| je 1b |
| // Clear the marker that we put for volatile fields. The x86 memory |
| // model doesn't require a barrier. |
| CLEAR_VOLATILE_MARKER %rax |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 6f |
| 5: |
| .if \wide |
| GET_WIDE_VREG rINSTq, rINSTq # rINST <- v[A] |
| .else |
| GET_VREG rINST, rINSTq # rINST <- v[A] |
| .endif |
| \store \rINST_reg, (%rax,%rdx,1) |
| lock addl $$0, (%rsp) |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 3: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 4b |
| 6: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 5b |
| .endm |
| |
| |
| .macro OP_IPUT_INTERNAL rINST_reg="rINST", store="movl", wide="0": |
| movzbq rINSTbl, %rcx # rcx <- BA |
| sarl $$4, %ecx # ecx <- B |
| GET_VREG %ecx, %rcx # vB (object we're operating on) |
| testl %ecx, %ecx # is object null? |
| je common_errNullObject |
| andb $$0xf, rINSTbl # rINST <- A |
| .if \wide |
| GET_WIDE_VREG rINSTq, rINSTq # rax<- fp[A]/fp[A+1] |
| .else |
| GET_VREG rINST, rINSTq # rINST <- v[A] |
| .endif |
| \store \rINST_reg, (%rcx,%rax,1) |
| .endm |
| |
| // Helper for instance field put. |
| .macro OP_IPUT rINST_reg="rINST", store="movl", wide="0": |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| OP_IPUT_INTERNAL \rINST_reg, \store, \wide |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_instance_field_offset |
| testl %eax, %eax |
| jns 1b |
| negl %eax |
| OP_IPUT_INTERNAL \rINST_reg, \store, \wide |
| lock addl $$0, (%rsp) |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| .endm |
| |
| // Helper for instance field get. |
| .macro OP_IGET load="movl", wide="0" |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl rINST, %ecx # rcx <- BA |
| sarl $$4, %ecx # ecx <- B |
| GET_VREG %ecx, %rcx # vB (object we're operating on) |
| testl %ecx, %ecx # is object null? |
| je common_errNullObject |
| andb $$0xf,rINSTbl # rINST <- A |
| .if \wide |
| movq (%rcx,%rax,1), %rax |
| SET_WIDE_VREG %rax, rINSTq # fp[A] <- value |
| .else |
| \load (%rcx,%rax,1), %eax |
| SET_VREG %eax, rINSTq # fp[A] <- value |
| .endif |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_instance_field_offset |
| testl %eax, %eax |
| jns 1b |
| negl %eax |
| jmp 1b |
| .endm |
| |
| %def entry(): |
| /* |
| * ArtMethod entry point. |
| * |
| * On entry: |
| * rdi ArtMethod* callee |
| * rest method parameters |
| */ |
| |
| OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| testq %rax, -STACK_OVERFLOW_RESERVED_BYTES(%rsp) |
| /* Spill callee save regs */ |
| SPILL_ALL_CALLEE_SAVES |
| |
| // TODO: Get shorty in a better way and remove below |
| PUSH rdi |
| PUSH rsi |
| PUSH rdx |
| PUSH rcx |
| PUSH r8 |
| PUSH r9 |
| |
| // Save xmm registers + alignment. |
| subq MACRO_LITERAL(8 * 8 + 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(8 * 8 + 8) |
| movq %xmm0, 0(%rsp) |
| movq %xmm1, 8(%rsp) |
| movq %xmm2, 16(%rsp) |
| movq %xmm3, 24(%rsp) |
| movq %xmm4, 32(%rsp) |
| movq %xmm5, 40(%rsp) |
| movq %xmm6, 48(%rsp) |
| movq %xmm7, 56(%rsp) |
| |
| call SYMBOL(NterpGetShorty) |
| // Save shorty in callee-save rbp. |
| movq %rax, %rbp |
| |
| // Restore xmm registers + alignment. |
| movq 0(%rsp), %xmm0 |
| movq 8(%rsp), %xmm1 |
| movq 16(%rsp), %xmm2 |
| movq 24(%rsp), %xmm3 |
| movq 32(%rsp), %xmm4 |
| movq 40(%rsp), %xmm5 |
| movq 48(%rsp), %xmm6 |
| movq 56(%rsp), %xmm7 |
| addq MACRO_LITERAL(8 * 8 + 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8 * 8 - 8) |
| |
| POP r9 |
| POP r8 |
| POP rcx |
| POP rdx |
| POP rsi |
| POP rdi |
| // TODO: Get shorty in a better way and remove above |
| |
| movq ART_METHOD_DATA_OFFSET_64(%rdi), rPC |
| |
| // Setup the stack for executing the method. |
| SETUP_STACK_FRAME rPC, rREFS, rREFS32, rFP, CFI_REFS, load_ins=1 |
| |
| // Setup the parameters |
| testl %r14d, %r14d |
| je .Lxmm_setup_finished |
| |
| subq %r14, %rbx |
| salq $$2, %rbx // rbx is now the offset for inputs into the registers array. |
| |
| testl $$ART_METHOD_IS_STATIC_FLAG, ART_METHOD_ACCESS_FLAGS_OFFSET(%rdi) |
| |
| // Available: rdi, r10, r14 |
| // Note the leaq below don't change the flags. |
| leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character |
| leaq (rFP, %rbx, 1), %rdi |
| leaq (rREFS, %rbx, 1), %rbx |
| jne .Lhandle_static_method |
| movl %esi, (%rdi) |
| movl %esi, (%rbx) |
| addq $$4, %rdi |
| addq $$4, %rbx |
| addq $$4, %r11 |
| movq $$0, %r14 |
| jmp .Lcontinue_setup_gprs |
| .Lhandle_static_method: |
| movq $$0, %r14 |
| LOOP_OVER_SHORTY_STORING_GPRS rsi, esi, r10, r14, rdi, rbx, .Lgpr_setup_finished |
| .Lcontinue_setup_gprs: |
| LOOP_OVER_SHORTY_STORING_GPRS rdx, edx, r10, r14, rdi, rbx, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_STORING_GPRS rcx, ecx, r10, r14, rdi, rbx, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_STORING_GPRS r8, r8d, r10, r14, rdi, rbx, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_STORING_GPRS r9, r9d, r10, r14, rdi, rbx, .Lgpr_setup_finished |
| LOOP_OVER_INTs r10, r14, rdi, rbx, r11, .Lgpr_setup_finished |
| .Lgpr_setup_finished: |
| leaq 1(%rbp), %r10 // shorty + 1 ; ie skip return arg character |
| movq $$0, %r14 // reset counter |
| LOOP_OVER_SHORTY_STORING_XMMS xmm0, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm1, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm2, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm3, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm4, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm5, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm6, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_STORING_XMMS xmm7, r10, r14, rdi, .Lxmm_setup_finished |
| LOOP_OVER_FPs r10, r14, rdi, r11, .Lxmm_setup_finished |
| .Lxmm_setup_finished: |
| CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) |
| |
| // Set rIBASE |
| leaq artNterpAsmInstructionStart(%rip), rIBASE |
| /* start executing the instruction at rPC */ |
| START_EXECUTING_INSTRUCTIONS |
| /* NOTE: no fallthrough */ |
| // cfi info continues, and covers the whole nterp implementation. |
| END ExecuteNterpImpl |
| |
| %def opcode_pre(): |
| |
| %def helpers(): |
| |
| %def footer(): |
| /* |
| * =========================================================================== |
| * Common subroutines and data |
| * =========================================================================== |
| */ |
| |
| .text |
| .align 2 |
| |
| // Note: mterp also uses the common_* names below for helpers, but that's OK |
| // as the C compiler compiled each interpreter separately. |
| common_errDivideByZero: |
| EXPORT_PC |
| call art_quick_throw_div_zero |
| |
| // Expect array in edi, index in esi. |
| common_errArrayIndex: |
| EXPORT_PC |
| movl MIRROR_ARRAY_LENGTH_OFFSET(%edi), %eax |
| movl %esi, %edi |
| movl %eax, %esi |
| call art_quick_throw_array_bounds |
| |
| common_errNullObject: |
| EXPORT_PC |
| call art_quick_throw_null_pointer_exception |
| |
| NterpCommonInvokeStatic: |
| COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, suffix="invokeStatic" |
| |
| NterpCommonInvokeStaticRange: |
| COMMON_INVOKE_RANGE is_static=1, is_interface=0, suffix="invokeStatic" |
| |
| NterpCommonInvokeInstance: |
| COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="invokeInstance" |
| |
| NterpCommonInvokeInstanceRange: |
| COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="invokeInstance" |
| |
| NterpCommonInvokeInterface: |
| COMMON_INVOKE_NON_RANGE is_static=0, is_interface=1, suffix="invokeInterface" |
| |
| NterpCommonInvokeInterfaceRange: |
| COMMON_INVOKE_RANGE is_static=0, is_interface=1, suffix="invokeInterface" |
| |
| NterpCommonInvokePolymorphic: |
| COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=0, is_polymorphic=1, suffix="invokePolymorphic" |
| |
| NterpCommonInvokePolymorphicRange: |
| COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_polymorphic=1, suffix="invokePolymorphic" |
| |
| NterpCommonInvokeCustom: |
| COMMON_INVOKE_NON_RANGE is_static=1, is_interface=0, is_string_init=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" |
| |
| NterpCommonInvokeCustomRange: |
| COMMON_INVOKE_RANGE is_static=1, is_interface=0, is_polymorphic=0, is_custom=1, suffix="invokeCustom" |
| |
| NterpHandleStringInit: |
| COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" |
| |
| NterpHandleStringInitRange: |
| COMMON_INVOKE_RANGE is_static=0, is_interface=0, is_string_init=1, suffix="stringInit" |
| |
| NterpNewInstance: |
| EXPORT_PC |
| // Fast-path which gets the class from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rdi, 2f |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 3f |
| 4: |
| callq *rSELF:THREAD_ALLOC_OBJECT_ENTRYPOINT_OFFSET |
| 1: |
| SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| call nterp_get_class_or_allocate_object |
| jmp 1b |
| 3: |
| // 07 is %rdi |
| call art_quick_read_barrier_mark_reg07 |
| jmp 4b |
| |
| NterpNewArray: |
| /* new-array vA, vB, class@CCCC */ |
| EXPORT_PC |
| // Fast-path which gets the class from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rdi, 2f |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 3f |
| 1: |
| movzbl rINSTbl,%esi |
| sarl $$4,%esi # esi<- B |
| GET_VREG %esi %rsi # esi<- vB (array length) |
| andb $$0xf,rINSTbl # rINST<- A |
| callq *rSELF:THREAD_ALLOC_ARRAY_ENTRYPOINT_OFFSET |
| SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| call nterp_get_class_or_allocate_object |
| movq %rax, %rdi |
| jmp 1b |
| 3: |
| // 07 is %rdi |
| call art_quick_read_barrier_mark_reg07 |
| jmp 1b |
| |
| NterpPutObjectInstanceField: |
| movzbq rINSTbl, %rbp # rcx <- BA |
| sarl $$4, %ebp # ebp <- B |
| GET_VREG %ebp, %rbp # vB (object we're operating on) |
| andb $$0xf, rINSTbl # rINST <- A |
| GET_VREG rINST, rINSTq # rINST <- v[A] |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| testl %ebp, %ebp # is object null? |
| je common_errNullObject |
| movl rINST, (%rbp,%rax,1) |
| testl rINST, rINST |
| je 4f |
| movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax |
| shrq $$CARD_TABLE_CARD_SHIFT, %rbp |
| movb %al, (%rax, %rbp, 1) |
| 4: |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq rINSTq, %rcx |
| call nterp_get_instance_field_offset |
| testl %eax, %eax |
| jns 1b |
| testl %ebp, %ebp # is object null? |
| je common_errNullObject |
| negl %eax |
| movl rINST, (%rbp,%rax,1) |
| testl rINST, rINST |
| je 5f |
| movq rSELF:THREAD_CARD_TABLE_OFFSET, %rax |
| shrq $$CARD_TABLE_CARD_SHIFT, %rbp |
| movb %al, (%rbp, %rax, 1) |
| 5: |
| lock addl $$0, (%rsp) |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| |
| NterpGetObjectInstanceField: |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl rINST, %ecx # rcx <- BA |
| sarl $$4, %ecx # ecx <- B |
| GET_VREG %ecx, %rcx # vB (object we're operating on) |
| testl %ecx, %ecx # is object null? |
| je common_errNullObject |
| testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%ecx) |
| movl (%rcx,%rax,1), %eax |
| jnz 3f |
| 4: |
| andb $$0xf,rINSTbl # rINST <- A |
| SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_instance_field_offset |
| testl %eax, %eax |
| jns 1b |
| // For volatile fields, we return a negative offset. Remove the sign |
| // and no need for any barrier thanks to the memory model. |
| negl %eax |
| jmp 1b |
| 3: |
| // reg00 is eax |
| call art_quick_read_barrier_mark_reg00 |
| jmp 4b |
| |
| NterpPutObjectStaticField: |
| GET_VREG %ebp, rINSTq |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 3f |
| 5: |
| movl %ebp, (%eax, %edx, 1) |
| testl %ebp, %ebp |
| je 4f |
| movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx |
| shrq $$CARD_TABLE_CARD_SHIFT, %rax |
| movb %cl, (%rax, %rcx, 1) |
| 4: |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq %rbp, %rcx |
| call nterp_get_static_field |
| testq MACRO_LITERAL(1), %rax |
| je 1b |
| CLEAR_VOLATILE_MARKER %rax |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 7f |
| 6: |
| movl %ebp, (%eax, %edx, 1) |
| testl %ebp, %ebp |
| je 8f |
| movq rSELF:THREAD_CARD_TABLE_OFFSET, %rcx |
| shrq $$CARD_TABLE_CARD_SHIFT, %rax |
| movb %cl, (%rax, %rcx, 1) |
| 8: |
| lock addl $$0, (%rsp) |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 3: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 5b |
| 7: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 6b |
| |
| NterpGetObjectStaticField: |
| // Fast-path which gets the field from thread-local cache. |
| FETCH_FROM_THREAD_CACHE %rax, 2f |
| 1: |
| movl ART_FIELD_OFFSET_OFFSET(%rax), %edx |
| movl ART_FIELD_DECLARING_CLASS_OFFSET(%rax), %eax |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 5f |
| 6: |
| testb $$READ_BARRIER_TEST_VALUE, GRAY_BYTE_OFFSET(%eax) |
| movl (%eax, %edx, 1), %eax |
| jnz 3f |
| 4: |
| SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 2: |
| EXPORT_PC |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| movq $$0, %rcx |
| call nterp_get_static_field |
| andq $$-2, %rax |
| jmp 1b |
| 3: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 4b |
| 5: |
| call art_quick_read_barrier_mark_reg00 |
| jmp 6b |
| |
| NterpGetBooleanStaticField: |
| OP_SGET load="movsbl", wide=0 |
| |
| NterpGetByteStaticField: |
| OP_SGET load="movsbl", wide=0 |
| |
| NterpGetCharStaticField: |
| OP_SGET load="movzwl", wide=0 |
| |
| NterpGetShortStaticField: |
| OP_SGET load="movswl", wide=0 |
| |
| NterpGetWideStaticField: |
| OP_SGET load="movq", wide=1 |
| |
| NterpGetIntStaticField: |
| OP_SGET load="movl", wide=0 |
| |
| NterpPutStaticField: |
| OP_SPUT rINST_reg=rINST, store="movl", wide=0 |
| |
| NterpPutBooleanStaticField: |
| NterpPutByteStaticField: |
| OP_SPUT rINST_reg=rINSTbl, store="movb", wide=0 |
| |
| NterpPutCharStaticField: |
| NterpPutShortStaticField: |
| OP_SPUT rINST_reg=rINSTw, store="movw", wide=0 |
| |
| NterpPutWideStaticField: |
| OP_SPUT rINST_reg=rINSTq, store="movq", wide=1 |
| |
| NterpPutInstanceField: |
| OP_IPUT rINST_reg=rINST, store="movl", wide=0 |
| |
| NterpPutBooleanInstanceField: |
| NterpPutByteInstanceField: |
| OP_IPUT rINST_reg=rINSTbl, store="movb", wide=0 |
| |
| NterpPutCharInstanceField: |
| NterpPutShortInstanceField: |
| OP_IPUT rINST_reg=rINSTw, store="movw", wide=0 |
| |
| NterpPutWideInstanceField: |
| OP_IPUT rINST_reg=rINSTq, store="movq", wide=1 |
| |
| NterpGetBooleanInstanceField: |
| OP_IGET load="movzbl", wide=0 |
| |
| NterpGetByteInstanceField: |
| OP_IGET load="movsbl", wide=0 |
| |
| NterpGetCharInstanceField: |
| OP_IGET load="movzwl", wide=0 |
| |
| NterpGetShortInstanceField: |
| OP_IGET load="movswl", wide=0 |
| |
| NterpGetWideInstanceField: |
| OP_IGET load="movq", wide=1 |
| |
| NterpGetInstanceField: |
| OP_IGET load="movl", wide=0 |
| |
| NterpInstanceOf: |
| /* instance-of vA, vB, class@CCCC */ |
| // Fast-path which gets the class from thread-local cache. |
| EXPORT_PC |
| FETCH_FROM_THREAD_CACHE %rsi, 2f |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 5f |
| 1: |
| movzbl rINSTbl,%edi |
| sarl $$4,%edi # edi<- B |
| GET_VREG %edi %rdi # edi<- vB (object) |
| andb $$0xf,rINSTbl # rINST<- A |
| testl %edi, %edi |
| je 3f |
| call art_quick_instance_of |
| SET_VREG %eax, rINSTq # fp[A] <- value |
| 4: |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 3: |
| SET_VREG %edi, rINSTq # fp[A] <-0 |
| jmp 4b |
| 2: |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| call nterp_get_class_or_allocate_object |
| movq %rax, %rsi |
| jmp 1b |
| 5: |
| // 06 is %rsi |
| call art_quick_read_barrier_mark_reg06 |
| jmp 1b |
| |
| NterpCheckCast: |
| // Fast-path which gets the class from thread-local cache. |
| EXPORT_PC |
| FETCH_FROM_THREAD_CACHE %rsi, 3f |
| cmpq $$0, rSELF:THREAD_READ_BARRIER_MARK_REG00_OFFSET |
| jne 4f |
| 1: |
| GET_VREG %edi, rINSTq |
| testl %edi, %edi |
| je 2f |
| call art_quick_check_instance_of |
| 2: |
| ADVANCE_PC_FETCH_AND_GOTO_NEXT 2 |
| 3: |
| movq rSELF:THREAD_SELF_OFFSET, %rdi |
| movq 0(%rsp), %rsi |
| movq rPC, %rdx |
| call nterp_get_class_or_allocate_object |
| movq %rax, %rsi |
| jmp 1b |
| 4: |
| // 06 is %rsi |
| call art_quick_read_barrier_mark_reg06 |
| jmp 1b |
| |
| NterpHandleHotnessOverflow: |
| leaq (rPC, rINSTq, 2), %rsi |
| movq rFP, %rdx |
| call nterp_hot_method |
| testq %rax, %rax |
| jne 1f |
| leaq (rPC, rINSTq, 2), rPC |
| FETCH_INST |
| GOTO_NEXT |
| 1: |
| // Drop the current frame. |
| movq -8(rREFS), %rsp |
| CFI_DEF_CFA(rsp, CALLEE_SAVES_SIZE) |
| |
| // Setup the new frame |
| movq OSR_DATA_FRAME_SIZE(%rax), %rcx |
| // Given stack size contains all callee saved registers, remove them. |
| subq $$CALLEE_SAVES_SIZE, %rcx |
| |
| // Remember CFA. |
| movq %rsp, %rbp |
| CFI_DEF_CFA_REGISTER(rbp) |
| |
| subq %rcx, %rsp |
| movq %rsp, %rdi // rdi := beginning of stack |
| leaq OSR_DATA_MEMORY(%rax), %rsi // rsi := memory to copy |
| rep movsb // while (rcx--) { *rdi++ = *rsi++ } |
| |
| // Fetch the native PC to jump to and save it in a callee-save register. |
| movq OSR_DATA_NATIVE_PC(%rax), %rbx |
| |
| // Free the memory holding OSR Data. |
| movq %rax, %rdi |
| call free |
| |
| // Jump to the compiled code. |
| jmp *%rbx |
| |
| NterpHandleInvokeInterfaceOnObjectMethodRange: |
| // First argument is the 'this' pointer. |
| movzwl 4(rPC), %r11d // arguments |
| movl (rFP, %r11, 4), %esi |
| // Note: if esi is null, this will be handled by our SIGSEGV handler. |
| movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx |
| movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi |
| jmp NterpCommonInvokeInstanceRange |
| |
| NterpHandleInvokeInterfaceOnObjectMethod: |
| // First argument is the 'this' pointer. |
| movzwl 4(rPC), %r11d // arguments |
| andq MACRO_LITERAL(0xf), %r11 |
| movl (rFP, %r11, 4), %esi |
| // Note: if esi is null, this will be handled by our SIGSEGV handler. |
| movl MIRROR_OBJECT_CLASS_OFFSET(%esi), %edx |
| movq MIRROR_CLASS_VTABLE_OFFSET_64(%edx, %eax, 8), %rdi |
| jmp NterpCommonInvokeInstance |
| |
| // This is the logical end of ExecuteNterpImpl, where the frame info applies. |
| // EndExecuteNterpImpl includes the methods below as we want the runtime to |
| // see them as part of the Nterp PCs. |
| .cfi_endproc |
| |
| nterp_to_nterp_static_non_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 |
| .cfi_endproc |
| |
| nterp_to_nterp_string_init_non_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 |
| .cfi_endproc |
| |
| nterp_to_nterp_instance_non_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 |
| .cfi_endproc |
| |
| nterp_to_nterp_static_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 |
| .cfi_endproc |
| |
| nterp_to_nterp_instance_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 |
| .cfi_endproc |
| |
| nterp_to_nterp_string_init_range: |
| .cfi_startproc |
| .cfi_def_cfa rsp, 8 |
| SETUP_STACK_FOR_INVOKE |
| SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 |
| .cfi_endproc |
| |
| // This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter |
| // entry point. |
| FUNCTION_TYPE(EndExecuteNterpImpl) |
| ASM_HIDDEN SYMBOL(EndExecuteNterpImpl) |
| .global SYMBOL(EndExecuteNterpImpl) |
| SYMBOL(EndExecuteNterpImpl): |
| |
| // Entrypoints into runtime. |
| NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField |
| NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset |
| NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray |
| NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange |
| NTERP_TRAMPOLINE nterp_get_class_or_allocate_object, NterpGetClassOrAllocateObject |
| NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod |
| NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod |
| NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject |
| |
| // gen_mterp.py will inline the following definitions |
| // within [ExecuteNterpImpl, EndExecuteNterpImpl). |
| %def instruction_end(): |
| |
| FUNCTION_TYPE(artNterpAsmInstructionEnd) |
| ASM_HIDDEN SYMBOL(artNterpAsmInstructionEnd) |
| .global SYMBOL(artNterpAsmInstructionEnd) |
| SYMBOL(artNterpAsmInstructionEnd): |
| // artNterpAsmInstructionEnd is used as landing pad for exception handling. |
| FETCH_INST |
| GOTO_NEXT |
| |
| %def instruction_start(): |
| |
| FUNCTION_TYPE(artNterpAsmInstructionStart) |
| ASM_HIDDEN SYMBOL(artNterpAsmInstructionStart) |
| .global SYMBOL(artNterpAsmInstructionStart) |
| SYMBOL(artNterpAsmInstructionStart) = .L_op_nop |
| .text |
| |
| %def default_helper_prefix(): |
| % return "nterp_" |
| |
| %def opcode_start(): |
| ENTRY nterp_${opcode} |
| %def opcode_end(): |
| END nterp_${opcode} |
| %def helper_start(name): |
| ENTRY ${name} |
| %def helper_end(name): |
| END ${name} |
