runtime/arch/x86_64/asm_support_x86_64.S - platform/art - Git at Google

 /*
  * Copyright (C) 2013 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #ifndef ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_
 #define ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_

 #include "asm_support_x86_64.h"
 #include "interpreter/cfi_asm_support.h"

 // Regular gas(1) & current clang/llvm assembler support named macro parameters.
 #define MACRO0(macro_name) .macro macro_name
 #define MACRO1(macro_name, macro_arg1) .macro macro_name macro_arg1
 #define MACRO2(macro_name, macro_arg1, macro_arg2) .macro macro_name macro_arg1, macro_arg2
 #define MACRO3(macro_name, macro_arg1, macro_arg2, macro_arg3) .macro macro_name macro_arg1, macro_arg2, macro_arg3
 #define MACRO4(macro_name, macro_arg1, macro_arg2, macro_arg3, macro_arg4) .macro macro_name macro_arg1, macro_arg2, macro_arg3, macro_arg4
 #define END_MACRO .endm

 #if defined(__clang__)
     // Clang/llvm does not support .altmacro. However, the clang/llvm preprocessor doesn't
     // separate the backslash and parameter by a space. Everything just works.
     #define RAW_VAR(name) \name
     #define VAR(name) \name
     #define CALLVAR(name) SYMBOL(\name)
     #define PLT_VAR(name) \name@PLT
     #define REG_VAR(name) %\name
     #define CALL_MACRO(name) \name
 #else
     // Regular gas(1) uses \argument_name for macro arguments.
     // We need to turn on alternate macro syntax so we can use & instead or the preprocessor
     // will screw us by inserting a space between the \ and the name. Even in this mode there's
     // no special meaning to $, so literals are still just $x. The use of altmacro means % is a
     // special character meaning care needs to be taken when passing registers as macro
     // arguments.
     .altmacro
     #define RAW_VAR(name) name&
     #define VAR(name) name&
     #define CALLVAR(name) SYMBOL(name&)
     #define PLT_VAR(name) name&@PLT
     #define REG_VAR(name) %name
     #define CALL_MACRO(name) name&
 #endif

 #define LITERAL(value) $value
 #if defined(__APPLE__)
     #define MACRO_LITERAL(value) $(value)
 #else
     #define MACRO_LITERAL(value) $value
 #endif

 #if defined(__APPLE__)
     #define FUNCTION_TYPE(name)
     #define SIZE(name)
 #else
     #define FUNCTION_TYPE(name) .type name, @function
     #define SIZE(name) .size name, .-name
 #endif

     // CFI support.
 #if !defined(__APPLE__)
     #define CFI_STARTPROC .cfi_startproc
     #define CFI_ENDPROC .cfi_endproc
     #define CFI_ADJUST_CFA_OFFSET(size) .cfi_adjust_cfa_offset size
     #define CFI_DEF_CFA(reg,size) .cfi_def_cfa reg,size
     #define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg
     #define CFI_RESTORE(reg) .cfi_restore reg
     #define CFI_REL_OFFSET(reg,size) .cfi_rel_offset reg,size
     #define CFI_REGISTER(orig_reg, current_reg) .cfi_register orig_reg, current_reg
     #define CFI_REMEMBER_STATE .cfi_remember_state
     // The spec is not clear whether the CFA is part of the saved state and tools
     // differ in the behaviour, so explicitly set the CFA to avoid any ambiguity.
     // The restored CFA state should match the CFA state during CFI_REMEMBER_STATE.
     // `objdump -Wf libart.so | egrep "_cfa|_state"` is useful to audit the opcodes.
     MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off)
         .cfi_restore_state
         .cfi_def_cfa \reg,\off
     END_MACRO
     #define CFI_RESTORE_STATE .cfi_restore_state
 #else
     // Mac OS' doesn't like cfi_* directives.
     #define CFI_STARTPROC
     #define CFI_ENDPROC
     #define CFI_ADJUST_CFA_OFFSET(size)
     #define CFI_DEF_CFA(reg,size)
     #define CFI_DEF_CFA_REGISTER(reg)
     #define CFI_RESTORE(reg)
     #define CFI_REL_OFFSET(reg,size)
     #define CFI_REGISTER(orig_reg, current_reg)
     #define CFI_REMEMBER_STATE
     #define CFI_RESTORE_STATE_AND_DEF_CFA(off)
     MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off)
     END_MACRO
     #define CFI_RESTORE_STATE
 #endif

 // The register numbers are a bit mixed up for x86-64.
 #define CFI_REG_rax 0
 #define CFI_REG_rcx 2
 #define CFI_REG_rdx 1
 #define CFI_REG_rbx 3
 #define CFI_REG_rsp 7
 #define CFI_REG_rbp 6
 #define CFI_REG_rsi 4
 #define CFI_REG_rdi 5
 #define CFI_REG_r8 8
 #define CFI_REG_r9 9
 #define CFI_REG_r10 10
 #define CFI_REG_r11 11
 #define CFI_REG_r12 12
 #define CFI_REG_r13 13
 #define CFI_REG_r14 14
 #define CFI_REG_r15 15
 #define CFI_REG_rip 16

 #define CFI_REG(reg) CFI_REG_##reg

 MACRO3(CFI_EXPRESSION_BREG, n, b, offset)
     .if (-0x40 <= (\offset)) && ((\offset) < 0x40)
         CFI_EXPRESSION_BREG_1(\n, \b, \offset)
     .elseif (-0x2000 <= (\offset)) && ((\offset) < 0x2000)
         CFI_EXPRESSION_BREG_2(\n, \b, \offset)
     .else
         .error "Unsupported offset"
     .endif
 END_MACRO

 MACRO3(CFI_DEF_CFA_BREG_PLUS_UCONST, reg, offset, size)
     .if ((\size) < 0)
         .error "Size should be positive"
     .endif
     .if (((\offset) < -0x40) || ((\offset) >= 0x40))
         .error "Unsupported offset"
     .endif
     .if ((\size) < 0x80)
         CFI_DEF_CFA_BREG_PLUS_UCONST_1_1(\reg, \offset, \size)
     .elseif ((\size) < 0x4000)
         CFI_DEF_CFA_BREG_PLUS_UCONST_1_2(\reg, \offset, \size)
     .else
         .error "Unsupported size"
     .endif
 END_MACRO

     // Symbols.
 #if !defined(__APPLE__)
     #define SYMBOL(name) name
     #define PLT_SYMBOL(name) name ## @PLT
 #else
     #define SYMBOL(name) _ ## name
     #define PLT_SYMBOL(name) _ ## name
 #endif

 // Directive to hide a function symbol.
 #if defined(__APPLE__)
     #define ASM_HIDDEN .private_extern
 #else
     #define ASM_HIDDEN .hidden
 #endif

     /* Cache alignment for function entry */
 MACRO0(ALIGN_FUNCTION_ENTRY)
     // ART-compiled functions have OatQuickMethodHeader but assembly funtions do not.
     // Prefix the assembly code with 0xFFs, which means there is no method header.
     .byte 0xFF, 0xFF, 0xFF, 0xFF
     // Cache alignment for function entry.
     .balign 16, 0xFF
 END_MACRO

 // TODO: we might need to use SYMBOL() here to add the underscore prefix
 // for mac builds.
 MACRO2(DEFINE_FUNCTION_CUSTOM_CFA, c_name, cfa_offset)
     FUNCTION_TYPE(SYMBOL(\c_name))
     ASM_HIDDEN CALLVAR(c_name)
     .globl CALLVAR(c_name)
     ALIGN_FUNCTION_ENTRY
 CALLVAR(c_name):
     CFI_STARTPROC
     // Ensure we get an appropriate starting CFA.
     CFI_DEF_CFA(rsp, RAW_VAR(cfa_offset))
 END_MACRO

 MACRO1(DEFINE_FUNCTION, c_name)
     DEFINE_FUNCTION_CUSTOM_CFA RAW_VAR(c_name), __SIZEOF_POINTER__
 END_MACRO

 MACRO1(END_FUNCTION, c_name)
     CFI_ENDPROC
     SIZE(SYMBOL(\c_name))
 END_MACRO

 MACRO1(PUSH, reg)
     pushq REG_VAR(reg)
     CFI_ADJUST_CFA_OFFSET(8)
     CFI_REL_OFFSET(REG_VAR(reg), 0)
 END_MACRO

 MACRO1(POP, reg)
     popq REG_VAR(reg)
     CFI_ADJUST_CFA_OFFSET(-8)
     CFI_RESTORE(REG_VAR(reg))
 END_MACRO

 // Arguments do not need .cfi_rel_offset as they are caller-saved and
 // therefore cannot hold caller's variables or unwinding data.
 MACRO1(PUSH_ARG, reg)
     pushq REG_VAR(reg)
     CFI_ADJUST_CFA_OFFSET(8)
 END_MACRO

 MACRO1(POP_ARG, reg)
     popq REG_VAR(reg)
     CFI_ADJUST_CFA_OFFSET(-8)
 END_MACRO

 MACRO3(SAVE_REG_BASE, base, reg, offset)
     movq REG_VAR(reg), RAW_VAR(offset)(REG_VAR(base))
     CFI_REL_OFFSET(REG_VAR(reg), RAW_VAR(offset))
 END_MACRO

 MACRO3(RESTORE_REG_BASE, base, reg, offset)
     movq RAW_VAR(offset)(REG_VAR(base)), REG_VAR(reg)
     CFI_RESTORE(REG_VAR(reg))
 END_MACRO

 MACRO1(INCREASE_FRAME, frame_adjustment)
     subq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp
     CFI_ADJUST_CFA_OFFSET((RAW_VAR(frame_adjustment)))
 END_MACRO

 MACRO1(DECREASE_FRAME, frame_adjustment)
     addq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp
     CFI_ADJUST_CFA_OFFSET(-(RAW_VAR(frame_adjustment)))
 END_MACRO

 MACRO1(UNIMPLEMENTED,name)
     FUNCTION_TYPE(SYMBOL(\name))
     ASM_HIDDEN VAR(name)
     .globl VAR(name)
     ALIGN_FUNCTION_ENTRY
 VAR(name):
     CFI_STARTPROC
     int3
     int3
     CFI_ENDPROC
     SIZE(SYMBOL(\name))
 END_MACRO

 MACRO0(UNREACHABLE)
     int3
 END_MACRO

 MACRO0(UNTESTED)
     int3
 END_MACRO

 MACRO1(LOAD_RUNTIME_INSTANCE, reg)
     movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), REG_VAR(reg)
     movq (REG_VAR(reg)), REG_VAR(reg)
 END_MACRO

 // Macros to poison (negate) the reference for heap poisoning.
 MACRO1(POISON_HEAP_REF, rRef)
 #ifdef USE_HEAP_POISONING
     negl REG_VAR(rRef)
 #endif  // USE_HEAP_POISONING
 END_MACRO

 // Macros to unpoison (negate) the reference for heap poisoning.
 MACRO1(UNPOISON_HEAP_REF, rRef)
 #ifdef USE_HEAP_POISONING
     negl REG_VAR(rRef)
 #endif  // USE_HEAP_POISONING
 END_MACRO

     /*
      * Macro that sets up the callee save frame to conform with
      * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly)
      */
 MACRO0(SETUP_SAVE_REFS_ONLY_FRAME)
 #if defined(__APPLE__)
     int3
     int3
 #else
     // R10 := Runtime::Current()
     LOAD_RUNTIME_INSTANCE r10
     // Save callee and GPR args, mixed together to agree with core spills bitmap.
     PUSH r15  // Callee save.
     PUSH r14  // Callee save.
     PUSH r13  // Callee save.
     PUSH r12  // Callee save.
     PUSH rbp  // Callee save.
     PUSH rbx  // Callee save.
     // Create space for FPR args, plus space for ArtMethod*.
     INCREASE_FRAME 8 + 4 * 8
     // Save FPRs.
     movq %xmm12, 8(%rsp)
     movq %xmm13, 16(%rsp)
     movq %xmm14, 24(%rsp)
     movq %xmm15, 32(%rsp)
     // R10 := ArtMethod* for refs only callee save frame method.
     movq RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET(%r10), %r10
     // Store ArtMethod* to bottom of stack.
     movq %r10, 0(%rsp)
     // Store rsp as the stop quick frame.
     movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET

     // Ugly compile-time check, but we only have the preprocessor.
     // Last +8: implicit return address pushed on stack when caller made call.
 #if (FRAME_SIZE_SAVE_REFS_ONLY != 6 * 8 + 4 * 8 + 8 + 8)
 #error "FRAME_SIZE_SAVE_REFS_ONLY(X86_64) size not as expected."
 #endif
 #endif  // __APPLE__
 END_MACRO

 MACRO0(RESTORE_SAVE_REFS_ONLY_FRAME)
     movq 8(%rsp), %xmm12
     movq 16(%rsp), %xmm13
     movq 24(%rsp), %xmm14
     movq 32(%rsp), %xmm15
     DECREASE_FRAME 8 + 4*8
     // TODO: optimize by not restoring callee-saves restored by the ABI
     POP rbx
     POP rbp
     POP r12
     POP r13
     POP r14
     POP r15
 END_MACRO

     /*
      * Macro that sets up the callee save frame to conform with
      * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs), except for storing the method.
      */
 MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY)
     // Save callee and GPR args, mixed together to agree with core spills bitmap.
     PUSH r15      // Callee save.
     PUSH r14      // Callee save.
     PUSH r13      // Callee save.
     PUSH r12      // Callee save.
     PUSH_ARG r9   // Quick arg 5.
     PUSH_ARG r8   // Quick arg 4.
     PUSH_ARG rsi  // Quick arg 1.
     PUSH rbp      // Callee save.
     PUSH rbx      // Callee save.
     PUSH_ARG rdx  // Quick arg 2.
     PUSH_ARG rcx  // Quick arg 3.
     // Create space for FPR args and create 2 slots for ArtMethod*.
     INCREASE_FRAME 16 + 12 * 8
     // Save FPRs.
     movq %xmm0, 16(%rsp)
     movq %xmm1, 24(%rsp)
     movq %xmm2, 32(%rsp)
     movq %xmm3, 40(%rsp)
     movq %xmm4, 48(%rsp)
     movq %xmm5, 56(%rsp)
     movq %xmm6, 64(%rsp)
     movq %xmm7, 72(%rsp)
     movq %xmm12, 80(%rsp)
     movq %xmm13, 88(%rsp)
     movq %xmm14, 96(%rsp)
     movq %xmm15, 104(%rsp)

     // Ugly compile-time check, but we only have the preprocessor.
     // Last +8: implicit return address pushed on stack when caller made call.
 #if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 11 * 8 + 12 * 8 + 16 + 8)
 #error "FRAME_SIZE_SAVE_REFS_AND_ARGS(X86_64) size not as expected."
 #endif
 END_MACRO

 MACRO0(RESTORE_SAVE_REFS_AND_ARGS_FRAME)
     // Restore FPRs.
     movq 16(%rsp), %xmm0
     movq 24(%rsp), %xmm1
     movq 32(%rsp), %xmm2
     movq 40(%rsp), %xmm3
     movq 48(%rsp), %xmm4
     movq 56(%rsp), %xmm5
     movq 64(%rsp), %xmm6
     movq 72(%rsp), %xmm7
     movq 80(%rsp), %xmm12
     movq 88(%rsp), %xmm13
     movq 96(%rsp), %xmm14
     movq 104(%rsp), %xmm15
     DECREASE_FRAME 80 + 4 * 8
     // Restore callee and GPR args, mixed together to agree with core spills bitmap.
     POP_ARG rcx
     POP_ARG rdx
     POP rbx
     POP rbp
     POP_ARG rsi
     POP_ARG r8
     POP_ARG r9
     POP r12
     POP r13
     POP r14
     POP r15
 END_MACRO

     /*
      * Macro that sets up the callee save frame to conform with
      * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves)
      */
 MACRO0(SETUP_SAVE_ALL_CALLEE_SAVES_FRAME)
 #if defined(__APPLE__)
     int3
     int3
 #else
     // R10 := Runtime::Current()
     LOAD_RUNTIME_INSTANCE r10
     // Save callee save registers to agree with core spills bitmap.
     PUSH r15  // Callee save.
     PUSH r14  // Callee save.
     PUSH r13  // Callee save.
     PUSH r12  // Callee save.
     PUSH rbp  // Callee save.
     PUSH rbx  // Callee save.
     // Create space for FPR args, plus space for ArtMethod*.
     INCREASE_FRAME 4 * 8 + 8
     // Save FPRs.
     movq %xmm12, 8(%rsp)
     movq %xmm13, 16(%rsp)
     movq %xmm14, 24(%rsp)
     movq %xmm15, 32(%rsp)
     // R10 := ArtMethod* for save all callee save frame method.
     movq RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET(%r10), %r10
     // Store ArtMethod* to bottom of stack.
     movq %r10, 0(%rsp)
     // Store rsp as the top quick frame.
     movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET

     // Ugly compile-time check, but we only have the preprocessor.
     // Last +8: implicit return address pushed on stack when caller made call.
 #if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 6 * 8 + 4 * 8 + 8 + 8)
 #error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(X86_64) size not as expected."
 #endif
 #endif  // __APPLE__
 END_MACRO

 MACRO0(SETUP_FP_CALLEE_SAVE_FRAME)
     // Create space for ART FP callee-saved registers
     INCREASE_FRAME 4 * 8
     movq %xmm12, 0(%rsp)
     movq %xmm13, 8(%rsp)
     movq %xmm14, 16(%rsp)
     movq %xmm15, 24(%rsp)
 END_MACRO

 MACRO0(RESTORE_FP_CALLEE_SAVE_FRAME)
     // Restore ART FP callee-saved registers
     movq 0(%rsp), %xmm12
     movq 8(%rsp), %xmm13
     movq 16(%rsp), %xmm14
     movq 24(%rsp), %xmm15
     DECREASE_FRAME 4 * 8
 END_MACRO

     /*
      * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending
      * exception is Thread::Current()->exception_ when the runtime method frame is ready.
      */
 MACRO0(DELIVER_PENDING_EXCEPTION_FRAME_READY)
     // (Thread*) setup
     movq %gs:THREAD_SELF_OFFSET, %rdi
     call SYMBOL(artDeliverPendingExceptionFromCode)  // artDeliverPendingExceptionFromCode(Thread*)
     UNREACHABLE
 END_MACRO
     /*
      * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending
      * exception is Thread::Current()->exception_.
      */
 MACRO0(DELIVER_PENDING_EXCEPTION)
     SETUP_SAVE_ALL_CALLEE_SAVES_FRAME        // save callee saves for throw
     DELIVER_PENDING_EXCEPTION_FRAME_READY
 END_MACRO

 MACRO0(RETURN_OR_DELIVER_PENDING_EXCEPTION)
     movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field
     testq %rcx, %rcx               // rcx == 0 ?
     jnz 1f                         // if rcx != 0 goto 1
     ret                            // return
 1:                                 // deliver exception on current thread
     DELIVER_PENDING_EXCEPTION
 END_MACRO

 // Locking is needed for both managed code and JNI stubs.
 MACRO3(LOCK_OBJECT_FAST_PATH, obj, tmp, slow_lock)
 1:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax  // EAX := lock word
     movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp)  // tmp: thread id.
     xorl %eax, REG_VAR(tmp)               // tmp: thread id with count 0 + read barrier bits.
     testl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %eax  // Test the non-gc bits.
     jnz  2f                               // Check if unlocked.
     // Unlocked case - store tmp: original lock word plus thread id, preserved read barrier bits.
     lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
     jnz  1b                               // cmpxchg failed retry
     ret
 2:  // EAX: original lock word, tmp: thread id ^ EAX
                                           // Check lock word state and thread id together,
     testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \
           REG_VAR(tmp)
     jne  \slow_lock                       // Slow path if either of the two high bits are set.
                                           // Increment the recursive lock count.
     leal LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp)
     testl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_MASK_SHIFTED), REG_VAR(tmp)
     je   \slow_lock                       // If count overflowed, go to slow lock.
     // Update lockword for recursive lock, cmpxchg necessary for read barrier bits.
                                           // EAX: old val, tmp: new val.
     lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
     jnz  1b                               // cmpxchg failed retry
     ret
 END_MACRO

 // Unlocking is needed for both managed code and JNI stubs.
 MACRO4(UNLOCK_OBJECT_FAST_PATH, obj, tmp, saved_rax, slow_unlock)
 1:
     movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax  // EAX := lock word
     movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp)  // tmp := thread id
     xorl %eax, REG_VAR(tmp)               // tmp := thread id ^ lock word
     test LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), REG_VAR(tmp)
     jnz  2f                               // Check if simply locked.
     // Transition to unlocked.
 #ifndef USE_READ_BARRIER
     movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
 #else
     lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
     jnz  1b                               // cmpxchg failed retry
 #endif
     .ifnc \saved_rax, none
         movq REG_VAR(saved_rax), %rax     // Restore RAX.
     .endif
     ret
 2:  // EAX: original lock word, tmp: lock_word ^ thread id
                                           // Check lock word state and thread id together.
     testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \
           REG_VAR(tmp)
     jnz  \slow_unlock
     // Update lockword for recursive unlock, cmpxchg necessary for read barrier bits.
                                           // tmp: new lock word with decremented count.
     leal -LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp)
 #ifndef USE_READ_BARRIER
                                           // EAX: new lock word with decremented count.
     movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
 #else
     lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
     jnz  1b                               // cmpxchg failed retry
 #endif
     .ifnc \saved_rax, none
         movq REG_VAR(saved_rax), %rax     // Restore RAX.
     .endif
     ret
 END_MACRO

 #endif  // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_
	/*
	* Copyright (C) 2013 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#ifndef ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_
	#define ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_

	#include "asm_support_x86_64.h"
	#include "interpreter/cfi_asm_support.h"

	// Regular gas(1) & current clang/llvm assembler support named macro parameters.
	#define MACRO0(macro_name) .macro macro_name
	#define MACRO1(macro_name, macro_arg1) .macro macro_name macro_arg1
	#define MACRO2(macro_name, macro_arg1, macro_arg2) .macro macro_name macro_arg1, macro_arg2
	#define MACRO3(macro_name, macro_arg1, macro_arg2, macro_arg3) .macro macro_name macro_arg1, macro_arg2, macro_arg3
	#define MACRO4(macro_name, macro_arg1, macro_arg2, macro_arg3, macro_arg4) .macro macro_name macro_arg1, macro_arg2, macro_arg3, macro_arg4
	#define END_MACRO .endm

	#if defined(__clang__)
	// Clang/llvm does not support .altmacro. However, the clang/llvm preprocessor doesn't
	// separate the backslash and parameter by a space. Everything just works.
	#define RAW_VAR(name) \name
	#define VAR(name) \name
	#define CALLVAR(name) SYMBOL(\name)
	#define PLT_VAR(name) \name@PLT
	#define REG_VAR(name) %\name
	#define CALL_MACRO(name) \name
	#else
	// Regular gas(1) uses \argument_name for macro arguments.
	// We need to turn on alternate macro syntax so we can use & instead or the preprocessor
	// will screw us by inserting a space between the \ and the name. Even in this mode there's
	// no special meaning to $, so literals are still just $x. The use of altmacro means % is a
	// special character meaning care needs to be taken when passing registers as macro
	// arguments.
	.altmacro
	#define RAW_VAR(name) name&
	#define VAR(name) name&
	#define CALLVAR(name) SYMBOL(name&)
	#define PLT_VAR(name) name&@PLT
	#define REG_VAR(name) %name
	#define CALL_MACRO(name) name&
	#endif

	#define LITERAL(value) $value
	#if defined(__APPLE__)
	#define MACRO_LITERAL(value) $(value)
	#else
	#define MACRO_LITERAL(value) $value
	#endif

	#if defined(__APPLE__)
	#define FUNCTION_TYPE(name)
	#define SIZE(name)
	#else
	#define FUNCTION_TYPE(name) .type name, @function
	#define SIZE(name) .size name, .-name
	#endif

	// CFI support.
	#if !defined(__APPLE__)
	#define CFI_STARTPROC .cfi_startproc
	#define CFI_ENDPROC .cfi_endproc
	#define CFI_ADJUST_CFA_OFFSET(size) .cfi_adjust_cfa_offset size
	#define CFI_DEF_CFA(reg,size) .cfi_def_cfa reg,size
	#define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg
	#define CFI_RESTORE(reg) .cfi_restore reg
	#define CFI_REL_OFFSET(reg,size) .cfi_rel_offset reg,size
	#define CFI_REGISTER(orig_reg, current_reg) .cfi_register orig_reg, current_reg
	#define CFI_REMEMBER_STATE .cfi_remember_state
	// The spec is not clear whether the CFA is part of the saved state and tools
	// differ in the behaviour, so explicitly set the CFA to avoid any ambiguity.
	// The restored CFA state should match the CFA state during CFI_REMEMBER_STATE.
	// `objdump -Wf libart.so \| egrep "_cfa\|_state"` is useful to audit the opcodes.
	MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off)
	.cfi_restore_state
	.cfi_def_cfa \reg,\off
	END_MACRO
	#define CFI_RESTORE_STATE .cfi_restore_state
	#else
	// Mac OS' doesn't like cfi_* directives.
	#define CFI_STARTPROC
	#define CFI_ENDPROC
	#define CFI_ADJUST_CFA_OFFSET(size)
	#define CFI_DEF_CFA(reg,size)
	#define CFI_DEF_CFA_REGISTER(reg)
	#define CFI_RESTORE(reg)
	#define CFI_REL_OFFSET(reg,size)
	#define CFI_REGISTER(orig_reg, current_reg)
	#define CFI_REMEMBER_STATE
	#define CFI_RESTORE_STATE_AND_DEF_CFA(off)
	MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off)
	END_MACRO
	#define CFI_RESTORE_STATE
	#endif

	// The register numbers are a bit mixed up for x86-64.
	#define CFI_REG_rax 0
	#define CFI_REG_rcx 2
	#define CFI_REG_rdx 1
	#define CFI_REG_rbx 3
	#define CFI_REG_rsp 7
	#define CFI_REG_rbp 6
	#define CFI_REG_rsi 4
	#define CFI_REG_rdi 5
	#define CFI_REG_r8 8
	#define CFI_REG_r9 9
	#define CFI_REG_r10 10
	#define CFI_REG_r11 11
	#define CFI_REG_r12 12
	#define CFI_REG_r13 13
	#define CFI_REG_r14 14
	#define CFI_REG_r15 15
	#define CFI_REG_rip 16

	#define CFI_REG(reg) CFI_REG_##reg

	MACRO3(CFI_EXPRESSION_BREG, n, b, offset)
	.if (-0x40 <= (\offset)) && ((\offset) < 0x40)
	CFI_EXPRESSION_BREG_1(\n, \b, \offset)
	.elseif (-0x2000 <= (\offset)) && ((\offset) < 0x2000)
	CFI_EXPRESSION_BREG_2(\n, \b, \offset)
	.else
	.error "Unsupported offset"
	.endif
	END_MACRO

	MACRO3(CFI_DEF_CFA_BREG_PLUS_UCONST, reg, offset, size)
	.if ((\size) < 0)
	.error "Size should be positive"
	.endif
	.if (((\offset) < -0x40) \|\| ((\offset) >= 0x40))
	.error "Unsupported offset"
	.endif
	.if ((\size) < 0x80)
	CFI_DEF_CFA_BREG_PLUS_UCONST_1_1(\reg, \offset, \size)
	.elseif ((\size) < 0x4000)
	CFI_DEF_CFA_BREG_PLUS_UCONST_1_2(\reg, \offset, \size)
	.else
	.error "Unsupported size"
	.endif
	END_MACRO

	// Symbols.
	#if !defined(__APPLE__)
	#define SYMBOL(name) name
	#define PLT_SYMBOL(name) name ## @PLT
	#else
	#define SYMBOL(name) _ ## name
	#define PLT_SYMBOL(name) _ ## name
	#endif

	// Directive to hide a function symbol.
	#if defined(__APPLE__)
	#define ASM_HIDDEN .private_extern
	#else
	#define ASM_HIDDEN .hidden
	#endif

	/* Cache alignment for function entry */
	MACRO0(ALIGN_FUNCTION_ENTRY)
	// ART-compiled functions have OatQuickMethodHeader but assembly funtions do not.
	// Prefix the assembly code with 0xFFs, which means there is no method header.
	.byte 0xFF, 0xFF, 0xFF, 0xFF
	// Cache alignment for function entry.
	.balign 16, 0xFF
	END_MACRO

	// TODO: we might need to use SYMBOL() here to add the underscore prefix
	// for mac builds.
	MACRO2(DEFINE_FUNCTION_CUSTOM_CFA, c_name, cfa_offset)
	FUNCTION_TYPE(SYMBOL(\c_name))
	ASM_HIDDEN CALLVAR(c_name)
	.globl CALLVAR(c_name)
	ALIGN_FUNCTION_ENTRY
	CALLVAR(c_name):
	CFI_STARTPROC
	// Ensure we get an appropriate starting CFA.
	CFI_DEF_CFA(rsp, RAW_VAR(cfa_offset))
	END_MACRO

	MACRO1(DEFINE_FUNCTION, c_name)
	DEFINE_FUNCTION_CUSTOM_CFA RAW_VAR(c_name), __SIZEOF_POINTER__
	END_MACRO

	MACRO1(END_FUNCTION, c_name)
	CFI_ENDPROC
	SIZE(SYMBOL(\c_name))
	END_MACRO

	MACRO1(PUSH, reg)
	pushq REG_VAR(reg)
	CFI_ADJUST_CFA_OFFSET(8)
	CFI_REL_OFFSET(REG_VAR(reg), 0)
	END_MACRO

	MACRO1(POP, reg)
	popq REG_VAR(reg)
	CFI_ADJUST_CFA_OFFSET(-8)
	CFI_RESTORE(REG_VAR(reg))
	END_MACRO

	// Arguments do not need .cfi_rel_offset as they are caller-saved and
	// therefore cannot hold caller's variables or unwinding data.
	MACRO1(PUSH_ARG, reg)
	pushq REG_VAR(reg)
	CFI_ADJUST_CFA_OFFSET(8)
	END_MACRO

	MACRO1(POP_ARG, reg)
	popq REG_VAR(reg)
	CFI_ADJUST_CFA_OFFSET(-8)
	END_MACRO

	MACRO3(SAVE_REG_BASE, base, reg, offset)
	movq REG_VAR(reg), RAW_VAR(offset)(REG_VAR(base))
	CFI_REL_OFFSET(REG_VAR(reg), RAW_VAR(offset))
	END_MACRO

	MACRO3(RESTORE_REG_BASE, base, reg, offset)
	movq RAW_VAR(offset)(REG_VAR(base)), REG_VAR(reg)
	CFI_RESTORE(REG_VAR(reg))
	END_MACRO

	MACRO1(INCREASE_FRAME, frame_adjustment)
	subq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp
	CFI_ADJUST_CFA_OFFSET((RAW_VAR(frame_adjustment)))
	END_MACRO

	MACRO1(DECREASE_FRAME, frame_adjustment)
	addq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp
	CFI_ADJUST_CFA_OFFSET(-(RAW_VAR(frame_adjustment)))
	END_MACRO

	MACRO1(UNIMPLEMENTED,name)
	FUNCTION_TYPE(SYMBOL(\name))
	ASM_HIDDEN VAR(name)
	.globl VAR(name)
	ALIGN_FUNCTION_ENTRY
	VAR(name):
	CFI_STARTPROC
	int3
	int3
	CFI_ENDPROC
	SIZE(SYMBOL(\name))
	END_MACRO

	MACRO0(UNREACHABLE)
	int3
	END_MACRO

	MACRO0(UNTESTED)
	int3
	END_MACRO

	MACRO1(LOAD_RUNTIME_INSTANCE, reg)
	movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), REG_VAR(reg)
	movq (REG_VAR(reg)), REG_VAR(reg)
	END_MACRO

	// Macros to poison (negate) the reference for heap poisoning.
	MACRO1(POISON_HEAP_REF, rRef)
	#ifdef USE_HEAP_POISONING
	negl REG_VAR(rRef)
	#endif // USE_HEAP_POISONING
	END_MACRO

	// Macros to unpoison (negate) the reference for heap poisoning.
	MACRO1(UNPOISON_HEAP_REF, rRef)
	#ifdef USE_HEAP_POISONING
	negl REG_VAR(rRef)
	#endif // USE_HEAP_POISONING
	END_MACRO

	/*
	* Macro that sets up the callee save frame to conform with
	* Runtime::CreateCalleeSaveMethod(kSaveRefsOnly)
	*/
	MACRO0(SETUP_SAVE_REFS_ONLY_FRAME)
	#if defined(__APPLE__)
	int3
	int3
	#else
	// R10 := Runtime::Current()
	LOAD_RUNTIME_INSTANCE r10
	// Save callee and GPR args, mixed together to agree with core spills bitmap.
	PUSH r15 // Callee save.
	PUSH r14 // Callee save.
	PUSH r13 // Callee save.
	PUSH r12 // Callee save.
	PUSH rbp // Callee save.
	PUSH rbx // Callee save.
	// Create space for FPR args, plus space for ArtMethod*.
	INCREASE_FRAME 8 + 4 * 8
	// Save FPRs.
	movq %xmm12, 8(%rsp)
	movq %xmm13, 16(%rsp)
	movq %xmm14, 24(%rsp)
	movq %xmm15, 32(%rsp)
	// R10 := ArtMethod* for refs only callee save frame method.
	movq RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET(%r10), %r10
	// Store ArtMethod* to bottom of stack.
	movq %r10, 0(%rsp)
	// Store rsp as the stop quick frame.
	movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET

	// Ugly compile-time check, but we only have the preprocessor.
	// Last +8: implicit return address pushed on stack when caller made call.
	#if (FRAME_SIZE_SAVE_REFS_ONLY != 6 * 8 + 4 * 8 + 8 + 8)
	#error "FRAME_SIZE_SAVE_REFS_ONLY(X86_64) size not as expected."
	#endif
	#endif // __APPLE__
	END_MACRO

	MACRO0(RESTORE_SAVE_REFS_ONLY_FRAME)
	movq 8(%rsp), %xmm12
	movq 16(%rsp), %xmm13
	movq 24(%rsp), %xmm14
	movq 32(%rsp), %xmm15
	DECREASE_FRAME 8 + 4*8
	// TODO: optimize by not restoring callee-saves restored by the ABI
	POP rbx
	POP rbp
	POP r12
	POP r13
	POP r14
	POP r15
	END_MACRO

	/*
	* Macro that sets up the callee save frame to conform with
	* Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs), except for storing the method.
	*/
	MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY)
	// Save callee and GPR args, mixed together to agree with core spills bitmap.
	PUSH r15 // Callee save.
	PUSH r14 // Callee save.
	PUSH r13 // Callee save.
	PUSH r12 // Callee save.
	PUSH_ARG r9 // Quick arg 5.
	PUSH_ARG r8 // Quick arg 4.
	PUSH_ARG rsi // Quick arg 1.
	PUSH rbp // Callee save.
	PUSH rbx // Callee save.
	PUSH_ARG rdx // Quick arg 2.
	PUSH_ARG rcx // Quick arg 3.
	// Create space for FPR args and create 2 slots for ArtMethod*.
	INCREASE_FRAME 16 + 12 * 8
	// Save FPRs.
	movq %xmm0, 16(%rsp)
	movq %xmm1, 24(%rsp)
	movq %xmm2, 32(%rsp)
	movq %xmm3, 40(%rsp)
	movq %xmm4, 48(%rsp)
	movq %xmm5, 56(%rsp)
	movq %xmm6, 64(%rsp)
	movq %xmm7, 72(%rsp)
	movq %xmm12, 80(%rsp)
	movq %xmm13, 88(%rsp)
	movq %xmm14, 96(%rsp)
	movq %xmm15, 104(%rsp)

	// Ugly compile-time check, but we only have the preprocessor.
	// Last +8: implicit return address pushed on stack when caller made call.
	#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 11 * 8 + 12 * 8 + 16 + 8)
	#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(X86_64) size not as expected."
	#endif
	END_MACRO

	MACRO0(RESTORE_SAVE_REFS_AND_ARGS_FRAME)
	// Restore FPRs.
	movq 16(%rsp), %xmm0
	movq 24(%rsp), %xmm1
	movq 32(%rsp), %xmm2
	movq 40(%rsp), %xmm3
	movq 48(%rsp), %xmm4
	movq 56(%rsp), %xmm5
	movq 64(%rsp), %xmm6
	movq 72(%rsp), %xmm7
	movq 80(%rsp), %xmm12
	movq 88(%rsp), %xmm13
	movq 96(%rsp), %xmm14
	movq 104(%rsp), %xmm15
	DECREASE_FRAME 80 + 4 * 8
	// Restore callee and GPR args, mixed together to agree with core spills bitmap.
	POP_ARG rcx
	POP_ARG rdx
	POP rbx
	POP rbp
	POP_ARG rsi
	POP_ARG r8
	POP_ARG r9
	POP r12
	POP r13
	POP r14
	POP r15
	END_MACRO

	/*
	* Macro that sets up the callee save frame to conform with
	* Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves)
	*/
	MACRO0(SETUP_SAVE_ALL_CALLEE_SAVES_FRAME)
	#if defined(__APPLE__)
	int3
	int3
	#else
	// R10 := Runtime::Current()
	LOAD_RUNTIME_INSTANCE r10
	// Save callee save registers to agree with core spills bitmap.
	PUSH r15 // Callee save.
	PUSH r14 // Callee save.
	PUSH r13 // Callee save.
	PUSH r12 // Callee save.
	PUSH rbp // Callee save.
	PUSH rbx // Callee save.
	// Create space for FPR args, plus space for ArtMethod*.
	INCREASE_FRAME 4 * 8 + 8
	// Save FPRs.
	movq %xmm12, 8(%rsp)
	movq %xmm13, 16(%rsp)
	movq %xmm14, 24(%rsp)
	movq %xmm15, 32(%rsp)
	// R10 := ArtMethod* for save all callee save frame method.
	movq RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET(%r10), %r10
	// Store ArtMethod* to bottom of stack.
	movq %r10, 0(%rsp)
	// Store rsp as the top quick frame.
	movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET

	// Ugly compile-time check, but we only have the preprocessor.
	// Last +8: implicit return address pushed on stack when caller made call.
	#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 6 * 8 + 4 * 8 + 8 + 8)
	#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(X86_64) size not as expected."
	#endif
	#endif // __APPLE__
	END_MACRO

	MACRO0(SETUP_FP_CALLEE_SAVE_FRAME)
	// Create space for ART FP callee-saved registers
	INCREASE_FRAME 4 * 8
	movq %xmm12, 0(%rsp)
	movq %xmm13, 8(%rsp)
	movq %xmm14, 16(%rsp)
	movq %xmm15, 24(%rsp)
	END_MACRO

	MACRO0(RESTORE_FP_CALLEE_SAVE_FRAME)
	// Restore ART FP callee-saved registers
	movq 0(%rsp), %xmm12
	movq 8(%rsp), %xmm13
	movq 16(%rsp), %xmm14
	movq 24(%rsp), %xmm15
	DECREASE_FRAME 4 * 8
	END_MACRO

	/*
	* Macro that calls through to artDeliverPendingExceptionFromCode, where the pending
	* exception is Thread::Current()->exception_ when the runtime method frame is ready.
	*/
	MACRO0(DELIVER_PENDING_EXCEPTION_FRAME_READY)
	// (Thread*) setup
	movq %gs:THREAD_SELF_OFFSET, %rdi
	call SYMBOL(artDeliverPendingExceptionFromCode) // artDeliverPendingExceptionFromCode(Thread*)
	UNREACHABLE
	END_MACRO
	/*
	* Macro that calls through to artDeliverPendingExceptionFromCode, where the pending
	* exception is Thread::Current()->exception_.
	*/
	MACRO0(DELIVER_PENDING_EXCEPTION)
	SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save callee saves for throw
	DELIVER_PENDING_EXCEPTION_FRAME_READY
	END_MACRO

	MACRO0(RETURN_OR_DELIVER_PENDING_EXCEPTION)
	movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field
	testq %rcx, %rcx // rcx == 0 ?
	jnz 1f // if rcx != 0 goto 1
	ret // return
	1: // deliver exception on current thread
	DELIVER_PENDING_EXCEPTION
	END_MACRO

	// Locking is needed for both managed code and JNI stubs.
	MACRO3(LOCK_OBJECT_FAST_PATH, obj, tmp, slow_lock)
	1:
	movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax // EAX := lock word
	movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp) // tmp: thread id.
	xorl %eax, REG_VAR(tmp) // tmp: thread id with count 0 + read barrier bits.
	testl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %eax // Test the non-gc bits.
	jnz 2f // Check if unlocked.
	// Unlocked case - store tmp: original lock word plus thread id, preserved read barrier bits.
	lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	jnz 1b // cmpxchg failed retry
	ret
	2: // EAX: original lock word, tmp: thread id ^ EAX
	// Check lock word state and thread id together,
	testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED \| LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \
	REG_VAR(tmp)
	jne \slow_lock // Slow path if either of the two high bits are set.
	// Increment the recursive lock count.
	leal LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp)
	testl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_MASK_SHIFTED), REG_VAR(tmp)
	je \slow_lock // If count overflowed, go to slow lock.
	// Update lockword for recursive lock, cmpxchg necessary for read barrier bits.
	// EAX: old val, tmp: new val.
	lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	jnz 1b // cmpxchg failed retry
	ret
	END_MACRO

	// Unlocking is needed for both managed code and JNI stubs.
	MACRO4(UNLOCK_OBJECT_FAST_PATH, obj, tmp, saved_rax, slow_unlock)
	1:
	movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax // EAX := lock word
	movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp) // tmp := thread id
	xorl %eax, REG_VAR(tmp) // tmp := thread id ^ lock word
	test LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), REG_VAR(tmp)
	jnz 2f // Check if simply locked.
	// Transition to unlocked.
	#ifndef USE_READ_BARRIER
	movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	#else
	lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	jnz 1b // cmpxchg failed retry
	#endif
	.ifnc \saved_rax, none
	movq REG_VAR(saved_rax), %rax // Restore RAX.
	.endif
	ret
	2: // EAX: original lock word, tmp: lock_word ^ thread id
	// Check lock word state and thread id together.
	testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED \| LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \
	REG_VAR(tmp)
	jnz \slow_unlock
	// Update lockword for recursive unlock, cmpxchg necessary for read barrier bits.
	// tmp: new lock word with decremented count.
	leal -LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp)
	#ifndef USE_READ_BARRIER
	// EAX: new lock word with decremented count.
	movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	#else
	lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj))
	jnz 1b // cmpxchg failed retry
	#endif
	.ifnc \saved_rax, none
	movq REG_VAR(saved_rax), %rax // Restore RAX.
	.endif
	ret
	END_MACRO

	#endif // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_