| / |
| / Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved. |
| / DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| / |
| / This code is free software; you can redistribute it and/or modify it |
| / under the terms of the GNU General Public License version 2 only, as |
| / published by the Free Software Foundation. |
| / |
| / This code is distributed in the hope that it will be useful, but WITHOUT |
| / ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| / FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| / version 2 for more details (a copy is included in the LICENSE file that |
| / accompanied this code). |
| / |
| / You should have received a copy of the GNU General Public License version |
| / 2 along with this work; if not, write to the Free Software Foundation, |
| / Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| / |
| / Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| / or visit www.oracle.com if you need additional information or have any |
| / questions. |
| / |
| |
| .globl fs_load |
| .globl fs_thread |
| |
| // NOTE WELL! The _Copy functions are called directly |
| // from server-compiler-generated code via CallLeafNoFP, |
| // which means that they *must* either not use floating |
| // point or use it in the same manner as does the server |
| // compiler. |
| |
| .globl _Copy_arrayof_conjoint_bytes |
| .globl _Copy_conjoint_jshorts_atomic |
| .globl _Copy_arrayof_conjoint_jshorts |
| .globl _Copy_conjoint_jints_atomic |
| .globl _Copy_arrayof_conjoint_jints |
| .globl _Copy_conjoint_jlongs_atomic |
| .globl _Copy_arrayof_conjoint_jlongs |
| |
| .section .text,"ax" |
| |
| / Fast thread accessors, used by threadLS_solaris_amd64.cpp |
| .align 16 |
| fs_load: |
| movq %fs:(%rdi),%rax |
| ret |
| |
| .align 16 |
| fs_thread: |
| movq %fs:0x0,%rax |
| ret |
| |
| .globl SpinPause |
| .align 16 |
| SpinPause: |
| rep |
| nop |
| movq $1, %rax |
| ret |
| |
| |
| / Support for void Copy::arrayof_conjoint_bytes(void* from, |
| / void* to, |
| / size_t count) |
| / rdi - from |
| / rsi - to |
| / rdx - count, treated as ssize_t |
| / |
| .align 16 |
| _Copy_arrayof_conjoint_bytes: |
| movq %rdx,%r8 / byte count |
| shrq $3,%rdx / qword count |
| cmpq %rdi,%rsi |
| leaq -1(%rdi,%r8,1),%rax / from + bcount*1 - 1 |
| jbe acb_CopyRight |
| cmpq %rax,%rsi |
| jbe acb_CopyLeft |
| acb_CopyRight: |
| leaq -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8 |
| leaq -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8 |
| negq %rdx |
| jmp 7f |
| .align 16 |
| 1: movq 8(%rax,%rdx,8),%rsi |
| movq %rsi,8(%rcx,%rdx,8) |
| addq $1,%rdx |
| jnz 1b |
| 2: testq $4,%r8 / check for trailing dword |
| jz 3f |
| movl 8(%rax),%esi / copy trailing dword |
| movl %esi,8(%rcx) |
| addq $4,%rax |
| addq $4,%rcx / original %rsi is trashed, so we |
| / can't use it as a base register |
| 3: testq $2,%r8 / check for trailing word |
| jz 4f |
| movw 8(%rax),%si / copy trailing word |
| movw %si,8(%rcx) |
| addq $2,%rcx |
| 4: testq $1,%r8 / check for trailing byte |
| jz 5f |
| movb -1(%rdi,%r8,1),%al / copy trailing byte |
| movb %al,8(%rcx) |
| 5: ret |
| .align 16 |
| 6: movq -24(%rax,%rdx,8),%rsi |
| movq %rsi,-24(%rcx,%rdx,8) |
| movq -16(%rax,%rdx,8),%rsi |
| movq %rsi,-16(%rcx,%rdx,8) |
| movq -8(%rax,%rdx,8),%rsi |
| movq %rsi,-8(%rcx,%rdx,8) |
| movq (%rax,%rdx,8),%rsi |
| movq %rsi,(%rcx,%rdx,8) |
| 7: addq $4,%rdx |
| jle 6b |
| subq $4,%rdx |
| jl 1b |
| jmp 2b |
| acb_CopyLeft: |
| testq $1,%r8 / check for trailing byte |
| jz 1f |
| movb -1(%rdi,%r8,1),%cl / copy trailing byte |
| movb %cl,-1(%rsi,%r8,1) |
| subq $1,%r8 / adjust for possible trailing word |
| 1: testq $2,%r8 / check for trailing word |
| jz 2f |
| movw -2(%rdi,%r8,1),%cx / copy trailing word |
| movw %cx,-2(%rsi,%r8,1) |
| 2: testq $4,%r8 / check for trailing dword |
| jz 5f |
| movl (%rdi,%rdx,8),%ecx / copy trailing dword |
| movl %ecx,(%rsi,%rdx,8) |
| jmp 5f |
| .align 16 |
| 3: movq -8(%rdi,%rdx,8),%rcx |
| movq %rcx,-8(%rsi,%rdx,8) |
| subq $1,%rdx |
| jnz 3b |
| ret |
| .align 16 |
| 4: movq 24(%rdi,%rdx,8),%rcx |
| movq %rcx,24(%rsi,%rdx,8) |
| movq 16(%rdi,%rdx,8),%rcx |
| movq %rcx,16(%rsi,%rdx,8) |
| movq 8(%rdi,%rdx,8),%rcx |
| movq %rcx,8(%rsi,%rdx,8) |
| movq (%rdi,%rdx,8),%rcx |
| movq %rcx,(%rsi,%rdx,8) |
| 5: subq $4,%rdx |
| jge 4b |
| addq $4,%rdx |
| jg 3b |
| ret |
| |
| / Support for void Copy::arrayof_conjoint_jshorts(void* from, |
| / void* to, |
| / size_t count) |
| / Equivalent to |
| / conjoint_jshorts_atomic |
| / |
| / If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we |
| / let the hardware handle it. The tow or four words within dwords |
| / or qwords that span cache line boundaries will still be loaded |
| / and stored atomically. |
| / |
| / rdi - from |
| / rsi - to |
| / rdx - count, treated as ssize_t |
| / |
| .align 16 |
| _Copy_arrayof_conjoint_jshorts: |
| _Copy_conjoint_jshorts_atomic: |
| movq %rdx,%r8 / word count |
| shrq $2,%rdx / qword count |
| cmpq %rdi,%rsi |
| leaq -2(%rdi,%r8,2),%rax / from + wcount*2 - 2 |
| jbe acs_CopyRight |
| cmpq %rax,%rsi |
| jbe acs_CopyLeft |
| acs_CopyRight: |
| leaq -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8 |
| leaq -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8 |
| negq %rdx |
| jmp 6f |
| 1: movq 8(%rax,%rdx,8),%rsi |
| movq %rsi,8(%rcx,%rdx,8) |
| addq $1,%rdx |
| jnz 1b |
| 2: testq $2,%r8 / check for trailing dword |
| jz 3f |
| movl 8(%rax),%esi / copy trailing dword |
| movl %esi,8(%rcx) |
| addq $4,%rcx / original %rsi is trashed, so we |
| / can't use it as a base register |
| 3: testq $1,%r8 / check for trailing word |
| jz 4f |
| movw -2(%rdi,%r8,2),%si / copy trailing word |
| movw %si,8(%rcx) |
| 4: ret |
| .align 16 |
| 5: movq -24(%rax,%rdx,8),%rsi |
| movq %rsi,-24(%rcx,%rdx,8) |
| movq -16(%rax,%rdx,8),%rsi |
| movq %rsi,-16(%rcx,%rdx,8) |
| movq -8(%rax,%rdx,8),%rsi |
| movq %rsi,-8(%rcx,%rdx,8) |
| movq (%rax,%rdx,8),%rsi |
| movq %rsi,(%rcx,%rdx,8) |
| 6: addq $4,%rdx |
| jle 5b |
| subq $4,%rdx |
| jl 1b |
| jmp 2b |
| acs_CopyLeft: |
| testq $1,%r8 / check for trailing word |
| jz 1f |
| movw -2(%rdi,%r8,2),%cx / copy trailing word |
| movw %cx,-2(%rsi,%r8,2) |
| 1: testq $2,%r8 / check for trailing dword |
| jz 4f |
| movl (%rdi,%rdx,8),%ecx / copy trailing dword |
| movl %ecx,(%rsi,%rdx,8) |
| jmp 4f |
| 2: movq -8(%rdi,%rdx,8),%rcx |
| movq %rcx,-8(%rsi,%rdx,8) |
| subq $1,%rdx |
| jnz 2b |
| ret |
| .align 16 |
| 3: movq 24(%rdi,%rdx,8),%rcx |
| movq %rcx,24(%rsi,%rdx,8) |
| movq 16(%rdi,%rdx,8),%rcx |
| movq %rcx,16(%rsi,%rdx,8) |
| movq 8(%rdi,%rdx,8),%rcx |
| movq %rcx,8(%rsi,%rdx,8) |
| movq (%rdi,%rdx,8),%rcx |
| movq %rcx,(%rsi,%rdx,8) |
| 4: subq $4,%rdx |
| jge 3b |
| addq $4,%rdx |
| jg 2b |
| ret |
| |
| / Support for void Copy::arrayof_conjoint_jints(jint* from, |
| / jint* to, |
| / size_t count) |
| / Equivalent to |
| / conjoint_jints_atomic |
| / |
| / If 'from' and/or 'to' are aligned on 4-byte boundaries, we let |
| / the hardware handle it. The two dwords within qwords that span |
| / cache line boundaries will still be loaded and stored atomically. |
| / |
| / rdi - from |
| / rsi - to |
| / rdx - count, treated as ssize_t |
| / |
| .align 16 |
| _Copy_arrayof_conjoint_jints: |
| _Copy_conjoint_jints_atomic: |
| movq %rdx,%r8 / dword count |
| shrq %rdx / qword count |
| cmpq %rdi,%rsi |
| leaq -4(%rdi,%r8,4),%rax / from + dcount*4 - 4 |
| jbe aci_CopyRight |
| cmpq %rax,%rsi |
| jbe aci_CopyLeft |
| aci_CopyRight: |
| leaq -8(%rdi,%rdx,8),%rax / from + qcount*8 - 8 |
| leaq -8(%rsi,%rdx,8),%rcx / to + qcount*8 - 8 |
| negq %rdx |
| jmp 5f |
| .align 16 |
| 1: movq 8(%rax,%rdx,8),%rsi |
| movq %rsi,8(%rcx,%rdx,8) |
| addq $1,%rdx |
| jnz 1b |
| 2: testq $1,%r8 / check for trailing dword |
| jz 3f |
| movl 8(%rax),%esi / copy trailing dword |
| movl %esi,8(%rcx) |
| 3: ret |
| .align 16 |
| 4: movq -24(%rax,%rdx,8),%rsi |
| movq %rsi,-24(%rcx,%rdx,8) |
| movq -16(%rax,%rdx,8),%rsi |
| movq %rsi,-16(%rcx,%rdx,8) |
| movq -8(%rax,%rdx,8),%rsi |
| movq %rsi,-8(%rcx,%rdx,8) |
| movq (%rax,%rdx,8),%rsi |
| movq %rsi,(%rcx,%rdx,8) |
| 5: addq $4,%rdx |
| jle 4b |
| subq $4,%rdx |
| jl 1b |
| jmp 2b |
| aci_CopyLeft: |
| testq $1,%r8 / check for trailing dword |
| jz 3f |
| movl -4(%rdi,%r8,4),%ecx / copy trailing dword |
| movl %ecx,-4(%rsi,%r8,4) |
| jmp 3f |
| 1: movq -8(%rdi,%rdx,8),%rcx |
| movq %rcx,-8(%rsi,%rdx,8) |
| subq $1,%rdx |
| jnz 1b |
| ret |
| .align 16 |
| 2: movq 24(%rdi,%rdx,8),%rcx |
| movq %rcx,24(%rsi,%rdx,8) |
| movq 16(%rdi,%rdx,8),%rcx |
| movq %rcx,16(%rsi,%rdx,8) |
| movq 8(%rdi,%rdx,8),%rcx |
| movq %rcx,8(%rsi,%rdx,8) |
| movq (%rdi,%rdx,8),%rcx |
| movq %rcx,(%rsi,%rdx,8) |
| 3: subq $4,%rdx |
| jge 2b |
| addq $4,%rdx |
| jg 1b |
| ret |
| |
| / Support for void Copy::arrayof_conjoint_jlongs(jlong* from, |
| / jlong* to, |
| / size_t count) |
| / Equivalent to |
| / conjoint_jlongs_atomic |
| / arrayof_conjoint_oops |
| / conjoint_oops_atomic |
| / |
| / rdi - from |
| / rsi - to |
| / rdx - count, treated as ssize_t |
| / |
| .align 16 |
| _Copy_arrayof_conjoint_jlongs: |
| _Copy_conjoint_jlongs_atomic: |
| cmpq %rdi,%rsi |
| leaq -8(%rdi,%rdx,8),%rax / from + count*8 - 8 |
| jbe acl_CopyRight |
| cmpq %rax,%rsi |
| jbe acl_CopyLeft |
| acl_CopyRight: |
| leaq -8(%rsi,%rdx,8),%rcx / to + count*8 - 8 |
| negq %rdx |
| jmp 3f |
| 1: movq 8(%rax,%rdx,8),%rsi |
| movq %rsi,8(%rcx,%rdx,8) |
| addq $1,%rdx |
| jnz 1b |
| ret |
| .align 16 |
| 2: movq -24(%rax,%rdx,8),%rsi |
| movq %rsi,-24(%rcx,%rdx,8) |
| movq -16(%rax,%rdx,8),%rsi |
| movq %rsi,-16(%rcx,%rdx,8) |
| movq -8(%rax,%rdx,8),%rsi |
| movq %rsi,-8(%rcx,%rdx,8) |
| movq (%rax,%rdx,8),%rsi |
| movq %rsi,(%rcx,%rdx,8) |
| 3: addq $4,%rdx |
| jle 2b |
| subq $4,%rdx |
| jl 1b |
| ret |
| 4: movq -8(%rdi,%rdx,8),%rcx |
| movq %rcx,-8(%rsi,%rdx,8) |
| subq $1,%rdx |
| jnz 4b |
| ret |
| .align 16 |
| 5: movq 24(%rdi,%rdx,8),%rcx |
| movq %rcx,24(%rsi,%rdx,8) |
| movq 16(%rdi,%rdx,8),%rcx |
| movq %rcx,16(%rsi,%rdx,8) |
| movq 8(%rdi,%rdx,8),%rcx |
| movq %rcx,8(%rsi,%rdx,8) |
| movq (%rdi,%rdx,8),%rcx |
| movq %rcx,(%rsi,%rdx,8) |
| acl_CopyLeft: |
| subq $4,%rdx |
| jge 5b |
| addq $4,%rdx |
| jg 4b |
| ret |
