8261649: AArch64: Optimize LSE atomics in C++ code
Reviewed-by: adinn
diff --git a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
index dc35fc4..ac12ba9 100644
--- a/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/atomic_aarch64.hpp
@@ -42,5 +42,8 @@
extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_impl;
extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_4_impl;
extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_1_relaxed_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_4_relaxed_impl;
+extern aarch64_atomic_stub_t aarch64_atomic_cmpxchg_8_relaxed_impl;
#endif // CPU_AARCH64_ATOMIC_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
index 813f466..852c4f7 100644
--- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp
@@ -4687,87 +4687,167 @@
}
#ifdef LINUX
+
// ARMv8.1 LSE versions of the atomic stubs used by Atomic::PlatformXX.
//
// If LSE is in use, generate LSE versions of all the stubs. The
// non-LSE versions are in atomic_aarch64.S.
- void generate_atomic_entry_points() {
+ // class AtomicStubMark records the entry point of a stub and the
+ // stub pointer which will point to it. The stub pointer is set to
+ // the entry point when ~AtomicStubMark() is called, which must be
+ // after ICache::invalidate_range. This ensures safe publication of
+ // the generated code.
+ class AtomicStubMark {
+ address _entry_point;
+ aarch64_atomic_stub_t *_stub;
+ MacroAssembler *_masm;
+ public:
+ AtomicStubMark(MacroAssembler *masm, aarch64_atomic_stub_t *stub) {
+ _masm = masm;
+ __ align(32);
+ _entry_point = __ pc();
+ _stub = stub;
+ }
+ ~AtomicStubMark() {
+ *_stub = (aarch64_atomic_stub_t)_entry_point;
+ }
+ };
+
+ // NB: For memory_order_conservative we need a trailing membar after
+ // LSE atomic operations but not a leading membar.
+ //
+ // We don't need a leading membar because a clause in the Arm ARM
+ // says:
+ //
+ // Barrier-ordered-before
+ //
+ // Barrier instructions order prior Memory effects before subsequent
+ // Memory effects generated by the same Observer. A read or a write
+ // RW1 is Barrier-ordered-before a read or a write RW 2 from the same
+ // Observer if and only if RW1 appears in program order before RW 2
+ // and [ ... ] at least one of RW 1 and RW 2 is generated by an atomic
+ // instruction with both Acquire and Release semantics.
+ //
+ // All the atomic instructions {ldaddal, swapal, casal} have Acquire
+ // and Release semantics, therefore we don't need a leading
+ // barrier. However, there is no corresponding Barrier-ordered-after
+ // relationship, therefore we need a trailing membar to prevent a
+ // later store or load from being reordered with the store in an
+ // atomic instruction.
+ //
+ // This was checked by using the herd7 consistency model simulator
+ // (http://diy.inria.fr/) with this test case:
+ //
+ // AArch64 LseCas
+ // { 0:X1=x; 0:X2=y; 1:X1=x; 1:X2=y; }
+ // P0 | P1;
+ // LDR W4, [X2] | MOV W3, #0;
+ // DMB LD | MOV W4, #1;
+ // LDR W3, [X1] | CASAL W3, W4, [X1];
+ // | DMB ISH;
+ // | STR W4, [X2];
+ // exists
+ // (0:X3=0 /\ 0:X4=1)
+ //
+ // If X3 == 0 && X4 == 1, the store to y in P1 has been reordered
+ // with the store to x in P1. Without the DMB in P1 this may happen.
+ //
+ // At the time of writing we don't know of any AArch64 hardware that
+ // reorders stores in this way, but the Reference Manual permits it.
+
+ void gen_cas_entry(Assembler::operand_size size,
+ atomic_memory_order order) {
+ Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
+ exchange_val = c_rarg2;
+ bool acquire, release;
+ switch (order) {
+ case memory_order_relaxed:
+ acquire = false;
+ release = false;
+ break;
+ default:
+ acquire = true;
+ release = true;
+ break;
+ }
+ __ mov(prev, compare_val);
+ __ lse_cas(prev, exchange_val, ptr, size, acquire, release, /*not_pair*/true);
+ if (order == memory_order_conservative) {
+ __ membar(Assembler::StoreStore|Assembler::StoreLoad);
+ }
+ if (size == Assembler::xword) {
+ __ mov(r0, prev);
+ } else {
+ __ movw(r0, prev);
+ }
+ __ ret(lr);
+ }
+
+ void gen_ldaddal_entry(Assembler::operand_size size) {
+ Register prev = r2, addr = c_rarg0, incr = c_rarg1;
+ __ ldaddal(size, incr, prev, addr);
+ __ membar(Assembler::StoreStore|Assembler::StoreLoad);
+ if (size == Assembler::xword) {
+ __ mov(r0, prev);
+ } else {
+ __ movw(r0, prev);
+ }
+ __ ret(lr);
+ }
+
+ void gen_swpal_entry(Assembler::operand_size size) {
+ Register prev = r2, addr = c_rarg0, incr = c_rarg1;
+ __ swpal(size, incr, prev, addr);
+ __ membar(Assembler::StoreStore|Assembler::StoreLoad);
+ if (size == Assembler::xword) {
+ __ mov(r0, prev);
+ } else {
+ __ movw(r0, prev);
+ }
+ __ ret(lr);
+ }
+
+ void generate_atomic_entry_points() {
if (! UseLSE) {
return;
}
__ align(CodeEntryAlignment);
StubCodeMark mark(this, "StubRoutines", "atomic entry points");
+ address first_entry = __ pc();
- __ align(32);
- aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r2, addr = c_rarg0, incr = c_rarg1;
- __ atomic_addal(prev, incr, addr);
- __ mov(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r2, addr = c_rarg0, incr = c_rarg1;
- __ atomic_addalw(prev, incr, addr);
- __ movw(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r2, addr = c_rarg0, newv = c_rarg1;
- __ atomic_xchglw(prev, newv, addr);
- __ movw(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r2, addr = c_rarg0, newv = c_rarg1;
- __ atomic_xchgl(prev, newv, addr);
- __ mov(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
- exchange_val = c_rarg2;
- __ cmpxchg(ptr, compare_val, exchange_val,
- MacroAssembler::byte,
- /*acquire*/false, /*release*/false, /*weak*/false,
- prev);
- __ movw(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
- exchange_val = c_rarg2;
- __ cmpxchg(ptr, compare_val, exchange_val,
- MacroAssembler::word,
- /*acquire*/false, /*release*/false, /*weak*/false,
- prev);
- __ movw(r0, prev);
- __ ret(lr);
- }
- __ align(32);
- aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)__ pc();
- {
- Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
- exchange_val = c_rarg2;
- __ cmpxchg(ptr, compare_val, exchange_val,
- MacroAssembler::xword,
- /*acquire*/false, /*release*/false, /*weak*/false,
- prev);
- __ mov(r0, prev);
- __ ret(lr);
- }
+ // All memory_order_conservative
+ AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
+ gen_ldaddal_entry(Assembler::word);
+ AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
+ gen_ldaddal_entry(Assembler::xword);
+
+ AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
+ gen_swpal_entry(Assembler::word);
+ AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
+ gen_swpal_entry(Assembler::xword);
+
+ // CAS, memory_order_conservative
+ AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
+ gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
+ AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
+ gen_cas_entry(MacroAssembler::word, memory_order_conservative);
+ AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
+ gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
+
+ // CAS, memory_order_relaxed
+ AtomicStubMark mark_cmpxchg_1_relaxed
+ (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
+ gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
+ AtomicStubMark mark_cmpxchg_4_relaxed
+ (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
+ gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
+ AtomicStubMark mark_cmpxchg_8_relaxed
+ (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
+ gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
+
+ ICache::invalidate_range(first_entry, __ pc() - first_entry);
}
#endif // LINUX
@@ -5854,9 +5934,7 @@
&StubRoutines::_safefetchN_continuation_pc);
#ifdef LINUX
-#if 0 // JDK-8261660: disabled for now.
generate_atomic_entry_points();
-#endif
#endif // LINUX
@@ -5883,19 +5961,22 @@
// Define pointers to atomic stubs and initialize them to point to the
// code in atomic_aarch64.S.
-#define DEFAULT_ATOMIC_OP(OPNAME, SIZE) \
- extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## _default_impl \
+#define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED) \
+ extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
(volatile void *ptr, uint64_t arg1, uint64_t arg2); \
- aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## _impl \
- = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## _default_impl;
+ aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
+ = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
-DEFAULT_ATOMIC_OP(fetch_add, 4)
-DEFAULT_ATOMIC_OP(fetch_add, 8)
-DEFAULT_ATOMIC_OP(xchg, 4)
-DEFAULT_ATOMIC_OP(xchg, 8)
-DEFAULT_ATOMIC_OP(cmpxchg, 1)
-DEFAULT_ATOMIC_OP(cmpxchg, 4)
-DEFAULT_ATOMIC_OP(cmpxchg, 8)
+DEFAULT_ATOMIC_OP(fetch_add, 4, )
+DEFAULT_ATOMIC_OP(fetch_add, 8, )
+DEFAULT_ATOMIC_OP(xchg, 4, )
+DEFAULT_ATOMIC_OP(xchg, 8, )
+DEFAULT_ATOMIC_OP(cmpxchg, 1, )
+DEFAULT_ATOMIC_OP(cmpxchg, 4, )
+DEFAULT_ATOMIC_OP(cmpxchg, 8, )
+DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
+DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
+DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
#undef DEFAULT_ATOMIC_OP
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
index 512b51c..c6e4dd0 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
@@ -26,44 +26,96 @@
.globl aarch64_atomic_fetch_add_8_default_impl
.align 5
aarch64_atomic_fetch_add_8_default_impl:
+ prfm pstl1strm, [x0]
0: ldaxr x2, [x0]
add x8, x2, x1
stlxr w9, x8, [x0]
cbnz w9, 0b
+ dmb ish
mov x0, x2
ret
.globl aarch64_atomic_fetch_add_4_default_impl
.align 5
aarch64_atomic_fetch_add_4_default_impl:
+ prfm pstl1strm, [x0]
0: ldaxr w2, [x0]
add w8, w2, w1
stlxr w9, w8, [x0]
cbnz w9, 0b
+ dmb ish
mov w0, w2
ret
.globl aarch64_atomic_xchg_4_default_impl
.align 5
aarch64_atomic_xchg_4_default_impl:
+ prfm pstl1strm, [x0]
0: ldaxr w2, [x0]
stlxr w8, w1, [x0]
cbnz w8, 0b
+ dmb ish
mov w0, w2
ret
.globl aarch64_atomic_xchg_8_default_impl
.align 5
aarch64_atomic_xchg_8_default_impl:
+ prfm pstl1strm, [x0]
0: ldaxr x2, [x0]
stlxr w8, x1, [x0]
cbnz w8, 0b
+ dmb ish
mov x0, x2
ret
.globl aarch64_atomic_cmpxchg_1_default_impl
.align 5
aarch64_atomic_cmpxchg_1_default_impl:
+ dmb ish
+ prfm pstl1strm, [x0]
+0: ldxrb w3, [x0]
+ eor w8, w3, w1
+ tst x8, #0xff
+ b.ne 1f
+ stxrb w8, w2, [x0]
+ cbnz w8, 0b
+1: mov w0, w3
+ dmb ish
+ ret
+
+ .globl aarch64_atomic_cmpxchg_4_default_impl
+ .align 5
+aarch64_atomic_cmpxchg_4_default_impl:
+ dmb ish
+ prfm pstl1strm, [x0]
+0: ldxr w3, [x0]
+ cmp w3, w1
+ b.ne 1f
+ stxr w8, w2, [x0]
+ cbnz w8, 0b
+1: mov w0, w3
+ dmb ish
+ ret
+
+ .globl aarch64_atomic_cmpxchg_8_default_impl
+ .align 5
+aarch64_atomic_cmpxchg_8_default_impl:
+ dmb ish
+ prfm pstl1strm, [x0]
+0: ldxr x3, [x0]
+ cmp x3, x1
+ b.ne 1f
+ stxr w8, x2, [x0]
+ cbnz w8, 0b
+1: mov x0, x3
+ dmb ish
+ ret
+
+ .globl aarch64_atomic_cmpxchg_1_relaxed_default_impl
+ .align 5
+aarch64_atomic_cmpxchg_1_relaxed_default_impl:
+ prfm pstl1strm, [x0]
0: ldxrb w3, [x0]
eor w8, w3, w1
tst x8, #0xff
@@ -73,9 +125,10 @@
1: mov w0, w3
ret
- .globl aarch64_atomic_cmpxchg_4_default_impl
+ .globl aarch64_atomic_cmpxchg_4_relaxed_default_impl
.align 5
-aarch64_atomic_cmpxchg_4_default_impl:
+aarch64_atomic_cmpxchg_4_relaxed_default_impl:
+ prfm pstl1strm, [x0]
0: ldxr w3, [x0]
cmp w3, w1
b.ne 1f
@@ -84,9 +137,10 @@
1: mov w0, w3
ret
- .globl aarch64_atomic_cmpxchg_8_default_impl
+ .globl aarch64_atomic_cmpxchg_8_relaxed_default_impl
.align 5
-aarch64_atomic_cmpxchg_8_default_impl:
+aarch64_atomic_cmpxchg_8_relaxed_default_impl:
+ prfm pstl1strm, [x0]
0: ldxr x3, [x0]
cmp x3, x1
b.ne 1f
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
index 8048eae..ce4ae8f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.hpp
@@ -90,7 +90,6 @@
STATIC_ASSERT(4 == sizeof(D));
D old_value
= atomic_fastcall(aarch64_atomic_fetch_add_4_impl, dest, add_value);
- FULL_MEM_BARRIER;
return old_value;
}
@@ -103,7 +102,6 @@
STATIC_ASSERT(8 == sizeof(D));
D old_value
= atomic_fastcall(aarch64_atomic_fetch_add_8_impl, dest, add_value);
- FULL_MEM_BARRIER;
return old_value;
}
@@ -114,7 +112,6 @@
atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
T old_value = atomic_fastcall(aarch64_atomic_xchg_4_impl, dest, exchange_value);
- FULL_MEM_BARRIER;
return old_value;
}
@@ -125,7 +122,6 @@
atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
T old_value = atomic_fastcall(aarch64_atomic_xchg_8_impl, dest, exchange_value);
- FULL_MEM_BARRIER;
return old_value;
}
@@ -136,18 +132,15 @@
T compare_value,
atomic_memory_order order) const {
STATIC_ASSERT(1 == sizeof(T));
- aarch64_atomic_stub_t stub = aarch64_atomic_cmpxchg_1_impl;
- if (order == memory_order_relaxed) {
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- return old_value;
- } else {
- FULL_MEM_BARRIER;
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- FULL_MEM_BARRIER;
- return old_value;
+ aarch64_atomic_stub_t stub;
+ switch (order) {
+ case memory_order_relaxed:
+ stub = aarch64_atomic_cmpxchg_1_relaxed_impl; break;
+ default:
+ stub = aarch64_atomic_cmpxchg_1_impl; break;
}
+
+ return atomic_fastcall(stub, dest, compare_value, exchange_value);
}
template<>
@@ -157,18 +150,15 @@
T compare_value,
atomic_memory_order order) const {
STATIC_ASSERT(4 == sizeof(T));
- aarch64_atomic_stub_t stub = aarch64_atomic_cmpxchg_4_impl;
- if (order == memory_order_relaxed) {
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- return old_value;
- } else {
- FULL_MEM_BARRIER;
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- FULL_MEM_BARRIER;
- return old_value;
+ aarch64_atomic_stub_t stub;
+ switch (order) {
+ case memory_order_relaxed:
+ stub = aarch64_atomic_cmpxchg_4_relaxed_impl; break;
+ default:
+ stub = aarch64_atomic_cmpxchg_4_impl; break;
}
+
+ return atomic_fastcall(stub, dest, compare_value, exchange_value);
}
template<>
@@ -178,18 +168,15 @@
T compare_value,
atomic_memory_order order) const {
STATIC_ASSERT(8 == sizeof(T));
- aarch64_atomic_stub_t stub = aarch64_atomic_cmpxchg_8_impl;
- if (order == memory_order_relaxed) {
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- return old_value;
- } else {
- FULL_MEM_BARRIER;
- T old_value = atomic_fastcall(stub, dest,
- compare_value, exchange_value);
- FULL_MEM_BARRIER;
- return old_value;
+ aarch64_atomic_stub_t stub;
+ switch (order) {
+ case memory_order_relaxed:
+ stub = aarch64_atomic_cmpxchg_8_relaxed_impl; break;
+ default:
+ stub = aarch64_atomic_cmpxchg_8_impl; break;
}
+
+ return atomic_fastcall(stub, dest, compare_value, exchange_value);
}
#endif // OS_CPU_LINUX_AARCH64_VM_ATOMIC_LINUX_AARCH64_HPP
