Use entrypoint switching to reduce code size of GcRoot read barrier
Set the read barrier mark register entrypoints to null when the GC
is not marking. The compiler uses this to avoid needing to load the
is_gc_marking boolean.
Code size results on ritzperf CC:
arm32: 13439400 -> 13242792 (-1.5%)
arm64: 16380544 -> 16208512 (-1.05%)
Implemented for arm32 and arm64. TODO: Consider implementing on x86.
Bug: 32638713
Bug: 29516974
Test: test-art-host + run ritzperf
Change-Id: I527ca5dc4cd43950ba43b872d0ac81e1eb5791eb
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 8ca8b8a..74a74ee 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -620,8 +620,10 @@
// reference (different from `ref`) in `obj.field`).
class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM {
public:
- ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location ref)
- : SlowPathCodeARM(instruction), ref_(ref) {
+ ReadBarrierMarkSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
@@ -676,10 +678,15 @@
//
// rX <- ReadBarrierMarkRegX(rX)
//
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ if (entrypoint_.IsValid()) {
+ arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
+ __ blx(entrypoint_.AsRegister<Register>());
+ } else {
+ int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
+ // This runtime call does not require a stack map.
+ arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ }
__ b(GetExitLabel());
}
@@ -687,6 +694,9 @@
// The location (register) of the marked object reference.
const Location ref_;
+ // The location of the entrypoint if already loaded.
+ const Location entrypoint_;
+
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM);
};
@@ -6829,8 +6839,9 @@
// Baker's read barrier are used:
//
// root = obj.field;
- // if (Thread::Current()->GetIsGcMarking()) {
- // root = ReadBarrier::Mark(root)
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp != null) {
+ // root = temp(root)
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
@@ -6844,14 +6855,23 @@
"have different sizes.");
// Slow path marking the GC root `root`.
+ Location temp = Location::RegisterLocation(LR);
SlowPathCodeARM* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root);
+ new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
+ instruction,
+ root,
+ /*entrypoint*/ temp);
codegen_->AddSlowPath(slow_path);
- // IP = Thread::Current()->GetIsGcMarking()
- __ LoadFromOffset(
- kLoadWord, IP, TR, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value());
- __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel());
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
} else {
// GC root loaded through a slow path for read barriers other
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 6f55b42..7860138 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -607,10 +607,16 @@
// probably still be a from-space reference (unless it gets updated by
// another thread, or if another thread installed another object
// reference (different from `ref`) in `obj.field`).
+// If entrypoint is a valid location it is assumed to already be holding the entrypoint. The case
+// where the entrypoint is passed in is for the GcRoot read barrier.
class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 {
public:
- ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location ref)
- : SlowPathCodeARM64(instruction), ref_(ref) {
+ ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : SlowPathCodeARM64(instruction),
+ ref_(ref),
+ entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
@@ -665,10 +671,16 @@
//
// rX <- ReadBarrierMarkRegX(rX)
//
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
- // This runtime call does not require a stack map.
- arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ if (entrypoint_.IsValid()) {
+ arm64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
+ __ Blr(XRegisterFrom(entrypoint_));
+ } else {
+ // Entrypoint is not already loaded, load from the thread.
+ int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
+ // This runtime call does not require a stack map.
+ arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ }
__ B(GetExitLabel());
}
@@ -676,6 +688,9 @@
// The location (register) of the marked object reference.
const Location ref_;
+ // The location of the entrypoint if it is already loaded.
+ const Location entrypoint_;
+
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64);
};
@@ -5371,8 +5386,9 @@
// Baker's read barrier are used:
//
// root = obj.field;
- // if (Thread::Current()->GetIsGcMarking()) {
- // root = ReadBarrier::Mark(root)
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp != null) {
+ // root = temp(root)
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
@@ -5389,16 +5405,22 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- SlowPathCodeARM64* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root);
- codegen_->AddSlowPath(slow_path);
+ Register temp = lr;
- MacroAssembler* masm = GetVIXLAssembler();
- UseScratchRegisterScope temps(masm);
- Register temp = temps.AcquireW();
- // temp = Thread::Current()->GetIsGcMarking()
- __ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
+ // Slow path marking the GC root `root`. The entrypoint will alrady be loaded in temp.
+ SlowPathCodeARM64* slow_path =
+ new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction,
+ root,
+ LocationFrom(temp));
+ codegen_->AddSlowPath(slow_path);
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp, MemOperand(tr, entry_point_offset));
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
__ Cbnz(temp, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
} else {
diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc
index 01b3f34..5bd6b56 100644
--- a/runtime/arch/arm/entrypoints_init_arm.cc
+++ b/runtime/arch/arm/entrypoints_init_arm.cc
@@ -67,6 +67,21 @@
// Long long arithmetics - REM_LONG[_2ADDR] and DIV_LONG[_2ADDR]
extern "C" int64_t __aeabi_ldivmod(int64_t, int64_t);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -123,18 +138,7 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
- qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04;
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg12 = nullptr; // Cannot use register 12 (IP) to pass arguments.
qpoints->pReadBarrierMarkReg13 = nullptr; // Cannot use register 13 (SP) to pass arguments.
qpoints->pReadBarrierMarkReg14 = nullptr; // Cannot use register 14 (LR) to pass arguments.
diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc
index 3c77672..e7c9fef 100644
--- a/runtime/arch/arm64/entrypoints_init_arm64.cc
+++ b/runtime/arch/arm64/entrypoints_init_arm64.cc
@@ -70,6 +70,47 @@
extern "C" mirror::Object* art_quick_read_barrier_mark_reg28(mirror::Object*);
extern "C" mirror::Object* art_quick_read_barrier_mark_reg29(mirror::Object*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ // ARM64 is the architecture with the largest number of core
+ // registers (32) that supports the read barrier configuration.
+ // Because registers 30 (LR) and 31 (SP/XZR) cannot be used to pass
+ // arguments, only define ReadBarrierMarkRegX entrypoints for the
+ // first 30 registers. This limitation is not a problem on other
+ // supported architectures (ARM, x86 and x86-64) either, as they
+ // have less core registers (resp. 16, 8 and 16). (We may have to
+ // revise that design choice if read barrier support is added for
+ // MIPS and/or MIPS64.)
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+ qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr;
+ qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr;
+ qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr;
+ qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr;
+ qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr;
+ qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr;
+ qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr;
+ qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr;
+ qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr;
+ qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr;
+ qpoints->pReadBarrierMarkReg23 = is_marking ? art_quick_read_barrier_mark_reg23 : nullptr;
+ qpoints->pReadBarrierMarkReg24 = is_marking ? art_quick_read_barrier_mark_reg24 : nullptr;
+ qpoints->pReadBarrierMarkReg25 = is_marking ? art_quick_read_barrier_mark_reg25 : nullptr;
+ qpoints->pReadBarrierMarkReg26 = is_marking ? art_quick_read_barrier_mark_reg26 : nullptr;
+ qpoints->pReadBarrierMarkReg27 = is_marking ? art_quick_read_barrier_mark_reg27 : nullptr;
+ qpoints->pReadBarrierMarkReg28 = is_marking ? art_quick_read_barrier_mark_reg28 : nullptr;
+ qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -126,45 +167,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- // ARM64 is the architecture with the largest number of core
- // registers (32) that supports the read barrier configuration.
- // Because registers 30 (LR) and 31 (SP/XZR) cannot be used to pass
- // arguments, only define ReadBarrierMarkRegX entrypoints for the
- // first 30 registers. This limitation is not a problem on other
- // supported architectures (ARM, x86 and x86-64) either, as they
- // have less core registers (resp. 16, 8 and 16). (We may have to
- // revise that design choice if read barrier support is added for
- // MIPS and/or MIPS64.)
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
- qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04;
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
- qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12;
- qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13;
- qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14;
- qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15;
qpoints->pReadBarrierMarkReg16 = nullptr; // IP0 is used as a temp by the asm stub.
- qpoints->pReadBarrierMarkReg17 = art_quick_read_barrier_mark_reg17;
- qpoints->pReadBarrierMarkReg18 = art_quick_read_barrier_mark_reg18;
- qpoints->pReadBarrierMarkReg19 = art_quick_read_barrier_mark_reg19;
- qpoints->pReadBarrierMarkReg20 = art_quick_read_barrier_mark_reg20;
- qpoints->pReadBarrierMarkReg21 = art_quick_read_barrier_mark_reg21;
- qpoints->pReadBarrierMarkReg22 = art_quick_read_barrier_mark_reg22;
- qpoints->pReadBarrierMarkReg23 = art_quick_read_barrier_mark_reg23;
- qpoints->pReadBarrierMarkReg24 = art_quick_read_barrier_mark_reg24;
- qpoints->pReadBarrierMarkReg25 = art_quick_read_barrier_mark_reg25;
- qpoints->pReadBarrierMarkReg26 = art_quick_read_barrier_mark_reg26;
- qpoints->pReadBarrierMarkReg27 = art_quick_read_barrier_mark_reg27;
- qpoints->pReadBarrierMarkReg28 = art_quick_read_barrier_mark_reg28;
- qpoints->pReadBarrierMarkReg29 = art_quick_read_barrier_mark_reg29;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierSlow = artReadBarrierSlow;
qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
};
diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc
index e3230f6..8c4df0f 100644
--- a/runtime/arch/mips/entrypoints_init_mips.cc
+++ b/runtime/arch/mips/entrypoints_init_mips.cc
@@ -59,6 +59,9 @@
extern "C" int64_t __divdi3(int64_t, int64_t);
extern "C" int64_t __moddi3(int64_t, int64_t);
+// No read barrier entrypoints for marking registers.
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
// Note: MIPS has asserts checking for the type of entrypoint. Don't move it
// to InitDefaultEntryPoints().
diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc
index 43b73f1..0bd4e14 100644
--- a/runtime/arch/mips64/entrypoints_init_mips64.cc
+++ b/runtime/arch/mips64/entrypoints_init_mips64.cc
@@ -59,6 +59,9 @@
extern "C" int64_t __divdi3(int64_t, int64_t);
extern "C" int64_t __moddi3(int64_t, int64_t);
+// No read barrier entrypoints for marking registers.
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc
index 877df8f..94fea69 100644
--- a/runtime/arch/x86/entrypoints_init_x86.cc
+++ b/runtime/arch/x86/entrypoints_init_x86.cc
@@ -44,6 +44,16 @@
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -87,14 +97,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (ESP) to pass arguments.
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
// x86 has only 8 core registers.
qpoints->pReadBarrierMarkReg08 = nullptr;
qpoints->pReadBarrierMarkReg09 = nullptr;
diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
index 59c9dfe..6b66e62 100644
--- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc
+++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
@@ -55,6 +55,24 @@
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+ qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr;
+ qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr;
+ qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr;
+ qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
#if defined(__APPLE__)
UNUSED(jpoints, qpoints);
@@ -101,22 +119,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (RSP) to pass arguments.
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
- qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12;
- qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13;
- qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14;
- qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15;
// x86-64 has only 16 core registers.
qpoints->pReadBarrierMarkReg16 = nullptr;
qpoints->pReadBarrierMarkReg17 = nullptr;
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 11d6849..2e72ada 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -246,7 +246,7 @@
Thread* self = Thread::Current();
CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc)
<< thread->GetState() << " thread " << thread << " self " << self;
- thread->SetIsGcMarking(true);
+ thread->SetIsGcMarkingAndUpdateEntrypoints(true);
if (use_tlab_ && thread->HasTlab()) {
if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) {
// This must come before the revoke.
@@ -746,7 +746,7 @@
// Disable the thread-local is_gc_marking flag.
// Note a thread that has just started right before this checkpoint may have already this flag
// set to false, which is ok.
- thread->SetIsGcMarking(false);
+ thread->SetIsGcMarkingAndUpdateEntrypoints(false);
// If thread is a running mutator, then act on behalf of the garbage collector.
// See the code in ThreadList::RunCheckpoint.
concurrent_copying_->GetBarrier().Pass(self);
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 8ce9661..23c077c 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -116,6 +116,13 @@
}
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking);
+
+void Thread::SetIsGcMarkingAndUpdateEntrypoints(bool is_marking) {
+ CHECK(kUseReadBarrier);
+ tls32_.is_gc_marking = is_marking;
+ UpdateReadBarrierEntrypoints(&tlsPtr_.quick_entrypoints, is_marking);
+}
void Thread::InitTlsEntryPoints() {
// Insert a placeholder so we can easily tell if we call an unimplemented entry point.
diff --git a/runtime/thread.h b/runtime/thread.h
index f3001be..faa77e1 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -845,10 +845,7 @@
return tls32_.is_gc_marking;
}
- void SetIsGcMarking(bool is_marking) {
- CHECK(kUseReadBarrier);
- tls32_.is_gc_marking = is_marking;
- }
+ void SetIsGcMarkingAndUpdateEntrypoints(bool is_marking);
bool GetWeakRefAccessEnabled() const {
CHECK(kUseReadBarrier);
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index eba6666..8a3bb15 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -1242,7 +1242,7 @@
// Initialize according to the state of the CC collector.
bool is_gc_marking =
Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking();
- self->SetIsGcMarking(is_gc_marking);
+ self->SetIsGcMarkingAndUpdateEntrypoints(is_gc_marking);
bool weak_ref_access_enabled =
Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsWeakRefAccessEnabled();
self->SetWeakRefAccessEnabled(weak_ref_access_enabled);
