ARM64: Remove all uses of BlockPoolsScope.
BlockPoolsScope should not be used because it is a VIXL scope
for VIXL internal usage only. In arm64 backend the intent was to
block pools between a particular instruction (Ldr, Str, Blr, etc)
and a subsequent MaybeRecordImplicitNullCheck or RecordPcInfo call.
However pools should be emitted at the opening of a scope if this
is required to satisfy branch/ldr ranges. This is not done by the
BlockPoolsScope, so proper scopes are now used now:
ExactAssemblyScope and EmissionCheckScope.
Test: test-art-host
Test: test-art-target
Bug: 34850123
Change-Id: I30365ad63c644cf9dd85d5a3c2118f9c57be9d20
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 598be47..ec72af9 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -34,6 +34,9 @@
#include "utils/stack_checks.h"
using namespace vixl::aarch64; // NOLINT(build/namespaces)
+using vixl::ExactAssemblyScope;
+using vixl::CodeBufferCheckScope;
+using vixl::EmissionCheckScope;
#ifdef __
#error "ARM64 Codegen VIXL macro-assembler macro already defined."
@@ -590,10 +593,9 @@
// We are about to use the assembler to place literals directly. Make sure we have enough
// underlying code buffer and we have generated the jump table with right size.
- vixl::CodeBufferCheckScope scope(codegen->GetVIXLAssembler(),
- num_entries * sizeof(int32_t),
- vixl::CodeBufferCheckScope::kReserveBufferSpace,
- vixl::CodeBufferCheckScope::kExactSize);
+ EmissionCheckScope scope(codegen->GetVIXLAssembler(),
+ num_entries * sizeof(int32_t),
+ CodeBufferCheckScope::kExactSize);
__ Bind(&table_start_);
const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors();
@@ -1254,7 +1256,6 @@
void CodeGeneratorARM64::GenerateFrameEntry() {
MacroAssembler* masm = GetVIXLAssembler();
- BlockPoolsScope block_pools(masm);
__ Bind(&frame_entry_label_);
bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
@@ -1263,8 +1264,14 @@
Register temp = temps.AcquireX();
DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
__ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
- __ Ldr(wzr, MemOperand(temp, 0));
- RecordPcInfo(nullptr, 0);
+ {
+ // Ensure that between load and RecordPcInfo there are no pools emitted.
+ ExactAssemblyScope eas(GetVIXLAssembler(),
+ kInstructionSize,
+ CodeBufferCheckScope::kExactSize);
+ __ ldr(wzr, MemOperand(temp, 0));
+ RecordPcInfo(nullptr, 0);
+ }
}
if (!HasEmptyFrame()) {
@@ -1299,7 +1306,6 @@
}
void CodeGeneratorARM64::GenerateFrameExit() {
- BlockPoolsScope block_pools(GetVIXLAssembler());
GetAssembler()->cfi().RememberState();
if (!HasEmptyFrame()) {
int frame_size = GetFrameSize();
@@ -1626,7 +1632,6 @@
const MemOperand& src,
bool needs_null_check) {
MacroAssembler* masm = GetVIXLAssembler();
- BlockPoolsScope block_pools(masm);
UseScratchRegisterScope temps(masm);
Register temp_base = temps.AcquireX();
Primitive::Type type = instruction->GetType();
@@ -1636,58 +1641,79 @@
// TODO(vixl): Let the MacroAssembler handle MemOperand.
__ Add(temp_base, src.GetBaseRegister(), OperandFromMemOperand(src));
- MemOperand base = MemOperand(temp_base);
- switch (type) {
- case Primitive::kPrimBoolean:
- __ Ldarb(Register(dst), base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- break;
- case Primitive::kPrimByte:
- __ Ldarb(Register(dst), base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
- break;
- case Primitive::kPrimChar:
- __ Ldarh(Register(dst), base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- break;
- case Primitive::kPrimShort:
- __ Ldarh(Register(dst), base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
- break;
- case Primitive::kPrimInt:
- case Primitive::kPrimNot:
- case Primitive::kPrimLong:
- DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
- __ Ldar(Register(dst), base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- break;
- case Primitive::kPrimFloat:
- case Primitive::kPrimDouble: {
- DCHECK(dst.IsFPRegister());
- DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ MemOperand base = MemOperand(temp_base);
+ switch (type) {
+ case Primitive::kPrimBoolean:
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldarb(Register(dst), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ break;
+ case Primitive::kPrimByte:
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldarb(Register(dst), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
+ break;
+ case Primitive::kPrimChar:
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldarh(Register(dst), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ break;
+ case Primitive::kPrimShort:
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldarh(Register(dst), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
+ break;
+ case Primitive::kPrimInt:
+ case Primitive::kPrimNot:
+ case Primitive::kPrimLong:
+ DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldar(Register(dst), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ break;
+ case Primitive::kPrimFloat:
+ case Primitive::kPrimDouble: {
+ DCHECK(dst.IsFPRegister());
+ DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
- Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
- __ Ldar(temp, base);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ ldar(temp, base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
+ __ Fmov(FPRegister(dst), temp);
+ break;
}
- __ Fmov(FPRegister(dst), temp);
- break;
+ case Primitive::kPrimVoid:
+ LOG(FATAL) << "Unreachable type " << type;
}
- case Primitive::kPrimVoid:
- LOG(FATAL) << "Unreachable type " << type;
}
}
@@ -1716,9 +1742,12 @@
}
}
-void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
+void CodeGeneratorARM64::StoreRelease(HInstruction* instruction,
+ Primitive::Type type,
CPURegister src,
- const MemOperand& dst) {
+ const MemOperand& dst,
+ bool needs_null_check) {
+ MacroAssembler* masm = GetVIXLAssembler();
UseScratchRegisterScope temps(GetVIXLAssembler());
Register temp_base = temps.AcquireX();
@@ -1729,20 +1758,39 @@
Operand op = OperandFromMemOperand(dst);
__ Add(temp_base, dst.GetBaseRegister(), op);
MemOperand base = MemOperand(temp_base);
+ // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted.
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
- __ Stlrb(Register(src), base);
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ stlrb(Register(src), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
break;
case Primitive::kPrimChar:
case Primitive::kPrimShort:
- __ Stlrh(Register(src), base);
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ stlrh(Register(src), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
break;
case Primitive::kPrimInt:
case Primitive::kPrimNot:
case Primitive::kPrimLong:
DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
- __ Stlr(Register(src), base);
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ stlr(Register(src), base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble: {
@@ -1756,8 +1804,13 @@
temp_src = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
__ Fmov(temp_src, FPRegister(src));
}
-
- __ Stlr(temp_src, base);
+ {
+ ExactAssemblyScope eas(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ stlr(temp_src, base);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
break;
}
case Primitive::kPrimVoid:
@@ -1770,9 +1823,15 @@
uint32_t dex_pc,
SlowPathCode* slow_path) {
ValidateInvokeRuntime(entrypoint, instruction, slow_path);
- GenerateInvokeRuntime(GetThreadOffset<kArm64PointerSize>(entrypoint).Int32Value());
- if (EntrypointRequiresStackMap(entrypoint)) {
- RecordPcInfo(instruction, dex_pc, slow_path);
+
+ __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArm64PointerSize>(entrypoint).Int32Value()));
+ {
+ // Ensure the pc position is recorded immediately after the `blr` instruction.
+ ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize);
+ __ blr(lr);
+ if (EntrypointRequiresStackMap(entrypoint)) {
+ RecordPcInfo(instruction, dex_pc, slow_path);
+ }
}
}
@@ -1780,11 +1839,6 @@
HInstruction* instruction,
SlowPathCode* slow_path) {
ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path);
- GenerateInvokeRuntime(entry_point_offset);
-}
-
-void CodeGeneratorARM64::GenerateInvokeRuntime(int32_t entry_point_offset) {
- BlockPoolsScope block_pools(GetVIXLAssembler());
__ Ldr(lr, MemOperand(tr, entry_point_offset));
__ Blr(lr);
}
@@ -1951,7 +2005,6 @@
Location out = locations->Out();
uint32_t offset = field_info.GetFieldOffset().Uint32Value();
Primitive::Type field_type = field_info.GetFieldType();
- BlockPoolsScope block_pools(GetVIXLAssembler());
MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
if (field_type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
@@ -1978,6 +2031,8 @@
codegen_->LoadAcquire(
instruction, OutputCPURegister(instruction), field, /* needs_null_check */ true);
} else {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
codegen_->Load(field_type, OutputCPURegister(instruction), field);
codegen_->MaybeRecordImplicitNullCheck(instruction);
}
@@ -2007,7 +2062,6 @@
const FieldInfo& field_info,
bool value_can_be_null) {
DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
- BlockPoolsScope block_pools(GetVIXLAssembler());
Register obj = InputRegisterAt(instruction, 0);
CPURegister value = InputCPURegisterOrZeroRegAt(instruction, 1);
@@ -2029,9 +2083,11 @@
}
if (field_info.IsVolatile()) {
- codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ codegen_->StoreRelease(
+ instruction, field_type, source, HeapOperand(obj, offset), /* needs_null_check */ true);
} else {
+ // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
codegen_->Store(field_type, source, HeapOperand(obj, offset));
codegen_->MaybeRecordImplicitNullCheck(instruction);
}
@@ -2317,10 +2373,7 @@
masm->GetCursorAddress<vixl::aarch64::Instruction*>() - kInstructionSize;
if (prev->IsLoadOrStore()) {
// Make sure we emit only exactly one nop.
- vixl::CodeBufferCheckScope scope(masm,
- kInstructionSize,
- vixl::CodeBufferCheckScope::kReserveBufferSpace,
- vixl::CodeBufferCheckScope::kExactSize);
+ ExactAssemblyScope scope(masm, kInstructionSize, CodeBufferCheckScope::kExactSize);
__ nop();
}
}
@@ -2376,8 +2429,6 @@
instruction->IsStringCharAt();
MacroAssembler* masm = GetVIXLAssembler();
UseScratchRegisterScope temps(masm);
- // Block pools between `Load` and `MaybeRecordImplicitNullCheck`.
- BlockPoolsScope block_pools(masm);
// The read barrier instrumentation of object ArrayGet instructions
// does not support the HIntermediateAddress instruction.
@@ -2399,15 +2450,21 @@
if (maybe_compressed_char_at) {
uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
length = temps.AcquireW();
- if (instruction->GetArray()->IsIntermediateAddress()) {
- DCHECK_LT(count_offset, offset);
- int64_t adjusted_offset = static_cast<int64_t>(count_offset) - static_cast<int64_t>(offset);
- // Note that `adjusted_offset` is negative, so this will be a LDUR.
- __ Ldr(length, MemOperand(obj.X(), adjusted_offset));
- } else {
- __ Ldr(length, HeapOperand(obj, count_offset));
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+
+ if (instruction->GetArray()->IsIntermediateAddress()) {
+ DCHECK_LT(count_offset, offset);
+ int64_t adjusted_offset =
+ static_cast<int64_t>(count_offset) - static_cast<int64_t>(offset);
+ // Note that `adjusted_offset` is negative, so this will be a LDUR.
+ __ Ldr(length, MemOperand(obj.X(), adjusted_offset));
+ } else {
+ __ Ldr(length, HeapOperand(obj, count_offset));
+ }
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
}
- codegen_->MaybeRecordImplicitNullCheck(instruction);
}
if (index.IsConstant()) {
if (maybe_compressed_char_at) {
@@ -2457,6 +2514,8 @@
}
}
if (!maybe_compressed_char_at) {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
codegen_->Load(type, OutputCPURegister(instruction), source);
codegen_->MaybeRecordImplicitNullCheck(instruction);
}
@@ -2484,9 +2543,12 @@
void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction);
vixl::aarch64::Register out = OutputRegister(instruction);
- BlockPoolsScope block_pools(GetVIXLAssembler());
- __ Ldr(out, HeapOperand(InputRegisterAt(instruction, 0), offset));
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ __ Ldr(out, HeapOperand(InputRegisterAt(instruction, 0), offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
// Mask out compression flag from String's array length.
if (mirror::kUseStringCompression && instruction->IsStringLength()) {
__ Lsr(out.W(), out.W(), 1u);
@@ -2527,7 +2589,6 @@
size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
MemOperand destination = HeapOperand(array);
MacroAssembler* masm = GetVIXLAssembler();
- BlockPoolsScope block_pools(masm);
if (!needs_write_barrier) {
DCHECK(!may_need_runtime_call_for_type_check);
@@ -2554,8 +2615,12 @@
LSL,
Primitive::ComponentSizeShift(value_type));
}
- codegen_->Store(value_type, value, destination);
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ {
+ // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ codegen_->Store(value_type, value, destination);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
} else {
DCHECK(!instruction->GetArray()->IsIntermediateAddress());
vixl::aarch64::Label done;
@@ -2588,8 +2653,13 @@
if (!index.IsConstant()) {
__ Add(temp, array, offset);
}
- __ Str(wzr, destination);
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ {
+ // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools
+ // emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ __ Str(wzr, destination);
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
__ B(&done);
__ Bind(&non_zero);
}
@@ -2604,8 +2674,12 @@
Register temp2 = temps.AcquireSameSizeAs(array);
// /* HeapReference<Class> */ temp = array->klass_
- __ Ldr(temp, HeapOperand(array, class_offset));
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ __ Ldr(temp, HeapOperand(array, class_offset));
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
GetAssembler()->MaybeUnpoisonHeapReference(temp);
// /* HeapReference<Class> */ temp = temp->component_type_
@@ -2646,10 +2720,14 @@
if (!index.IsConstant()) {
__ Add(temp, array, offset);
}
- __ Str(source, destination);
+ {
+ // Ensure that between store and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ __ Str(source, destination);
- if (!may_need_runtime_call_for_type_check) {
- codegen_->MaybeRecordImplicitNullCheck(instruction);
+ if (!may_need_runtime_call_for_type_check) {
+ codegen_->MaybeRecordImplicitNullCheck(instruction);
+ }
}
}
@@ -3944,19 +4022,25 @@
// art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
MacroAssembler* masm = GetVIXLAssembler();
UseScratchRegisterScope scratch_scope(masm);
- BlockPoolsScope block_pools(masm);
scratch_scope.Exclude(ip1);
__ Mov(ip1, invoke->GetDexMethodIndex());
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
if (receiver.IsStackSlot()) {
__ Ldr(temp.W(), StackOperandFrom(receiver));
- // /* HeapReference<Class> */ temp = temp->klass_
- __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
+ {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ // /* HeapReference<Class> */ temp = temp->klass_
+ __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
+ codegen_->MaybeRecordImplicitNullCheck(invoke);
+ }
} else {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
// /* HeapReference<Class> */ temp = receiver->klass_
__ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
+ codegen_->MaybeRecordImplicitNullCheck(invoke);
}
- codegen_->MaybeRecordImplicitNullCheck(invoke);
+
// Instead of simply (possibly) unpoisoning `temp` here, we should
// emit a read barrier for the previous class reference load.
// However this is not required in practice, as this is an
@@ -3973,10 +4057,16 @@
__ Ldr(temp, MemOperand(temp, method_offset));
// lr = temp->GetEntryPoint();
__ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
- // lr();
- __ Blr(lr);
- DCHECK(!codegen_->IsLeafMethod());
- codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
+
+ {
+ // Ensure the pc position is recorded immediately after the `blr` instruction.
+ ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize);
+
+ // lr();
+ __ blr(lr);
+ DCHECK(!codegen_->IsLeafMethod());
+ codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
+ }
}
void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
@@ -4088,8 +4178,16 @@
__ Ldr(lr, MemOperand(
XRegisterFrom(callee_method),
ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize).Int32Value()));
- // lr()
- __ Blr(lr);
+ {
+ // To ensure that the pc position is recorded immediately after the `blr` instruction
+ // BLR must be the last instruction emitted in this function.
+ // Recording the pc will occur right after returning from this function.
+ ExactAssemblyScope eas(GetVIXLAssembler(),
+ kInstructionSize,
+ CodeBufferCheckScope::kExactSize);
+ // lr()
+ __ blr(lr);
+ }
break;
}
@@ -4109,12 +4207,15 @@
Offset class_offset = mirror::Object::ClassOffset();
Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
- BlockPoolsScope block_pools(GetVIXLAssembler());
-
DCHECK(receiver.IsRegister());
- // /* HeapReference<Class> */ temp = receiver->klass_
- __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset));
- MaybeRecordImplicitNullCheck(invoke);
+
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ // /* HeapReference<Class> */ temp = receiver->klass_
+ __ Ldr(temp.W(), HeapOperandFrom(LocationFrom(receiver), class_offset));
+ MaybeRecordImplicitNullCheck(invoke);
+ }
// Instead of simply (possibly) unpoisoning `temp` here, we should
// emit a read barrier for the previous class reference load.
// intermediate/temporary reference and because the current
@@ -4126,8 +4227,14 @@
__ Ldr(temp, MemOperand(temp, method_offset));
// lr = temp->GetEntryPoint();
__ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
- // lr();
- __ Blr(lr);
+ {
+ // To ensure that the pc position is recorded immediately after the `blr` instruction
+ // BLR should be the last instruction emitted in this function.
+ // Recording the pc will occur right after returning from this function.
+ ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize);
+ // lr();
+ __ blr(lr);
+ }
}
void LocationsBuilderARM64::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
@@ -4340,7 +4447,9 @@
return;
}
- BlockPoolsScope block_pools(GetVIXLAssembler());
+ // Ensure that between the BLR (emitted by GenerateStaticOrDirectCall) and RecordPcInfo there
+ // are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
LocationSummary* locations = invoke->GetLocations();
codegen_->GenerateStaticOrDirectCall(
invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
@@ -4352,6 +4461,9 @@
return;
}
+ // Ensure that between the BLR (emitted by GenerateVirtualCall) and RecordPcInfo there
+ // are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
DCHECK(!codegen_->IsLeafMethod());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
@@ -4817,8 +4929,15 @@
MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
__ Ldr(XRegisterFrom(temp), MemOperand(tr, QUICK_ENTRY_POINT(pNewEmptyString)));
__ Ldr(lr, MemOperand(XRegisterFrom(temp), code_offset.Int32Value()));
- __ Blr(lr);
- codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
+
+ {
+ // Ensure the pc position is recorded immediately after the `blr` instruction.
+ ExactAssemblyScope eas(GetVIXLAssembler(),
+ kInstructionSize,
+ CodeBufferCheckScope::kExactSize);
+ __ blr(lr);
+ codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
+ }
} else {
codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc());
CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>();
@@ -4862,11 +4981,13 @@
if (CanMoveNullCheckToUser(instruction)) {
return;
}
-
- BlockPoolsScope block_pools(GetVIXLAssembler());
- Location obj = instruction->GetLocations()->InAt(0);
- __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
- RecordPcInfo(instruction, instruction->GetDexPc());
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ Location obj = instruction->GetLocations()->InAt(0);
+ __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
+ RecordPcInfo(instruction, instruction->GetDexPc());
+ }
}
void CodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
@@ -5603,10 +5724,14 @@
DCHECK(obj.IsW());
uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- // /* int32_t */ monitor = obj->monitor_
- __ Ldr(temp, HeapOperand(obj, monitor_offset));
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ // /* int32_t */ monitor = obj->monitor_
+ __ Ldr(temp, HeapOperand(obj, monitor_offset));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
}
// /* LockWord */ lock_word = LockWord(monitor)
static_assert(sizeof(LockWord) == sizeof(int32_t),
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index f6cb90a..5faf29a 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -43,6 +43,11 @@
// Use a local definition to prevent copying mistakes.
static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
+// These constants are used as an approximate margin when emission of veneer and literal pools
+// must be blocked.
+static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
+static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes;
+
static const vixl::aarch64::Register kParameterCoreRegisters[] = {
vixl::aarch64::x1,
vixl::aarch64::x2,
@@ -486,9 +491,11 @@
vixl::aarch64::CPURegister dst,
const vixl::aarch64::MemOperand& src,
bool needs_null_check);
- void StoreRelease(Primitive::Type type,
+ void StoreRelease(HInstruction* instruction,
+ Primitive::Type type,
vixl::aarch64::CPURegister src,
- const vixl::aarch64::MemOperand& dst);
+ const vixl::aarch64::MemOperand& dst,
+ bool needs_null_check);
// Generate code to invoke a runtime entry point.
void InvokeRuntime(QuickEntrypointEnum entrypoint,
@@ -502,8 +509,6 @@
HInstruction* instruction,
SlowPathCode* slow_path);
- void GenerateInvokeRuntime(int32_t entry_point_offset);
-
ParallelMoveResolverARM64* GetMoveResolver() OVERRIDE { return &move_resolver_; }
bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE {
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index bbf826c..1047d3b 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -115,13 +115,18 @@
MoveArguments(invoke_, codegen);
- if (invoke_->IsInvokeStaticOrDirect()) {
- codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
- LocationFrom(kArtMethodRegister));
- } else {
- codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
+ {
+ // Ensure that between the BLR (emitted by Generate*Call) and RecordPcInfo there
+ // are no pools emitted.
+ vixl::EmissionCheckScope guard(codegen->GetVIXLAssembler(), kInvokeCodeMarginSizeInBytes);
+ if (invoke_->IsInvokeStaticOrDirect()) {
+ codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
+ LocationFrom(kArtMethodRegister));
+ } else {
+ codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
+ }
+ codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
}
- codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
// Copy the result back to the expected output.
Location out = invoke_->GetLocations()->Out();
@@ -980,11 +985,12 @@
CreateIntIntIntIntToVoid(arena_, invoke);
}
-static void GenUnsafePut(LocationSummary* locations,
+static void GenUnsafePut(HInvoke* invoke,
Primitive::Type type,
bool is_volatile,
bool is_ordered,
CodeGeneratorARM64* codegen) {
+ LocationSummary* locations = invoke->GetLocations();
MacroAssembler* masm = codegen->GetVIXLAssembler();
Register base = WRegisterFrom(locations->InAt(1)); // Object pointer.
@@ -1007,7 +1013,7 @@
}
if (is_volatile || is_ordered) {
- codegen->StoreRelease(type, source, mem_op);
+ codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check */ false);
} else {
codegen->Store(type, source, mem_op);
}
@@ -1020,63 +1026,63 @@
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimInt,
/* is_volatile */ false,
/* is_ordered */ false,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimInt,
/* is_volatile */ false,
/* is_ordered */ true,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimInt,
/* is_volatile */ true,
/* is_ordered */ false,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimNot,
/* is_volatile */ false,
/* is_ordered */ false,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimNot,
/* is_volatile */ false,
/* is_ordered */ true,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimNot,
/* is_volatile */ true,
/* is_ordered */ false,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimLong,
/* is_volatile */ false,
/* is_ordered */ false,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimLong,
/* is_volatile */ false,
/* is_ordered */ true,
codegen_);
}
void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
- GenUnsafePut(invoke->GetLocations(),
+ GenUnsafePut(invoke,
Primitive::kPrimLong,
/* is_volatile */ true,
/* is_ordered */ false,
@@ -2825,9 +2831,13 @@
}
__ Cbnz(temp0, slow_path->GetEntryLabel());
- // Fast path.
- __ Ldr(out, HeapOperand(obj, mirror::Reference::ReferentOffset().Int32Value()));
- codegen_->MaybeRecordImplicitNullCheck(invoke);
+ {
+ // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
+ vixl::EmissionCheckScope guard(codegen_->GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ // Fast path.
+ __ Ldr(out, HeapOperand(obj, mirror::Reference::ReferentOffset().Int32Value()));
+ codegen_->MaybeRecordImplicitNullCheck(invoke);
+ }
codegen_->GetAssembler()->MaybeUnpoisonHeapReference(out);
__ Bind(slow_path->GetExitLabel());
}
