blob: 05a76e1105063b1c9d3bfff6bbec9ed0e63e9b73 [file] [log] [blame]
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "code_generator_arm_vixl.h"
#include "arch/arm/instruction_set_features_arm.h"
#include "art_method.h"
#include "code_generator_utils.h"
#include "common_arm.h"
#include "compiled_method.h"
#include "entrypoints/quick/quick_entrypoints.h"
#include "gc/accounting/card_table.h"
#include "intrinsics_arm_vixl.h"
#include "mirror/array-inl.h"
#include "mirror/class-inl.h"
#include "thread.h"
#include "utils/arm/assembler_arm_vixl.h"
#include "utils/arm/managed_register_arm.h"
#include "utils/assembler.h"
#include "utils/stack_checks.h"
namespace art {
namespace arm {
namespace vixl32 = vixl::aarch32;
using namespace vixl32; // NOLINT(build/namespaces)
using helpers::DRegisterFrom;
using helpers::DWARFReg;
using helpers::HighDRegisterFrom;
using helpers::HighRegisterFrom;
using helpers::InputOperandAt;
using helpers::InputRegister;
using helpers::InputRegisterAt;
using helpers::InputSRegisterAt;
using helpers::InputVRegister;
using helpers::InputVRegisterAt;
using helpers::Int32ConstantFrom;
using helpers::Int64ConstantFrom;
using helpers::LocationFrom;
using helpers::LowRegisterFrom;
using helpers::LowSRegisterFrom;
using helpers::OutputRegister;
using helpers::OutputSRegister;
using helpers::OutputVRegister;
using helpers::RegisterFrom;
using helpers::SRegisterFrom;
using helpers::Uint64ConstantFrom;
using vixl::ExactAssemblyScope;
using vixl::CodeBufferCheckScope;
using RegisterList = vixl32::RegisterList;
static bool ExpectedPairLayout(Location location) {
// We expected this for both core and fpu register pairs.
return ((location.low() & 1) == 0) && (location.low() + 1 == location.high());
}
// Use a local definition to prevent copying mistakes.
static constexpr size_t kArmWordSize = static_cast<size_t>(kArmPointerSize);
static constexpr size_t kArmBitsPerWord = kArmWordSize * kBitsPerByte;
static constexpr int kCurrentMethodStackOffset = 0;
static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7;
#ifdef __
#error "ARM Codegen VIXL macro-assembler macro already defined."
#endif
// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
#define __ down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler()-> // NOLINT
#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value()
// Marker that code is yet to be, and must, be implemented.
#define TODO_VIXL32(level) LOG(level) << __PRETTY_FUNCTION__ << " unimplemented "
// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers,
// for each live D registers they treat two corresponding S registers as live ones.
//
// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build
// from a list of contiguous S registers a list of contiguous D registers (processing first/last
// S registers corner cases) and save/restore this new list treating them as D registers.
// - decreasing code size
// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is
// restored and then used in regular non SlowPath code as D register.
//
// For the following example (v means the S register is live):
// D names: | D0 | D1 | D2 | D4 | ...
// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ...
// Live? | | v | v | v | v | v | v | | ...
//
// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed
// as D registers.
//
// TODO(VIXL): All this code should be unnecessary once the VIXL AArch32 backend provides helpers
// for lists of floating-point registers.
static size_t SaveContiguousSRegisterList(size_t first,
size_t last,
CodeGenerator* codegen,
size_t stack_offset) {
static_assert(kSRegSizeInBytes == kArmWordSize, "Broken assumption on reg/word sizes.");
static_assert(kDRegSizeInBytes == 2 * kArmWordSize, "Broken assumption on reg/word sizes.");
DCHECK_LE(first, last);
if ((first == last) && (first == 0)) {
__ Vstr(vixl32::SRegister(first), MemOperand(sp, stack_offset));
return stack_offset + kSRegSizeInBytes;
}
if (first % 2 == 1) {
__ Vstr(vixl32::SRegister(first++), MemOperand(sp, stack_offset));
stack_offset += kSRegSizeInBytes;
}
bool save_last = false;
if (last % 2 == 0) {
save_last = true;
--last;
}
if (first < last) {
vixl32::DRegister d_reg = vixl32::DRegister(first / 2);
DCHECK_EQ((last - first + 1) % 2, 0u);
size_t number_of_d_regs = (last - first + 1) / 2;
if (number_of_d_regs == 1) {
__ Vstr(d_reg, MemOperand(sp, stack_offset));
} else if (number_of_d_regs > 1) {
UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler());
vixl32::Register base = sp;
if (stack_offset != 0) {
base = temps.Acquire();
__ Add(base, sp, Operand::From(stack_offset));
}
__ Vstm(F64, base, NO_WRITE_BACK, DRegisterList(d_reg, number_of_d_regs));
}
stack_offset += number_of_d_regs * kDRegSizeInBytes;
}
if (save_last) {
__ Vstr(vixl32::SRegister(last + 1), MemOperand(sp, stack_offset));
stack_offset += kSRegSizeInBytes;
}
return stack_offset;
}
static size_t RestoreContiguousSRegisterList(size_t first,
size_t last,
CodeGenerator* codegen,
size_t stack_offset) {
static_assert(kSRegSizeInBytes == kArmWordSize, "Broken assumption on reg/word sizes.");
static_assert(kDRegSizeInBytes == 2 * kArmWordSize, "Broken assumption on reg/word sizes.");
DCHECK_LE(first, last);
if ((first == last) && (first == 0)) {
__ Vldr(vixl32::SRegister(first), MemOperand(sp, stack_offset));
return stack_offset + kSRegSizeInBytes;
}
if (first % 2 == 1) {
__ Vldr(vixl32::SRegister(first++), MemOperand(sp, stack_offset));
stack_offset += kSRegSizeInBytes;
}
bool restore_last = false;
if (last % 2 == 0) {
restore_last = true;
--last;
}
if (first < last) {
vixl32::DRegister d_reg = vixl32::DRegister(first / 2);
DCHECK_EQ((last - first + 1) % 2, 0u);
size_t number_of_d_regs = (last - first + 1) / 2;
if (number_of_d_regs == 1) {
__ Vldr(d_reg, MemOperand(sp, stack_offset));
} else if (number_of_d_regs > 1) {
UseScratchRegisterScope temps(down_cast<CodeGeneratorARMVIXL*>(codegen)->GetVIXLAssembler());
vixl32::Register base = sp;
if (stack_offset != 0) {
base = temps.Acquire();
__ Add(base, sp, Operand::From(stack_offset));
}
__ Vldm(F64, base, NO_WRITE_BACK, DRegisterList(d_reg, number_of_d_regs));
}
stack_offset += number_of_d_regs * kDRegSizeInBytes;
}
if (restore_last) {
__ Vldr(vixl32::SRegister(last + 1), MemOperand(sp, stack_offset));
stack_offset += kSRegSizeInBytes;
}
return stack_offset;
}
void SlowPathCodeARMVIXL::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
size_t orig_offset = stack_offset;
const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
for (uint32_t i : LowToHighBits(core_spills)) {
// If the register holds an object, update the stack mask.
if (locations->RegisterContainsObject(i)) {
locations->SetStackBit(stack_offset / kVRegSize);
}
DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
saved_core_stack_offsets_[i] = stack_offset;
stack_offset += kArmWordSize;
}
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
arm_codegen->GetAssembler()->StoreRegisterList(core_spills, orig_offset);
uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
orig_offset = stack_offset;
for (uint32_t i : LowToHighBits(fp_spills)) {
DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
saved_fpu_stack_offsets_[i] = stack_offset;
stack_offset += kArmWordSize;
}
stack_offset = orig_offset;
while (fp_spills != 0u) {
uint32_t begin = CTZ(fp_spills);
uint32_t tmp = fp_spills + (1u << begin);
fp_spills &= tmp; // Clear the contiguous range of 1s.
uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined.
stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset);
}
DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
}
void SlowPathCodeARMVIXL::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
size_t orig_offset = stack_offset;
const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true);
for (uint32_t i : LowToHighBits(core_spills)) {
DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
stack_offset += kArmWordSize;
}
// TODO(VIXL): Check the coherency of stack_offset after this with a test.
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
arm_codegen->GetAssembler()->LoadRegisterList(core_spills, orig_offset);
uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false);
while (fp_spills != 0u) {
uint32_t begin = CTZ(fp_spills);
uint32_t tmp = fp_spills + (1u << begin);
fp_spills &= tmp; // Clear the contiguous range of 1s.
uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined.
stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset);
}
DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
}
class NullCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit NullCheckSlowPathARMVIXL(HNullCheck* instruction) : SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
if (instruction_->CanThrowIntoCatchBlock()) {
// Live registers will be restored in the catch block if caught.
SaveLiveRegisters(codegen, instruction_->GetLocations());
}
arm_codegen->InvokeRuntime(kQuickThrowNullPointer,
instruction_,
instruction_->GetDexPc(),
this);
CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
}
bool IsFatal() const OVERRIDE { return true; }
const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARMVIXL);
};
class DivZeroCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit DivZeroCheckSlowPathARMVIXL(HDivZeroCheck* instruction)
: SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
}
bool IsFatal() const OVERRIDE { return true; }
const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARMVIXL);
};
class SuspendCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
SuspendCheckSlowPathARMVIXL(HSuspendCheck* instruction, HBasicBlock* successor)
: SlowPathCodeARMVIXL(instruction), successor_(successor) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickTestSuspend, void, void>();
if (successor_ == nullptr) {
__ B(GetReturnLabel());
} else {
__ B(arm_codegen->GetLabelOf(successor_));
}
}
vixl32::Label* GetReturnLabel() {
DCHECK(successor_ == nullptr);
return &return_label_;
}
HBasicBlock* GetSuccessor() const {
return successor_;
}
const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARMVIXL"; }
private:
// If not null, the block to branch to after the suspend check.
HBasicBlock* const successor_;
// If `successor_` is null, the label to branch to after the suspend check.
vixl32::Label return_label_;
DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARMVIXL);
};
class BoundsCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit BoundsCheckSlowPathARMVIXL(HBoundsCheck* instruction)
: SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
LocationSummary* locations = instruction_->GetLocations();
__ Bind(GetEntryLabel());
if (instruction_->CanThrowIntoCatchBlock()) {
// Live registers will be restored in the catch block if caught.
SaveLiveRegisters(codegen, instruction_->GetLocations());
}
// We're moving two locations to locations that could overlap, so we need a parallel
// move resolver.
InvokeRuntimeCallingConventionARMVIXL calling_convention;
codegen->EmitParallelMoves(
locations->InAt(0),
LocationFrom(calling_convention.GetRegisterAt(0)),
Primitive::kPrimInt,
locations->InAt(1),
LocationFrom(calling_convention.GetRegisterAt(1)),
Primitive::kPrimInt);
QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt()
? kQuickThrowStringBounds
: kQuickThrowArrayBounds;
arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>();
CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
}
bool IsFatal() const OVERRIDE { return true; }
const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARMVIXL);
};
class LoadClassSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
LoadClassSlowPathARMVIXL(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit)
: SlowPathCodeARMVIXL(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) {
DCHECK(at->IsLoadClass() || at->IsClinitCheck());
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConventionARMVIXL calling_convention;
dex::TypeIndex type_index = cls_->GetTypeIndex();
__ Mov(calling_convention.GetRegisterAt(0), type_index.index_);
QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage
: kQuickInitializeType;
arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this);
if (do_clinit_) {
CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
} else {
CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
}
// Move the class to the desired location.
Location out = locations->Out();
if (out.IsValid()) {
DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
arm_codegen->Move32(locations->Out(), LocationFrom(r0));
}
RestoreLiveRegisters(codegen, locations);
// For HLoadClass/kBssEntry, store the resolved Class to the BSS entry.
DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_);
if (cls_ == instruction_ && cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry) {
DCHECK(out.IsValid());
// TODO: Change art_quick_initialize_type/art_quick_initialize_static_storage to
// kSaveEverything and use a temporary for the .bss entry address in the fast path,
// so that we can avoid another calculation here.
CodeGeneratorARMVIXL::PcRelativePatchInfo* labels =
arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index);
arm_codegen->EmitMovwMovtPlaceholder(labels, ip);
__ Str(OutputRegister(cls_), MemOperand(ip));
}
__ B(GetExitLabel());
}
const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARMVIXL"; }
private:
// The class this slow path will load.
HLoadClass* const cls_;
// The dex PC of `at_`.
const uint32_t dex_pc_;
// Whether to initialize the class.
const bool do_clinit_;
DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARMVIXL);
};
class LoadStringSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit LoadStringSlowPathARMVIXL(HLoadString* instruction)
: SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
HLoadString* load = instruction_->AsLoadString();
const dex::StringIndex string_index = load->GetStringIndex();
vixl32::Register out = OutputRegister(load);
vixl32::Register temp = RegisterFrom(locations->GetTemp(0));
constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier);
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConventionARMVIXL calling_convention;
// In the unlucky case that the `temp` is R0, we preserve the address in `out` across
// the kSaveEverything call (or use `out` for the address after non-kSaveEverything call).
bool temp_is_r0 = (temp.Is(calling_convention.GetRegisterAt(0)));
vixl32::Register entry_address = temp_is_r0 ? out : temp;
DCHECK(!entry_address.Is(calling_convention.GetRegisterAt(0)));
if (call_saves_everything_except_r0 && temp_is_r0) {
__ Mov(entry_address, temp);
}
__ Mov(calling_convention.GetRegisterAt(0), string_index.index_);
arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
// Store the resolved String to the .bss entry.
if (call_saves_everything_except_r0) {
// The string entry address was preserved in `entry_address` thanks to kSaveEverything.
__ Str(r0, MemOperand(entry_address));
} else {
// For non-Baker read barrier, we need to re-calculate the address of the string entry.
CodeGeneratorARMVIXL::PcRelativePatchInfo* labels =
arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index);
arm_codegen->EmitMovwMovtPlaceholder(labels, out);
__ Str(r0, MemOperand(entry_address));
}
arm_codegen->Move32(locations->Out(), LocationFrom(r0));
RestoreLiveRegisters(codegen, locations);
__ B(GetExitLabel());
}
const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARMVIXL);
};
class TypeCheckSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
TypeCheckSlowPathARMVIXL(HInstruction* instruction, bool is_fatal)
: SlowPathCodeARMVIXL(instruction), is_fatal_(is_fatal) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
DCHECK(instruction_->IsCheckCast()
|| !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
if (!is_fatal_) {
SaveLiveRegisters(codegen, locations);
}
// We're moving two locations to locations that could overlap, so we need a parallel
// move resolver.
InvokeRuntimeCallingConventionARMVIXL calling_convention;
codegen->EmitParallelMoves(locations->InAt(0),
LocationFrom(calling_convention.GetRegisterAt(0)),
Primitive::kPrimNot,
locations->InAt(1),
LocationFrom(calling_convention.GetRegisterAt(1)),
Primitive::kPrimNot);
if (instruction_->IsInstanceOf()) {
arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial,
instruction_,
instruction_->GetDexPc(),
this);
CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>();
arm_codegen->Move32(locations->Out(), LocationFrom(r0));
} else {
DCHECK(instruction_->IsCheckCast());
arm_codegen->InvokeRuntime(kQuickCheckInstanceOf,
instruction_,
instruction_->GetDexPc(),
this);
CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>();
}
if (!is_fatal_) {
RestoreLiveRegisters(codegen, locations);
__ B(GetExitLabel());
}
}
const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARMVIXL"; }
bool IsFatal() const OVERRIDE { return is_fatal_; }
private:
const bool is_fatal_;
DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARMVIXL);
};
class DeoptimizationSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit DeoptimizationSlowPathARMVIXL(HDeoptimize* instruction)
: SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
__ Bind(GetEntryLabel());
arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickDeoptimize, void, void>();
}
const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARMVIXL);
};
class ArraySetSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
explicit ArraySetSlowPathARMVIXL(HInstruction* instruction) : SlowPathCodeARMVIXL(instruction) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConventionARMVIXL calling_convention;
HParallelMove parallel_move(codegen->GetGraph()->GetArena());
parallel_move.AddMove(
locations->InAt(0),
LocationFrom(calling_convention.GetRegisterAt(0)),
Primitive::kPrimNot,
nullptr);
parallel_move.AddMove(
locations->InAt(1),
LocationFrom(calling_convention.GetRegisterAt(1)),
Primitive::kPrimInt,
nullptr);
parallel_move.AddMove(
locations->InAt(2),
LocationFrom(calling_convention.GetRegisterAt(2)),
Primitive::kPrimNot,
nullptr);
codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
RestoreLiveRegisters(codegen, locations);
__ B(GetExitLabel());
}
const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARMVIXL"; }
private:
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARMVIXL);
};
// Slow path marking an object reference `ref` during a read
// barrier. The field `obj.field` in the object `obj` holding this
// reference does not get updated by this slow path after marking (see
// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that).
//
// This means that after the execution of this slow path, `ref` will
// always be up-to-date, but `obj.field` may not; i.e., after the
// flip, `ref` will be a to-space reference, but `obj.field` will
// probably still be a from-space reference (unless it gets updated by
// another thread, or if another thread installed another object
// reference (different from `ref`) in `obj.field`).
class ReadBarrierMarkSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction,
Location ref,
Location entrypoint = Location::NoLocation())
: SlowPathCodeARMVIXL(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; }
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
vixl32::Register ref_reg = RegisterFrom(ref_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
DCHECK(instruction_->IsInstanceFieldGet() ||
instruction_->IsStaticFieldGet() ||
instruction_->IsArrayGet() ||
instruction_->IsArraySet() ||
instruction_->IsLoadClass() ||
instruction_->IsLoadString() ||
instruction_->IsInstanceOf() ||
instruction_->IsCheckCast() ||
(instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
(instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking slow path: "
<< instruction_->DebugName();
// The read barrier instrumentation of object ArrayGet
// instructions does not support the HIntermediateAddress
// instruction.
DCHECK(!(instruction_->IsArrayGet() &&
instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
__ Bind(GetEntryLabel());
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
DCHECK(!ref_reg.Is(sp));
DCHECK(!ref_reg.Is(lr));
DCHECK(!ref_reg.Is(pc));
// IP is used internally by the ReadBarrierMarkRegX entry point
// as a temporary, it cannot be the entry point's input/output.
DCHECK(!ref_reg.Is(ip));
DCHECK(ref_reg.IsRegister()) << ref_reg;
// "Compact" slow path, saving two moves.
//
// Instead of using the standard runtime calling convention (input
// and output in R0):
//
// R0 <- ref
// R0 <- ReadBarrierMark(R0)
// ref <- R0
//
// we just use rX (the register containing `ref`) as input and output
// of a dedicated entrypoint:
//
// rX <- ReadBarrierMarkRegX(rX)
//
if (entrypoint_.IsValid()) {
arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
__ Blx(RegisterFrom(entrypoint_));
} else {
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
__ B(GetExitLabel());
}
private:
// The location (register) of the marked object reference.
const Location ref_;
// The location of the entrypoint if already loaded.
const Location entrypoint_;
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL);
};
// Slow path marking an object reference `ref` during a read barrier,
// and if needed, atomically updating the field `obj.field` in the
// object `obj` holding this reference after marking (contrary to
// ReadBarrierMarkSlowPathARM above, which never tries to update
// `obj.field`).
//
// This means that after the execution of this slow path, both `ref`
// and `obj.field` will be up-to-date; i.e., after the flip, both will
// hold the same to-space reference (unless another thread installed
// another object reference (different from `ref`) in `obj.field`).
class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction,
Location ref,
vixl32::Register obj,
Location field_offset,
vixl32::Register temp1,
vixl32::Register temp2)
: SlowPathCodeARMVIXL(instruction),
ref_(ref),
obj_(obj),
field_offset_(field_offset),
temp1_(temp1),
temp2_(temp2) {
DCHECK(kEmitCompilerReadBarrier);
}
const char* GetDescription() const OVERRIDE {
return "ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL";
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
vixl32::Register ref_reg = RegisterFrom(ref_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
// This slow path is only used by the UnsafeCASObject intrinsic.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
DCHECK(field_offset_.IsRegisterPair()) << field_offset_;
__ Bind(GetEntryLabel());
// Save the old reference.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK(!temp1_.Is(ip));
__ Mov(temp1_, ref_reg);
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
DCHECK(!ref_reg.Is(sp));
DCHECK(!ref_reg.Is(lr));
DCHECK(!ref_reg.Is(pc));
// IP is used internally by the ReadBarrierMarkRegX entry point
// as a temporary, it cannot be the entry point's input/output.
DCHECK(!ref_reg.Is(ip));
DCHECK(ref_reg.IsRegister()) << ref_reg;
// "Compact" slow path, saving two moves.
//
// Instead of using the standard runtime calling convention (input
// and output in R0):
//
// R0 <- ref
// R0 <- ReadBarrierMark(R0)
// ref <- R0
//
// we just use rX (the register containing `ref`) as input and output
// of a dedicated entrypoint:
//
// rX <- ReadBarrierMarkRegX(rX)
//
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
// If the new reference is different from the old reference,
// update the field in the holder (`*(obj_ + field_offset_)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDREX/SUBS/ITNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
vixl32::Label done;
__ Cmp(temp1_, ref_reg);
__ B(eq, &done, /* far_target */ false);
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
// using a strong compare-and-set (CAS) operation with relaxed
// memory synchronization ordering, where the expected value is
// the old reference and the desired value is the new reference.
UseScratchRegisterScope temps(arm_codegen->GetVIXLAssembler());
// Convenience aliases.
vixl32::Register base = obj_;
// The UnsafeCASObject intrinsic uses a register pair as field
// offset ("long offset"), of which only the low part contains
// data.
vixl32::Register offset = LowRegisterFrom(field_offset_);
vixl32::Register expected = temp1_;
vixl32::Register value = ref_reg;
vixl32::Register tmp_ptr = temps.Acquire(); // Pointer to actual memory.
vixl32::Register tmp = temp2_; // Value in memory.
__ Add(tmp_ptr, base, offset);
if (kPoisonHeapReferences) {
arm_codegen->GetAssembler()->PoisonHeapReference(expected);
if (value.Is(expected)) {
// Do not poison `value`, as it is the same register as
// `expected`, which has just been poisoned.
} else {
arm_codegen->GetAssembler()->PoisonHeapReference(value);
}
}
// do {
// tmp = [r_ptr] - expected;
// } while (tmp == 0 && failure([r_ptr] <- r_new_value));
vixl32::Label loop_head, exit_loop;
__ Bind(&loop_head);
__ Ldrex(tmp, MemOperand(tmp_ptr));
__ Subs(tmp, tmp, expected);
{
ExactAssemblyScope aas(arm_codegen->GetVIXLAssembler(),
2 * kMaxInstructionSizeInBytes,
CodeBufferCheckScope::kMaximumSize);
__ it(ne);
__ clrex(ne);
}
__ B(ne, &exit_loop, /* far_target */ false);
__ Strex(tmp, value, MemOperand(tmp_ptr));
__ Cmp(tmp, 1);
__ B(eq, &loop_head, /* far_target */ false);
__ Bind(&exit_loop);
if (kPoisonHeapReferences) {
arm_codegen->GetAssembler()->UnpoisonHeapReference(expected);
if (value.Is(expected)) {
// Do not unpoison `value`, as it is the same register as
// `expected`, which has just been unpoisoned.
} else {
arm_codegen->GetAssembler()->UnpoisonHeapReference(value);
}
}
__ Bind(&done);
__ B(GetExitLabel());
}
private:
// The location (register) of the marked object reference.
const Location ref_;
// The register containing the object holding the marked object reference field.
const vixl32::Register obj_;
// The location of the offset of the marked reference field within `obj_`.
Location field_offset_;
const vixl32::Register temp1_;
const vixl32::Register temp2_;
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL);
};
// Slow path generating a read barrier for a heap reference.
class ReadBarrierForHeapReferenceSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
ReadBarrierForHeapReferenceSlowPathARMVIXL(HInstruction* instruction,
Location out,
Location ref,
Location obj,
uint32_t offset,
Location index)
: SlowPathCodeARMVIXL(instruction),
out_(out),
ref_(ref),
obj_(obj),
offset_(offset),
index_(index) {
DCHECK(kEmitCompilerReadBarrier);
// If `obj` is equal to `out` or `ref`, it means the initial object
// has been overwritten by (or after) the heap object reference load
// to be instrumented, e.g.:
//
// __ LoadFromOffset(kLoadWord, out, out, offset);
// codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
//
// In that case, we have lost the information about the original
// object, and the emitted read barrier cannot work properly.
DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
LocationSummary* locations = instruction_->GetLocations();
vixl32::Register reg_out = RegisterFrom(out_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode()));
DCHECK(instruction_->IsInstanceFieldGet() ||
instruction_->IsStaticFieldGet() ||
instruction_->IsArrayGet() ||
instruction_->IsInstanceOf() ||
instruction_->IsCheckCast() ||
(instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified())
<< "Unexpected instruction in read barrier for heap reference slow path: "
<< instruction_->DebugName();
// The read barrier instrumentation of object ArrayGet
// instructions does not support the HIntermediateAddress
// instruction.
DCHECK(!(instruction_->IsArrayGet() &&
instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
// We may have to change the index's value, but as `index_` is a
// constant member (like other "inputs" of this slow path),
// introduce a copy of it, `index`.
Location index = index_;
if (index_.IsValid()) {
// Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics.
if (instruction_->IsArrayGet()) {
// Compute the actual memory offset and store it in `index`.
vixl32::Register index_reg = RegisterFrom(index_);
DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg.GetCode()));
if (codegen->IsCoreCalleeSaveRegister(index_reg.GetCode())) {
// We are about to change the value of `index_reg` (see the
// calls to art::arm::Thumb2Assembler::Lsl and
// art::arm::Thumb2Assembler::AddConstant below), but it has
// not been saved by the previous call to
// art::SlowPathCode::SaveLiveRegisters, as it is a
// callee-save register --
// art::SlowPathCode::SaveLiveRegisters does not consider
// callee-save registers, as it has been designed with the
// assumption that callee-save registers are supposed to be
// handled by the called function. So, as a callee-save
// register, `index_reg` _would_ eventually be saved onto
// the stack, but it would be too late: we would have
// changed its value earlier. Therefore, we manually save
// it here into another freely available register,
// `free_reg`, chosen of course among the caller-save
// registers (as a callee-save `free_reg` register would
// exhibit the same problem).
//
// Note we could have requested a temporary register from
// the register allocator instead; but we prefer not to, as
// this is a slow path, and we know we can find a
// caller-save register that is available.
vixl32::Register free_reg = FindAvailableCallerSaveRegister(codegen);
__ Mov(free_reg, index_reg);
index_reg = free_reg;
index = LocationFrom(index_reg);
} else {
// The initial register stored in `index_` has already been
// saved in the call to art::SlowPathCode::SaveLiveRegisters
// (as it is not a callee-save register), so we can freely
// use it.
}
// Shifting the index value contained in `index_reg` by the scale
// factor (2) cannot overflow in practice, as the runtime is
// unable to allocate object arrays with a size larger than
// 2^26 - 1 (that is, 2^28 - 4 bytes).
__ Lsl(index_reg, index_reg, TIMES_4);
static_assert(
sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
"art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
__ Add(index_reg, index_reg, offset_);
} else {
// In the case of the UnsafeGetObject/UnsafeGetObjectVolatile
// intrinsics, `index_` is not shifted by a scale factor of 2
// (as in the case of ArrayGet), as it is actually an offset
// to an object field within an object.
DCHECK(instruction_->IsInvoke()) << instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
(instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
<< instruction_->AsInvoke()->GetIntrinsic();
DCHECK_EQ(offset_, 0U);
DCHECK(index_.IsRegisterPair());
// UnsafeGet's offset location is a register pair, the low
// part contains the correct offset.
index = index_.ToLow();
}
}
// We're moving two or three locations to locations that could
// overlap, so we need a parallel move resolver.
InvokeRuntimeCallingConventionARMVIXL calling_convention;
HParallelMove parallel_move(codegen->GetGraph()->GetArena());
parallel_move.AddMove(ref_,
LocationFrom(calling_convention.GetRegisterAt(0)),
Primitive::kPrimNot,
nullptr);
parallel_move.AddMove(obj_,
LocationFrom(calling_convention.GetRegisterAt(1)),
Primitive::kPrimNot,
nullptr);
if (index.IsValid()) {
parallel_move.AddMove(index,
LocationFrom(calling_convention.GetRegisterAt(2)),
Primitive::kPrimInt,
nullptr);
codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
} else {
codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
__ Mov(calling_convention.GetRegisterAt(2), offset_);
}
arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this);
CheckEntrypointTypes<
kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
arm_codegen->Move32(out_, LocationFrom(r0));
RestoreLiveRegisters(codegen, locations);
__ B(GetExitLabel());
}
const char* GetDescription() const OVERRIDE {
return "ReadBarrierForHeapReferenceSlowPathARMVIXL";
}
private:
vixl32::Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
uint32_t ref = RegisterFrom(ref_).GetCode();
uint32_t obj = RegisterFrom(obj_).GetCode();
for (uint32_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
return vixl32::Register(i);
}
}
// We shall never fail to find a free caller-save register, as
// there are more than two core caller-save registers on ARM
// (meaning it is possible to find one which is different from
// `ref` and `obj`).
DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
LOG(FATAL) << "Could not find a free caller-save register";
UNREACHABLE();
}
const Location out_;
const Location ref_;
const Location obj_;
const uint32_t offset_;
// An additional location containing an index to an array.
// Only used for HArrayGet and the UnsafeGetObject &
// UnsafeGetObjectVolatile intrinsics.
const Location index_;
DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARMVIXL);
};
// Slow path generating a read barrier for a GC root.
class ReadBarrierForRootSlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
ReadBarrierForRootSlowPathARMVIXL(HInstruction* instruction, Location out, Location root)
: SlowPathCodeARMVIXL(instruction), out_(out), root_(root) {
DCHECK(kEmitCompilerReadBarrier);
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
vixl32::Register reg_out = RegisterFrom(out_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out.GetCode()));
DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
<< "Unexpected instruction in read barrier for GC root slow path: "
<< instruction_->DebugName();
__ Bind(GetEntryLabel());
SaveLiveRegisters(codegen, locations);
InvokeRuntimeCallingConventionARMVIXL calling_convention;
CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
arm_codegen->Move32(LocationFrom(calling_convention.GetRegisterAt(0)), root_);
arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow,
instruction_,
instruction_->GetDexPc(),
this);
CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
arm_codegen->Move32(out_, LocationFrom(r0));
RestoreLiveRegisters(codegen, locations);
__ B(GetExitLabel());
}
const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARMVIXL"; }
private:
const Location out_;
const Location root_;
DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARMVIXL);
};
inline vixl32::Condition ARMCondition(IfCondition cond) {
switch (cond) {
case kCondEQ: return eq;
case kCondNE: return ne;
case kCondLT: return lt;
case kCondLE: return le;
case kCondGT: return gt;
case kCondGE: return ge;
case kCondB: return lo;
case kCondBE: return ls;
case kCondA: return hi;
case kCondAE: return hs;
}
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
// Maps signed condition to unsigned condition.
inline vixl32::Condition ARMUnsignedCondition(IfCondition cond) {
switch (cond) {
case kCondEQ: return eq;
case kCondNE: return ne;
// Signed to unsigned.
case kCondLT: return lo;
case kCondLE: return ls;
case kCondGT: return hi;
case kCondGE: return hs;
// Unsigned remain unchanged.
case kCondB: return lo;
case kCondBE: return ls;
case kCondA: return hi;
case kCondAE: return hs;
}
LOG(FATAL) << "Unreachable";
UNREACHABLE();
}
inline vixl32::Condition ARMFPCondition(IfCondition cond, bool gt_bias) {
// The ARM condition codes can express all the necessary branches, see the
// "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual.
// There is no dex instruction or HIR that would need the missing conditions
// "equal or unordered" or "not equal".
switch (cond) {
case kCondEQ: return eq;
case kCondNE: return ne /* unordered */;
case kCondLT: return gt_bias ? cc : lt /* unordered */;
case kCondLE: return gt_bias ? ls : le /* unordered */;
case kCondGT: return gt_bias ? hi /* unordered */ : gt;
case kCondGE: return gt_bias ? cs /* unordered */ : ge;
default:
LOG(FATAL) << "UNREACHABLE";
UNREACHABLE();
}
}
void CodeGeneratorARMVIXL::DumpCoreRegister(std::ostream& stream, int reg) const {
stream << vixl32::Register(reg);
}
void CodeGeneratorARMVIXL::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
stream << vixl32::SRegister(reg);
}
static uint32_t ComputeSRegisterListMask(const SRegisterList& regs) {
uint32_t mask = 0;
for (uint32_t i = regs.GetFirstSRegister().GetCode();
i <= regs.GetLastSRegister().GetCode();
++i) {
mask |= (1 << i);
}
return mask;
}
// Saves the register in the stack. Returns the size taken on stack.
size_t CodeGeneratorARMVIXL::SaveCoreRegister(size_t stack_index ATTRIBUTE_UNUSED,
uint32_t reg_id ATTRIBUTE_UNUSED) {
TODO_VIXL32(FATAL);
return 0;
}
// Restores the register from the stack. Returns the size taken on stack.
size_t CodeGeneratorARMVIXL::RestoreCoreRegister(size_t stack_index ATTRIBUTE_UNUSED,
uint32_t reg_id ATTRIBUTE_UNUSED) {
TODO_VIXL32(FATAL);
return 0;
}
size_t CodeGeneratorARMVIXL::SaveFloatingPointRegister(size_t stack_index ATTRIBUTE_UNUSED,
uint32_t reg_id ATTRIBUTE_UNUSED) {
TODO_VIXL32(FATAL);
return 0;
}
size_t CodeGeneratorARMVIXL::RestoreFloatingPointRegister(size_t stack_index ATTRIBUTE_UNUSED,
uint32_t reg_id ATTRIBUTE_UNUSED) {
TODO_VIXL32(FATAL);
return 0;
}
#undef __
CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph,
const ArmInstructionSetFeatures& isa_features,
const CompilerOptions& compiler_options,
OptimizingCompilerStats* stats)
: CodeGenerator(graph,
kNumberOfCoreRegisters,
kNumberOfSRegisters,
kNumberOfRegisterPairs,
kCoreCalleeSaves.GetList(),
ComputeSRegisterListMask(kFpuCalleeSaves),
compiler_options,
stats),
block_labels_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
location_builder_(graph, this),
instruction_visitor_(graph, this),
move_resolver_(graph->GetArena(), this),
assembler_(graph->GetArena()),
isa_features_(isa_features),
uint32_literals_(std::less<uint32_t>(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_type_patches_(TypeReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_address_patches_(std::less<uint32_t>(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Always save the LR register to mimic Quick.
AddAllocatedRegister(Location::RegisterLocation(LR));
// Give d14 and d15 as scratch registers to VIXL.
// They are removed from the register allocator in `SetupBlockedRegisters()`.
// TODO(VIXL): We need two scratch D registers for `EmitSwap` when swapping two double stack
// slots. If that is sufficiently rare, and we have pressure on FP registers, we could instead
// spill in `EmitSwap`. But if we actually are guaranteed to have 32 D registers, we could give
// d30 and d31 to VIXL to avoid removing registers from the allocator. If that is the case, we may
// also want to investigate giving those 14 other D registers to the allocator.
GetVIXLAssembler()->GetScratchVRegisterList()->Combine(d14);
GetVIXLAssembler()->GetScratchVRegisterList()->Combine(d15);
}
void JumpTableARMVIXL::EmitTable(CodeGeneratorARMVIXL* codegen) {
uint32_t num_entries = switch_instr_->GetNumEntries();
DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold);
// We are about to use the assembler to place literals directly. Make sure we have enough
// underlying code buffer and we have generated a jump table of the right size, using
// codegen->GetVIXLAssembler()->GetBuffer().Align();
ExactAssemblyScope aas(codegen->GetVIXLAssembler(),
num_entries * sizeof(int32_t),
CodeBufferCheckScope::kMaximumSize);
// TODO(VIXL): Check that using lower case bind is fine here.
codegen->GetVIXLAssembler()->bind(&table_start_);
for (uint32_t i = 0; i < num_entries; i++) {
codegen->GetVIXLAssembler()->place(bb_addresses_[i].get());
}
}
void JumpTableARMVIXL::FixTable(CodeGeneratorARMVIXL* codegen) {
uint32_t num_entries = switch_instr_->GetNumEntries();
DCHECK_GE(num_entries, kPackedSwitchCompareJumpThreshold);
const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors();
for (uint32_t i = 0; i < num_entries; i++) {
vixl32::Label* target_label = codegen->GetLabelOf(successors[i]);
DCHECK(target_label->IsBound());
int32_t jump_offset = target_label->GetLocation() - table_start_.GetLocation();
// When doing BX to address we need to have lower bit set to 1 in T32.
if (codegen->GetVIXLAssembler()->IsUsingT32()) {
jump_offset++;
}
DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min());
DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max());
bb_addresses_[i].get()->UpdateValue(jump_offset, codegen->GetVIXLAssembler()->GetBuffer());
}
}
void CodeGeneratorARMVIXL::FixJumpTables() {
for (auto&& jump_table : jump_tables_) {
jump_table->FixTable(this);
}
}
#define __ reinterpret_cast<ArmVIXLAssembler*>(GetAssembler())->GetVIXLAssembler()-> // NOLINT
void CodeGeneratorARMVIXL::Finalize(CodeAllocator* allocator) {
FixJumpTables();
GetAssembler()->FinalizeCode();
CodeGenerator::Finalize(allocator);
}
void CodeGeneratorARMVIXL::SetupBlockedRegisters() const {
// Stack register, LR and PC are always reserved.
blocked_core_registers_[SP] = true;
blocked_core_registers_[LR] = true;
blocked_core_registers_[PC] = true;
// Reserve thread register.
blocked_core_registers_[TR] = true;
// Reserve temp register.
blocked_core_registers_[IP] = true;
// Registers s28-s31 (d14-d15) are left to VIXL for scratch registers.
// (They are given to the `MacroAssembler` in `CodeGeneratorARMVIXL::CodeGeneratorARMVIXL`.)
blocked_fpu_registers_[28] = true;
blocked_fpu_registers_[29] = true;
blocked_fpu_registers_[30] = true;
blocked_fpu_registers_[31] = true;
if (GetGraph()->IsDebuggable()) {
// Stubs do not save callee-save floating point registers. If the graph
// is debuggable, we need to deal with these registers differently. For
// now, just block them.
for (uint32_t i = kFpuCalleeSaves.GetFirstSRegister().GetCode();
i <= kFpuCalleeSaves.GetLastSRegister().GetCode();
++i) {
blocked_fpu_registers_[i] = true;
}
}
}
InstructionCodeGeneratorARMVIXL::InstructionCodeGeneratorARMVIXL(HGraph* graph,
CodeGeneratorARMVIXL* codegen)
: InstructionCodeGenerator(graph, codegen),
assembler_(codegen->GetAssembler()),
codegen_(codegen) {}
void CodeGeneratorARMVIXL::ComputeSpillMask() {
core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_;
DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved";
// There is no easy instruction to restore just the PC on thumb2. We spill and
// restore another arbitrary register.
core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister.GetCode());
fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_;
// We use vpush and vpop for saving and restoring floating point registers, which take
// a SRegister and the number of registers to save/restore after that SRegister. We
// therefore update the `fpu_spill_mask_` to also contain those registers not allocated,
// but in the range.
if (fpu_spill_mask_ != 0) {
uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_);
uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_);
for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) {
fpu_spill_mask_ |= (1 << i);
}
}
}
void CodeGeneratorARMVIXL::GenerateFrameEntry() {
bool skip_overflow_check =
IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm);
DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
__ Bind(&frame_entry_label_);
if (HasEmptyFrame()) {
return;
}
if (!skip_overflow_check) {
UseScratchRegisterScope temps(GetVIXLAssembler());
vixl32::Register temp = temps.Acquire();
__ Sub(temp, sp, Operand::From(GetStackOverflowReservedBytes(kArm)));
// The load must immediately precede RecordPcInfo.
ExactAssemblyScope aas(GetVIXLAssembler(),
vixl32::kMaxInstructionSizeInBytes,
CodeBufferCheckScope::kMaximumSize);
__ ldr(temp, MemOperand(temp));
RecordPcInfo(nullptr, 0);
}
__ Push(RegisterList(core_spill_mask_));
GetAssembler()->cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_));
GetAssembler()->cfi().RelOffsetForMany(DWARFReg(kMethodRegister),
0,
core_spill_mask_,
kArmWordSize);
if (fpu_spill_mask_ != 0) {
uint32_t first = LeastSignificantBit(fpu_spill_mask_);
// Check that list is contiguous.
DCHECK_EQ(fpu_spill_mask_ >> CTZ(fpu_spill_mask_), ~0u >> (32 - POPCOUNT(fpu_spill_mask_)));
__ Vpush(SRegisterList(vixl32::SRegister(first), POPCOUNT(fpu_spill_mask_)));
GetAssembler()->cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_));
GetAssembler()->cfi().RelOffsetForMany(DWARFReg(s0), 0, fpu_spill_mask_, kArmWordSize);
}
if (GetGraph()->HasShouldDeoptimizeFlag()) {
UseScratchRegisterScope temps(GetVIXLAssembler());
vixl32::Register temp = temps.Acquire();
// Initialize should_deoptimize flag to 0.
__ Mov(temp, 0);
GetAssembler()->StoreToOffset(kStoreWord, temp, sp, -kShouldDeoptimizeFlagSize);
}
int adjust = GetFrameSize() - FrameEntrySpillSize();
__ Sub(sp, sp, adjust);
GetAssembler()->cfi().AdjustCFAOffset(adjust);
// Save the current method if we need it. Note that we do not
// do this in HCurrentMethod, as the instruction might have been removed
// in the SSA graph.
if (RequiresCurrentMethod()) {
GetAssembler()->StoreToOffset(kStoreWord, kMethodRegister, sp, 0);
}
}
void CodeGeneratorARMVIXL::GenerateFrameExit() {
if (HasEmptyFrame()) {
__ Bx(lr);
return;
}
GetAssembler()->cfi().RememberState();
int adjust = GetFrameSize() - FrameEntrySpillSize();
__ Add(sp, sp, adjust);
GetAssembler()->cfi().AdjustCFAOffset(-adjust);
if (fpu_spill_mask_ != 0) {
uint32_t first = LeastSignificantBit(fpu_spill_mask_);
// Check that list is contiguous.
DCHECK_EQ(fpu_spill_mask_ >> CTZ(fpu_spill_mask_), ~0u >> (32 - POPCOUNT(fpu_spill_mask_)));
__ Vpop(SRegisterList(vixl32::SRegister(first), POPCOUNT(fpu_spill_mask_)));
GetAssembler()->cfi().AdjustCFAOffset(
-static_cast<int>(kArmWordSize) * POPCOUNT(fpu_spill_mask_));
GetAssembler()->cfi().RestoreMany(DWARFReg(vixl32::SRegister(0)), fpu_spill_mask_);
}
// Pop LR into PC to return.
DCHECK_NE(core_spill_mask_ & (1 << kLrCode), 0U);
uint32_t pop_mask = (core_spill_mask_ & (~(1 << kLrCode))) | 1 << kPcCode;
__ Pop(RegisterList(pop_mask));
GetAssembler()->cfi().RestoreState();
GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
}
void CodeGeneratorARMVIXL::Bind(HBasicBlock* block) {
__ Bind(GetLabelOf(block));
}
Location InvokeDexCallingConventionVisitorARMVIXL::GetNextLocation(Primitive::Type type) {
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
case Primitive::kPrimChar:
case Primitive::kPrimShort:
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
uint32_t index = gp_index_++;
uint32_t stack_index = stack_index_++;
if (index < calling_convention.GetNumberOfRegisters()) {
return LocationFrom(calling_convention.GetRegisterAt(index));
} else {
return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index));
}
}
case Primitive::kPrimLong: {
uint32_t index = gp_index_;
uint32_t stack_index = stack_index_;
gp_index_ += 2;
stack_index_ += 2;
if (index + 1 < calling_convention.GetNumberOfRegisters()) {
if (calling_convention.GetRegisterAt(index).Is(r1)) {
// Skip R1, and use R2_R3 instead.
gp_index_++;
index++;
}
}
if (index + 1 < calling_convention.GetNumberOfRegisters()) {
DCHECK_EQ(calling_convention.GetRegisterAt(index).GetCode() + 1,
calling_convention.GetRegisterAt(index + 1).GetCode());
return LocationFrom(calling_convention.GetRegisterAt(index),
calling_convention.GetRegisterAt(index + 1));
} else {
return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index));
}
}
case Primitive::kPrimFloat: {
uint32_t stack_index = stack_index_++;
if (float_index_ % 2 == 0) {
float_index_ = std::max(double_index_, float_index_);
}
if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) {
return LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
} else {
return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index));
}
}
case Primitive::kPrimDouble: {
double_index_ = std::max(double_index_, RoundUp(float_index_, 2));
uint32_t stack_index = stack_index_;
stack_index_ += 2;
if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) {
uint32_t index = double_index_;
double_index_ += 2;
Location result = LocationFrom(
calling_convention.GetFpuRegisterAt(index),
calling_convention.GetFpuRegisterAt(index + 1));
DCHECK(ExpectedPairLayout(result));
return result;
} else {
return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index));
}
}
case Primitive::kPrimVoid:
LOG(FATAL) << "Unexpected parameter type " << type;
break;
}
return Location::NoLocation();
}
Location InvokeDexCallingConventionVisitorARMVIXL::GetReturnLocation(Primitive::Type type) const {
switch (type) {
case Primitive::kPrimBoolean:
case Primitive::kPrimByte:
case Primitive::kPrimChar:
case Primitive::kPrimShort:
case Primitive::kPrimInt:
case Primitive::kPrimNot: {
return LocationFrom(r0);
}
case Primitive::kPrimFloat: {
return LocationFrom(s0);
}
case Primitive::kPrimLong: {
return LocationFrom(r0, r1);
}
case Primitive::kPrimDouble: {
return LocationFrom(s0, s1);
}
case Primitive::kPrimVoid:
return Location::NoLocation();
}
UNREACHABLE();
}
Location InvokeDexCallingConventionVisitorARMVIXL::GetMethodLocation() const {
return LocationFrom(kMethodRegister);
}
void CodeGeneratorARMVIXL::Move32(Location destination, Location source) {
if (source.Equals(destination)) {
return;
}
if (destination.IsRegister()) {
if (source.IsRegister()) {
__ Mov(RegisterFrom(destination), RegisterFrom(source));
} else if (source.IsFpuRegister()) {
__ Vmov(RegisterFrom(destination), SRegisterFrom(source));
} else {
GetAssembler()->LoadFromOffset(kLoadWord,
RegisterFrom(destination),
sp,
source.GetStackIndex());
}
} else if (destination.IsFpuRegister()) {
if (source.IsRegister()) {
__ Vmov(SRegisterFrom(destination), RegisterFrom(source));
} else if (source.IsFpuRegister()) {
__ Vmov(SRegisterFrom(destination), SRegisterFrom(source));
} else {
GetAssembler()->LoadSFromOffset(SRegisterFrom(destination), sp, source.GetStackIndex());
}
} else {
DCHECK(destination.IsStackSlot()) << destination;
if (source.IsRegister()) {
GetAssembler()->StoreToOffset(kStoreWord,
RegisterFrom(source),
sp,
destination.GetStackIndex());
} else if (source.IsFpuRegister()) {
GetAssembler()->StoreSToOffset(SRegisterFrom(source), sp, destination.GetStackIndex());
} else {
DCHECK(source.IsStackSlot()) << source;
UseScratchRegisterScope temps(GetVIXLAssembler());
vixl32::Register temp = temps.Acquire();
GetAssembler()->LoadFromOffset(kLoadWord, temp, sp, source.GetStackIndex());
GetAssembler()->StoreToOffset(kStoreWord, temp, sp, destination.GetStackIndex());
}
}
}
void CodeGeneratorARMVIXL::MoveConstant(Location location, int32_t value) {
DCHECK(location.IsRegister());
__ Mov(RegisterFrom(location), value);
}
void CodeGeneratorARMVIXL::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
// TODO(VIXL): Maybe refactor to have the 'move' implementation here and use it in
// `ParallelMoveResolverARMVIXL::EmitMove`, as is done in the `arm64` backend.
HParallelMove move(GetGraph()->GetArena());
move.AddMove(src, dst, dst_type, nullptr);
GetMoveResolver()->EmitNativeCode(&move);
}
void CodeGeneratorARMVIXL::AddLocationAsTemp(Location location, LocationSummary* locations) {
if (location.IsRegister()) {
locations->AddTemp(location);
} else if (location.IsRegisterPair()) {
locations->AddTemp(LocationFrom(LowRegisterFrom(location)));
locations->AddTemp(LocationFrom(HighRegisterFrom(location)));
} else {
UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
}
}
void CodeGeneratorARMVIXL::InvokeRuntime(QuickEntrypointEnum entrypoint,
HInstruction* instruction,
uint32_t dex_pc,
SlowPathCode* slow_path) {
ValidateInvokeRuntime(entrypoint, instruction, slow_path);
__ Ldr(lr, MemOperand(tr, GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()));
// Ensure the pc position is recorded immediately after the `blx` instruction.
// blx in T32 has only 16bit encoding that's why a stricter check for the scope is used.
ExactAssemblyScope aas(GetVIXLAssembler(),
vixl32::k16BitT32InstructionSizeInBytes,
CodeBufferCheckScope::kExactSize);
__ blx(lr);
if (EntrypointRequiresStackMap(entrypoint)) {
RecordPcInfo(instruction, dex_pc, slow_path);
}
}
void CodeGeneratorARMVIXL::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
HInstruction* instruction,
SlowPathCode* slow_path) {
ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path);
__ Ldr(lr, MemOperand(tr, entry_point_offset));
__ Blx(lr);
}
void InstructionCodeGeneratorARMVIXL::HandleGoto(HInstruction* got, HBasicBlock* successor) {
DCHECK(!successor->IsExitBlock());
HBasicBlock* block = got->GetBlock();
HInstruction* previous = got->GetPrevious();
HLoopInformation* info = block->GetLoopInformation();
if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
GenerateSuspendCheck(info->GetSuspendCheck(), successor);
return;
}
if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
}
if (!codegen_->GoesToNextBlock(block, successor)) {
__ B(codegen_->GetLabelOf(successor));
}
}
void LocationsBuilderARMVIXL::VisitGoto(HGoto* got) {
got->SetLocations(nullptr);
}
void InstructionCodeGeneratorARMVIXL::VisitGoto(HGoto* got) {
HandleGoto(got, got->GetSuccessor());
}
void LocationsBuilderARMVIXL::VisitTryBoundary(HTryBoundary* try_boundary) {
try_boundary->SetLocations(nullptr);
}
void InstructionCodeGeneratorARMVIXL::VisitTryBoundary(HTryBoundary* try_boundary) {
HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
if (!successor->IsExitBlock()) {
HandleGoto(try_boundary, successor);
}
}
void LocationsBuilderARMVIXL::VisitExit(HExit* exit) {
exit->SetLocations(nullptr);
}
void InstructionCodeGeneratorARMVIXL::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
}
void InstructionCodeGeneratorARMVIXL::GenerateVcmp(HInstruction* instruction) {
Primitive::Type type = instruction->InputAt(0)->GetType();
Location lhs_loc = instruction->GetLocations()->InAt(0);
Location rhs_loc = instruction->GetLocations()->InAt(1);
if (rhs_loc.IsConstant()) {
// 0.0 is the only immediate that can be encoded directly in
// a VCMP instruction.
//
// Both the JLS (section 15.20.1) and the JVMS (section 6.5)
// specify that in a floating-point comparison, positive zero
// and negative zero are considered equal, so we can use the
// literal 0.0 for both cases here.
//
// Note however that some methods (Float.equal, Float.compare,
// Float.compareTo, Double.equal, Double.compare,
// Double.compareTo, Math.max, Math.min, StrictMath.max,
// StrictMath.min) consider 0.0 to be (strictly) greater than
// -0.0. So if we ever translate calls to these methods into a
// HCompare instruction, we must handle the -0.0 case with
// care here.
DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
if (type == Primitive::kPrimFloat) {
__ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0);
} else {
DCHECK_EQ(type, Primitive::kPrimDouble);
__ Vcmp(F64, DRegisterFrom(lhs_loc), 0.0);
}
} else {
if (type == Primitive::kPrimFloat) {
__ Vcmp(InputSRegisterAt(instruction, 0), InputSRegisterAt(instruction, 1));
} else {
DCHECK_EQ(type, Primitive::kPrimDouble);
__ Vcmp(DRegisterFrom(lhs_loc), DRegisterFrom(rhs_loc));
}
}
}
void InstructionCodeGeneratorARMVIXL::GenerateFPJumps(HCondition* cond,
vixl32::Label* true_label,
vixl32::Label* false_label ATTRIBUTE_UNUSED) {
// To branch on the result of the FP compare we transfer FPSCR to APSR (encoded as PC in VMRS).
__ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR);
__ B(ARMFPCondition(cond->GetCondition(), cond->IsGtBias()), true_label);
}
void InstructionCodeGeneratorARMVIXL::GenerateLongComparesAndJumps(HCondition* cond,
vixl32::Label* true_label,
vixl32::Label* false_label) {
LocationSummary* locations = cond->GetLocations();
Location left = locations->InAt(0);
Location right = locations->InAt(1);
IfCondition if_cond = cond->GetCondition();
vixl32::Register left_high = HighRegisterFrom(left);
vixl32::Register left_low = LowRegisterFrom(left);
IfCondition true_high_cond = if_cond;
IfCondition false_high_cond = cond->GetOppositeCondition();
vixl32::Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part
// Set the conditions for the test, remembering that == needs to be
// decided using the low words.
// TODO: consider avoiding jumps with temporary and CMP low+SBC high
switch (if_cond) {
case kCondEQ:
case kCondNE:
// Nothing to do.
break;
case kCondLT:
false_high_cond = kCondGT;
break;
case kCondLE:
true_high_cond = kCondLT;
break;
case kCondGT:
false_high_cond = kCondLT;
break;
case kCondGE:
true_high_cond = kCondGT;
break;
case kCondB:
false_high_cond = kCondA;
break;
case kCondBE:
true_high_cond = kCondB;
break;
case kCondA:
false_high_cond = kCondB;
break;
case kCondAE:
true_high_cond = kCondA;
break;
}
if (right.IsConstant()) {
int64_t value = Int64ConstantFrom(right);
int32_t val_low = Low32Bits(value);
int32_t val_high = High32Bits(value);
__ Cmp(left_high, val_high);
if (if_cond == kCondNE) {
__ B(ARMCondition(true_high_cond), true_label);
} else if (if_cond == kCondEQ) {
__ B(ARMCondition(false_high_cond), false_label);
} else {
__ B(ARMCondition(true_high_cond), true_label);
__ B(ARMCondition(false_high_cond), false_label);
}
// Must be equal high, so compare the lows.
__ Cmp(left_low, val_low);
} else {
vixl32::Register right_high = HighRegisterFrom(right);
vixl32::Register right_low = LowRegisterFrom(right);
__ Cmp(left_high, right_high);
if (if_cond == kCondNE) {
__ B(ARMCondition(true_high_cond), true_label);
} else if (if_cond == kCondEQ) {
__ B(ARMCondition(false_high_cond), false_label);
} else {
__ B(ARMCondition(true_high_cond), true_label);
__ B(ARMCondition(false_high_cond), false_label);
}
// Must be equal high, so compare the lows.
__ Cmp(left_low, right_low);
}
// The last comparison might be unsigned.
// TODO: optimize cases where this is always true/false
__ B(final_condition, true_label);
}
void InstructionCodeGeneratorARMVIXL::GenerateCompareTestAndBranch(HCondition* condition,
vixl32::Label* true_target_in,
vixl32::Label* false_target_in) {
// Generated branching requires both targets to be explicit. If either of the
// targets is nullptr (fallthrough) use and bind `fallthrough` instead.
vixl32::Label fallthrough;
vixl32::Label* true_target = (true_target_in == nullptr) ? &fallthrough : true_target_in;
vixl32::Label* false_target = (false_target_in == nullptr) ? &fallthrough : false_target_in;
Primitive::Type type = condition->InputAt(0)->GetType();
switch (type) {
case Primitive::kPrimLong:
GenerateLongComparesAndJumps(condition, true_target, false_target);
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
GenerateVcmp(condition);
GenerateFPJumps(condition, true_target, false_target);
break;
default:
LOG(FATAL) << "Unexpected compare type " << type;
}
if (false_target != &fallthrough) {
__ B(false_target);
}
if (true_target_in == nullptr || false_target_in == nullptr) {
__ Bind(&fallthrough);
}
}
void InstructionCodeGeneratorARMVIXL::GenerateTestAndBranch(HInstruction* instruction,
size_t condition_input_index,
vixl32::Label* true_target,
vixl32::Label* false_target,
bool far_target) {
HInstruction* cond = instruction->InputAt(condition_input_index);
if (true_target == nullptr && false_target == nullptr) {
// Nothing to do. The code always falls through.
return;
} else if (cond->IsIntConstant()) {
// Constant condition, statically compared against "true" (integer value 1).
if (cond->AsIntConstant()->IsTrue()) {
if (true_target != nullptr) {
__ B(true_target);
}
} else {
DCHECK(cond->AsIntConstant()->IsFalse()) << Int32ConstantFrom(cond);
if (false_target != nullptr) {
__ B(false_target);
}
}
return;
}
// The following code generates these patterns:
// (1) true_target == nullptr && false_target != nullptr
// - opposite condition true => branch to false_target
// (2) true_target != nullptr && false_target == nullptr
// - condition true => branch to true_target
// (3) true_target != nullptr && false_target != nullptr
// - condition true => branch to true_target
// - branch to false_target
if (IsBooleanValueOrMaterializedCondition(cond)) {
// Condition has been materialized, compare the output to 0.
if (kIsDebugBuild) {
Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
DCHECK(cond_val.IsRegister());
}
if (true_target == nullptr) {
__ CompareAndBranchIfZero(InputRegisterAt(instruction, condition_input_index),
false_target,
far_target);
} else {
__ CompareAndBranchIfNonZero(InputRegisterAt(instruction, condition_input_index),
true_target,
far_target);
}
} else {
// Condition has not been materialized. Use its inputs as the comparison and
// its condition as the branch condition.
HCondition* condition = cond->AsCondition();
// If this is a long or FP comparison that has been folded into
// the HCondition, generate the comparison directly.
Primitive::Type type = condition->InputAt(0)->GetType();
if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
GenerateCompareTestAndBranch(condition, true_target, false_target);
return;
}
LocationSummary* locations = cond->GetLocations();
DCHECK(locations->InAt(0).IsRegister());
vixl32::Register left = InputRegisterAt(cond, 0);
Location right = locations->InAt(1);
if (right.IsRegister()) {
__ Cmp(left, InputRegisterAt(cond, 1));
} else {
DCHECK(right.IsConstant());
__ Cmp(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
}
if (true_target == nullptr) {
__ B(ARMCondition(condition->GetOppositeCondition()), false_target);
} else {
__ B(ARMCondition(condition->GetCondition()), true_target);
}
}
// If neither branch falls through (case 3), the conditional branch to `true_target`
// was already emitted (case 2) and we need to emit a jump to `false_target`.
if (true_target != nullptr && false_target != nullptr) {
__ B(false_target);
}
}
void LocationsBuilderARMVIXL::VisitIf(HIf* if_instr) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARMVIXL::VisitIf(HIf* if_instr) {
HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
vixl32::Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
nullptr : codegen_->GetLabelOf(true_successor);
vixl32::Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
nullptr : codegen_->GetLabelOf(false_successor);
GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
}
void LocationsBuilderARMVIXL::VisitDeoptimize(HDeoptimize* deoptimize) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
locations->SetInAt(0, Location::RequiresRegister());
}
}
void InstructionCodeGeneratorARMVIXL::VisitDeoptimize(HDeoptimize* deoptimize) {
SlowPathCodeARMVIXL* slow_path =
deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARMVIXL>(deoptimize);
GenerateTestAndBranch(deoptimize,
/* condition_input_index */ 0,
slow_path->GetEntryLabel(),
/* false_target */ nullptr);
}
void LocationsBuilderARMVIXL::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) {
LocationSummary* locations = new (GetGraph()->GetArena())
LocationSummary(flag, LocationSummary::kNoCall);
locations->SetOut(Location::RequiresRegister());
}
void InstructionCodeGeneratorARMVIXL::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) {
GetAssembler()->LoadFromOffset(kLoadWord,
OutputRegister(flag),
sp,
codegen_->GetStackOffsetOfShouldDeoptimizeFlag());
}
void LocationsBuilderARMVIXL::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
if (Primitive::IsFloatingPointType(select->GetType())) {
locations->SetInAt(0, Location::RequiresFpuRegister());
locations->SetInAt(1, Location::RequiresFpuRegister());
} else {
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
}
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
locations->SetInAt(2, Location::RequiresRegister());
}
locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARMVIXL::VisitSelect(HSelect* select) {
LocationSummary* locations = select->GetLocations();
vixl32::Label false_target;
GenerateTestAndBranch(select,
/* condition_input_index */ 2,
/* true_target */ nullptr,
&false_target,
/* far_target */ false);
codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
__ Bind(&false_target);
}
void LocationsBuilderARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo* info) {
new (GetGraph()->GetArena()) LocationSummary(info);
}
void InstructionCodeGeneratorARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo*) {
// MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile.
}
void CodeGeneratorARMVIXL::GenerateNop() {
__ Nop();
}
void LocationsBuilderARMVIXL::HandleCondition(HCondition* cond) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
// Handle the long/FP comparisons made in instruction simplification.
switch (cond->InputAt(0)->GetType()) {
case Primitive::kPrimLong:
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
if (!cond->IsEmittedAtUseSite()) {
locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
}
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
locations->SetInAt(0, Location::RequiresFpuRegister());
locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1)));
if (!cond->IsEmittedAtUseSite()) {
locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
break;
default:
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
if (!cond->IsEmittedAtUseSite()) {
locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
}
}
void InstructionCodeGeneratorARMVIXL::HandleCondition(HCondition* cond) {
if (cond->IsEmittedAtUseSite()) {
return;
}
Location right = cond->GetLocations()->InAt(1);
vixl32::Register out = OutputRegister(cond);
vixl32::Label true_label, false_label;
switch (cond->InputAt(0)->GetType()) {
default: {
// Integer case.
if (right.IsRegister()) {
__ Cmp(InputRegisterAt(cond, 0), InputOperandAt(cond, 1));
} else {
DCHECK(right.IsConstant());
__ Cmp(InputRegisterAt(cond, 0),
CodeGenerator::GetInt32ValueOf(right.GetConstant()));
}
ExactAssemblyScope aas(GetVIXLAssembler(),
3 * vixl32::kMaxInstructionSizeInBytes,
CodeBufferCheckScope::kMaximumSize);
__ ite(ARMCondition(cond->GetCondition()));
__ mov(ARMCondition(cond->GetCondition()), OutputRegister(cond), 1);
__ mov(ARMCondition(cond->GetOppositeCondition()), OutputRegister(cond), 0);
return;
}
case Primitive::kPrimLong:
GenerateLongComparesAndJumps(cond, &true_label, &false_label);
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
GenerateVcmp(cond);
GenerateFPJumps(cond, &true_label, &false_label);
break;
}
// Convert the jumps into the result.
vixl32::Label done_label;
// False case: result = 0.
__ Bind(&false_label);
__ Mov(out, 0);
__ B(&done_label);
// True case: result = 1.
__ Bind(&true_label);
__ Mov(out, 1);
__ Bind(&done_label);
}
void LocationsBuilderARMVIXL::VisitEqual(HEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitEqual(HEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitNotEqual(HNotEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitNotEqual(HNotEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitLessThan(HLessThan* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitLessThan(HLessThan* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitGreaterThan(HGreaterThan* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitGreaterThan(HGreaterThan* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitBelow(HBelow* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitBelow(HBelow* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitBelowOrEqual(HBelowOrEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitBelowOrEqual(HBelowOrEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitAbove(HAbove* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitAbove(HAbove* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitAboveOrEqual(HAboveOrEqual* comp) {
HandleCondition(comp);
}
void InstructionCodeGeneratorARMVIXL::VisitAboveOrEqual(HAboveOrEqual* comp) {
HandleCondition(comp);
}
void LocationsBuilderARMVIXL::VisitIntConstant(HIntConstant* constant) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
locations->SetOut(Location::ConstantLocation(constant));
}
void InstructionCodeGeneratorARMVIXL::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
// Will be generated at use site.
}
void LocationsBuilderARMVIXL::VisitNullConstant(HNullConstant* constant) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
locations->SetOut(Location::ConstantLocation(constant));
}
void InstructionCodeGeneratorARMVIXL::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
// Will be generated at use site.
}
void LocationsBuilderARMVIXL::VisitLongConstant(HLongConstant* constant) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
locations->SetOut(Location::ConstantLocation(constant));
}
void InstructionCodeGeneratorARMVIXL::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
// Will be generated at use site.
}
void LocationsBuilderARMVIXL::VisitFloatConstant(HFloatConstant* constant) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
locations->SetOut(Location::ConstantLocation(constant));
}
void InstructionCodeGeneratorARMVIXL::VisitFloatConstant(
HFloatConstant* constant ATTRIBUTE_UNUSED) {
// Will be generated at use site.
}
void LocationsBuilderARMVIXL::VisitDoubleConstant(HDoubleConstant* constant) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
locations->SetOut(Location::ConstantLocation(constant));
}
void InstructionCodeGeneratorARMVIXL::VisitDoubleConstant(
HDoubleConstant* constant ATTRIBUTE_UNUSED) {
// Will be generated at use site.
}
void LocationsBuilderARMVIXL::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
memory_barrier->SetLocations(nullptr);
}
void InstructionCodeGeneratorARMVIXL::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
}
void LocationsBuilderARMVIXL::VisitReturnVoid(HReturnVoid* ret) {
ret->SetLocations(nullptr);
}
void InstructionCodeGeneratorARMVIXL::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
codegen_->GenerateFrameExit();
}
void LocationsBuilderARMVIXL::VisitReturn(HReturn* ret) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType()));
}
void InstructionCodeGeneratorARMVIXL::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) {
codegen_->GenerateFrameExit();
}
void LocationsBuilderARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
// The trampoline uses the same calling convention as dex calling conventions,
// except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
// the method_idx.
HandleInvoke(invoke);
}
void InstructionCodeGeneratorARMVIXL::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
}
void LocationsBuilderARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
// Explicit clinit checks triggered by static invokes must have been pruned by
// art::PrepareForRegisterAllocation.
DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_);
if (intrinsic.TryDispatch(invoke)) {
if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
}
return;
}
HandleInvoke(invoke);
// For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
if (invoke->HasPcRelativeDexCache()) {
invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
}
}
static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARMVIXL* codegen) {
if (invoke->GetLocations()->Intrinsified()) {
IntrinsicCodeGeneratorARMVIXL intrinsic(codegen);
intrinsic.Dispatch(invoke);
return true;
}
return false;
}
void InstructionCodeGeneratorARMVIXL::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
// Explicit clinit checks triggered by static invokes must have been pruned by
// art::PrepareForRegisterAllocation.
DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
return;
}
LocationSummary* locations = invoke->GetLocations();
codegen_->GenerateStaticOrDirectCall(
invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
}
void LocationsBuilderARMVIXL::HandleInvoke(HInvoke* invoke) {
InvokeDexCallingConventionVisitorARMVIXL calling_convention_visitor;
CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
}
void LocationsBuilderARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) {
IntrinsicLocationsBuilderARMVIXL intrinsic(codegen_);
if (intrinsic.TryDispatch(invoke)) {
return;
}
HandleInvoke(invoke);
}
void InstructionCodeGeneratorARMVIXL::VisitInvokeVirtual(HInvokeVirtual* invoke) {
if (TryGenerateIntrinsicCode(invoke, codegen_)) {
return;
}
codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
DCHECK(!codegen_->IsLeafMethod());
}
void LocationsBuilderARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) {
HandleInvoke(invoke);
// Add the hidden argument.
invoke->GetLocations()->AddTemp(LocationFrom(r12));
}
void InstructionCodeGeneratorARMVIXL::VisitInvokeInterface(HInvokeInterface* invoke) {
// TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
LocationSummary* locations = invoke->GetLocations();
vixl32::Register temp = RegisterFrom(locations->GetTemp(0));
vixl32::Register hidden_reg = RegisterFrom(locations->GetTemp(1));
Location receiver = locations->InAt(0);
uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
DCHECK(!receiver.IsStackSlot());
// Ensure the pc position is recorded immediately after the `ldr` instruction.
{
ExactAssemblyScope aas(GetVIXLAssembler(),
vixl32::kMaxInstructionSizeInBytes,
CodeBufferCheckScope::kMaximumSize);
// /* HeapReference<Class> */ temp = receiver->klass_
__ ldr(temp, MemOperand(RegisterFrom(receiver), class_offset));
codegen_->MaybeRecordImplicitNullCheck(invoke);
}
// Instead of simply (possibly) unpoisoning `temp` here, we should
// emit a read barrier for the previous class reference load.
// However this is not required in practice, as this is an
// intermediate/temporary reference and because the current
// concurrent copying collector keeps the from-space memory
// intact/accessible until the end of the marking phase (the
// concurrent copying collector may not in the future).
GetAssembler()->MaybeUnpoisonHeapReference(temp);
GetAssembler()->LoadFromOffset(kLoadWord,
temp,
temp,
mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value());
uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement(
invoke->GetImtIndex(), kArmPointerSize));
// temp = temp->GetImtEntryAt(method_offset);
GetAssembler()->LoadFromOffset(kLoadWord, temp, temp, method_offset);
uint32_t entry_point =
ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value();
// LR = temp->GetEntryPoint();
GetAssembler()->LoadFromOffset(kLoadWord, lr, temp, entry_point);
// Set the hidden (in r12) argument. It is done here, right before a BLX to prevent other
// instruction from clobbering it as they might use r12 as a scratch register.
DCHECK(hidden_reg.Is(r12));
{
// The VIXL macro assembler may clobber any of the scratch registers that are available to it,
// so it checks if the application is using them (by passing them to the macro assembler
// methods). The following application of UseScratchRegisterScope corrects VIXL's notion of
// what is available, and is the opposite of the standard usage: Instead of requesting a
// temporary location, it imposes an external constraint (i.e. a specific register is reserved
// for the hidden argument). Note that this works even if VIXL needs a scratch register itself
// (to materialize the constant), since the destination register becomes available for such use
// internally for the duration of the macro instruction.
UseScratchRegisterScope temps(GetVIXLAssembler());
temps.Exclude(hidden_reg);
__ Mov(hidden_reg, invoke->GetDexMethodIndex());
}
{
// Ensure the pc position is recorded immediately after the `blx` instruction.
// blx in T32 has only 16bit encoding that's why a stricter check for the scope is used.
ExactAssemblyScope aas(GetVIXLAssembler(),
vixl32::k16BitT32InstructionSizeInBytes,
CodeBufferCheckScope::kExactSize);
// LR();
__ blx(lr);
codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
DCHECK(!codegen_->IsLeafMethod());
}
}
void LocationsBuilderARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
HandleInvoke(invoke);
}
void InstructionCodeGeneratorARMVIXL::VisitInvokePolymorphic(HInvokePolymorphic* invoke) {
codegen_->GenerateInvokePolymorphicCall(invoke);
}
void LocationsBuilderARMVIXL::VisitNeg(HNeg* neg) {
LocationSummary* locations =
new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
switch (neg->GetResultType()) {
case Primitive::kPrimInt: {
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
break;
}
case Primitive::kPrimLong: {
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
break;
}
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
locations->SetInAt(0, Location::RequiresFpuRegister());
locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
break;
default:
LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
}
}
void InstructionCodeGeneratorARMVIXL::VisitNeg(HNeg* neg) {
LocationSummary* locations = neg->GetLocations();
Location out = locations->Out();
Location in = locations->InAt(0);
switch (neg->GetResultType()) {
case Primitive::kPrimInt:
__ Rsb(OutputRegister(neg), InputRegisterAt(neg, 0), 0);
break;
case Primitive::kPrimLong:
// out.lo = 0 - in.lo (and update the carry/borrow (C) flag)
__ Rsbs(LowRegisterFrom(out), LowRegisterFrom(in), 0);
// We cannot emit an RSC (Reverse Subtract with Carry)
// instruction here, as it does not exist in the Thumb-2
// instruction set. We use the following approach
// using SBC and SUB instead.
//
// out.hi = -C
__ Sbc(HighRegisterFrom(out), HighRegisterFrom(out), HighRegisterFrom(out));
// out.hi = out.hi - in.hi
__ Sub(HighRegisterFrom(out), HighRegisterFrom(out), HighRegisterFrom(in));
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
__ Vneg(OutputVRegister(neg), InputVRegister(neg));
break;
default:
LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
}
}
void LocationsBuilderARMVIXL::VisitTypeConversion(HTypeConversion* conversion) {
Primitive::Type result_type = conversion->GetResultType();
Primitive::Type input_type = conversion->GetInputType();
DCHECK_NE(result_type, input_type);
// The float-to-long, double-to-long and long-to-float type conversions
// rely on a call to the runtime.
LocationSummary::CallKind call_kind =
(((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
&& result_type == Primitive::kPrimLong)
|| (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat))
? LocationSummary::kCallOnMainOnly
: LocationSummary::kNoCall;
LocationSummary* locations =
new (GetGraph()->GetArena