Compile link-time thunks in codegen.
Prepare for experimenting with Baker read barrier marking
introspection entrypoints for JIT.
Test: m test-art-host-gtest
Test: Compare compiled boot*.oat before and after (no diff).
Test: Pixel 2 XL boots.
Bug: 36141117
Change-Id: Idb413a31b158db4bf89a8707ea46dd167a06f110
diff --git a/compiler/cfi_test.h b/compiler/cfi_test.h
index 29ff235..581edaa 100644
--- a/compiler/cfi_test.h
+++ b/compiler/cfi_test.h
@@ -37,8 +37,8 @@
class CFITest : public dwarf::DwarfTest {
public:
void GenerateExpected(FILE* f, InstructionSet isa, const char* isa_str,
- const std::vector<uint8_t>& actual_asm,
- const std::vector<uint8_t>& actual_cfi) {
+ ArrayRef<const uint8_t> actual_asm,
+ ArrayRef<const uint8_t> actual_cfi) {
std::vector<std::string> lines;
// Print the raw bytes.
fprintf(f, "static constexpr uint8_t expected_asm_%s[] = {", isa_str);
@@ -50,11 +50,18 @@
// Pretty-print CFI opcodes.
constexpr bool is64bit = false;
dwarf::DebugFrameOpCodeWriter<> initial_opcodes;
- dwarf::WriteCIE(is64bit, dwarf::Reg(8),
- initial_opcodes, kCFIFormat, &debug_frame_data_);
+ dwarf::WriteCIE(is64bit, dwarf::Reg(8), initial_opcodes, kCFIFormat, &debug_frame_data_);
std::vector<uintptr_t> debug_frame_patches;
- dwarf::WriteFDE(is64bit, 0, 0, 0, actual_asm.size(), ArrayRef<const uint8_t>(actual_cfi),
- kCFIFormat, 0, &debug_frame_data_, &debug_frame_patches);
+ dwarf::WriteFDE(is64bit,
+ /* section_address */ 0,
+ /* cie_address */ 0,
+ /* code_address */ 0,
+ actual_asm.size(),
+ actual_cfi,
+ kCFIFormat,
+ /* buffer_address */ 0,
+ &debug_frame_data_,
+ &debug_frame_patches);
ReformatCfi(Objdump(false, "-W"), &lines);
// Pretty-print assembly.
const uint8_t* asm_base = actual_asm.data();
@@ -142,7 +149,7 @@
}
// Pretty-print byte array. 12 bytes per line.
- static void HexDump(FILE* f, const std::vector<uint8_t>& data) {
+ static void HexDump(FILE* f, ArrayRef<const uint8_t> data) {
for (size_t i = 0; i < data.size(); i++) {
fprintf(f, i % 12 == 0 ? "\n " : " "); // Whitespace.
fprintf(f, "0x%02X,", data[i]);
diff --git a/compiler/driver/compiled_method_storage.cc b/compiler/driver/compiled_method_storage.cc
index a26a985..aa8277e 100644
--- a/compiler/driver/compiled_method_storage.cc
+++ b/compiler/driver/compiled_method_storage.cc
@@ -161,6 +161,46 @@
SwapSpace* const swap_space_;
};
+class CompiledMethodStorage::ThunkMapKey {
+ public:
+ ThunkMapKey(linker::LinkerPatch::Type type, uint32_t custom_value1, uint32_t custom_value2)
+ : type_(type), custom_value1_(custom_value1), custom_value2_(custom_value2) {}
+
+ bool operator<(const ThunkMapKey& other) const {
+ if (custom_value1_ != other.custom_value1_) {
+ return custom_value1_ < other.custom_value1_;
+ }
+ if (custom_value2_ != other.custom_value2_) {
+ return custom_value2_ < other.custom_value2_;
+ }
+ return type_ < other.type_;
+ }
+
+ private:
+ linker::LinkerPatch::Type type_;
+ uint32_t custom_value1_;
+ uint32_t custom_value2_;
+};
+
+class CompiledMethodStorage::ThunkMapValue {
+ public:
+ ThunkMapValue(std::vector<uint8_t, SwapAllocator<uint8_t>>&& code,
+ const std::string& debug_name)
+ : code_(std::move(code)), debug_name_(debug_name) {}
+
+ ArrayRef<const uint8_t> GetCode() const {
+ return ArrayRef<const uint8_t>(code_);
+ }
+
+ const std::string& GetDebugName() const {
+ return debug_name_;
+ }
+
+ private:
+ std::vector<uint8_t, SwapAllocator<uint8_t>> code_;
+ std::string debug_name_;
+};
+
CompiledMethodStorage::CompiledMethodStorage(int swap_fd)
: swap_space_(swap_fd == -1 ? nullptr : new SwapSpace(swap_fd, 10 * MB)),
dedupe_enabled_(true),
@@ -171,7 +211,9 @@
LengthPrefixedArrayAlloc<uint8_t>(swap_space_.get())),
dedupe_cfi_info_("dedupe cfi info", LengthPrefixedArrayAlloc<uint8_t>(swap_space_.get())),
dedupe_linker_patches_("dedupe cfi info",
- LengthPrefixedArrayAlloc<linker::LinkerPatch>(swap_space_.get())) {
+ LengthPrefixedArrayAlloc<linker::LinkerPatch>(swap_space_.get())),
+ thunk_map_lock_("thunk_map_lock"),
+ thunk_map_(std::less<ThunkMapKey>(), SwapAllocator<ThunkMapValueType>(swap_space_.get())) {
}
CompiledMethodStorage::~CompiledMethodStorage() {
@@ -237,4 +279,55 @@
ReleaseArrayIfNotDeduplicated(linker_patches);
}
+CompiledMethodStorage::ThunkMapKey CompiledMethodStorage::GetThunkMapKey(
+ const linker::LinkerPatch& linker_patch) {
+ uint32_t custom_value1 = 0u;
+ uint32_t custom_value2 = 0u;
+ switch (linker_patch.GetType()) {
+ case linker::LinkerPatch::Type::kBakerReadBarrierBranch:
+ custom_value1 = linker_patch.GetBakerCustomValue1();
+ custom_value2 = linker_patch.GetBakerCustomValue2();
+ break;
+ case linker::LinkerPatch::Type::kCallRelative:
+ // No custom values.
+ break;
+ default:
+ LOG(FATAL) << "Unexpected patch type: " << linker_patch.GetType();
+ UNREACHABLE();
+ }
+ return ThunkMapKey(linker_patch.GetType(), custom_value1, custom_value2);
+}
+
+ArrayRef<const uint8_t> CompiledMethodStorage::GetThunkCode(const linker::LinkerPatch& linker_patch,
+ /*out*/ std::string* debug_name) {
+ ThunkMapKey key = GetThunkMapKey(linker_patch);
+ MutexLock lock(Thread::Current(), thunk_map_lock_);
+ auto it = thunk_map_.find(key);
+ if (it != thunk_map_.end()) {
+ const ThunkMapValue& value = it->second;
+ if (debug_name != nullptr) {
+ *debug_name = value.GetDebugName();
+ }
+ return value.GetCode();
+ } else {
+ if (debug_name != nullptr) {
+ *debug_name = std::string();
+ }
+ return ArrayRef<const uint8_t>();
+ }
+}
+
+void CompiledMethodStorage::SetThunkCode(const linker::LinkerPatch& linker_patch,
+ ArrayRef<const uint8_t> code,
+ const std::string& debug_name) {
+ DCHECK(!code.empty());
+ ThunkMapKey key = GetThunkMapKey(linker_patch);
+ std::vector<uint8_t, SwapAllocator<uint8_t>> code_copy(
+ code.begin(), code.end(), SwapAllocator<uint8_t>(swap_space_.get()));
+ ThunkMapValue value(std::move(code_copy), debug_name);
+ MutexLock lock(Thread::Current(), thunk_map_lock_);
+ // Note: Multiple threads can try and compile the same thunk, so this may not create a new entry.
+ thunk_map_.emplace(key, std::move(value));
+}
+
} // namespace art
diff --git a/compiler/driver/compiled_method_storage.h b/compiler/driver/compiled_method_storage.h
index 249f06c..1634fac 100644
--- a/compiler/driver/compiled_method_storage.h
+++ b/compiler/driver/compiled_method_storage.h
@@ -18,6 +18,7 @@
#define ART_COMPILER_DRIVER_COMPILED_METHOD_STORAGE_H_
#include <iosfwd>
+#include <map>
#include <memory>
#include "base/array_ref.h"
@@ -67,7 +68,29 @@
const ArrayRef<const linker::LinkerPatch>& linker_patches);
void ReleaseLinkerPatches(const LengthPrefixedArray<linker::LinkerPatch>* linker_patches);
+ // Returns the code associated with the given patch.
+ // If the code has not been set, returns empty data.
+ // If `debug_name` is not null, stores the associated debug name in `*debug_name`.
+ ArrayRef<const uint8_t> GetThunkCode(const linker::LinkerPatch& linker_patch,
+ /*out*/ std::string* debug_name = nullptr);
+
+ // Sets the code and debug name associated with the given patch.
+ void SetThunkCode(const linker::LinkerPatch& linker_patch,
+ ArrayRef<const uint8_t> code,
+ const std::string& debug_name);
+
private:
+ class ThunkMapKey;
+ class ThunkMapValue;
+ using ThunkMapValueType = std::pair<const ThunkMapKey, ThunkMapValue>;
+ using ThunkMap = std::map<ThunkMapKey,
+ ThunkMapValue,
+ std::less<ThunkMapKey>,
+ SwapAllocator<ThunkMapValueType>>;
+ static_assert(std::is_same<ThunkMapValueType, ThunkMap::value_type>::value, "Value type check.");
+
+ static ThunkMapKey GetThunkMapKey(const linker::LinkerPatch& linker_patch);
+
template <typename T, typename DedupeSetType>
const LengthPrefixedArray<T>* AllocateOrDeduplicateArray(const ArrayRef<const T>& data,
DedupeSetType* dedupe_set);
@@ -102,6 +125,9 @@
ArrayDedupeSet<uint8_t> dedupe_cfi_info_;
ArrayDedupeSet<linker::LinkerPatch> dedupe_linker_patches_;
+ Mutex thunk_map_lock_;
+ ThunkMap thunk_map_ GUARDED_BY(thunk_map_lock_);
+
DISALLOW_COPY_AND_ASSIGN(CompiledMethodStorage);
};
diff --git a/compiler/jni/jni_cfi_test.cc b/compiler/jni/jni_cfi_test.cc
index 236b5c0..38f9548 100644
--- a/compiler/jni/jni_cfi_test.cc
+++ b/compiler/jni/jni_cfi_test.cc
@@ -94,7 +94,11 @@
const std::vector<uint8_t>& actual_cfi = *(jni_asm->cfi().data());
if (kGenerateExpected) {
- GenerateExpected(stdout, isa, isa_str, actual_asm, actual_cfi);
+ GenerateExpected(stdout,
+ isa,
+ isa_str,
+ ArrayRef<const uint8_t>(actual_asm),
+ ArrayRef<const uint8_t>(actual_cfi));
} else {
EXPECT_EQ(expected_asm, actual_asm);
EXPECT_EQ(expected_cfi, actual_cfi);
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc
index 6e0286a..7cb8ae5 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.cc
+++ b/compiler/linker/arm/relative_patcher_arm_base.cc
@@ -30,8 +30,9 @@
class ArmBaseRelativePatcher::ThunkData {
public:
- ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset)
- : code_(std::move(code)),
+ ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset)
+ : code_(code),
+ debug_name_(debug_name),
offsets_(),
max_next_offset_(max_next_offset),
pending_offset_(0u) {
@@ -45,7 +46,11 @@
}
ArrayRef<const uint8_t> GetCode() const {
- return ArrayRef<const uint8_t>(code_);
+ return code_;
+ }
+
+ const std::string& GetDebugName() const {
+ return debug_name_;
}
bool NeedsNextThunk() const {
@@ -142,10 +147,11 @@
}
private:
- std::vector<uint8_t> code_; // The code of the thunk.
- std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
- uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
- uint32_t pending_offset_; // The index of the next offset to write.
+ const ArrayRef<const uint8_t> code_; // The code of the thunk.
+ const std::string debug_name_; // The debug name of the thunk.
+ std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written.
+ uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed.
+ uint32_t pending_offset_; // The index of the next offset to write.
};
class ArmBaseRelativePatcher::PendingThunkComparator {
@@ -239,14 +245,13 @@
std::vector<debug::MethodDebugInfo> result;
result.reserve(number_of_thunks);
for (auto&& entry : thunks_) {
- const ThunkKey& key = entry.first;
const ThunkData& data = entry.second;
size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset);
if (start == data.NumberOfThunks()) {
continue;
}
// Get the base name to use for the first occurrence of the thunk.
- std::string base_name = GetThunkDebugName(key);
+ std::string base_name = data.GetDebugName();
for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) {
debug::MethodDebugInfo info = {};
if (i == 0u) {
@@ -267,9 +272,11 @@
return result;
}
-ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
+ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
InstructionSet instruction_set)
- : provider_(provider),
+ : thunk_provider_(thunk_provider),
+ target_provider_(target_provider),
instruction_set_(instruction_set),
thunks_(),
unprocessed_method_call_patches_(),
@@ -398,7 +405,7 @@
unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod());
if (method_call_thunk_ == nullptr) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset));
+ auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset));
method_call_thunk_ = &it->second;
AddUnreservedThunk(method_call_thunk_);
} else {
@@ -409,7 +416,7 @@
auto lb = thunks_.lower_bound(key);
if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) {
uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key);
- auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset));
+ auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset));
AddUnreservedThunk(&it->second);
} else {
old_data = &lb->second;
@@ -477,7 +484,7 @@
break;
}
} else {
- auto result = provider_->FindMethodOffset(target_method);
+ auto result = target_provider_->FindMethodOffset(target_method);
if (!result.first) {
break;
}
@@ -518,5 +525,14 @@
GetInstructionSetAlignment(instruction_set_));
}
+inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch(
+ const LinkerPatch& patch, uint32_t max_next_offset) {
+ ArrayRef<const uint8_t> code;
+ std::string debug_name;
+ thunk_provider_->GetThunkCode(patch, &code, &debug_name);
+ DCHECK(!code.empty());
+ return ThunkData(code, debug_name, max_next_offset);
+}
+
} // namespace linker
} // namespace art
diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h
index ee09bf9..963d669 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.h
+++ b/compiler/linker/arm/relative_patcher_arm_base.h
@@ -37,7 +37,8 @@
std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE;
protected:
- ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider,
+ ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
InstructionSet instruction_set);
~ArmBaseRelativePatcher();
@@ -94,8 +95,6 @@
uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset,
uint32_t target_offset);
- virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0;
- virtual std::string GetThunkDebugName(const ThunkKey& key) = 0;
virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0;
virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0;
@@ -108,8 +107,10 @@
void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref);
uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key);
+ ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset);
- RelativePatcherTargetProvider* const provider_;
+ RelativePatcherThunkProvider* const thunk_provider_;
+ RelativePatcherTargetProvider* const target_provider_;
const InstructionSet instruction_set_;
// The data for all thunks.
diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc
index 7875517..7400d11 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2.cc
@@ -48,8 +48,9 @@
constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement;
constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement;
-Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider)
- : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) {
+Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider)
+ : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) {
}
void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code,
@@ -110,62 +111,6 @@
uint32_t insn = GetInsn32(code, literal_offset);
DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
- if (kIsDebugBuild) {
- const uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- // Check that the next instruction matches the expected LDR.
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- if (width == BakerReadBarrierWidth::kWide) {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
- // LDR (immediate), encoding T3, with correct base_reg.
- CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16));
- } else {
- DCHECK_GE(code->size() - literal_offset, 6u);
- uint32_t next_insn = GetInsn16(code, literal_offset + 4u);
- // LDR (immediate), encoding T1, with correct base_reg.
- CheckValidReg(next_insn & 0x7u); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3));
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn32(code, literal_offset + 4u);
- // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]).
- CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16));
- CheckValidReg(next_insn & 0xf); // Check index register
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- if (width == BakerReadBarrierWidth::kWide) {
- DCHECK_GE(literal_offset, 4u);
- uint32_t prev_insn = GetInsn32(code, literal_offset - 4u);
- // LDR (immediate), encoding T3, with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12));
- } else {
- DCHECK_GE(literal_offset, 2u);
- uint32_t prev_insn = GetInsn16(code, literal_offset - 2u);
- // LDR (immediate), encoding T1, with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg);
- }
- break;
- }
- default:
- LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType());
- UNREACHABLE();
- }
- }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - (patch_offset + kPcDisplacement);
@@ -178,250 +123,6 @@
SetInsn32(code, literal_offset, insn);
}
-#define __ assembler.GetVIXLAssembler()->
-
-static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler,
- vixl::aarch32::Register base_reg,
- vixl::aarch32::MemOperand& lock_word,
- vixl::aarch32::Label* slow_path,
- int32_t raw_ldr_offset) {
- using namespace vixl::aarch32; // NOLINT(build/namespaces)
- // Load the lock word containing the rb_state.
- __ Ldr(ip, lock_word);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted));
- __ B(ne, slow_path, /* is_far_target */ false);
- __ Add(lr, lr, raw_ldr_offset);
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- __ Add(base_reg, base_reg, Operand(ip, LSR, 32));
- __ Bx(lr); // And return back to the function.
- // Note: The fake dependency is unnecessary for the slow path.
-}
-
-// Load the read barrier introspection entrypoint in register `entrypoint`
-static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler,
- vixl::aarch32::Register entrypoint) {
- using vixl::aarch32::MemOperand;
- using vixl::aarch32::ip;
- // Thread Register.
- const vixl::aarch32::Register tr = vixl::aarch32::r9;
-
- // The register where the read barrier introspection entrypoint is loaded
- // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4).
- DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister);
- // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection.
- DCHECK_EQ(ip.GetCode(), 12u);
- const int32_t entry_point_offset =
- Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode());
- __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
-}
-
-void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler,
- uint32_t encoded_data) {
- using namespace vixl::aarch32; // NOLINT(build/namespaces)
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- // Check if the holder is gray and, if not, add fake dependency to the base register
- // and return to the LDR instruction to load the reference. Otherwise, use introspection
- // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister)
- // that performs further checks on the reference and marks it if needed.
- Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data));
- CheckValidReg(holder_reg.GetCode());
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- // If base_reg differs from holder_reg, the offset was too large and we must have
- // emitted an explicit null check before the load. Otherwise, we need to null-check
- // the holder as we do not necessarily do that check before going to the thunk.
- vixl::aarch32::Label throw_npe;
- if (holder_reg.Is(base_reg)) {
- __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false);
- }
- vixl::aarch32::Label slow_path;
- MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
- const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide)
- ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET
- : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET;
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
- __ Bind(&slow_path);
- const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
- raw_ldr_offset;
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- if (width == BakerReadBarrierWidth::kWide) {
- MemOperand ldr_half_address(lr, ldr_offset + 2);
- __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12".
- __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12.
- __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference.
- } else {
- MemOperand ldr_address(lr, ldr_offset);
- __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1.
- __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint
- ep_reg, // for narrow LDR.
- Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET));
- __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4.
- __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference.
- }
- // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
- __ Bx(ep_reg); // Jump to the entrypoint.
- if (holder_reg.Is(base_reg)) {
- // Add null check slow path. The stack map is at the address pointed to by LR.
- __ Bind(&throw_npe);
- int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value();
- __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset));
- __ Bx(ip);
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- vixl::aarch32::Label slow_path;
- int32_t data_offset =
- mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
- MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
- DCHECK_LT(lock_word.GetOffsetImmediate(), 0);
- const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET;
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
- __ Bind(&slow_path);
- const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
- raw_ldr_offset;
- MemOperand ldr_address(lr, ldr_offset + 2);
- __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm",
- // i.e. Rm+32 because the scale in imm2 is 2.
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create
- // a switch case target based on the index register.
- __ Mov(ip, base_reg); // Move the base register to ip0.
- __ Bx(ep_reg); // Jump to the entrypoint's array switch case.
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
- // and it does not have a forwarding address), call the correct introspection entrypoint;
- // otherwise return the reference (or the extracted forwarding address).
- // There is no gray bit check for GC roots.
- Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(root_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip);
- vixl::aarch32::Label return_label, not_marked, forwarding_address;
- __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false);
- MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value());
- __ Ldr(ip, lock_word);
- __ Tst(ip, LockWord::kMarkBitStateMaskShifted);
- __ B(eq, ¬_marked);
- __ Bind(&return_label);
- __ Bx(lr);
- __ Bind(¬_marked);
- static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3,
- "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in "
- " the highest bits and the 'forwarding address' state to have all bits set");
- __ Cmp(ip, Operand(0xc0000000));
- __ B(hs, &forwarding_address);
- Register ep_reg(kBakerCcEntrypointRegister);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
- // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister
- // to art_quick_read_barrier_mark_introspection_gc_roots.
- int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide)
- ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET
- : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET;
- __ Add(ep_reg, ep_reg, Operand(entrypoint_offset));
- __ Mov(ip, root_reg);
- __ Bx(ep_reg);
- __ Bind(&forwarding_address);
- __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift);
- __ Bx(lr);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
-}
-
-std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) {
- ArenaPool pool;
- ArenaAllocator allocator(&pool);
- arm::ArmVIXLAssembler assembler(&allocator);
-
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- // The thunk just uses the entry point in the ArtMethod. This works even for calls
- // to the generic JNI and interpreter trampolines.
- assembler.LoadFromOffset(
- arm::kLoadWord,
- vixl::aarch32::pc,
- vixl::aarch32::r0,
- ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value());
- __ Bkpt(0);
- break;
- case ThunkType::kBakerReadBarrier:
- CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
- break;
- }
-
- assembler.FinalizeCode();
- std::vector<uint8_t> thunk_code(assembler.CodeSize());
- MemoryRegion code(thunk_code.data(), thunk_code.size());
- assembler.FinalizeInstructions(code);
- return thunk_code;
-}
-
-std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- return "MethodCallThunk";
-
- case ThunkType::kBakerReadBarrier: {
- uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- std::ostringstream oss;
- oss << "BakerReadBarrierThunk";
- switch (kind) {
- case BakerReadBarrierKind::kField:
- oss << "Field";
- if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
- oss << "Wide";
- }
- oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
- << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
- break;
- case BakerReadBarrierKind::kArray:
- oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
- break;
- case BakerReadBarrierKind::kGcRoot:
- oss << "GcRoot";
- if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
- oss << "Wide";
- }
- oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- }
- return oss.str();
- }
- }
-}
-
-#undef __
-
uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h
index 68386c0..68610d6 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.h
+++ b/compiler/linker/arm/relative_patcher_thumb2.h
@@ -19,8 +19,6 @@
#include "arch/arm/registers_arm.h"
#include "base/array_ref.h"
-#include "base/bit_field.h"
-#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -33,42 +31,8 @@
class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- static constexpr uint32_t kBakerCcEntrypointRegister = 4u;
-
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
- uint32_t holder_reg,
- bool narrow) {
- CheckValidReg(base_reg);
- CheckValidReg(holder_reg);
- DCHECK(!narrow || base_reg < 8u) << base_reg;
- BakerReadBarrierWidth width =
- narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(holder_reg) |
- BakerReadBarrierWidthField::Encode(width);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- CheckValidReg(base_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
- BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
- CheckValidReg(root_reg);
- DCHECK(!narrow || root_reg < 8u) << root_reg;
- BakerReadBarrierWidth width =
- narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
- BakerReadBarrierFirstRegField::Encode(root_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) |
- BakerReadBarrierWidthField::Encode(width);
- }
-
- explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider);
+ explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider);
void PatchCall(std::vector<uint8_t>* code,
uint32_t literal_offset,
@@ -83,48 +47,10 @@
uint32_t patch_offset) OVERRIDE;
protected:
- std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
- std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
- static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u;
-
- enum class BakerReadBarrierKind : uint8_t {
- kField, // Field get or array get with constant offset (i.e. constant index).
- kArray, // Array get with index in register.
- kGcRoot, // GC root load.
- kLast = kGcRoot
- };
-
- enum class BakerReadBarrierWidth : uint8_t {
- kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled).
- kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled).
- kLast = kNarrow
- };
-
- static constexpr size_t kBitsForBakerReadBarrierKind =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
- static constexpr size_t kBitsForRegister = 4u;
- using BakerReadBarrierKindField =
- BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
- using BakerReadBarrierFirstRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
- using BakerReadBarrierSecondRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
- static constexpr size_t kBitsForBakerReadBarrierWidth =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast));
- using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth,
- kBitsForBakerReadBarrierKind + 2 * kBitsForRegister,
- kBitsForBakerReadBarrierWidth>;
-
- static void CheckValidReg(uint32_t reg) {
- DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg;
- }
-
- void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data);
-
void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value);
static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset);
diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc
index 2c22a35..e7b11bd 100644
--- a/compiler/linker/arm/relative_patcher_thumb2_test.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc
@@ -16,12 +16,15 @@
#include "linker/arm/relative_patcher_thumb2.h"
+#include "arch/arm/instruction_set_features_arm.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
+#include "optimizing/code_generator_arm_vixl.h"
+#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -189,9 +192,42 @@
return result.second - 1 /* thumb mode */;
}
+ std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
+ /*out*/ std::string* debug_name = nullptr) {
+ OptimizingUnitTestHelper helper;
+ HGraph* graph = helper.CreateGraph();
+ std::string error_msg;
+ ArmFeaturesUniquePtr features =
+ ArmInstructionSetFeatures::FromVariant("default", &error_msg);
+ CompilerOptions options;
+ arm::CodeGeneratorARMVIXL codegen(graph, *features, options);
+ ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
+ codegen.EmitThunkCode(patch, &code, debug_name);
+ return std::vector<uint8_t>(code.begin(), code.end());
+ }
+
+ void AddCompiledMethod(
+ MethodReference method_ref,
+ const ArrayRef<const uint8_t>& code,
+ const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
+ RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
+
+ // Make sure the ThunkProvider has all the necessary thunks.
+ for (const LinkerPatch& patch : patches) {
+ if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative) {
+ std::string debug_name;
+ std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
+ thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
+ }
+ }
+ }
+
std::vector<uint8_t> CompileMethodCallThunk() {
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
- return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
+ /* target_dex_file*/ nullptr,
+ /* target_method_idx */ 0u);
+ return CompileThunk(patch);
}
uint32_t MethodCallThunkSize() {
@@ -228,27 +264,38 @@
void TestStringReference(uint32_t string_offset);
void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
+ uint32_t holder_reg,
+ bool narrow) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
+ return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
+ }
+
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
uint32_t holder_reg,
bool narrow) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
+ return CompileThunk(patch);
}
uint32_t GetOutputInsn32(uint32_t offset) {
@@ -594,7 +641,7 @@
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ false);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -696,7 +743,7 @@
const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
base_reg, holder_reg, /* narrow */ true);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
@@ -809,7 +856,7 @@
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -877,7 +924,7 @@
constexpr uint32_t kLiteralOffset1 = 4u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -907,7 +954,7 @@
constexpr uint32_t kLiteralOffset1 = 6u;
const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
@@ -993,7 +1040,7 @@
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1074,8 +1121,7 @@
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset,
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1134,8 +1180,7 @@
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset,
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1182,8 +1227,7 @@
patches.reserve(num_patches);
const uint32_t ldr =
kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
- uint32_t encoded_data =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
+ uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
for (size_t i = 0; i != num_patches; ++i) {
PushBackInsn(&code, ldr);
PushBackInsn(&code, kBneWPlus0);
@@ -1264,10 +1308,8 @@
ldr1, kBneWPlus0, // First GC root LDR with read barrier.
ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
- uint32_t encoded_data2 =
- Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
+ uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
+ uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc
index b268204..135e39d 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64.cc
@@ -82,9 +82,10 @@
} // anonymous namespace
-Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
+Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
const Arm64InstructionSetFeatures* features)
- : ArmBaseRelativePatcher(provider, InstructionSet::kArm64),
+ : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64),
fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()),
reserved_adrp_thunks_(0u),
processed_adrp_thunks_(0u) {
@@ -313,44 +314,6 @@
uint32_t insn = GetInsn(code, literal_offset);
DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched)
ThunkKey key = GetBakerThunkKey(patch);
- if (kIsDebugBuild) {
- const uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- // Check that the next instruction matches the expected LDR.
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn(code, literal_offset + 4u);
- // LDR (immediate) with correct base_reg.
- CheckValidReg(next_insn & 0x1fu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5));
- break;
- }
- case BakerReadBarrierKind::kArray: {
- DCHECK_GE(code->size() - literal_offset, 8u);
- uint32_t next_insn = GetInsn(code, literal_offset + 4u);
- // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL),
- // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2].
- CheckValidReg(next_insn & 0x1fu); // Check destination register.
- const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5));
- CheckValidReg((next_insn >> 16) & 0x1f); // Check index register
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- DCHECK_GE(literal_offset, 4u);
- uint32_t prev_insn = GetInsn(code, literal_offset - 4u);
- // LDR (immediate) with correct root_reg.
- const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
- CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
- }
uint32_t target_offset = GetThunkTargetOffset(key, patch_offset);
DCHECK_ALIGNED(target_offset, 4u);
uint32_t disp = target_offset - patch_offset;
@@ -359,216 +322,6 @@
SetInsn(code, literal_offset, insn);
}
-#define __ assembler.GetVIXLAssembler()->
-
-static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler,
- vixl::aarch64::Register base_reg,
- vixl::aarch64::MemOperand& lock_word,
- vixl::aarch64::Label* slow_path) {
- using namespace vixl::aarch64; // NOLINT(build/namespaces)
- // Load the lock word containing the rb_state.
- __ Ldr(ip0.W(), lock_word);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path);
- static_assert(
- BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET,
- "Field and array LDR offsets must be the same to reuse the same code.");
- // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning).
- static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
- "Field LDR must be 1 instruction (4B) before the return address label; "
- " 2 instructions (8B) for heap poisoning.");
- __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- __ Add(base_reg, base_reg, Operand(ip0, LSR, 32));
- __ Br(lr); // And return back to the function.
- // Note: The fake dependency is unnecessary for the slow path.
-}
-
-// Load the read barrier introspection entrypoint in register `entrypoint`.
-static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler,
- vixl::aarch64::Register entrypoint) {
- using vixl::aarch64::MemOperand;
- using vixl::aarch64::ip0;
- // Thread Register.
- const vixl::aarch64::Register tr = vixl::aarch64::x19;
-
- // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection.
- DCHECK_EQ(ip0.GetCode(), 16u);
- const int32_t entry_point_offset =
- Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode());
- __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
-}
-
-void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler,
- uint32_t encoded_data) {
- using namespace vixl::aarch64; // NOLINT(build/namespaces)
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- switch (kind) {
- case BakerReadBarrierKind::kField: {
- // Check if the holder is gray and, if not, add fake dependency to the base register
- // and return to the LDR instruction to load the reference. Otherwise, use introspection
- // to load the reference and call the entrypoint (in IP1) that performs further checks
- // on the reference and marks it if needed.
- auto base_reg =
- Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- auto holder_reg =
- Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data));
- CheckValidReg(holder_reg.GetCode());
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- // If base_reg differs from holder_reg, the offset was too large and we must have
- // emitted an explicit null check before the load. Otherwise, we need to null-check
- // the holder as we do not necessarily do that check before going to the thunk.
- vixl::aarch64::Label throw_npe;
- if (holder_reg.Is(base_reg)) {
- __ Cbz(holder_reg.W(), &throw_npe);
- }
- vixl::aarch64::Label slow_path;
- MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
- __ Bind(&slow_path);
- MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
- __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset.
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset.
- __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference.
- // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
- __ Br(ip1); // Jump to the entrypoint.
- if (holder_reg.Is(base_reg)) {
- // Add null check slow path. The stack map is at the address pointed to by LR.
- __ Bind(&throw_npe);
- int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value();
- __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset));
- __ Br(ip0);
- }
- break;
- }
- case BakerReadBarrierKind::kArray: {
- auto base_reg =
- Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(base_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- vixl::aarch64::Label slow_path;
- int32_t data_offset =
- mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
- MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
- DCHECK_LT(lock_word.GetOffset(), 0);
- EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
- __ Bind(&slow_path);
- MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
- __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset.
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set).
- __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create
- // a switch case target based on the index register.
- __ Mov(ip0, base_reg); // Move the base register to ip0.
- __ Br(ip1); // Jump to the entrypoint's array switch case.
- break;
- }
- case BakerReadBarrierKind::kGcRoot: {
- // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
- // and it does not have a forwarding address), call the correct introspection entrypoint;
- // otherwise return the reference (or the extracted forwarding address).
- // There is no gray bit check for GC roots.
- auto root_reg =
- Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
- CheckValidReg(root_reg.GetCode());
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
- temps.Exclude(ip0, ip1);
- vixl::aarch64::Label return_label, not_marked, forwarding_address;
- __ Cbz(root_reg, &return_label);
- MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value());
- __ Ldr(ip0.W(), lock_word);
- __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked);
- __ Bind(&return_label);
- __ Br(lr);
- __ Bind(¬_marked);
- __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1));
- __ B(&forwarding_address, mi);
- LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
- // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to
- // art_quick_read_barrier_mark_introspection_gc_roots.
- __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET));
- __ Mov(ip0.W(), root_reg);
- __ Br(ip1);
- __ Bind(&forwarding_address);
- __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift);
- __ Br(lr);
- break;
- }
- default:
- LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
- UNREACHABLE();
- }
-}
-
-std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) {
- ArenaPool pool;
- ArenaAllocator allocator(&pool);
- arm64::Arm64Assembler assembler(&allocator);
-
- switch (key.GetType()) {
- case ThunkType::kMethodCall: {
- // The thunk just uses the entry point in the ArtMethod. This works even for calls
- // to the generic JNI and interpreter trampolines.
- Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset(
- kArm64PointerSize).Int32Value());
- assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0));
- break;
- }
- case ThunkType::kBakerReadBarrier: {
- CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1());
- break;
- }
- }
-
- // Ensure we emit the literal pool.
- assembler.FinalizeCode();
- std::vector<uint8_t> thunk_code(assembler.CodeSize());
- MemoryRegion code(thunk_code.data(), thunk_code.size());
- assembler.FinalizeInstructions(code);
- return thunk_code;
-}
-
-std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) {
- switch (key.GetType()) {
- case ThunkType::kMethodCall:
- return "MethodCallThunk";
-
- case ThunkType::kBakerReadBarrier: {
- uint32_t encoded_data = key.GetCustomValue1();
- BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
- std::ostringstream oss;
- oss << "BakerReadBarrierThunk";
- switch (kind) {
- case BakerReadBarrierKind::kField:
- oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
- << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
- break;
- case BakerReadBarrierKind::kArray:
- oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- case BakerReadBarrierKind::kGcRoot:
- oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
- DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data));
- break;
- }
- return oss.str();
- }
- }
-}
-
-#undef __
-
uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) {
switch (key.GetType()) {
case ThunkType::kMethodCall:
diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h
index 8ba5997..9dc289d 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.h
+++ b/compiler/linker/arm64/relative_patcher_arm64.h
@@ -18,8 +18,6 @@
#define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_
#include "base/array_ref.h"
-#include "base/bit_field.h"
-#include "base/bit_utils.h"
#include "linker/arm/relative_patcher_arm_base.h"
namespace art {
@@ -32,29 +30,8 @@
class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher {
public:
- static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
- CheckValidReg(base_reg);
- CheckValidReg(holder_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(holder_reg);
- }
-
- static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
- CheckValidReg(base_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
- BakerReadBarrierFirstRegField::Encode(base_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
- }
-
- static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
- CheckValidReg(root_reg);
- return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
- BakerReadBarrierFirstRegField::Encode(root_reg) |
- BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg);
- }
-
- Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
+ Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider,
const Arm64InstructionSetFeatures* features);
uint32_t ReserveSpace(uint32_t offset,
@@ -75,37 +52,10 @@
uint32_t patch_offset) OVERRIDE;
protected:
- std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE;
- std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE;
uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE;
uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE;
private:
- static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u;
-
- enum class BakerReadBarrierKind : uint8_t {
- kField, // Field get or array get with constant offset (i.e. constant index).
- kArray, // Array get with index in register.
- kGcRoot, // GC root load.
- kLast = kGcRoot
- };
-
- static constexpr size_t kBitsForBakerReadBarrierKind =
- MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
- static constexpr size_t kBitsForRegister = 5u;
- using BakerReadBarrierKindField =
- BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
- using BakerReadBarrierFirstRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>;
- using BakerReadBarrierSecondRegField =
- BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>;
-
- static void CheckValidReg(uint32_t reg) {
- DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg;
- }
-
- void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data);
-
static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp);
static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset,
diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc
index 05459a2..393733d 100644
--- a/compiler/linker/arm64/relative_patcher_arm64_test.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc
@@ -16,12 +16,15 @@
#include "linker/arm64/relative_patcher_arm64.h"
+#include "arch/arm64/instruction_set_features_arm64.h"
#include "base/casts.h"
#include "linker/relative_patcher_test.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
#include "mirror/object.h"
#include "oat_quick_method_header.h"
+#include "optimizing/code_generator_arm64.h"
+#include "optimizing/optimizing_unit_test.h"
namespace art {
namespace linker {
@@ -168,9 +171,42 @@
return result.second;
}
+ std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
+ /*out*/ std::string* debug_name = nullptr) {
+ OptimizingUnitTestHelper helper;
+ HGraph* graph = helper.CreateGraph();
+ std::string error_msg;
+ Arm64FeaturesUniquePtr features =
+ Arm64InstructionSetFeatures::FromVariant("default", &error_msg);
+ CompilerOptions options;
+ arm64::CodeGeneratorARM64 codegen(graph, *features, options);
+ ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
+ codegen.EmitThunkCode(patch, &code, debug_name);
+ return std::vector<uint8_t>(code.begin(), code.end());
+ }
+
+ void AddCompiledMethod(
+ MethodReference method_ref,
+ const ArrayRef<const uint8_t>& code,
+ const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
+ RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
+
+ // Make sure the ThunkProvider has all the necessary thunks.
+ for (const LinkerPatch& patch : patches) {
+ if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative) {
+ std::string debug_name;
+ std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
+ thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
+ }
+ }
+ }
+
std::vector<uint8_t> CompileMethodCallThunk() {
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey();
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
+ /* target_dex_file*/ nullptr,
+ /* target_method_idx */ 0u);
+ return CompileThunk(patch);
}
uint32_t MethodCallThunkSize() {
@@ -475,25 +511,34 @@
TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
}
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
+ return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg);
+ }
+
std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) {
const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
+ return CompileThunk(patch);
}
std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) {
LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
- 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg));
- ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch);
- return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key);
+ /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg));
+ return CompileThunk(patch);
}
uint32_t GetOutputInsn(uint32_t offset) {
@@ -919,8 +964,7 @@
const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr});
ASSERT_EQ(kMethodCodeSize, raw_code.size());
ArrayRef<const uint8_t> code(raw_code);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
};
@@ -1005,8 +1049,7 @@
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1066,8 +1109,7 @@
constexpr uint32_t kLiteralOffset1 = 0;
const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1096,8 +1138,7 @@
constexpr uint32_t kLiteralOffset1 = 4;
const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
ArrayRef<const uint8_t> code1(raw_code1);
- uint32_t encoded_data =
- Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
+ uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
const LinkerPatch patches1[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
};
@@ -1170,7 +1211,7 @@
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1247,7 +1288,7 @@
ArrayRef<const uint8_t> code(raw_code);
const LinkerPatch patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(
- kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)),
+ kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)),
};
AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
}
@@ -1343,8 +1384,8 @@
kNopInsn, kNopInsn, // Padding before second GC root read barrier.
ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier.
});
- uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
- uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
+ uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
+ uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
const LinkerPatch last_method_patches[] = {
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h
index 710d8a6..7b35fd9 100644
--- a/compiler/linker/linker_patch.h
+++ b/compiler/linker/linker_patch.h
@@ -141,7 +141,7 @@
static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset,
uint32_t custom_value1 = 0u,
uint32_t custom_value2 = 0u) {
- LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr);
+ LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr);
patch.baker_custom_value1_ = custom_value1;
patch.baker_custom_value2_ = custom_value2;
return patch;
diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc
index 13877f8..b82d152 100644
--- a/compiler/linker/relative_patcher.cc
+++ b/compiler/linker/relative_patcher.cc
@@ -43,7 +43,8 @@
std::unique_ptr<RelativePatcher> RelativePatcher::Create(
InstructionSet instruction_set,
const InstructionSetFeatures* features,
- RelativePatcherTargetProvider* provider) {
+ RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider) {
class RelativePatcherNone FINAL : public RelativePatcher {
public:
RelativePatcherNone() { }
@@ -92,7 +93,8 @@
};
UNUSED(features);
- UNUSED(provider);
+ UNUSED(thunk_provider);
+ UNUSED(target_provider);
switch (instruction_set) {
#ifdef ART_ENABLE_CODEGEN_x86
case InstructionSet::kX86:
@@ -106,12 +108,15 @@
case InstructionSet::kArm:
// Fall through: we generate Thumb2 code for "arm".
case InstructionSet::kThumb2:
- return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider));
+ return std::unique_ptr<RelativePatcher>(
+ new Thumb2RelativePatcher(thunk_provider, target_provider));
#endif
#ifdef ART_ENABLE_CODEGEN_arm64
case InstructionSet::kArm64:
return std::unique_ptr<RelativePatcher>(
- new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures()));
+ new Arm64RelativePatcher(thunk_provider,
+ target_provider,
+ features->AsArm64InstructionSetFeatures()));
#endif
#ifdef ART_ENABLE_CODEGEN_mips
case InstructionSet::kMips:
diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h
index b58e3df..06c7e70 100644
--- a/compiler/linker/relative_patcher.h
+++ b/compiler/linker/relative_patcher.h
@@ -39,6 +39,27 @@
class OutputStream;
/**
+ * @class RelativePatcherThunkProvider
+ * @brief Interface for providing method offsets for relative call targets.
+ */
+class RelativePatcherThunkProvider {
+ public:
+ /**
+ * Get the code and debug name of a thunk needed by the given linker patch.
+ *
+ * @param patch The patch for which we need to retrieve the thunk code.
+ * @param code A variable to receive the code of the thunk. This code must not be empty.
+ * @param debug_name A variable to receive the debug name of the thunk.
+ */
+ virtual void GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) = 0;
+
+ protected:
+ virtual ~RelativePatcherThunkProvider() { }
+};
+
+/**
* @class RelativePatcherTargetProvider
* @brief Interface for providing method offsets for relative call targets.
*/
@@ -70,8 +91,10 @@
class RelativePatcher {
public:
static std::unique_ptr<RelativePatcher> Create(
- InstructionSet instruction_set, const InstructionSetFeatures* features,
- RelativePatcherTargetProvider* provider);
+ InstructionSet instruction_set,
+ const InstructionSetFeatures* features,
+ RelativePatcherThunkProvider* thunk_provider,
+ RelativePatcherTargetProvider* target_provider);
virtual ~RelativePatcher() { }
diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h
index d21f279..af8dc4d 100644
--- a/compiler/linker/relative_patcher_test.h
+++ b/compiler/linker/relative_patcher_test.h
@@ -58,7 +58,10 @@
instruction_set_(instruction_set),
features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)),
method_offset_map_(),
- patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)),
+ patcher_(RelativePatcher::Create(instruction_set,
+ features_.get(),
+ &thunk_provider_,
+ &method_offset_map_)),
bss_begin_(0u),
compiled_method_refs_(),
compiled_methods_(),
@@ -248,6 +251,72 @@
LOG(ERROR) << " " << diff_indicator_str;
}
+ class ThunkProvider : public RelativePatcherThunkProvider {
+ public:
+ ThunkProvider() {}
+
+ void SetThunkCode(const LinkerPatch& patch,
+ ArrayRef<const uint8_t> code,
+ const std::string& debug_name) {
+ thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name));
+ }
+
+ void GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) OVERRIDE {
+ auto it = thunk_map_.find(ThunkKey(patch));
+ CHECK(it != thunk_map_.end());
+ const ThunkValue& value = it->second;
+ CHECK(code != nullptr);
+ *code = value.GetCode();
+ CHECK(debug_name != nullptr);
+ *debug_name = value.GetDebugName();
+ }
+
+ private:
+ class ThunkKey {
+ public:
+ explicit ThunkKey(const LinkerPatch& patch)
+ : type_(patch.GetType()),
+ custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
+ ? patch.GetBakerCustomValue1() : 0u),
+ custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch
+ ? patch.GetBakerCustomValue2() : 0u) {
+ CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == LinkerPatch::Type::kCallRelative);
+ }
+
+ bool operator<(const ThunkKey& other) const {
+ if (custom_value1_ != other.custom_value1_) {
+ return custom_value1_ < other.custom_value1_;
+ }
+ if (custom_value2_ != other.custom_value2_) {
+ return custom_value2_ < other.custom_value2_;
+ }
+ return type_ < other.type_;
+ }
+
+ private:
+ const LinkerPatch::Type type_;
+ const uint32_t custom_value1_;
+ const uint32_t custom_value2_;
+ };
+
+ class ThunkValue {
+ public:
+ ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name)
+ : code_(code.begin(), code.end()), debug_name_(debug_name) {}
+ ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); }
+ const std::string& GetDebugName() const { return debug_name_; }
+
+ private:
+ const std::vector<uint8_t> code_;
+ const std::string debug_name_;
+ };
+
+ std::map<ThunkKey, ThunkValue> thunk_map_;
+ };
+
// Map method reference to assinged offset.
// Wrap the map in a class implementing RelativePatcherTargetProvider.
class MethodOffsetMap FINAL : public RelativePatcherTargetProvider {
@@ -272,6 +341,7 @@
std::string error_msg_;
InstructionSet instruction_set_;
std::unique_ptr<const InstructionSetFeatures> features_;
+ ThunkProvider thunk_provider_;
MethodOffsetMap method_offset_map_;
std::unique_ptr<RelativePatcher> patcher_;
uint32_t bss_begin_;
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index c2ae764..231017f 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -449,6 +449,18 @@
// No linker patches by default.
}
+bool CodeGenerator::NeedsThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED) const {
+ // Code generators that create patches requiring thunk compilation should override this function.
+ return false;
+}
+
+void CodeGenerator::EmitThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED,
+ /*out*/ ArenaVector<uint8_t>* code ATTRIBUTE_UNUSED,
+ /*out*/ std::string* debug_name ATTRIBUTE_UNUSED) {
+ // Code generators that create patches requiring thunk compilation should override this function.
+ LOG(FATAL) << "Unexpected call to EmitThunkCode().";
+}
+
void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots,
size_t maximum_safepoint_spill_size,
size_t number_of_out_slots,
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index 3bd5e14..a86b271 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -21,6 +21,7 @@
#include "arch/instruction_set_features.h"
#include "base/arena_containers.h"
#include "base/arena_object.h"
+#include "base/array_ref.h"
#include "base/bit_field.h"
#include "base/bit_utils.h"
#include "base/enums.h"
@@ -74,6 +75,7 @@
virtual ~CodeAllocator() {}
virtual uint8_t* Allocate(size_t size) = 0;
+ virtual ArrayRef<const uint8_t> GetMemory() const = 0;
private:
DISALLOW_COPY_AND_ASSIGN(CodeAllocator);
@@ -210,6 +212,10 @@
virtual void Initialize() = 0;
virtual void Finalize(CodeAllocator* allocator);
virtual void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches);
+ virtual bool NeedsThunkCode(const linker::LinkerPatch& patch) const;
+ virtual void EmitThunkCode(const linker::LinkerPatch& patch,
+ /*out*/ ArenaVector<uint8_t>* code,
+ /*out*/ std::string* debug_name);
virtual void GenerateFrameEntry() = 0;
virtual void GenerateFrameExit() = 0;
virtual void Bind(HBasicBlock* block) = 0;
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 273346a..31887d9 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -30,7 +30,6 @@
#include "heap_poisoning.h"
#include "intrinsics.h"
#include "intrinsics_arm64.h"
-#include "linker/arm64/relative_patcher_arm64.h"
#include "linker/linker_patch.h"
#include "lock_word.h"
#include "mirror/array-inl.h"
@@ -1425,6 +1424,62 @@
__ FinalizeCode();
CodeGenerator::Finalize(allocator);
+
+ // Verify Baker read barrier linker patches.
+ if (kIsDebugBuild) {
+ ArrayRef<const uint8_t> code = allocator->GetMemory();
+ for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) {
+ DCHECK(info.label.IsBound());
+ uint32_t literal_offset = info.label.GetLocation();
+ DCHECK_ALIGNED(literal_offset, 4u);
+
+ auto GetInsn = [&code](uint32_t offset) {
+ DCHECK_ALIGNED(offset, 4u);
+ return
+ (static_cast<uint32_t>(code[offset + 0]) << 0) +
+ (static_cast<uint32_t>(code[offset + 1]) << 8) +
+ (static_cast<uint32_t>(code[offset + 2]) << 16)+
+ (static_cast<uint32_t>(code[offset + 3]) << 24);
+ };
+
+ const uint32_t encoded_data = info.custom_data;
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ // Check that the next instruction matches the expected LDR.
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ DCHECK_GE(code.size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn(literal_offset + 4u);
+ // LDR (immediate) with correct base_reg.
+ CheckValidReg(next_insn & 0x1fu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5));
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ DCHECK_GE(code.size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn(literal_offset + 4u);
+ // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL),
+ // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2].
+ CheckValidReg(next_insn & 0x1fu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5));
+ CheckValidReg((next_insn >> 16) & 0x1f); // Check index register
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ DCHECK_GE(literal_offset, 4u);
+ uint32_t prev_insn = GetInsn(literal_offset - 4u);
+ // LDR (immediate) with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+ }
+ }
}
void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
@@ -4814,6 +4869,44 @@
DCHECK_EQ(size, linker_patches->size());
}
+bool CodeGeneratorARM64::NeedsThunkCode(const linker::LinkerPatch& patch) const {
+ return patch.GetType() == linker::LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == linker::LinkerPatch::Type::kCallRelative;
+}
+
+void CodeGeneratorARM64::EmitThunkCode(const linker::LinkerPatch& patch,
+ /*out*/ ArenaVector<uint8_t>* code,
+ /*out*/ std::string* debug_name) {
+ Arm64Assembler assembler(GetGraph()->GetAllocator());
+ switch (patch.GetType()) {
+ case linker::LinkerPatch::Type::kCallRelative: {
+ // The thunk just uses the entry point in the ArtMethod. This works even for calls
+ // to the generic JNI and interpreter trampolines.
+ Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset(
+ kArm64PointerSize).Int32Value());
+ assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0));
+ if (GetCompilerOptions().GenerateAnyDebugInfo()) {
+ *debug_name = "MethodCallThunk";
+ }
+ break;
+ }
+ case linker::LinkerPatch::Type::kBakerReadBarrierBranch: {
+ DCHECK_EQ(patch.GetBakerCustomValue2(), 0u);
+ CompileBakerReadBarrierThunk(assembler, patch.GetBakerCustomValue1(), debug_name);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected patch type " << patch.GetType();
+ UNREACHABLE();
+ }
+
+ // Ensure we emit the literal pool if any.
+ assembler.FinalizeCode();
+ code->resize(assembler.CodeSize());
+ MemoryRegion code_region(code->data(), code->size());
+ assembler.FinalizeInstructions(code_region);
+}
+
vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value) {
return uint32_literals_.GetOrCreate(
value,
@@ -4954,12 +5047,12 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
Register current_method = InputRegisterAt(cls, 0);
- GenerateGcRootFieldLoad(cls,
- out_loc,
- current_method,
- ArtMethod::DeclaringClassOffset().Int32Value(),
- /* fixup_label */ nullptr,
- read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls,
+ out_loc,
+ current_method,
+ ArtMethod::DeclaringClassOffset().Int32Value(),
+ /* fixup_label */ nullptr,
+ read_barrier_option);
break;
}
case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: {
@@ -5006,12 +5099,12 @@
vixl::aarch64::Label* ldr_label =
codegen_->NewBssEntryTypePatch(dex_file, type_index, adrp_label);
// /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */
- GenerateGcRootFieldLoad(cls,
- out_loc,
- temp,
- /* offset placeholder */ 0u,
- ldr_label,
- read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls,
+ out_loc,
+ temp,
+ /* offset placeholder */ 0u,
+ ldr_label,
+ read_barrier_option);
generate_null_check = true;
break;
}
@@ -5019,12 +5112,12 @@
__ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(),
cls->GetTypeIndex(),
cls->GetClass()));
- GenerateGcRootFieldLoad(cls,
- out_loc,
- out.X(),
- /* offset */ 0,
- /* fixup_label */ nullptr,
- read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls,
+ out_loc,
+ out.X(),
+ /* offset */ 0,
+ /* fixup_label */ nullptr,
+ read_barrier_option);
break;
}
case HLoadClass::LoadKind::kRuntimeCall:
@@ -5167,12 +5260,12 @@
vixl::aarch64::Label* ldr_label =
codegen_->NewStringBssEntryPatch(dex_file, string_index, adrp_label);
// /* GcRoot<mirror::String> */ out = *(base_address + offset) /* PC-relative */
- GenerateGcRootFieldLoad(load,
- out_loc,
- temp,
- /* offset placeholder */ 0u,
- ldr_label,
- kCompilerReadBarrierOption);
+ codegen_->GenerateGcRootFieldLoad(load,
+ out_loc,
+ temp,
+ /* offset placeholder */ 0u,
+ ldr_label,
+ kCompilerReadBarrierOption);
SlowPathCodeARM64* slow_path =
new (codegen_->GetScopedAllocator()) LoadStringSlowPathARM64(load);
codegen_->AddSlowPath(slow_path);
@@ -5185,12 +5278,12 @@
__ Ldr(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(),
load->GetStringIndex(),
load->GetString()));
- GenerateGcRootFieldLoad(load,
- out_loc,
- out.X(),
- /* offset */ 0,
- /* fixup_label */ nullptr,
- kCompilerReadBarrierOption);
+ codegen_->GenerateGcRootFieldLoad(load,
+ out_loc,
+ out.X(),
+ /* offset */ 0,
+ /* fixup_label */ nullptr,
+ kCompilerReadBarrierOption);
return;
}
default:
@@ -6139,7 +6232,7 @@
}
}
-void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(
+void CodeGeneratorARM64::GenerateGcRootFieldLoad(
HInstruction* instruction,
Location root,
Register obj,
@@ -6173,9 +6266,8 @@
DCHECK(temps.IsAvailable(ip0));
DCHECK(temps.IsAvailable(ip1));
temps.Exclude(ip0, ip1);
- uint32_t custom_data =
- linker::Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg.GetCode());
- vixl::aarch64::Label* cbnz_label = codegen_->NewBakerReadBarrierPatch(custom_data);
+ uint32_t custom_data = EncodeBakerReadBarrierGcRootData(root_reg.GetCode());
+ vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data);
EmissionCheckScope guard(GetVIXLAssembler(), 3 * vixl::aarch64::kInstructionSize);
vixl::aarch64::Label return_address;
@@ -6204,14 +6296,14 @@
// Slow path marking the GC root `root`. The entrypoint will
// be loaded by the slow path code.
SlowPathCodeARM64* slow_path =
- new (codegen_->GetScopedAllocator()) ReadBarrierMarkSlowPathARM64(instruction, root);
- codegen_->AddSlowPath(slow_path);
+ new (GetScopedAllocator()) ReadBarrierMarkSlowPathARM64(instruction, root);
+ AddSlowPath(slow_path);
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
if (fixup_label == nullptr) {
__ Ldr(root_reg, MemOperand(obj, offset));
} else {
- codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj);
+ EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj);
}
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
@@ -6231,10 +6323,10 @@
if (fixup_label == nullptr) {
__ Add(root_reg.X(), obj.X(), offset);
} else {
- codegen_->EmitAddPlaceholder(fixup_label, root_reg.X(), obj.X());
+ EmitAddPlaceholder(fixup_label, root_reg.X(), obj.X());
}
// /* mirror::Object* */ root = root->Read()
- codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
+ GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
@@ -6242,12 +6334,12 @@
if (fixup_label == nullptr) {
__ Ldr(root_reg, MemOperand(obj, offset));
} else {
- codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj.X());
+ EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj.X());
}
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
- codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
+ MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__);
}
void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -6296,9 +6388,7 @@
DCHECK(temps.IsAvailable(ip0));
DCHECK(temps.IsAvailable(ip1));
temps.Exclude(ip0, ip1);
- uint32_t custom_data = linker::Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(
- base.GetCode(),
- obj.GetCode());
+ uint32_t custom_data = EncodeBakerReadBarrierFieldData(base.GetCode(), obj.GetCode());
vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data);
{
@@ -6383,8 +6473,7 @@
DCHECK(temps.IsAvailable(ip0));
DCHECK(temps.IsAvailable(ip1));
temps.Exclude(ip0, ip1);
- uint32_t custom_data =
- linker::Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(temp.GetCode());
+ uint32_t custom_data = EncodeBakerReadBarrierArrayData(temp.GetCode());
vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data);
__ Add(temp.X(), obj.X(), Operand(data_offset));
@@ -6744,5 +6833,176 @@
#undef __
#undef QUICK_ENTRY_POINT
+#define __ assembler.GetVIXLAssembler()->
+
+static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler,
+ vixl::aarch64::Register base_reg,
+ vixl::aarch64::MemOperand& lock_word,
+ vixl::aarch64::Label* slow_path) {
+ // Load the lock word containing the rb_state.
+ __ Ldr(ip0.W(), lock_word);
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path);
+ static_assert(
+ BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET,
+ "Field and array LDR offsets must be the same to reuse the same code.");
+ // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning).
+ static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4),
+ "Field LDR must be 1 instruction (4B) before the return address label; "
+ " 2 instructions (8B) for heap poisoning.");
+ __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ __ Add(base_reg, base_reg, Operand(ip0, LSR, 32));
+ __ Br(lr); // And return back to the function.
+ // Note: The fake dependency is unnecessary for the slow path.
+}
+
+// Load the read barrier introspection entrypoint in register `entrypoint`.
+static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler,
+ vixl::aarch64::Register entrypoint) {
+ // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection.
+ DCHECK_EQ(ip0.GetCode(), 16u);
+ const int32_t entry_point_offset =
+ Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode());
+ __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
+}
+
+void CodeGeneratorARM64::CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
+ uint32_t encoded_data,
+ /*out*/ std::string* debug_name) {
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ // Check if the holder is gray and, if not, add fake dependency to the base register
+ // and return to the LDR instruction to load the reference. Otherwise, use introspection
+ // to load the reference and call the entrypoint (in IP1) that performs further checks
+ // on the reference and marks it if needed.
+ auto base_reg =
+ Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ auto holder_reg =
+ Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data));
+ CheckValidReg(holder_reg.GetCode());
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ // If base_reg differs from holder_reg, the offset was too large and we must have
+ // emitted an explicit null check before the load. Otherwise, we need to null-check
+ // the holder as we do not necessarily do that check before going to the thunk.
+ vixl::aarch64::Label throw_npe;
+ if (holder_reg.Is(base_reg)) {
+ __ Cbz(holder_reg.W(), &throw_npe);
+ }
+ vixl::aarch64::Label slow_path;
+ MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
+ __ Bind(&slow_path);
+ MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET);
+ __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset.
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset.
+ __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference.
+ // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
+ __ Br(ip1); // Jump to the entrypoint.
+ if (holder_reg.Is(base_reg)) {
+ // Add null check slow path. The stack map is at the address pointed to by LR.
+ __ Bind(&throw_npe);
+ int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value();
+ __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset));
+ __ Br(ip0);
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ auto base_reg =
+ Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ vixl::aarch64::Label slow_path;
+ int32_t data_offset =
+ mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
+ MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
+ DCHECK_LT(lock_word.GetOffset(), 0);
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path);
+ __ Bind(&slow_path);
+ MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET);
+ __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset.
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set).
+ __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create
+ // a switch case target based on the index register.
+ __ Mov(ip0, base_reg); // Move the base register to ip0.
+ __ Br(ip1); // Jump to the entrypoint's array switch case.
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
+ // and it does not have a forwarding address), call the correct introspection entrypoint;
+ // otherwise return the reference (or the extracted forwarding address).
+ // There is no gray bit check for GC roots.
+ auto root_reg =
+ Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(root_reg.GetCode());
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip0, ip1);
+ vixl::aarch64::Label return_label, not_marked, forwarding_address;
+ __ Cbz(root_reg, &return_label);
+ MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value());
+ __ Ldr(ip0.W(), lock_word);
+ __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked);
+ __ Bind(&return_label);
+ __ Br(lr);
+ __ Bind(¬_marked);
+ __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1));
+ __ B(&forwarding_address, mi);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1);
+ // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to
+ // art_quick_read_barrier_mark_introspection_gc_roots.
+ __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET));
+ __ Mov(ip0.W(), root_reg);
+ __ Br(ip1);
+ __ Bind(&forwarding_address);
+ __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift);
+ __ Br(lr);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+
+ if (GetCompilerOptions().GenerateAnyDebugInfo()) {
+ std::ostringstream oss;
+ oss << "BakerReadBarrierThunk";
+ switch (kind) {
+ case BakerReadBarrierKind::kField:
+ oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
+ << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
+ break;
+ case BakerReadBarrierKind::kArray:
+ oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ case BakerReadBarrierKind::kGcRoot:
+ oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ }
+ *debug_name = oss.str();
+ }
+}
+
+#undef __
+
} // namespace arm64
} // namespace art
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 6a52eec..aa343b1 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -18,6 +18,7 @@
#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
#include "arch/arm64/quick_method_frame_info_arm64.h"
+#include "base/bit_field.h"
#include "code_generator.h"
#include "common_arm64.h"
#include "dex/dex_file_types.h"
@@ -36,6 +37,11 @@
#pragma GCC diagnostic pop
namespace art {
+
+namespace linker {
+class Arm64RelativePatcherTest;
+} // namespace linker
+
namespace arm64 {
class CodeGeneratorARM64;
@@ -309,17 +315,6 @@
uint32_t offset,
Location maybe_temp,
ReadBarrierOption read_barrier_option);
- // Generate a GC root reference load:
- //
- // root <- *(obj + offset)
- //
- // while honoring read barriers based on read_barrier_option.
- void GenerateGcRootFieldLoad(HInstruction* instruction,
- Location root,
- vixl::aarch64::Register obj,
- uint32_t offset,
- vixl::aarch64::Label* fixup_label,
- ReadBarrierOption read_barrier_option);
// Generate a floating-point comparison.
void GenerateFcmp(HInstruction* instruction);
@@ -641,9 +636,24 @@
vixl::aarch64::Register base);
void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) OVERRIDE;
+ bool NeedsThunkCode(const linker::LinkerPatch& patch) const OVERRIDE;
+ void EmitThunkCode(const linker::LinkerPatch& patch,
+ /*out*/ ArenaVector<uint8_t>* code,
+ /*out*/ std::string* debug_name) OVERRIDE;
void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE;
+ // Generate a GC root reference load:
+ //
+ // root <- *(obj + offset)
+ //
+ // while honoring read barriers based on read_barrier_option.
+ void GenerateGcRootFieldLoad(HInstruction* instruction,
+ Location root,
+ vixl::aarch64::Register obj,
+ uint32_t offset,
+ vixl::aarch64::Label* fixup_label,
+ ReadBarrierOption read_barrier_option);
// Fast path implementation of ReadBarrier::Barrier for a heap
// reference field load when Baker's read barriers are used.
void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -778,6 +788,62 @@
void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE;
private:
+ // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
+
+ enum class BakerReadBarrierKind : uint8_t {
+ kField, // Field get or array get with constant offset (i.e. constant index).
+ kArray, // Array get with index in register.
+ kGcRoot, // GC root load.
+ kLast = kGcRoot
+ };
+
+ static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u;
+
+ static constexpr size_t kBitsForBakerReadBarrierKind =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
+ static constexpr size_t kBakerReadBarrierBitsForRegister =
+ MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
+ using BakerReadBarrierKindField =
+ BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
+ using BakerReadBarrierFirstRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
+ using BakerReadBarrierSecondRegField =
+ BitField<uint32_t,
+ kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
+ kBakerReadBarrierBitsForRegister>;
+
+ static void CheckValidReg(uint32_t reg) {
+ DCHECK(reg < vixl::aarch64::lr.GetCode() &&
+ reg != vixl::aarch64::ip0.GetCode() &&
+ reg != vixl::aarch64::ip1.GetCode()) << reg;
+ }
+
+ static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
+ CheckValidReg(base_reg);
+ CheckValidReg(holder_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(holder_reg);
+ }
+
+ static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ CheckValidReg(base_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
+ }
+
+ static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
+ CheckValidReg(root_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
+ BakerReadBarrierFirstRegField::Encode(root_reg) |
+ BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
+ }
+
+ void CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
+ uint32_t encoded_data,
+ /*out*/ std::string* debug_name);
+
using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>;
using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>;
using StringToLiteralMap = ArenaSafeMap<StringReference,
@@ -854,6 +920,7 @@
// Patches for class literals in JIT compiled code.
TypeToLiteralMap jit_class_patches_;
+ friend class linker::Arm64RelativePatcherTest;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
};
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index b38a006..15d9526 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -94,9 +94,6 @@
constexpr bool kBakerReadBarrierLinkTimeThunksEnableForArrays = true;
constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true;
-// The reserved entrypoint register for link-time generated thunks.
-const vixl32::Register kBakerCcEntrypointRegister = r4;
-
// Using a base helps identify when we hit Marking Register check breakpoints.
constexpr int kMarkingRegisterCheckBreakCodeBaseCode = 0x10;
@@ -116,8 +113,6 @@
DCHECK(temps->IsAvailable(ip));
temps->Exclude(ip);
DCHECK(!temps->IsAvailable(kBakerCcEntrypointRegister));
- DCHECK_EQ(kBakerCcEntrypointRegister.GetCode(),
- linker::Thumb2RelativePatcher::kBakerCcEntrypointRegister);
DCHECK_NE(instruction->GetLocations()->GetTempCount(), 0u);
DCHECK(RegisterFrom(instruction->GetLocations()->GetTemp(
instruction->GetLocations()->GetTempCount() - 1u)).Is(kBakerCcEntrypointRegister));
@@ -2422,6 +2417,80 @@
FixJumpTables();
GetAssembler()->FinalizeCode();
CodeGenerator::Finalize(allocator);
+
+ // Verify Baker read barrier linker patches.
+ if (kIsDebugBuild) {
+ ArrayRef<const uint8_t> code = allocator->GetMemory();
+ for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) {
+ DCHECK(info.label.IsBound());
+ uint32_t literal_offset = info.label.GetLocation();
+ DCHECK_ALIGNED(literal_offset, 2u);
+
+ auto GetInsn16 = [&code](uint32_t offset) {
+ DCHECK_ALIGNED(offset, 2u);
+ return (static_cast<uint32_t>(code[offset + 0]) << 0) +
+ (static_cast<uint32_t>(code[offset + 1]) << 8);
+ };
+ auto GetInsn32 = [=](uint32_t offset) {
+ return (GetInsn16(offset) << 16) + (GetInsn16(offset + 2u) << 0);
+ };
+
+ uint32_t encoded_data = info.custom_data;
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ // Check that the next instruction matches the expected LDR.
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ if (width == BakerReadBarrierWidth::kWide) {
+ DCHECK_GE(code.size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn32(literal_offset + 4u);
+ // LDR (immediate), encoding T3, with correct base_reg.
+ CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16));
+ } else {
+ DCHECK_GE(code.size() - literal_offset, 6u);
+ uint32_t next_insn = GetInsn16(literal_offset + 4u);
+ // LDR (immediate), encoding T1, with correct base_reg.
+ CheckValidReg(next_insn & 0x7u); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3));
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ DCHECK_GE(code.size() - literal_offset, 8u);
+ uint32_t next_insn = GetInsn32(literal_offset + 4u);
+ // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]).
+ CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register.
+ const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16));
+ CheckValidReg(next_insn & 0xf); // Check index register
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ if (width == BakerReadBarrierWidth::kWide) {
+ DCHECK_GE(literal_offset, 4u);
+ uint32_t prev_insn = GetInsn32(literal_offset - 4u);
+ // LDR (immediate), encoding T3, with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12));
+ } else {
+ DCHECK_GE(literal_offset, 2u);
+ uint32_t prev_insn = GetInsn16(literal_offset - 2u);
+ // LDR (immediate), encoding T1, with correct root_reg.
+ const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data);
+ CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg);
+ }
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+ }
+ }
}
void CodeGeneratorARMVIXL::SetupBlockedRegisters() const {
@@ -7413,11 +7482,11 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
vixl32::Register current_method = InputRegisterAt(cls, 0);
- GenerateGcRootFieldLoad(cls,
- out_loc,
- current_method,
- ArtMethod::DeclaringClassOffset().Int32Value(),
- read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls,
+ out_loc,
+ current_method,
+ ArtMethod::DeclaringClassOffset().Int32Value(),
+ read_barrier_option);
break;
}
case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: {
@@ -7448,7 +7517,7 @@
CodeGeneratorARMVIXL::PcRelativePatchInfo* labels =
codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex());
codegen_->EmitMovwMovtPlaceholder(labels, out);
- GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option);
generate_null_check = true;
break;
}
@@ -7457,7 +7526,7 @@
cls->GetTypeIndex(),
cls->GetClass()));
// /* GcRoot<mirror::Class> */ out = *out
- GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option);
+ codegen_->GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option);
break;
}
case HLoadClass::LoadKind::kRuntimeCall:
@@ -7665,7 +7734,8 @@
CodeGeneratorARMVIXL::PcRelativePatchInfo* labels =
codegen_->NewStringBssEntryPatch(load->GetDexFile(), load->GetStringIndex());
codegen_->EmitMovwMovtPlaceholder(labels, out);
- GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption);
+ codegen_->GenerateGcRootFieldLoad(
+ load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption);
LoadStringSlowPathARMVIXL* slow_path =
new (codegen_->GetScopedAllocator()) LoadStringSlowPathARMVIXL(load);
codegen_->AddSlowPath(slow_path);
@@ -7679,7 +7749,8 @@
load->GetStringIndex(),
load->GetString()));
// /* GcRoot<mirror::String> */ out = *out
- GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption);
+ codegen_->GenerateGcRootFieldLoad(
+ load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption);
return;
}
default:
@@ -8730,7 +8801,7 @@
}
}
-void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad(
+void CodeGeneratorARMVIXL::GenerateGcRootFieldLoad(
HInstruction* instruction,
Location root,
vixl32::Register obj,
@@ -8761,9 +8832,8 @@
UseScratchRegisterScope temps(GetVIXLAssembler());
ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction);
bool narrow = CanEmitNarrowLdr(root_reg, obj, offset);
- uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(
- root_reg.GetCode(), narrow);
- vixl32::Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data);
+ uint32_t custom_data = EncodeBakerReadBarrierGcRootData(root_reg.GetCode(), narrow);
+ vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data);
vixl::EmissionCheckScope guard(GetVIXLAssembler(), 4 * vixl32::kMaxInstructionSizeInBytes);
vixl32::Label return_address;
@@ -8774,7 +8844,7 @@
DCHECK_LT(offset, kReferenceLoadMinFarOffset);
ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset();
__ ldr(EncodingSize(narrow ? Narrow : Wide), root_reg, MemOperand(obj, offset));
- EmitPlaceholderBne(codegen_, bne_label);
+ EmitPlaceholderBne(this, bne_label);
__ Bind(&return_address);
DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(),
narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET
@@ -8794,8 +8864,8 @@
// Slow path marking the GC root `root`. The entrypoint will
// be loaded by the slow path code.
SlowPathCodeARMVIXL* slow_path =
- new (codegen_->GetScopedAllocator()) ReadBarrierMarkSlowPathARMVIXL(instruction, root);
- codegen_->AddSlowPath(slow_path);
+ new (GetScopedAllocator()) ReadBarrierMarkSlowPathARMVIXL(instruction, root);
+ AddSlowPath(slow_path);
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset);
@@ -8816,7 +8886,7 @@
// /* GcRoot<mirror::Object>* */ root = obj + offset
__ Add(root_reg, obj, offset);
// /* mirror::Object* */ root = root->Read()
- codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
+ GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
@@ -8825,7 +8895,7 @@
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
- codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 18);
+ MaybeGenerateMarkingRegisterCheck(/* code */ 18);
}
void CodeGeneratorARMVIXL::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) {
@@ -8886,8 +8956,7 @@
}
UseScratchRegisterScope temps(GetVIXLAssembler());
ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction);
- uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(
- base.GetCode(), obj.GetCode(), narrow);
+ uint32_t custom_data = EncodeBakerReadBarrierFieldData(base.GetCode(), obj.GetCode(), narrow);
vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data);
{
@@ -8973,8 +9042,7 @@
UseScratchRegisterScope temps(GetVIXLAssembler());
ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction);
- uint32_t custom_data =
- linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg.GetCode());
+ uint32_t custom_data = EncodeBakerReadBarrierArrayData(data_reg.GetCode());
vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data);
__ Add(data_reg, obj, Operand(data_offset));
@@ -9111,7 +9179,7 @@
void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction,
Location ref,
- vixl::aarch32::Register obj,
+ vixl32::Register obj,
uint32_t offset,
Location index,
ScaleFactor scale_factor,
@@ -9451,7 +9519,7 @@
return &patches->back();
}
-vixl::aarch32::Label* CodeGeneratorARMVIXL::NewBakerReadBarrierPatch(uint32_t custom_data) {
+vixl32::Label* CodeGeneratorARMVIXL::NewBakerReadBarrierPatch(uint32_t custom_data) {
baker_read_barrier_patches_.emplace_back(custom_data);
return &baker_read_barrier_patches_.back().label;
}
@@ -9548,6 +9616,45 @@
DCHECK_EQ(size, linker_patches->size());
}
+bool CodeGeneratorARMVIXL::NeedsThunkCode(const linker::LinkerPatch& patch) const {
+ return patch.GetType() == linker::LinkerPatch::Type::kBakerReadBarrierBranch ||
+ patch.GetType() == linker::LinkerPatch::Type::kCallRelative;
+}
+
+void CodeGeneratorARMVIXL::EmitThunkCode(const linker::LinkerPatch& patch,
+ /*out*/ ArenaVector<uint8_t>* code,
+ /*out*/ std::string* debug_name) {
+ arm::ArmVIXLAssembler assembler(GetGraph()->GetAllocator());
+ switch (patch.GetType()) {
+ case linker::LinkerPatch::Type::kCallRelative:
+ // The thunk just uses the entry point in the ArtMethod. This works even for calls
+ // to the generic JNI and interpreter trampolines.
+ assembler.LoadFromOffset(
+ arm::kLoadWord,
+ vixl32::pc,
+ vixl32::r0,
+ ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value());
+ assembler.GetVIXLAssembler()->Bkpt(0);
+ if (GetCompilerOptions().GenerateAnyDebugInfo()) {
+ *debug_name = "MethodCallThunk";
+ }
+ break;
+ case linker::LinkerPatch::Type::kBakerReadBarrierBranch:
+ DCHECK_EQ(patch.GetBakerCustomValue2(), 0u);
+ CompileBakerReadBarrierThunk(assembler, patch.GetBakerCustomValue1(), debug_name);
+ break;
+ default:
+ LOG(FATAL) << "Unexpected patch type " << patch.GetType();
+ UNREACHABLE();
+ }
+
+ // Ensure we emit the literal pool if any.
+ assembler.FinalizeCode();
+ code->resize(assembler.CodeSize());
+ MemoryRegion code_region(code->data(), code->size());
+ assembler.FinalizeInstructions(code_region);
+}
+
VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateUint32Literal(
uint32_t value,
Uint32ToLiteralMap* map) {
@@ -9792,5 +9899,210 @@
#undef QUICK_ENTRY_POINT
#undef TODO_VIXL32
+#define __ assembler.GetVIXLAssembler()->
+
+static void EmitGrayCheckAndFastPath(ArmVIXLAssembler& assembler,
+ vixl32::Register base_reg,
+ vixl32::MemOperand& lock_word,
+ vixl32::Label* slow_path,
+ int32_t raw_ldr_offset) {
+ // Load the lock word containing the rb_state.
+ __ Ldr(ip, lock_word);
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted));
+ __ B(ne, slow_path, /* is_far_target */ false);
+ __ Add(lr, lr, raw_ldr_offset);
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ __ Add(base_reg, base_reg, Operand(ip, LSR, 32));
+ __ Bx(lr); // And return back to the function.
+ // Note: The fake dependency is unnecessary for the slow path.
+}
+
+// Load the read barrier introspection entrypoint in register `entrypoint`
+static void LoadReadBarrierMarkIntrospectionEntrypoint(ArmVIXLAssembler& assembler,
+ vixl32::Register entrypoint) {
+ // The register where the read barrier introspection entrypoint is loaded
+ // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4).
+ DCHECK(entrypoint.Is(kBakerCcEntrypointRegister));
+ // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection.
+ DCHECK_EQ(ip.GetCode(), 12u);
+ const int32_t entry_point_offset =
+ Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode());
+ __ Ldr(entrypoint, MemOperand(tr, entry_point_offset));
+}
+
+void CodeGeneratorARMVIXL::CompileBakerReadBarrierThunk(ArmVIXLAssembler& assembler,
+ uint32_t encoded_data,
+ /*out*/ std::string* debug_name) {
+ BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data);
+ switch (kind) {
+ case BakerReadBarrierKind::kField: {
+ // Check if the holder is gray and, if not, add fake dependency to the base register
+ // and return to the LDR instruction to load the reference. Otherwise, use introspection
+ // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister)
+ // that performs further checks on the reference and marks it if needed.
+ vixl32::Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ vixl32::Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data));
+ CheckValidReg(holder_reg.GetCode());
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ // If base_reg differs from holder_reg, the offset was too large and we must have
+ // emitted an explicit null check before the load. Otherwise, we need to null-check
+ // the holder as we do not necessarily do that check before going to the thunk.
+ vixl32::Label throw_npe;
+ if (holder_reg.Is(base_reg)) {
+ __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false);
+ }
+ vixl32::Label slow_path;
+ MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value());
+ const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide)
+ ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET
+ : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET;
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
+ __ Bind(&slow_path);
+ const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
+ raw_ldr_offset;
+ vixl32::Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ if (width == BakerReadBarrierWidth::kWide) {
+ MemOperand ldr_half_address(lr, ldr_offset + 2);
+ __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12".
+ __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12.
+ __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference.
+ } else {
+ MemOperand ldr_address(lr, ldr_offset);
+ __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1.
+ __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint
+ ep_reg, // for narrow LDR.
+ Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET));
+ __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4.
+ __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference.
+ }
+ // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference.
+ __ Bx(ep_reg); // Jump to the entrypoint.
+ if (holder_reg.Is(base_reg)) {
+ // Add null check slow path. The stack map is at the address pointed to by LR.
+ __ Bind(&throw_npe);
+ int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value();
+ __ Ldr(ip, MemOperand(/* Thread* */ vixl32::r9, offset));
+ __ Bx(ip);
+ }
+ break;
+ }
+ case BakerReadBarrierKind::kArray: {
+ vixl32::Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(base_reg.GetCode());
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ vixl32::Label slow_path;
+ int32_t data_offset =
+ mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
+ MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset);
+ DCHECK_LT(lock_word.GetOffsetImmediate(), 0);
+ const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET;
+ EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset);
+ __ Bind(&slow_path);
+ const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 +
+ raw_ldr_offset;
+ MemOperand ldr_address(lr, ldr_offset + 2);
+ __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm",
+ // i.e. Rm+32 because the scale in imm2 is 2.
+ vixl32::Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create
+ // a switch case target based on the index register.
+ __ Mov(ip, base_reg); // Move the base register to ip0.
+ __ Bx(ep_reg); // Jump to the entrypoint's array switch case.
+ break;
+ }
+ case BakerReadBarrierKind::kGcRoot: {
+ // Check if the reference needs to be marked and if so (i.e. not null, not marked yet
+ // and it does not have a forwarding address), call the correct introspection entrypoint;
+ // otherwise return the reference (or the extracted forwarding address).
+ // There is no gray bit check for GC roots.
+ vixl32::Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data));
+ CheckValidReg(root_reg.GetCode());
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data);
+ UseScratchRegisterScope temps(assembler.GetVIXLAssembler());
+ temps.Exclude(ip);
+ vixl32::Label return_label, not_marked, forwarding_address;
+ __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false);
+ MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value());
+ __ Ldr(ip, lock_word);
+ __ Tst(ip, LockWord::kMarkBitStateMaskShifted);
+ __ B(eq, ¬_marked);
+ __ Bind(&return_label);
+ __ Bx(lr);
+ __ Bind(¬_marked);
+ static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3,
+ "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in "
+ " the highest bits and the 'forwarding address' state to have all bits set");
+ __ Cmp(ip, Operand(0xc0000000));
+ __ B(hs, &forwarding_address);
+ vixl32::Register ep_reg(kBakerCcEntrypointRegister);
+ LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg);
+ // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister
+ // to art_quick_read_barrier_mark_introspection_gc_roots.
+ int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide)
+ ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET
+ : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET;
+ __ Add(ep_reg, ep_reg, Operand(entrypoint_offset));
+ __ Mov(ip, root_reg);
+ __ Bx(ep_reg);
+ __ Bind(&forwarding_address);
+ __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift);
+ __ Bx(lr);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind);
+ UNREACHABLE();
+ }
+
+ if (GetCompilerOptions().GenerateAnyDebugInfo()) {
+ std::ostringstream oss;
+ oss << "BakerReadBarrierThunk";
+ switch (kind) {
+ case BakerReadBarrierKind::kField:
+ oss << "Field";
+ if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
+ oss << "Wide";
+ }
+ oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data)
+ << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data);
+ break;
+ case BakerReadBarrierKind::kArray:
+ oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide);
+ break;
+ case BakerReadBarrierKind::kGcRoot:
+ oss << "GcRoot";
+ if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) {
+ oss << "Wide";
+ }
+ oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data);
+ DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg,
+ BakerReadBarrierSecondRegField::Decode(encoded_data));
+ break;
+ }
+ *debug_name = oss.str();
+ }
+}
+
+#undef __
+
} // namespace arm
} // namespace art
diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h
index 2114ea1..6b9919a 100644
--- a/compiler/optimizing/code_generator_arm_vixl.h
+++ b/compiler/optimizing/code_generator_arm_vixl.h
@@ -36,6 +36,11 @@
#pragma GCC diagnostic pop
namespace art {
+
+namespace linker {
+class Thumb2RelativePatcherTest;
+} // namespace linker
+
namespace arm {
// This constant is used as an approximate margin when emission of veneer and literal pools
@@ -108,6 +113,9 @@
static const size_t kRuntimeParameterFpuRegistersLengthVIXL =
arraysize(kRuntimeParameterFpuRegistersVIXL);
+// The reserved entrypoint register for link-time generated thunks.
+const vixl::aarch32::Register kBakerCcEntrypointRegister = vixl32::r4;
+
class LoadClassSlowPathARMVIXL;
class CodeGeneratorARMVIXL;
@@ -388,16 +396,6 @@
uint32_t offset,
Location maybe_temp,
ReadBarrierOption read_barrier_option);
- // Generate a GC root reference load:
- //
- // root <- *(obj + offset)
- //
- // while honoring read barriers based on read_barrier_option.
- void GenerateGcRootFieldLoad(HInstruction* instruction,
- Location root,
- vixl::aarch32::Register obj,
- uint32_t offset,
- ReadBarrierOption read_barrier_option);
void GenerateTestAndBranch(HInstruction* instruction,
size_t condition_input_index,
vixl::aarch32::Label* true_target,
@@ -606,6 +604,10 @@
Handle<mirror::Class> handle);
void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) OVERRIDE;
+ bool NeedsThunkCode(const linker::LinkerPatch& patch) const OVERRIDE;
+ void EmitThunkCode(const linker::LinkerPatch& patch,
+ /*out*/ ArenaVector<uint8_t>* code,
+ /*out*/ std::string* debug_name) OVERRIDE;
void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE;
@@ -613,6 +615,16 @@
// is added only for AOT compilation if link-time generated thunks for fields are enabled.
void MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations);
+ // Generate a GC root reference load:
+ //
+ // root <- *(obj + offset)
+ //
+ // while honoring read barriers based on read_barrier_option.
+ void GenerateGcRootFieldLoad(HInstruction* instruction,
+ Location root,
+ vixl::aarch32::Register obj,
+ uint32_t offset,
+ ReadBarrierOption read_barrier_option);
// Fast path implementation of ReadBarrier::Barrier for a heap
// reference field load when Baker's read barriers are used.
void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -767,6 +779,83 @@
vixl::aarch32::Register temp = vixl32::Register());
private:
+ // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
+
+ enum class BakerReadBarrierKind : uint8_t {
+ kField, // Field get or array get with constant offset (i.e. constant index).
+ kArray, // Array get with index in register.
+ kGcRoot, // GC root load.
+ kLast = kGcRoot
+ };
+
+ enum class BakerReadBarrierWidth : uint8_t {
+ kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled).
+ kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled).
+ kLast = kNarrow
+ };
+
+ static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* pc is invalid */ 15u;
+
+ static constexpr size_t kBitsForBakerReadBarrierKind =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
+ static constexpr size_t kBakerReadBarrierBitsForRegister =
+ MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
+ using BakerReadBarrierKindField =
+ BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
+ using BakerReadBarrierFirstRegField =
+ BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
+ using BakerReadBarrierSecondRegField =
+ BitField<uint32_t,
+ kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
+ kBakerReadBarrierBitsForRegister>;
+ static constexpr size_t kBitsForBakerReadBarrierWidth =
+ MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast));
+ using BakerReadBarrierWidthField =
+ BitField<BakerReadBarrierWidth,
+ kBitsForBakerReadBarrierKind + 2 * kBakerReadBarrierBitsForRegister,
+ kBitsForBakerReadBarrierWidth>;
+
+ static void CheckValidReg(uint32_t reg) {
+ DCHECK(reg < vixl::aarch32::ip.GetCode() && reg != kBakerCcEntrypointRegister.GetCode()) << reg;
+ }
+
+ static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
+ uint32_t holder_reg,
+ bool narrow) {
+ CheckValidReg(base_reg);
+ CheckValidReg(holder_reg);
+ DCHECK(!narrow || base_reg < 8u) << base_reg;
+ BakerReadBarrierWidth width =
+ narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(holder_reg) |
+ BakerReadBarrierWidthField::Encode(width);
+ }
+
+ static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
+ CheckValidReg(base_reg);
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
+ BakerReadBarrierFirstRegField::Encode(base_reg) |
+ BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) |
+ BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide);
+ }
+
+ static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
+ CheckValidReg(root_reg);
+ DCHECK(!narrow || root_reg < 8u) << root_reg;
+ BakerReadBarrierWidth width =
+ narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide;
+ return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
+ BakerReadBarrierFirstRegField::Encode(root_reg) |
+ BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) |
+ BakerReadBarrierWidthField::Encode(width);
+ }
+
+ void CompileBakerReadBarrierThunk(ArmVIXLAssembler& assembler,
+ uint32_t encoded_data,
+ /*out*/ std::string* debug_name);
+
vixl::aarch32::Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
vixl::aarch32::Register temp);
@@ -829,6 +918,7 @@
// Patches for class literals in JIT compiled code.
TypeToLiteralMap jit_class_patches_;
+ friend class linker::Thumb2RelativePatcherTest;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARMVIXL);
};
diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h
index c41c290..792cfb5 100644
--- a/compiler/optimizing/codegen_test_utils.h
+++ b/compiler/optimizing/codegen_test_utils.h
@@ -195,7 +195,9 @@
}
size_t GetSize() const { return size_; }
- uint8_t* GetMemory() const { return memory_.get(); }
+ ArrayRef<const uint8_t> GetMemory() const OVERRIDE {
+ return ArrayRef<const uint8_t>(memory_.get(), size_);
+ }
private:
size_t size_;
@@ -269,8 +271,8 @@
InstructionSet target_isa = codegen.GetInstructionSet();
typedef Expected (*fptr)();
- CommonCompilerTest::MakeExecutable(allocator.GetMemory(), allocator.GetSize());
- fptr f = reinterpret_cast<fptr>(allocator.GetMemory());
+ CommonCompilerTest::MakeExecutable(allocator.GetMemory().data(), allocator.GetMemory().size());
+ fptr f = reinterpret_cast<fptr>(reinterpret_cast<uintptr_t>(allocator.GetMemory().data()));
if (target_isa == InstructionSet::kThumb2) {
// For thumb we need the bottom bit set.
f = reinterpret_cast<fptr>(reinterpret_cast<uintptr_t>(f) + 1);
diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc
index d20b681..2e189fd 100644
--- a/compiler/optimizing/optimizing_cfi_test.cc
+++ b/compiler/optimizing/optimizing_cfi_test.cc
@@ -105,15 +105,15 @@
const std::vector<uint8_t>& expected_asm,
const std::vector<uint8_t>& expected_cfi) {
// Get the outputs.
- const std::vector<uint8_t>& actual_asm = code_allocator_.GetMemory();
+ ArrayRef<const uint8_t> actual_asm = code_allocator_.GetMemory();
Assembler* opt_asm = code_gen_->GetAssembler();
- const std::vector<uint8_t>& actual_cfi = *(opt_asm->cfi().data());
+ ArrayRef<const uint8_t> actual_cfi(*(opt_asm->cfi().data()));
if (kGenerateExpected) {
GenerateExpected(stdout, isa, isa_str, actual_asm, actual_cfi);
} else {
- EXPECT_EQ(expected_asm, actual_asm);
- EXPECT_EQ(expected_cfi, actual_cfi);
+ EXPECT_EQ(ArrayRef<const uint8_t>(expected_asm), actual_asm);
+ EXPECT_EQ(ArrayRef<const uint8_t>(expected_cfi), actual_cfi);
}
}
@@ -140,7 +140,7 @@
return memory_.data();
}
- const std::vector<uint8_t>& GetMemory() { return memory_; }
+ ArrayRef<const uint8_t> GetMemory() const OVERRIDE { return ArrayRef<const uint8_t>(memory_); }
private:
std::vector<uint8_t> memory_;
diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc
index e42dfc1..7916582 100644
--- a/compiler/optimizing/optimizing_compiler.cc
+++ b/compiler/optimizing/optimizing_compiler.cc
@@ -75,22 +75,18 @@
class CodeVectorAllocator FINAL : public CodeAllocator {
public:
explicit CodeVectorAllocator(ArenaAllocator* allocator)
- : memory_(allocator->Adapter(kArenaAllocCodeBuffer)),
- size_(0) {}
+ : memory_(allocator->Adapter(kArenaAllocCodeBuffer)) {}
virtual uint8_t* Allocate(size_t size) {
- size_ = size;
memory_.resize(size);
return &memory_[0];
}
- size_t GetSize() const { return size_; }
- const ArenaVector<uint8_t>& GetMemory() const { return memory_; }
+ ArrayRef<const uint8_t> GetMemory() const OVERRIDE { return ArrayRef<const uint8_t>(memory_); }
uint8_t* GetData() { return memory_.data(); }
private:
ArenaVector<uint8_t> memory_;
- size_t size_;
DISALLOW_COPY_AND_ASSIGN(CodeVectorAllocator);
};
@@ -719,7 +715,7 @@
CompiledMethod* compiled_method = CompiledMethod::SwapAllocCompiledMethod(
GetCompilerDriver(),
codegen->GetInstructionSet(),
- ArrayRef<const uint8_t>(code_allocator->GetMemory()),
+ code_allocator->GetMemory(),
// Follow Quick's behavior and set the frame size to zero if it is
// considered "empty" (see the definition of
// art::CodeGenerator::HasEmptyFrame).
@@ -731,6 +727,16 @@
ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data()),
ArrayRef<const linker::LinkerPatch>(linker_patches));
+ CompiledMethodStorage* storage = GetCompilerDriver()->GetCompiledMethodStorage();
+ for (const linker::LinkerPatch& patch : linker_patches) {
+ if (codegen->NeedsThunkCode(patch) && storage->GetThunkCode(patch).empty()) {
+ ArenaVector<uint8_t> code(allocator->Adapter());
+ std::string debug_name;
+ codegen->EmitThunkCode(patch, &code, &debug_name);
+ storage->SetThunkCode(patch, ArrayRef<const uint8_t>(code), debug_name);
+ }
+ }
+
return compiled_method;
}
@@ -1339,7 +1345,7 @@
codegen->GetCoreSpillMask(),
codegen->GetFpuSpillMask(),
code_allocator.GetMemory().data(),
- code_allocator.GetSize(),
+ code_allocator.GetMemory().size(),
data_size,
osr,
roots,
@@ -1369,7 +1375,7 @@
info.is_optimized = true;
info.is_code_address_text_relative = false;
info.code_address = code_address;
- info.code_size = code_allocator.GetSize();
+ info.code_size = code_allocator.GetMemory().size();
info.frame_size_in_bytes = method_header->GetFrameSizeInBytes();
info.code_info = stack_map_size == 0 ? nullptr : stack_map_data;
info.cfi = ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data());
@@ -1378,7 +1384,7 @@
Runtime::Current()->GetJit()->AddMemoryUsage(method, allocator.BytesUsed());
if (jit_logger != nullptr) {
- jit_logger->WriteLog(code, code_allocator.GetSize(), method);
+ jit_logger->WriteLog(code, code_allocator.GetMemory().size(), method);
}
if (kArenaAllocatorCountAllocations) {
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 6950b93..0c1d778 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -2061,7 +2061,9 @@
{
TimingLogger::ScopedTiming t2("dex2oat Write ELF", timings_);
- linker::MultiOatRelativePatcher patcher(instruction_set_, instruction_set_features_.get());
+ linker::MultiOatRelativePatcher patcher(instruction_set_,
+ instruction_set_features_.get(),
+ driver_->GetCompiledMethodStorage());
for (size_t i = 0, size = oat_files_.size(); i != size; ++i) {
std::unique_ptr<linker::ElfWriter>& elf_writer = elf_writers_[i];
std::unique_ptr<linker::OatWriter>& oat_writer = oat_writers_[i];
diff --git a/dex2oat/linker/image_test.h b/dex2oat/linker/image_test.h
index 7449191..476a843 100644
--- a/dex2oat/linker/image_test.h
+++ b/dex2oat/linker/image_test.h
@@ -299,7 +299,8 @@
DCHECK_EQ(vdex_files.size(), oat_files.size());
for (size_t i = 0, size = oat_files.size(); i != size; ++i) {
MultiOatRelativePatcher patcher(driver->GetInstructionSet(),
- driver->GetInstructionSetFeatures());
+ driver->GetInstructionSetFeatures(),
+ driver->GetCompiledMethodStorage());
OatWriter* const oat_writer = oat_writers[i].get();
ElfWriter* const elf_writer = elf_writers[i].get();
std::vector<const DexFile*> cur_dex_files(1u, class_path[i]);
diff --git a/dex2oat/linker/multi_oat_relative_patcher.cc b/dex2oat/linker/multi_oat_relative_patcher.cc
index 1abaf7d..1449d47 100644
--- a/dex2oat/linker/multi_oat_relative_patcher.cc
+++ b/dex2oat/linker/multi_oat_relative_patcher.cc
@@ -20,14 +20,28 @@
#include "base/bit_utils.h"
#include "globals.h"
+#include "driver/compiled_method_storage.h"
namespace art {
namespace linker {
+void MultiOatRelativePatcher::ThunkProvider::GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) {
+ *code = storage_->GetThunkCode(patch, debug_name);
+ DCHECK(!code->empty());
+}
+
+
MultiOatRelativePatcher::MultiOatRelativePatcher(InstructionSet instruction_set,
- const InstructionSetFeatures* features)
- : method_offset_map_(),
- relative_patcher_(RelativePatcher::Create(instruction_set, features, &method_offset_map_)),
+ const InstructionSetFeatures* features,
+ CompiledMethodStorage* storage)
+ : thunk_provider_(storage),
+ method_offset_map_(),
+ relative_patcher_(RelativePatcher::Create(instruction_set,
+ features,
+ &thunk_provider_,
+ &method_offset_map_)),
adjustment_(0u),
instruction_set_(instruction_set),
start_size_code_alignment_(0u),
diff --git a/dex2oat/linker/multi_oat_relative_patcher.h b/dex2oat/linker/multi_oat_relative_patcher.h
index bd33b95..60fcfe8 100644
--- a/dex2oat/linker/multi_oat_relative_patcher.h
+++ b/dex2oat/linker/multi_oat_relative_patcher.h
@@ -26,6 +26,7 @@
namespace art {
class CompiledMethod;
+class CompiledMethodStorage;
class InstructionSetFeatures;
namespace linker {
@@ -38,7 +39,9 @@
public:
using const_iterator = SafeMap<MethodReference, uint32_t>::const_iterator;
- MultiOatRelativePatcher(InstructionSet instruction_set, const InstructionSetFeatures* features);
+ MultiOatRelativePatcher(InstructionSet instruction_set,
+ const InstructionSetFeatures* features,
+ CompiledMethodStorage* storage);
// Mark the start of a new oat file (for statistics retrieval) and set the
// adjustment for a new oat file to apply to all relative offsets that are
@@ -129,6 +132,19 @@
uint32_t MiscThunksSize() const;
private:
+ class ThunkProvider : public RelativePatcherThunkProvider {
+ public:
+ explicit ThunkProvider(CompiledMethodStorage* storage)
+ : storage_(storage) {}
+
+ void GetThunkCode(const LinkerPatch& patch,
+ /*out*/ ArrayRef<const uint8_t>* code,
+ /*out*/ std::string* debug_name) OVERRIDE;
+
+ private:
+ CompiledMethodStorage* storage_;
+ };
+
// Map method reference to assigned offset.
// Wrap the map in a class implementing RelativePatcherTargetProvider.
class MethodOffsetMap : public RelativePatcherTargetProvider {
@@ -137,6 +153,7 @@
SafeMap<MethodReference, uint32_t> map;
};
+ ThunkProvider thunk_provider_;
MethodOffsetMap method_offset_map_;
std::unique_ptr<RelativePatcher> relative_patcher_;
uint32_t adjustment_;
diff --git a/dex2oat/linker/multi_oat_relative_patcher_test.cc b/dex2oat/linker/multi_oat_relative_patcher_test.cc
index ca9c5f1..05fe36a 100644
--- a/dex2oat/linker/multi_oat_relative_patcher_test.cc
+++ b/dex2oat/linker/multi_oat_relative_patcher_test.cc
@@ -122,7 +122,7 @@
MultiOatRelativePatcherTest()
: instruction_set_features_(InstructionSetFeatures::FromCppDefines()),
- patcher_(kRuntimeISA, instruction_set_features_.get()) {
+ patcher_(kRuntimeISA, instruction_set_features_.get(), /* storage */ nullptr) {
std::unique_ptr<MockPatcher> mock(new MockPatcher());
mock_ = mock.get();
patcher_.relative_patcher_ = std::move(mock);
diff --git a/dex2oat/linker/oat_writer_test.cc b/dex2oat/linker/oat_writer_test.cc
index 6e95393..ea4e210 100644
--- a/dex2oat/linker/oat_writer_test.cc
+++ b/dex2oat/linker/oat_writer_test.cc
@@ -213,7 +213,8 @@
class_linker->RegisterDexFile(*dex_file, nullptr);
}
MultiOatRelativePatcher patcher(compiler_driver_->GetInstructionSet(),
- instruction_set_features_.get());
+ instruction_set_features_.get(),
+ compiler_driver_->GetCompiledMethodStorage());
oat_writer.Initialize(compiler_driver_.get(), nullptr, dex_files);
oat_writer.PrepareLayout(&patcher);
elf_writer->PrepareDynamicSection(oat_writer.GetOatHeader().GetExecutableOffset(),
