Store ImtIndex in ArtMethod.
This avoids recalculation and reduces pressure on the thread local cache.
This halves the time we spend hashing from 2% to 1% (maps on device).
Test: ./art/test.py -b --host --64
Change-Id: I2407bd9c222de4ddc6eea938908a1ac6d7abc35b
diff --git a/runtime/art_method-inl.h b/runtime/art_method-inl.h
index f254116..c240017 100644
--- a/runtime/art_method-inl.h
+++ b/runtime/art_method-inl.h
@@ -31,6 +31,7 @@
#include "dex/invoke_type.h"
#include "dex/primitive.h"
#include "gc_root-inl.h"
+#include "imtable-inl.h"
#include "intrinsics_enum.h"
#include "jit/profiling_info.h"
#include "mirror/class-inl.h"
@@ -421,6 +422,31 @@
return CodeItemDebugInfoAccessor(*GetDexFile(), GetCodeItem(), GetDexMethodIndex());
}
+inline void ArtMethod::SetCounter(int16_t hotness_count) {
+ DCHECK(!IsAbstract()) << PrettyMethod();
+ hotness_count_ = hotness_count;
+}
+
+inline uint16_t ArtMethod::GetCounter() {
+ DCHECK(!IsAbstract()) << PrettyMethod();
+ return hotness_count_;
+}
+
+inline uint32_t ArtMethod::GetImtIndex() {
+ if (LIKELY(IsAbstract() && imt_index_ != 0)) {
+ uint16_t imt_index = ~imt_index_;
+ DCHECK_EQ(imt_index, ImTable::GetImtIndex(this)) << PrettyMethod();
+ return imt_index;
+ } else {
+ return ImTable::GetImtIndex(this);
+ }
+}
+
+inline void ArtMethod::CalculateAndSetImtIndex() {
+ DCHECK(IsAbstract()) << PrettyMethod();
+ imt_index_ = ~ImTable::GetImtIndex(this);
+}
+
} // namespace art
#endif // ART_RUNTIME_ART_METHOD_INL_H_
diff --git a/runtime/art_method.h b/runtime/art_method.h
index 5bbee92..cc214f7 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -650,24 +650,13 @@
void CopyFrom(ArtMethod* src, PointerSize image_pointer_size)
REQUIRES_SHARED(Locks::mutator_lock_);
- // Note, hotness_counter_ updates are non-atomic but it doesn't need to be precise. Also,
- // given that the counter is only 16 bits wide we can expect wrap-around in some
- // situations. Consumers of hotness_count_ must be able to deal with that.
- uint16_t IncrementCounter() {
- return ++hotness_count_;
- }
+ ALWAYS_INLINE void SetCounter(int16_t hotness_count) REQUIRES_SHARED(Locks::mutator_lock_);
- void ClearCounter() {
- hotness_count_ = 0;
- }
+ ALWAYS_INLINE uint16_t GetCounter() REQUIRES_SHARED(Locks::mutator_lock_);
- void SetCounter(int16_t hotness_count) {
- hotness_count_ = hotness_count;
- }
+ ALWAYS_INLINE uint32_t GetImtIndex() REQUIRES_SHARED(Locks::mutator_lock_);
- uint16_t GetCounter() const {
- return hotness_count_;
- }
+ void CalculateAndSetImtIndex() REQUIRES_SHARED(Locks::mutator_lock_);
static constexpr MemberOffset HotnessCountOffset() {
return MemberOffset(OFFSETOF_MEMBER(ArtMethod, hotness_count_));
@@ -772,9 +761,14 @@
// ifTable.
uint16_t method_index_;
- // The hotness we measure for this method. Not atomic, as we allow
- // missing increments: if the method is hot, we will see it eventually.
- uint16_t hotness_count_;
+ union {
+ // Non-abstract methods: The hotness we measure for this method. Not atomic,
+ // as we allow missing increments: if the method is hot, we will see it eventually.
+ uint16_t hotness_count_;
+ // Abstract methods: IMT index (bitwise negated) or zero if it was not cached.
+ // The negation is needed to distinguish zero index and missing cached entry.
+ uint16_t imt_index_;
+ };
// Fake padding field gets inserted here.
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 03cb95c..820dea3 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3634,6 +3634,10 @@
dex_file, dst->GetClassDef(), dex_method_idx);
}
dst->SetAccessFlags(access_flags);
+ // Must be done after SetAccessFlags since IsAbstract depends on it.
+ if (klass->IsInterface() && dst->IsAbstract()) {
+ dst->CalculateAndSetImtIndex();
+ }
}
void ClassLinker::AppendToBootClassPath(Thread* self, const DexFile& dex_file) {
@@ -6723,7 +6727,7 @@
// or interface methods in the IMT here they will not create extra conflicts since we compare
// names and signatures in SetIMTRef.
ArtMethod* interface_method = interface->GetVirtualMethod(j, image_pointer_size_);
- const uint32_t imt_index = ImTable::GetImtIndex(interface_method);
+ const uint32_t imt_index = interface_method->GetImtIndex();
// There is only any conflicts if all of the interface methods for an IMT slot don't have
// the same implementation method, keep track of this to avoid creating a conflict table in
@@ -6777,7 +6781,7 @@
}
DCHECK(implementation_method != nullptr);
ArtMethod* interface_method = interface->GetVirtualMethod(j, image_pointer_size_);
- const uint32_t imt_index = ImTable::GetImtIndex(interface_method);
+ const uint32_t imt_index = interface_method->GetImtIndex();
if (!imt[imt_index]->IsRuntimeMethod() ||
imt[imt_index] == unimplemented_method ||
imt[imt_index] == imt_conflict_method) {
@@ -7703,7 +7707,7 @@
auto* interface_method = iftable->GetInterface(i)->GetVirtualMethod(j, image_pointer_size_);
MethodNameAndSignatureComparator interface_name_comparator(
interface_method->GetInterfaceMethodIfProxy(image_pointer_size_));
- uint32_t imt_index = ImTable::GetImtIndex(interface_method);
+ uint32_t imt_index = interface_method->GetImtIndex();
ArtMethod** imt_ptr = &out_imt[imt_index];
// For each method listed in the interface's method list, find the
// matching method in our class's method list. We want to favor the
diff --git a/runtime/entrypoints/entrypoint_utils-inl.h b/runtime/entrypoints/entrypoint_utils-inl.h
index 2236e61..a18cca4 100644
--- a/runtime/entrypoints/entrypoint_utils-inl.h
+++ b/runtime/entrypoints/entrypoint_utils-inl.h
@@ -533,13 +533,7 @@
UNREACHABLE();
}
case kInterface: {
- size_t imt_index;
- InterpreterCache* tls_cache = self->GetInterpreterCache();
- if (UNLIKELY(!tls_cache->Get(resolved_method, &imt_index))) {
- imt_index = ImTable::GetImtIndex(resolved_method);
- tls_cache->Set(resolved_method, imt_index);
- }
- DCHECK_EQ(imt_index, ImTable::GetImtIndex(resolved_method));
+ size_t imt_index = resolved_method->GetImtIndex();
PointerSize pointer_size = class_linker->GetImagePointerSize();
ObjPtr<mirror::Class> klass = (*this_object)->GetClass();
ArtMethod* imt_method = klass->GetImt(pointer_size)->Get(imt_index, pointer_size);
diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
index 1472490..b6adcf0 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
@@ -2671,7 +2671,7 @@
DCHECK(!interface_method->IsRuntimeMethod());
// Look whether we have a match in the ImtConflictTable.
- uint32_t imt_index = ImTable::GetImtIndex(interface_method);
+ uint32_t imt_index = interface_method->GetImtIndex();
ArtMethod* conflict_method = imt->Get(imt_index, kRuntimePointerSize);
if (LIKELY(conflict_method->IsRuntimeMethod())) {
ImtConflictTable* current_table = conflict_method->GetImtConflictTable(kRuntimePointerSize);
diff --git a/runtime/image.cc b/runtime/image.cc
index 3023cef..f50c39c 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -26,7 +26,7 @@
namespace art {
const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '6', '9', '\0' }; // Remove boot oat extents.
+const uint8_t ImageHeader::kImageVersion[] = { '0', '7', '0', '\0' }; // Store ImtIndex.
ImageHeader::ImageHeader(uint32_t image_begin,
uint32_t image_size,
diff --git a/runtime/interpreter/interpreter_cache.h b/runtime/interpreter/interpreter_cache.h
index 355058f..003ea6c 100644
--- a/runtime/interpreter/interpreter_cache.h
+++ b/runtime/interpreter/interpreter_cache.h
@@ -38,7 +38,6 @@
// iget/iput: The field offset. The field must be non-volatile.
// sget/sput: The ArtField* pointer. The field must be non-volitile.
// invoke: The ArtMethod* pointer (before vtable indirection, etc).
-// ArtMethod*: The ImtIndex of the method.
//
// We ensure consistency of the cache by clearing it
// whenever any dex file is unloaded.
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index ff39a66..28978c5 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -892,7 +892,8 @@
}
}
-static void ClearMethodCounter(ArtMethod* method, bool was_warm) {
+static void ClearMethodCounter(ArtMethod* method, bool was_warm)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
if (was_warm) {
method->SetPreviouslyWarm();
}
@@ -1128,7 +1129,7 @@
return false;
}
- method->ClearCounter();
+ method->SetCounter(0);
Runtime::Current()->GetInstrumentation()->UpdateMethodsCode(
method, GetQuickToInterpreterBridge());
VLOG(jit)
diff --git a/runtime/jit/profile_saver.cc b/runtime/jit/profile_saver.cc
index e3248ea..c8d4728 100644
--- a/runtime/jit/profile_saver.cc
+++ b/runtime/jit/profile_saver.cc
@@ -362,7 +362,7 @@
}
// Visit all of the methods in the class to see which ones were executed.
for (ArtMethod& method : klass->GetMethods(kRuntimePointerSize)) {
- if (!method.IsNative()) {
+ if (!method.IsNative() && !method.IsAbstract()) {
DCHECK(!method.IsProxyMethod());
const uint16_t counter = method.GetCounter();
// Mark startup methods as hot if they have more than hot_method_sample_threshold
