Revert^2 "Hash-based DexCache field array."
Test: testrunner.py --host --interpreter
Bug: 30627598
This reverts commit 6374c58f2ea403b3a05fb27376110fe4d0fc8e3f.
Change-Id: I275508e288a85d3aa08f7405a1a4f362af43b775
diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc
index 562f97b..35aa1ee 100644
--- a/compiler/driver/compiler_driver_test.cc
+++ b/compiler/driver/compiler_driver_test.cc
@@ -133,9 +133,10 @@
<< " " << dex.GetMethodDeclaringClassDescriptor(dex.GetMethodId(i)) << " "
<< dex.GetMethodName(dex.GetMethodId(i));
}
- EXPECT_EQ(dex.NumFieldIds(), dex_cache->NumResolvedFields());
+ EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields()
+ || dex.NumFieldIds() == dex_cache->NumResolvedFields());
for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) {
- ArtField* field = cl->GetResolvedField(i, dex_cache);
+ ArtField* field = dex_cache->GetResolvedField(i, cl->GetImagePointerSize());
EXPECT_TRUE(field != nullptr) << "field_idx=" << i
<< " " << dex.GetFieldDeclaringClassDescriptor(dex.GetFieldId(i))
<< " " << dex.GetFieldName(dex.GetFieldId(i));
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index aa0791f..aa73456 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -968,11 +968,12 @@
<< Class::PrettyClass(declaring_class) << " not in class linker table";
}
}
- ArtField** resolved_fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields();
for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) {
- ArtField* field = mirror::DexCache::GetElementPtrSize(resolved_fields, i, target_ptr_size_);
+ auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, i, target_ptr_size_);
+ ArtField* field = pair.object;
if (field != nullptr && !KeepClass(field->GetDeclaringClass().Ptr())) {
- dex_cache->SetResolvedField(i, nullptr, target_ptr_size_);
+ dex_cache->ClearResolvedField(pair.index, target_ptr_size_);
}
}
// Clean the dex field. It might have been populated during the initialization phase, but
@@ -1594,7 +1595,7 @@
break;
}
case kBinDexCacheArray:
- bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment());
+ bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment(target_ptr_size_));
break;
case kBinImTable:
case kBinIMTConflictTable: {
@@ -2233,16 +2234,17 @@
mirror::DexCache::SetElementPtrSize(copy_methods, i, copy, target_ptr_size_);
}
}
- ArtField** orig_fields = orig_dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields();
if (orig_fields != nullptr) {
copy_dex_cache->SetFieldPtrWithSize<false>(mirror::DexCache::ResolvedFieldsOffset(),
NativeLocationInImage(orig_fields),
PointerSize::k64);
- ArtField** copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache);
+ mirror::FieldDexCacheType* copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache);
for (size_t i = 0, num = orig_dex_cache->NumResolvedFields(); i != num; ++i) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, i, target_ptr_size_);
- ArtField* copy = NativeLocationInImage(orig);
- mirror::DexCache::SetElementPtrSize(copy_fields, i, copy, target_ptr_size_);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(orig_fields, i, target_ptr_size_);
+ mirror::FieldDexCachePair copy(NativeLocationInImage(orig.object), orig.index);
+ mirror::DexCache::SetNativePairPtrSize(copy_fields, i, copy, target_ptr_size_);
}
}
mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes();
diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc
index 664b95a..583008b 100644
--- a/compiler/optimizing/inliner.cc
+++ b/compiler/optimizing/inliner.cc
@@ -1209,9 +1209,8 @@
// TODO: Needs null check.
return false;
}
- Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache()));
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
- HInstanceFieldGet* iget = CreateInstanceFieldGet(dex_cache, data.field_idx, obj);
+ HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj);
DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset);
DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile);
invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction);
@@ -1224,10 +1223,9 @@
// TODO: Needs null check.
return false;
}
- Handle<mirror::DexCache> dex_cache(handles_->NewHandle(resolved_method->GetDexCache()));
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg);
HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg);
- HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, data.field_idx, obj, value);
+ HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value);
DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset);
DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile);
invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
@@ -1261,24 +1259,19 @@
[](uint16_t index) { return index != DexFile::kDexNoIndex16; }));
// Create HInstanceFieldSet for each IPUT that stores non-zero data.
- Handle<mirror::DexCache> dex_cache;
HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u);
bool needs_constructor_barrier = false;
for (size_t i = 0; i != number_of_iputs; ++i) {
HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]);
if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) {
- if (dex_cache.GetReference() == nullptr) {
- dex_cache = handles_->NewHandle(resolved_method->GetDexCache());
- }
uint16_t field_index = iput_field_indexes[i];
- HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, field_index, obj, value);
+ bool is_final;
+ HInstanceFieldSet* iput =
+ CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final);
invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction);
// Check whether the field is final. If it is, we need to add a barrier.
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
- DCHECK(resolved_field != nullptr);
- if (resolved_field->IsFinal()) {
+ if (is_final) {
needs_constructor_barrier = true;
}
}
@@ -1297,12 +1290,13 @@
return true;
}
-HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj)
REQUIRES_SHARED(Locks::mutator_lock_) {
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* resolved_field =
+ class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
DCHECK(resolved_field != nullptr);
HInstanceFieldGet* iget = new (graph_->GetArena()) HInstanceFieldGet(
obj,
@@ -1312,12 +1306,13 @@
resolved_field->IsVolatile(),
field_index,
resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
- *dex_cache->GetDexFile(),
+ *referrer->GetDexFile(),
// Read barrier generates a runtime call in slow path and we need a valid
// dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
/* dex_pc */ 0);
if (iget->GetType() == Primitive::kPrimNot) {
// Use the same dex_cache that we used for field lookup as the hint_dex_cache.
+ Handle<mirror::DexCache> dex_cache = handles_->NewHandle(referrer->GetDexCache());
ReferenceTypePropagation rtp(graph_,
outer_compilation_unit_.GetClassLoader(),
dex_cache,
@@ -1328,14 +1323,21 @@
return iget;
}
-HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj,
- HInstruction* value)
+ HInstruction* value,
+ bool* is_final)
REQUIRES_SHARED(Locks::mutator_lock_) {
- PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
- ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* resolved_field =
+ class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false);
DCHECK(resolved_field != nullptr);
+ if (is_final != nullptr) {
+ // This information is needed only for constructors.
+ DCHECK(referrer->IsConstructor());
+ *is_final = resolved_field->IsFinal();
+ }
HInstanceFieldSet* iput = new (graph_->GetArena()) HInstanceFieldSet(
obj,
value,
@@ -1345,7 +1347,7 @@
resolved_field->IsVolatile(),
field_index,
resolved_field->GetDeclaringClass()->GetDexClassDefIndex(),
- *dex_cache->GetDexFile(),
+ *referrer->GetDexFile(),
// Read barrier generates a runtime call in slow path and we need a valid
// dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537.
/* dex_pc */ 0);
diff --git a/compiler/optimizing/inliner.h b/compiler/optimizing/inliner.h
index 8f8b268..a032042 100644
--- a/compiler/optimizing/inliner.h
+++ b/compiler/optimizing/inliner.h
@@ -107,14 +107,15 @@
REQUIRES_SHARED(Locks::mutator_lock_);
// Create a new HInstanceFieldGet.
- HInstanceFieldGet* CreateInstanceFieldGet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+ HInstanceFieldGet* CreateInstanceFieldGet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj);
// Create a new HInstanceFieldSet.
- HInstanceFieldSet* CreateInstanceFieldSet(Handle<mirror::DexCache> dex_cache,
- uint32_t field_index,
+ HInstanceFieldSet* CreateInstanceFieldSet(uint32_t field_index,
+ ArtMethod* referrer,
HInstruction* obj,
- HInstruction* value);
+ HInstruction* value,
+ bool* is_final = nullptr);
// Try inlining the invoke instruction using inline caches.
bool TryInlineFromInlineCache(
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index becb827..e767023 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -2210,13 +2210,13 @@
ScopedIndentation indent2(&state->vios_);
auto* resolved_fields = dex_cache->GetResolvedFields();
for (size_t i = 0, length = dex_cache->NumResolvedFields(); i < length; ++i) {
- auto* elem = mirror::DexCache::GetElementPtrSize(
- resolved_fields, i, image_pointer_size);
+ auto* elem = mirror::DexCache::GetNativePairPtrSize(
+ resolved_fields, i, image_pointer_size).object;
size_t run = 0;
for (size_t j = i + 1;
- j != length && elem == mirror::DexCache::GetElementPtrSize(resolved_fields,
- j,
- image_pointer_size);
+ j != length &&
+ elem == mirror::DexCache::GetNativePairPtrSize(
+ resolved_fields, j, image_pointer_size).object;
++j) {
++run;
}
diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc
index 18a6670..dfaae7d 100644
--- a/patchoat/patchoat.cc
+++ b/patchoat/patchoat.cc
@@ -534,17 +534,18 @@
mirror::DexCache::SetElementPtrSize(copy_methods, j, copy, pointer_size);
}
}
- ArtField** orig_fields = orig_dex_cache->GetResolvedFields();
- ArtField** relocated_fields = RelocatedAddressOfPointer(orig_fields);
+ mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* relocated_fields = RelocatedAddressOfPointer(orig_fields);
copy_dex_cache->SetField64<false>(
mirror::DexCache::ResolvedFieldsOffset(),
static_cast<int64_t>(reinterpret_cast<uintptr_t>(relocated_fields)));
if (orig_fields != nullptr) {
- ArtField** copy_fields = RelocatedCopyOf(orig_fields);
+ mirror::FieldDexCacheType* copy_fields = RelocatedCopyOf(orig_fields);
for (size_t j = 0, num = orig_dex_cache->NumResolvedFields(); j != num; ++j) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, j, pointer_size);
- ArtField* copy = RelocatedAddressOfPointer(orig);
- mirror::DexCache::SetElementPtrSize(copy_fields, j, copy, pointer_size);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(orig_fields, j, pointer_size);
+ mirror::FieldDexCachePair copy(RelocatedAddressOfPointer(orig.object), orig.index);
+ mirror::DexCache::SetNativePairPtrSize(copy_fields, j, copy, pointer_size);
}
}
mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes();
diff --git a/runtime/base/arena_allocator.cc b/runtime/base/arena_allocator.cc
index db43319..5aede38 100644
--- a/runtime/base/arena_allocator.cc
+++ b/runtime/base/arena_allocator.cc
@@ -15,6 +15,7 @@
*/
#include <algorithm>
+#include <cstddef>
#include <iomanip>
#include <numeric>
@@ -27,7 +28,7 @@
namespace art {
-static constexpr size_t kMemoryToolRedZoneBytes = 8;
+constexpr size_t kMemoryToolRedZoneBytes = 8;
constexpr size_t Arena::kDefaultSize;
template <bool kCount>
@@ -168,23 +169,75 @@
Arena::Arena() : bytes_allocated_(0), next_(nullptr) {
}
+class MallocArena FINAL : public Arena {
+ public:
+ explicit MallocArena(size_t size = Arena::kDefaultSize);
+ virtual ~MallocArena();
+ private:
+ static constexpr size_t RequiredOverallocation() {
+ return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment)
+ ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t)
+ : 0u;
+ }
+
+ uint8_t* unaligned_memory_;
+};
+
MallocArena::MallocArena(size_t size) {
- memory_ = reinterpret_cast<uint8_t*>(calloc(1, size));
- CHECK(memory_ != nullptr); // Abort on OOM.
- DCHECK_ALIGNED(memory_, ArenaAllocator::kAlignment);
+ // We need to guarantee kArenaAlignment aligned allocation for the new arena.
+ // TODO: Use std::aligned_alloc() when it becomes available with C++17.
+ constexpr size_t overallocation = RequiredOverallocation();
+ unaligned_memory_ = reinterpret_cast<uint8_t*>(calloc(1, size + overallocation));
+ CHECK(unaligned_memory_ != nullptr); // Abort on OOM.
+ DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t));
+ if (overallocation == 0u) {
+ memory_ = unaligned_memory_;
+ } else {
+ memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment);
+ if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) {
+ size_t head = memory_ - unaligned_memory_;
+ size_t tail = overallocation - head;
+ MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head);
+ MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail);
+ }
+ }
+ DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment);
size_ = size;
}
MallocArena::~MallocArena() {
- free(reinterpret_cast<void*>(memory_));
+ constexpr size_t overallocation = RequiredOverallocation();
+ if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) {
+ size_t head = memory_ - unaligned_memory_;
+ size_t tail = overallocation - head;
+ MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head);
+ MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail);
+ }
+ free(reinterpret_cast<void*>(unaligned_memory_));
}
+class MemMapArena FINAL : public Arena {
+ public:
+ MemMapArena(size_t size, bool low_4gb, const char* name);
+ virtual ~MemMapArena();
+ void Release() OVERRIDE;
+
+ private:
+ std::unique_ptr<MemMap> map_;
+};
+
MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) {
+ // Round up to a full page as that's the smallest unit of allocation for mmap()
+ // and we want to be able to use all memory that we actually allocate.
+ size = RoundUp(size, kPageSize);
std::string error_msg;
map_.reset(MemMap::MapAnonymous(
name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg));
CHECK(map_.get() != nullptr) << error_msg;
memory_ = map_->Begin();
+ static_assert(ArenaAllocator::kArenaAlignment <= kPageSize,
+ "Arena should not need stronger alignment than kPageSize.");
+ DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment);
size_ = map_->Size();
}
@@ -332,20 +385,7 @@
ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind);
uint8_t* ret;
if (UNLIKELY(rounded_bytes > static_cast<size_t>(end_ - ptr_))) {
- ret = AllocFromNewArena(rounded_bytes);
- uint8_t* noaccess_begin = ret + bytes;
- uint8_t* noaccess_end;
- if (ret == arena_head_->Begin()) {
- DCHECK(ptr_ - rounded_bytes == ret);
- noaccess_end = end_;
- } else {
- // We're still using the old arena but `ret` comes from a new one just after it.
- DCHECK(arena_head_->next_ != nullptr);
- DCHECK(ret == arena_head_->next_->Begin());
- DCHECK_EQ(rounded_bytes, arena_head_->next_->GetBytesAllocated());
- noaccess_end = arena_head_->next_->End();
- }
- MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin);
+ ret = AllocFromNewArenaWithMemoryTool(rounded_bytes);
} else {
ret = ptr_;
ptr_ += rounded_bytes;
@@ -356,6 +396,30 @@
return ret;
}
+void* ArenaAllocator::AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind) {
+ // We mark all memory for a newly retrieved arena as inaccessible and then
+ // mark only the actually allocated memory as defined. That leaves red zones
+ // and padding between allocations marked as inaccessible.
+ size_t rounded_bytes = bytes + kMemoryToolRedZoneBytes;
+ DCHECK_ALIGNED(rounded_bytes, 8); // `bytes` is 16-byte aligned, red zone is 8-byte aligned.
+ uintptr_t padding =
+ ((reinterpret_cast<uintptr_t>(ptr_) + 15u) & 15u) - reinterpret_cast<uintptr_t>(ptr_);
+ ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind);
+ uint8_t* ret;
+ if (UNLIKELY(padding + rounded_bytes > static_cast<size_t>(end_ - ptr_))) {
+ static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena.");
+ ret = AllocFromNewArenaWithMemoryTool(rounded_bytes);
+ } else {
+ ptr_ += padding; // Leave padding inaccessible.
+ ret = ptr_;
+ ptr_ += rounded_bytes;
+ }
+ MEMORY_TOOL_MAKE_DEFINED(ret, bytes);
+ // Check that the memory is already zeroed out.
+ DCHECK(std::all_of(ret, ret + bytes, [](uint8_t val) { return val == 0u; }));
+ return ret;
+}
+
ArenaAllocator::~ArenaAllocator() {
// Reclaim all the arenas by giving them back to the thread pool.
UpdateBytesAllocated();
@@ -386,6 +450,24 @@
return new_arena->Begin();
}
+uint8_t* ArenaAllocator::AllocFromNewArenaWithMemoryTool(size_t bytes) {
+ uint8_t* ret = AllocFromNewArena(bytes);
+ uint8_t* noaccess_begin = ret + bytes;
+ uint8_t* noaccess_end;
+ if (ret == arena_head_->Begin()) {
+ DCHECK(ptr_ - bytes == ret);
+ noaccess_end = end_;
+ } else {
+ // We're still using the old arena but `ret` comes from a new one just after it.
+ DCHECK(arena_head_->next_ != nullptr);
+ DCHECK(ret == arena_head_->next_->Begin());
+ DCHECK_EQ(bytes, arena_head_->next_->GetBytesAllocated());
+ noaccess_end = arena_head_->next_->End();
+ }
+ MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin);
+ return ret;
+}
+
bool ArenaAllocator::Contains(const void* ptr) const {
if (ptr >= begin_ && ptr < end_) {
return true;
@@ -398,7 +480,9 @@
return false;
}
-MemStats::MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena,
+MemStats::MemStats(const char* name,
+ const ArenaAllocatorStats* stats,
+ const Arena* first_arena,
ssize_t lost_bytes_adjustment)
: name_(name),
stats_(stats),
diff --git a/runtime/base/arena_allocator.h b/runtime/base/arena_allocator.h
index f92fbea..c39429c 100644
--- a/runtime/base/arena_allocator.h
+++ b/runtime/base/arena_allocator.h
@@ -34,7 +34,6 @@
class ArenaAllocator;
class ArenaStack;
class ScopedArenaAllocator;
-class MemMap;
class MemStats;
template <typename T>
@@ -244,22 +243,6 @@
DISALLOW_COPY_AND_ASSIGN(Arena);
};
-class MallocArena FINAL : public Arena {
- public:
- explicit MallocArena(size_t size = Arena::kDefaultSize);
- virtual ~MallocArena();
-};
-
-class MemMapArena FINAL : public Arena {
- public:
- MemMapArena(size_t size, bool low_4gb, const char* name);
- virtual ~MemMapArena();
- void Release() OVERRIDE;
-
- private:
- std::unique_ptr<MemMap> map_;
-};
-
class ArenaPool {
public:
explicit ArenaPool(bool use_malloc = true,
@@ -319,8 +302,31 @@
return ret;
}
+ // Returns zeroed memory.
+ void* AllocAlign16(size_t bytes, ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE {
+ // It is an error to request 16-byte aligned allocation of unaligned size.
+ DCHECK_ALIGNED(bytes, 16);
+ if (UNLIKELY(IsRunningOnMemoryTool())) {
+ return AllocWithMemoryToolAlign16(bytes, kind);
+ }
+ uintptr_t padding =
+ ((reinterpret_cast<uintptr_t>(ptr_) + 15u) & 15u) - reinterpret_cast<uintptr_t>(ptr_);
+ ArenaAllocatorStats::RecordAlloc(bytes, kind);
+ if (UNLIKELY(padding + bytes > static_cast<size_t>(end_ - ptr_))) {
+ static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena.");
+ return AllocFromNewArena(bytes);
+ }
+ ptr_ += padding;
+ uint8_t* ret = ptr_;
+ DCHECK_ALIGNED(ret, 16);
+ ptr_ += bytes;
+ return ret;
+ }
+
// Realloc never frees the input pointer, it is the caller's job to do this if necessary.
- void* Realloc(void* ptr, size_t ptr_size, size_t new_size,
+ void* Realloc(void* ptr,
+ size_t ptr_size,
+ size_t new_size,
ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE {
DCHECK_GE(new_size, ptr_size);
DCHECK_EQ(ptr == nullptr, ptr_size == 0u);
@@ -371,12 +377,17 @@
bool Contains(const void* ptr) const;
- static constexpr size_t kAlignment = 8;
+ // The alignment guaranteed for individual allocations.
+ static constexpr size_t kAlignment = 8u;
+
+ // The alignment required for the whole Arena rather than individual allocations.
+ static constexpr size_t kArenaAlignment = 16u;
private:
void* AllocWithMemoryTool(size_t bytes, ArenaAllocKind kind);
+ void* AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind);
uint8_t* AllocFromNewArena(size_t bytes);
-
+ uint8_t* AllocFromNewArenaWithMemoryTool(size_t bytes);
void UpdateBytesAllocated();
@@ -396,7 +407,9 @@
class MemStats {
public:
- MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena,
+ MemStats(const char* name,
+ const ArenaAllocatorStats* stats,
+ const Arena* first_arena,
ssize_t lost_bytes_adjustment = 0);
void Dump(std::ostream& os) const;
diff --git a/runtime/base/scoped_arena_allocator.h b/runtime/base/scoped_arena_allocator.h
index 55044b3..1a0eb5e 100644
--- a/runtime/base/scoped_arena_allocator.h
+++ b/runtime/base/scoped_arena_allocator.h
@@ -39,8 +39,6 @@
kFree,
};
-static constexpr size_t kArenaAlignment = 8;
-
// Holds a list of Arenas for use by ScopedArenaAllocator stack.
// The memory is returned to the ArenaPool when the ArenaStack is destroyed.
class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryTool {
@@ -67,6 +65,9 @@
return *(reinterpret_cast<ArenaFreeTag*>(ptr) - 1);
}
+ // The alignment guaranteed for individual allocations.
+ static constexpr size_t kAlignment = 8u;
+
private:
struct Peak;
struct Current;
@@ -89,8 +90,8 @@
if (UNLIKELY(IsRunningOnMemoryTool())) {
return AllocWithMemoryTool(bytes, kind);
}
- // Add kArenaAlignment for the free or used tag. Required to preserve alignment.
- size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kArenaAlignment : 0u), kArenaAlignment);
+ // Add kAlignment for the free or used tag. Required to preserve alignment.
+ size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kAlignment : 0u), kAlignment);
uint8_t* ptr = top_ptr_;
if (UNLIKELY(static_cast<size_t>(top_end_ - ptr) < rounded_bytes)) {
ptr = AllocateFromNextArena(rounded_bytes);
@@ -98,7 +99,7 @@
CurrentStats()->RecordAlloc(bytes, kind);
top_ptr_ = ptr + rounded_bytes;
if (kIsDebugBuild) {
- ptr += kArenaAlignment;
+ ptr += kAlignment;
ArenaTagForAllocation(ptr) = ArenaFreeTag::kUsed;
}
return ptr;
diff --git a/runtime/class_linker-inl.h b/runtime/class_linker-inl.h
index bd510ca..9ddc6cf 100644
--- a/runtime/class_linker-inl.h
+++ b/runtime/class_linker-inl.h
@@ -161,9 +161,15 @@
return resolved_method;
}
-inline ArtField* ClassLinker::GetResolvedField(uint32_t field_idx,
- ObjPtr<mirror::DexCache> dex_cache) {
- return dex_cache->GetResolvedField(field_idx, image_pointer_size_);
+inline ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx,
+ ArtMethod* referrer,
+ bool is_static) {
+ ObjPtr<mirror::DexCache> dex_cache = referrer->GetDexCache();
+ ArtField* field = dex_cache->GetResolvedField(field_idx, image_pointer_size_);
+ if (field == nullptr) {
+ field = LookupResolvedField(field_idx, dex_cache, referrer->GetClassLoader(), is_static);
+ }
+ return field;
}
inline ArtField* ClassLinker::ResolveField(uint32_t field_idx,
@@ -171,7 +177,8 @@
bool is_static) {
Thread::PoisonObjectPointersIfDebug();
ObjPtr<mirror::Class> declaring_class = referrer->GetDeclaringClass();
- ArtField* resolved_field = GetResolvedField(field_idx, referrer->GetDexCache());
+ ArtField* resolved_field =
+ referrer->GetDexCache()->GetResolvedField(field_idx, image_pointer_size_);
if (UNLIKELY(resolved_field == nullptr)) {
StackHandleScope<2> hs(Thread::Current());
Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache()));
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 5b7d566..952a7179 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1280,7 +1280,10 @@
num_types = dex_file->NumTypeIds();
}
const size_t num_methods = dex_file->NumMethodIds();
- const size_t num_fields = dex_file->NumFieldIds();
+ size_t num_fields = mirror::DexCache::kDexCacheFieldCacheSize;
+ if (dex_file->NumFieldIds() < num_fields) {
+ num_fields = dex_file->NumFieldIds();
+ }
size_t num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize;
if (dex_file->NumProtoIds() < num_method_types) {
num_method_types = dex_file->NumProtoIds();
@@ -1324,17 +1327,22 @@
dex_cache->SetResolvedMethods(methods);
}
if (num_fields != 0u) {
- ArtField** const fields =
- reinterpret_cast<ArtField**>(raw_arrays + layout.FieldsOffset());
- for (size_t j = 0; kIsDebugBuild && j < num_fields; ++j) {
- DCHECK(fields[j] == nullptr);
+ mirror::FieldDexCacheType* const image_resolved_fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* const fields =
+ reinterpret_cast<mirror::FieldDexCacheType*>(raw_arrays + layout.FieldsOffset());
+ for (size_t j = 0; j < num_fields; ++j) {
+ DCHECK_EQ(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).index,
+ 0u);
+ DCHECK(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).object ==
+ nullptr);
+ mirror::DexCache::SetNativePairPtrSize(
+ fields,
+ j,
+ mirror::DexCache::GetNativePairPtrSize(image_resolved_fields,
+ j,
+ image_pointer_size_),
+ image_pointer_size_);
}
- CopyNonNull(dex_cache->GetResolvedFields(),
- num_fields,
- fields,
- [] (const ArtField* field) {
- return field == nullptr;
- });
dex_cache->SetResolvedFields(fields);
}
if (num_method_types != 0u) {
@@ -8150,6 +8158,43 @@
return resolved;
}
+ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx,
+ ObjPtr<mirror::DexCache> dex_cache,
+ ObjPtr<mirror::ClassLoader> class_loader,
+ bool is_static) {
+ const DexFile& dex_file = *dex_cache->GetDexFile();
+ const DexFile::FieldId& field_id = dex_file.GetFieldId(field_idx);
+ ObjPtr<mirror::Class> klass = dex_cache->GetResolvedType(field_id.class_idx_);
+ if (klass == nullptr) {
+ klass = LookupResolvedType(dex_file, field_id.class_idx_, dex_cache, class_loader);
+ }
+ if (klass == nullptr) {
+ // The class has not been resolved yet, so the field is also unresolved.
+ return nullptr;
+ }
+ DCHECK(klass->IsResolved());
+ Thread* self = is_static ? Thread::Current() : nullptr;
+
+ // First try to find a field declared directly by `klass` by the field index.
+ ArtField* resolved_field = is_static
+ ? mirror::Class::FindStaticField(self, klass, dex_cache, field_idx)
+ : klass->FindInstanceField(dex_cache, field_idx);
+
+ if (resolved_field == nullptr) {
+ // If not found in `klass` by field index, search the class hierarchy using the name and type.
+ const char* name = dex_file.GetFieldName(field_id);
+ const char* type = dex_file.GetFieldTypeDescriptor(field_id);
+ resolved_field = is_static
+ ? mirror::Class::FindStaticField(self, klass, name, type)
+ : klass->FindInstanceField(name, type);
+ }
+
+ if (resolved_field != nullptr) {
+ dex_cache->SetResolvedField(field_idx, resolved_field, image_pointer_size_);
+ }
+ return resolved_field;
+}
+
ArtField* ClassLinker::ResolveField(const DexFile& dex_file,
uint32_t field_idx,
Handle<mirror::DexCache> dex_cache,
@@ -8210,9 +8255,8 @@
return nullptr;
}
- StringPiece name(dex_file.StringDataByIdx(field_id.name_idx_));
- StringPiece type(dex_file.StringDataByIdx(
- dex_file.GetTypeId(field_id.type_idx_).descriptor_idx_));
+ StringPiece name(dex_file.GetFieldName(field_id));
+ StringPiece type(dex_file.GetFieldTypeDescriptor(field_id));
resolved = mirror::Class::FindField(self, klass, name, type);
if (resolved != nullptr) {
dex_cache->SetResolvedField(field_idx, resolved, image_pointer_size_);
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index a5d26c7..6254acb 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -333,7 +333,7 @@
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!Locks::dex_lock_, !Roles::uninterruptible_);
- ArtField* GetResolvedField(uint32_t field_idx, ObjPtr<mirror::DexCache> dex_cache)
+ ArtField* LookupResolvedField(uint32_t field_idx, ArtMethod* referrer, bool is_static)
REQUIRES_SHARED(Locks::mutator_lock_);
ArtField* ResolveField(uint32_t field_idx, ArtMethod* referrer, bool is_static)
REQUIRES_SHARED(Locks::mutator_lock_)
@@ -842,6 +842,13 @@
REQUIRES(!Locks::classlinker_classes_lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
+ // Find a field by its field index.
+ ArtField* LookupResolvedField(uint32_t field_idx,
+ ObjPtr<mirror::DexCache> dex_cache,
+ ObjPtr<mirror::ClassLoader> class_loader,
+ bool is_static)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
void RegisterDexFileLocked(const DexFile& dex_file,
ObjPtr<mirror::DexCache> dex_cache,
ObjPtr<mirror::ClassLoader> class_loader)
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 010ef11..568f8d6 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -1259,17 +1259,18 @@
}
}
}
- ArtField** fields = dex_cache->GetResolvedFields();
+ mirror::FieldDexCacheType* fields = dex_cache->GetResolvedFields();
if (fields != nullptr) {
- ArtField** new_fields = fixup_adapter.ForwardObject(fields);
+ mirror::FieldDexCacheType* new_fields = fixup_adapter.ForwardObject(fields);
if (fields != new_fields) {
dex_cache->SetResolvedFields(new_fields);
}
for (size_t j = 0, num = dex_cache->NumResolvedFields(); j != num; ++j) {
- ArtField* orig = mirror::DexCache::GetElementPtrSize(new_fields, j, pointer_size);
- ArtField* copy = fixup_adapter.ForwardObject(orig);
- if (orig != copy) {
- mirror::DexCache::SetElementPtrSize(new_fields, j, copy, pointer_size);
+ mirror::FieldDexCachePair orig =
+ mirror::DexCache::GetNativePairPtrSize(new_fields, j, pointer_size);
+ mirror::FieldDexCachePair copy(fixup_adapter.ForwardObject(orig.object), orig.index);
+ if (orig.object != copy.object) {
+ mirror::DexCache::SetNativePairPtrSize(new_fields, j, copy, pointer_size);
}
}
}
diff --git a/runtime/image.cc b/runtime/image.cc
index 6dbf20b..b153ea0 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -25,7 +25,7 @@
namespace art {
const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '2', '\0' }; // hash-based DexCache types
+const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '3', '\0' }; // hash-based DexCache fields
ImageHeader::ImageHeader(uint32_t image_begin,
uint32_t image_size,
diff --git a/runtime/linear_alloc.cc b/runtime/linear_alloc.cc
index f91b0ed..e9db9b8 100644
--- a/runtime/linear_alloc.cc
+++ b/runtime/linear_alloc.cc
@@ -33,6 +33,11 @@
return allocator_.Alloc(size);
}
+void* LinearAlloc::AllocAlign16(Thread* self, size_t size) {
+ MutexLock mu(self, lock_);
+ return allocator_.AllocAlign16(size);
+}
+
size_t LinearAlloc::GetUsedMemory() const {
MutexLock mu(Thread::Current(), lock_);
return allocator_.BytesUsed();
diff --git a/runtime/linear_alloc.h b/runtime/linear_alloc.h
index df7f17d..384b2e3 100644
--- a/runtime/linear_alloc.h
+++ b/runtime/linear_alloc.h
@@ -29,6 +29,7 @@
explicit LinearAlloc(ArenaPool* pool);
void* Alloc(Thread* self, size_t size) REQUIRES(!lock_);
+ void* AllocAlign16(Thread* self, size_t size) REQUIRES(!lock_);
// Realloc never frees the input pointer, it is the caller's job to do this if necessary.
void* Realloc(Thread* self, void* ptr, size_t old_size, size_t new_size) REQUIRES(!lock_);
diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h
index 29bf6a0..582ecb2 100644
--- a/runtime/mirror/dex_cache-inl.h
+++ b/runtime/mirror/dex_cache-inl.h
@@ -24,6 +24,7 @@
#include "base/casts.h"
#include "base/enums.h"
#include "base/logging.h"
+#include "dex_file.h"
#include "gc_root.h"
#include "mirror/class.h"
#include "mirror/call_site.h"
@@ -36,6 +37,15 @@
namespace art {
namespace mirror {
+template <typename T>
+inline void NativeDexCachePair<T>::Initialize(std::atomic<NativeDexCachePair<T>>* dex_cache,
+ PointerSize pointer_size) {
+ NativeDexCachePair<T> first_elem;
+ first_elem.object = nullptr;
+ first_elem.index = InvalidIndexForSlot(0);
+ DexCache::SetNativePairPtrSize(dex_cache, 0, first_elem, pointer_size);
+}
+
inline uint32_t DexCache::ClassSize(PointerSize pointer_size) {
uint32_t vtable_entries = Object::kVTableLength + 5;
return Class::ComputeClassSize(true, vtable_entries, 0, 0, 0, 0, 0, pointer_size);
@@ -164,20 +174,36 @@
}
}
+inline uint32_t DexCache::FieldSlotIndex(uint32_t field_idx) {
+ DCHECK_LT(field_idx, GetDexFile()->NumFieldIds());
+ const uint32_t slot_idx = field_idx % kDexCacheFieldCacheSize;
+ DCHECK_LT(slot_idx, NumResolvedFields());
+ return slot_idx;
+}
+
inline ArtField* DexCache::GetResolvedField(uint32_t field_idx, PointerSize ptr_size) {
DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
- DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB.
- ArtField* field = GetElementPtrSize(GetResolvedFields(), field_idx, ptr_size);
- if (field == nullptr || field->GetDeclaringClass()->IsErroneous()) {
- return nullptr;
- }
- return field;
+ auto pair = GetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), ptr_size);
+ return pair.GetObjectForIndex(field_idx);
}
inline void DexCache::SetResolvedField(uint32_t field_idx, ArtField* field, PointerSize ptr_size) {
DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
- DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB.
- SetElementPtrSize(GetResolvedFields(), field_idx, field, ptr_size);
+ DCHECK(field != nullptr);
+ FieldDexCachePair pair(field, field_idx);
+ SetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), pair, ptr_size);
+}
+
+inline void DexCache::ClearResolvedField(uint32_t field_idx, PointerSize ptr_size) {
+ DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size);
+ uint32_t slot_idx = FieldSlotIndex(field_idx);
+ auto* resolved_fields = GetResolvedFields();
+ // This is racy but should only be called from the single-threaded ImageWriter.
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ if (GetNativePairPtrSize(resolved_fields, slot_idx, ptr_size).index == field_idx) {
+ FieldDexCachePair cleared(nullptr, FieldDexCachePair::InvalidIndexForSlot(slot_idx));
+ SetNativePairPtrSize(resolved_fields, slot_idx, cleared, ptr_size);
+ }
}
inline ArtMethod* DexCache::GetResolvedMethod(uint32_t method_idx, PointerSize ptr_size) {
@@ -225,6 +251,40 @@
}
}
+template <typename T>
+NativeDexCachePair<T> DexCache::GetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ PointerSize ptr_size) {
+ if (ptr_size == PointerSize::k64) {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair64>*>(pair_array);
+ ConversionPair64 value = AtomicLoadRelaxed16B(&array[idx]);
+ return NativeDexCachePair<T>(reinterpret_cast64<T*>(value.first),
+ dchecked_integral_cast<size_t>(value.second));
+ } else {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair32>*>(pair_array);
+ ConversionPair32 value = array[idx].load(std::memory_order_relaxed);
+ return NativeDexCachePair<T>(reinterpret_cast<T*>(value.first), value.second);
+ }
+}
+
+template <typename T>
+void DexCache::SetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ NativeDexCachePair<T> pair,
+ PointerSize ptr_size) {
+ if (ptr_size == PointerSize::k64) {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair64>*>(pair_array);
+ ConversionPair64 v(reinterpret_cast64<uint64_t>(pair.object), pair.index);
+ AtomicStoreRelease16B(&array[idx], v);
+ } else {
+ auto* array = reinterpret_cast<std::atomic<ConversionPair32>*>(pair_array);
+ ConversionPair32 v(
+ dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>(pair.object)),
+ dchecked_integral_cast<uint32_t>(pair.index));
+ array[idx].store(v, std::memory_order_release);
+ }
+}
+
template <typename T,
ReadBarrierOption kReadBarrierOption,
typename Visitor>
diff --git a/runtime/mirror/dex_cache.cc b/runtime/mirror/dex_cache.cc
index 1b8b391..c95d92e 100644
--- a/runtime/mirror/dex_cache.cc
+++ b/runtime/mirror/dex_cache.cc
@@ -52,8 +52,12 @@
dex_file->NumTypeIds() != 0u ||
dex_file->NumMethodIds() != 0u ||
dex_file->NumFieldIds() != 0u) {
+ static_assert(ArenaAllocator::kAlignment == 8, "Expecting arena alignment of 8.");
+ DCHECK(layout.Alignment() == 8u || layout.Alignment() == 16u);
// Zero-initialized.
- raw_arrays = reinterpret_cast<uint8_t*>(linear_alloc->Alloc(self, layout.Size()));
+ raw_arrays = (layout.Alignment() == 16u)
+ ? reinterpret_cast<uint8_t*>(linear_alloc->AllocAlign16(self, layout.Size()))
+ : reinterpret_cast<uint8_t*>(linear_alloc->Alloc(self, layout.Size()));
}
mirror::StringDexCacheType* strings = (dex_file->NumStringIds() == 0u) ? nullptr :
@@ -62,17 +66,21 @@
reinterpret_cast<mirror::TypeDexCacheType*>(raw_arrays + layout.TypesOffset());
ArtMethod** methods = (dex_file->NumMethodIds() == 0u) ? nullptr :
reinterpret_cast<ArtMethod**>(raw_arrays + layout.MethodsOffset());
- ArtField** fields = (dex_file->NumFieldIds() == 0u) ? nullptr :
- reinterpret_cast<ArtField**>(raw_arrays + layout.FieldsOffset());
+ mirror::FieldDexCacheType* fields = (dex_file->NumFieldIds() == 0u) ? nullptr :
+ reinterpret_cast<mirror::FieldDexCacheType*>(raw_arrays + layout.FieldsOffset());
- size_t num_strings = mirror::DexCache::kDexCacheStringCacheSize;
+ size_t num_strings = kDexCacheStringCacheSize;
if (dex_file->NumStringIds() < num_strings) {
num_strings = dex_file->NumStringIds();
}
- size_t num_types = mirror::DexCache::kDexCacheTypeCacheSize;
+ size_t num_types = kDexCacheTypeCacheSize;
if (dex_file->NumTypeIds() < num_types) {
num_types = dex_file->NumTypeIds();
}
+ size_t num_fields = kDexCacheFieldCacheSize;
+ if (dex_file->NumFieldIds() < num_fields) {
+ num_fields = dex_file->NumFieldIds();
+ }
// Note that we allocate the method type dex caches regardless of this flag,
// and we make sure here that they're not used by the runtime. This is in the
@@ -80,17 +88,17 @@
//
// If this needs to be mitigated in a production system running this code,
// DexCache::kDexCacheMethodTypeCacheSize can be set to zero.
- mirror::MethodTypeDexCacheType* method_types = nullptr;
+ MethodTypeDexCacheType* method_types = nullptr;
size_t num_method_types = 0;
- if (dex_file->NumProtoIds() < mirror::DexCache::kDexCacheMethodTypeCacheSize) {
+ if (dex_file->NumProtoIds() < kDexCacheMethodTypeCacheSize) {
num_method_types = dex_file->NumProtoIds();
} else {
- num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize;
+ num_method_types = kDexCacheMethodTypeCacheSize;
}
if (num_method_types > 0) {
- method_types = reinterpret_cast<mirror::MethodTypeDexCacheType*>(
+ method_types = reinterpret_cast<MethodTypeDexCacheType*>(
raw_arrays + layout.MethodTypesOffset());
}
@@ -98,13 +106,13 @@
? nullptr
: reinterpret_cast<GcRoot<mirror::CallSite>*>(raw_arrays + layout.CallSitesOffset());
- DCHECK_ALIGNED(raw_arrays, alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(raw_arrays, alignof(StringDexCacheType)) <<
"Expected raw_arrays to align to StringDexCacheType.";
- DCHECK_ALIGNED(layout.StringsOffset(), alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(layout.StringsOffset(), alignof(StringDexCacheType)) <<
"Expected StringsOffset() to align to StringDexCacheType.";
- DCHECK_ALIGNED(strings, alignof(mirror::StringDexCacheType)) <<
+ DCHECK_ALIGNED(strings, alignof(StringDexCacheType)) <<
"Expected strings to align to StringDexCacheType.";
- static_assert(alignof(mirror::StringDexCacheType) == 8u,
+ static_assert(alignof(StringDexCacheType) == 8u,
"Expected StringDexCacheType to have align of 8.");
if (kIsDebugBuild) {
// Sanity check to make sure all the dex cache arrays are empty. b/28992179
@@ -117,10 +125,11 @@
CHECK(types[i].load(std::memory_order_relaxed).object.IsNull());
}
for (size_t i = 0; i < dex_file->NumMethodIds(); ++i) {
- CHECK(mirror::DexCache::GetElementPtrSize(methods, i, image_pointer_size) == nullptr);
+ CHECK(GetElementPtrSize(methods, i, image_pointer_size) == nullptr);
}
- for (size_t i = 0; i < dex_file->NumFieldIds(); ++i) {
- CHECK(mirror::DexCache::GetElementPtrSize(fields, i, image_pointer_size) == nullptr);
+ for (size_t i = 0; i < num_fields; ++i) {
+ CHECK_EQ(GetNativePairPtrSize(fields, i, image_pointer_size).index, 0u);
+ CHECK(GetNativePairPtrSize(fields, i, image_pointer_size).object == nullptr);
}
for (size_t i = 0; i < num_method_types; ++i) {
CHECK_EQ(method_types[i].load(std::memory_order_relaxed).index, 0u);
@@ -136,6 +145,9 @@
if (types != nullptr) {
mirror::TypeDexCachePair::Initialize(types);
}
+ if (fields != nullptr) {
+ mirror::FieldDexCachePair::Initialize(fields, image_pointer_size);
+ }
if (method_types != nullptr) {
mirror::MethodTypeDexCachePair::Initialize(method_types);
}
@@ -148,7 +160,7 @@
methods,
dex_file->NumMethodIds(),
fields,
- dex_file->NumFieldIds(),
+ num_fields,
method_types,
num_method_types,
call_sites,
@@ -164,7 +176,7 @@
uint32_t num_resolved_types,
ArtMethod** resolved_methods,
uint32_t num_resolved_methods,
- ArtField** resolved_fields,
+ FieldDexCacheType* resolved_fields,
uint32_t num_resolved_fields,
MethodTypeDexCacheType* resolved_method_types,
uint32_t num_resolved_method_types,
@@ -218,5 +230,23 @@
SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(DexCache, location_), location);
}
+#if !defined(__aarch64__) && !defined(__x86_64__)
+static pthread_mutex_t dex_cache_slow_atomic_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+DexCache::ConversionPair64 DexCache::AtomicLoadRelaxed16B(std::atomic<ConversionPair64>* target) {
+ pthread_mutex_lock(&dex_cache_slow_atomic_mutex);
+ DexCache::ConversionPair64 value = *reinterpret_cast<ConversionPair64*>(target);
+ pthread_mutex_unlock(&dex_cache_slow_atomic_mutex);
+ return value;
+}
+
+void DexCache::AtomicStoreRelease16B(std::atomic<ConversionPair64>* target,
+ ConversionPair64 value) {
+ pthread_mutex_lock(&dex_cache_slow_atomic_mutex);
+ *reinterpret_cast<ConversionPair64*>(target) = value;
+ pthread_mutex_unlock(&dex_cache_slow_atomic_mutex);
+}
+#endif
+
} // namespace mirror
} // namespace art
diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h
index 0579198..35707ef 100644
--- a/runtime/mirror/dex_cache.h
+++ b/runtime/mirror/dex_cache.h
@@ -91,12 +91,44 @@
}
};
+template <typename T> struct PACKED(2 * __SIZEOF_POINTER__) NativeDexCachePair {
+ T* object;
+ size_t index;
+ // This is similar to DexCachePair except that we're storing a native pointer
+ // instead of a GC root. See DexCachePair for the details.
+ NativeDexCachePair(T* object, uint32_t index)
+ : object(object),
+ index(index) {}
+ NativeDexCachePair() : object(nullptr), index(0u) { }
+ NativeDexCachePair(const NativeDexCachePair<T>&) = default;
+ NativeDexCachePair& operator=(const NativeDexCachePair<T>&) = default;
+
+ static void Initialize(std::atomic<NativeDexCachePair<T>>* dex_cache, PointerSize pointer_size);
+
+ static uint32_t InvalidIndexForSlot(uint32_t slot) {
+ // Since the cache size is a power of two, 0 will always map to slot 0.
+ // Use 1 for slot 0 and 0 for all other slots.
+ return (slot == 0) ? 1u : 0u;
+ }
+
+ T* GetObjectForIndex(uint32_t idx) REQUIRES_SHARED(Locks::mutator_lock_) {
+ if (idx != index) {
+ return nullptr;
+ }
+ DCHECK(object != nullptr);
+ return object;
+ }
+};
+
using TypeDexCachePair = DexCachePair<Class>;
using TypeDexCacheType = std::atomic<TypeDexCachePair>;
using StringDexCachePair = DexCachePair<String>;
using StringDexCacheType = std::atomic<StringDexCachePair>;
+using FieldDexCachePair = NativeDexCachePair<ArtField>;
+using FieldDexCacheType = std::atomic<FieldDexCachePair>;
+
using MethodTypeDexCachePair = DexCachePair<MethodType>;
using MethodTypeDexCacheType = std::atomic<MethodTypeDexCachePair>;
@@ -116,6 +148,11 @@
static_assert(IsPowerOfTwo(kDexCacheStringCacheSize),
"String dex cache size is not a power of 2.");
+ // Size of field dex cache. Needs to be a power of 2 for entrypoint assumptions to hold.
+ static constexpr size_t kDexCacheFieldCacheSize = 1024;
+ static_assert(IsPowerOfTwo(kDexCacheFieldCacheSize),
+ "Field dex cache size is not a power of 2.");
+
// Size of method type dex cache. Needs to be a power of 2 for entrypoint assumptions
// to hold.
static constexpr size_t kDexCacheMethodTypeCacheSize = 1024;
@@ -130,6 +167,10 @@
return kDexCacheStringCacheSize;
}
+ static constexpr size_t StaticArtFieldSize() {
+ return kDexCacheFieldCacheSize;
+ }
+
static constexpr size_t StaticMethodTypeSize() {
return kDexCacheMethodTypeCacheSize;
}
@@ -255,6 +296,8 @@
// Pointer sized variant, used for patching.
ALWAYS_INLINE void SetResolvedField(uint32_t idx, ArtField* field, PointerSize ptr_size)
REQUIRES_SHARED(Locks::mutator_lock_);
+ ALWAYS_INLINE void ClearResolvedField(uint32_t idx, PointerSize ptr_size)
+ REQUIRES_SHARED(Locks::mutator_lock_);
MethodType* GetResolvedMethodType(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_);
@@ -299,11 +342,11 @@
SetFieldPtr<false>(ResolvedMethodsOffset(), resolved_methods);
}
- ArtField** GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) {
- return GetFieldPtr<ArtField**>(ResolvedFieldsOffset());
+ FieldDexCacheType* GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) {
+ return GetFieldPtr<FieldDexCacheType*>(ResolvedFieldsOffset());
}
- void SetResolvedFields(ArtField** resolved_fields)
+ void SetResolvedFields(FieldDexCacheType* resolved_fields)
ALWAYS_INLINE
REQUIRES_SHARED(Locks::mutator_lock_) {
SetFieldPtr<false>(ResolvedFieldsOffset(), resolved_fields);
@@ -376,6 +419,17 @@
template <typename PtrType>
static void SetElementPtrSize(PtrType* ptr_array, size_t idx, PtrType ptr, PointerSize ptr_size);
+ template <typename T>
+ static NativeDexCachePair<T> GetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ PointerSize ptr_size);
+
+ template <typename T>
+ static void SetNativePairPtrSize(std::atomic<NativeDexCachePair<T>>* pair_array,
+ size_t idx,
+ NativeDexCachePair<T> pair,
+ PointerSize ptr_size);
+
private:
void Init(const DexFile* dex_file,
ObjPtr<String> location,
@@ -385,7 +439,7 @@
uint32_t num_resolved_types,
ArtMethod** resolved_methods,
uint32_t num_resolved_methods,
- ArtField** resolved_fields,
+ FieldDexCacheType* resolved_fields,
uint32_t num_resolved_fields,
MethodTypeDexCacheType* resolved_method_types,
uint32_t num_resolved_method_types,
@@ -394,8 +448,22 @@
PointerSize pointer_size)
REQUIRES_SHARED(Locks::mutator_lock_);
+ // std::pair<> is not trivially copyable and as such it is unsuitable for atomic operations,
+ // so we use a custom pair class for loading and storing the NativeDexCachePair<>.
+ template <typename IntType>
+ struct PACKED(2 * sizeof(IntType)) ConversionPair {
+ ConversionPair(IntType f, IntType s) : first(f), second(s) { }
+ ConversionPair(const ConversionPair&) = default;
+ ConversionPair& operator=(const ConversionPair&) = default;
+ IntType first;
+ IntType second;
+ };
+ using ConversionPair32 = ConversionPair<uint32_t>;
+ using ConversionPair64 = ConversionPair<uint64_t>;
+
uint32_t StringSlotIndex(dex::StringIndex string_idx) REQUIRES_SHARED(Locks::mutator_lock_);
uint32_t TypeSlotIndex(dex::TypeIndex type_idx) REQUIRES_SHARED(Locks::mutator_lock_);
+ uint32_t FieldSlotIndex(uint32_t field_idx) REQUIRES_SHARED(Locks::mutator_lock_);
uint32_t MethodTypeSlotIndex(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_);
// Visit instance fields of the dex cache as well as its associated arrays.
@@ -406,12 +474,55 @@
void VisitReferences(ObjPtr<Class> klass, const Visitor& visitor)
REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_);
+ // Due to lack of 16-byte atomics support, we use hand-crafted routines.
+#if defined(__aarch64__)
+ // 16-byte atomics are supported on aarch64.
+ ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B(
+ std::atomic<ConversionPair64>* target) {
+ return target->load(std::memory_order_relaxed);
+ }
+
+ ALWAYS_INLINE static void AtomicStoreRelease16B(
+ std::atomic<ConversionPair64>* target, ConversionPair64 value) {
+ target->store(value, std::memory_order_release);
+ }
+#elif defined(__x86_64__)
+ ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B(
+ std::atomic<ConversionPair64>* target) {
+ uint64_t first, second;
+ __asm__ __volatile__(
+ "lock cmpxchg16b (%2)"
+ : "=&a"(first), "=&d"(second)
+ : "r"(target), "a"(0), "d"(0), "b"(0), "c"(0)
+ : "cc");
+ return ConversionPair64(first, second);
+ }
+
+ ALWAYS_INLINE static void AtomicStoreRelease16B(
+ std::atomic<ConversionPair64>* target, ConversionPair64 value) {
+ uint64_t first, second;
+ __asm__ __volatile__ (
+ "movq (%2), %%rax\n\t"
+ "movq 8(%2), %%rdx\n\t"
+ "1:\n\t"
+ "lock cmpxchg16b (%2)\n\t"
+ "jnz 1b"
+ : "=&a"(first), "=&d"(second)
+ : "r"(target), "b"(value.first), "c"(value.second)
+ : "cc");
+ }
+#else
+ static ConversionPair64 AtomicLoadRelaxed16B(std::atomic<ConversionPair64>* target);
+ static void AtomicStoreRelease16B(std::atomic<ConversionPair64>* target, ConversionPair64 value);
+#endif
+
HeapReference<Object> dex_;
HeapReference<String> location_;
uint64_t dex_file_; // const DexFile*
uint64_t resolved_call_sites_; // GcRoot<CallSite>* array with num_resolved_call_sites_
// elements.
- uint64_t resolved_fields_; // ArtField*, array with num_resolved_fields_ elements.
+ uint64_t resolved_fields_; // std::atomic<FieldDexCachePair>*, array with
+ // num_resolved_fields_ elements.
uint64_t resolved_method_types_; // std::atomic<MethodTypeDexCachePair>* array with
// num_resolved_method_types_ elements.
uint64_t resolved_methods_; // ArtMethod*, array with num_resolved_methods_ elements.
diff --git a/runtime/mirror/dex_cache_test.cc b/runtime/mirror/dex_cache_test.cc
index ef0aaaa..71a47f6 100644
--- a/runtime/mirror/dex_cache_test.cc
+++ b/runtime/mirror/dex_cache_test.cc
@@ -54,7 +54,8 @@
EXPECT_TRUE(dex_cache->StaticTypeSize() == dex_cache->NumResolvedTypes()
|| java_lang_dex_file_->NumTypeIds() == dex_cache->NumResolvedTypes());
EXPECT_EQ(java_lang_dex_file_->NumMethodIds(), dex_cache->NumResolvedMethods());
- EXPECT_EQ(java_lang_dex_file_->NumFieldIds(), dex_cache->NumResolvedFields());
+ EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields()
+ || java_lang_dex_file_->NumFieldIds() == dex_cache->NumResolvedFields());
EXPECT_TRUE(dex_cache->StaticMethodTypeSize() == dex_cache->NumResolvedMethodTypes()
|| java_lang_dex_file_->NumProtoIds() == dex_cache->NumResolvedMethodTypes());
}
diff --git a/runtime/mirror/field.cc b/runtime/mirror/field.cc
index f6b6489..54034c2 100644
--- a/runtime/mirror/field.cc
+++ b/runtime/mirror/field.cc
@@ -68,8 +68,16 @@
}
}
mirror::DexCache* const dex_cache = declaring_class->GetDexCache();
- ArtField* const art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize);
- CHECK(art_field != nullptr);
+ ArtField* art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize);
+ if (UNLIKELY(art_field == nullptr)) {
+ if (IsStatic()) {
+ art_field = declaring_class->FindDeclaredStaticField(dex_cache, GetDexFieldIndex());
+ } else {
+ art_field = declaring_class->FindInstanceField(dex_cache, GetDexFieldIndex());
+ }
+ CHECK(art_field != nullptr);
+ dex_cache->SetResolvedField(GetDexFieldIndex(), art_field, kRuntimePointerSize);
+ }
CHECK_EQ(declaring_class, art_field->GetDeclaringClass());
return art_field;
}
diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc
index 9b707f8..d81c13d 100644
--- a/runtime/native/dalvik_system_VMRuntime.cc
+++ b/runtime/native/dalvik_system_VMRuntime.cc
@@ -444,6 +444,7 @@
if (!kPreloadDexCachesCollectStats) {
return;
}
+ // TODO: Update for hash-based DexCache arrays.
ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
Thread* const self = Thread::Current();
for (const DexFile* dex_file : class_linker->GetBootClassPath()) {
@@ -463,7 +464,7 @@
}
}
for (size_t j = 0; j < dex_cache->NumResolvedFields(); j++) {
- ArtField* field = class_linker->GetResolvedField(j, dex_cache);
+ ArtField* field = dex_cache->GetResolvedField(j, class_linker->GetImagePointerSize());
if (field != nullptr) {
filled->num_fields++;
}
diff --git a/runtime/oat.h b/runtime/oat.h
index 1544121..df43107 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -32,7 +32,7 @@
class PACKED(4) OatHeader {
public:
static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' };
- static constexpr uint8_t kOatVersion[] = { '1', '1', '4', '\0' }; // hash-based DexCache types.
+ static constexpr uint8_t kOatVersion[] = { '1', '1', '5', '\0' }; // hash-based DexCache fields
static constexpr const char* kImageLocationKey = "image-location";
static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
diff --git a/runtime/quick/inline_method_analyser.cc b/runtime/quick/inline_method_analyser.cc
index b009b47..3347070 100644
--- a/runtime/quick/inline_method_analyser.cc
+++ b/runtime/quick/inline_method_analyser.cc
@@ -215,9 +215,8 @@
REQUIRES_SHARED(Locks::mutator_lock_) {
DCHECK(IsInstructionIPut(new_iput->Opcode()));
uint32_t field_index = new_iput->VRegC_22c();
- PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
- mirror::DexCache* dex_cache = method->GetDexCache();
- ArtField* field = dex_cache->GetResolvedField(field_index, pointer_size);
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* field = class_linker->LookupResolvedField(field_index, method, /* is_static */ false);
if (UNLIKELY(field == nullptr)) {
return false;
}
@@ -227,7 +226,9 @@
if (iputs[old_pos].field_index == DexFile::kDexNoIndex16) {
break;
}
- ArtField* f = dex_cache->GetResolvedField(iputs[old_pos].field_index, pointer_size);
+ ArtField* f = class_linker->LookupResolvedField(iputs[old_pos].field_index,
+ method,
+ /* is_static */ false);
DCHECK(f != nullptr);
if (f == field) {
auto back_it = std::copy(iputs + old_pos + 1, iputs + arraysize(iputs), iputs + old_pos);
@@ -732,9 +733,9 @@
if (method == nullptr) {
return false;
}
- mirror::DexCache* dex_cache = method->GetDexCache();
- PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize();
- ArtField* field = dex_cache->GetResolvedField(field_idx, pointer_size);
+ ObjPtr<mirror::DexCache> dex_cache = method->GetDexCache();
+ ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
+ ArtField* field = class_linker->LookupResolvedField(field_idx, method, /* is_static */ false);
if (field == nullptr || field->IsStatic()) {
return false;
}
diff --git a/runtime/utils/dex_cache_arrays_layout-inl.h b/runtime/utils/dex_cache_arrays_layout-inl.h
index f9a1405..95904af 100644
--- a/runtime/utils/dex_cache_arrays_layout-inl.h
+++ b/runtime/utils/dex_cache_arrays_layout-inl.h
@@ -51,7 +51,11 @@
: DexCacheArraysLayout(pointer_size, dex_file->GetHeader(), dex_file->NumCallSiteIds()) {
}
-constexpr size_t DexCacheArraysLayout::Alignment() {
+inline size_t DexCacheArraysLayout::Alignment() const {
+ return Alignment(pointer_size_);
+}
+
+inline constexpr size_t DexCacheArraysLayout::Alignment(PointerSize pointer_size) {
// mirror::Type/String/MethodTypeDexCacheType alignment is 8,
// i.e. higher than or equal to the pointer alignment.
static_assert(alignof(mirror::TypeDexCacheType) == 8,
@@ -60,8 +64,8 @@
"Expecting alignof(StringDexCacheType) == 8");
static_assert(alignof(mirror::MethodTypeDexCacheType) == 8,
"Expecting alignof(MethodTypeDexCacheType) == 8");
- // This is the same as alignof(MethodTypeDexCacheType).
- return alignof(mirror::StringDexCacheType);
+ // This is the same as alignof(FieldDexCacheType) for the given pointer size.
+ return 2u * static_cast<size_t>(pointer_size);
}
template <typename T>
@@ -100,8 +104,8 @@
}
inline size_t DexCacheArraysLayout::StringOffset(uint32_t string_idx) const {
- return strings_offset_ + ElementOffset(PointerSize::k64,
- string_idx % mirror::DexCache::kDexCacheStringCacheSize);
+ uint32_t string_hash = string_idx % mirror::DexCache::kDexCacheStringCacheSize;
+ return strings_offset_ + ElementOffset(PointerSize::k64, string_hash);
}
inline size_t DexCacheArraysLayout::StringsSize(size_t num_elements) const {
@@ -119,15 +123,20 @@
}
inline size_t DexCacheArraysLayout::FieldOffset(uint32_t field_idx) const {
- return fields_offset_ + ElementOffset(pointer_size_, field_idx);
+ uint32_t field_hash = field_idx % mirror::DexCache::kDexCacheFieldCacheSize;
+ return fields_offset_ + 2u * static_cast<size_t>(pointer_size_) * field_hash;
}
inline size_t DexCacheArraysLayout::FieldsSize(size_t num_elements) const {
- return ArraySize(pointer_size_, num_elements);
+ size_t cache_size = mirror::DexCache::kDexCacheFieldCacheSize;
+ if (num_elements < cache_size) {
+ cache_size = num_elements;
+ }
+ return 2u * static_cast<size_t>(pointer_size_) * num_elements;
}
inline size_t DexCacheArraysLayout::FieldsAlignment() const {
- return static_cast<size_t>(pointer_size_);
+ return 2u * static_cast<size_t>(pointer_size_);
}
inline size_t DexCacheArraysLayout::MethodTypesSize(size_t num_elements) const {
diff --git a/runtime/utils/dex_cache_arrays_layout.h b/runtime/utils/dex_cache_arrays_layout.h
index ed677ed..377a374 100644
--- a/runtime/utils/dex_cache_arrays_layout.h
+++ b/runtime/utils/dex_cache_arrays_layout.h
@@ -57,7 +57,9 @@
return size_;
}
- static constexpr size_t Alignment();
+ size_t Alignment() const;
+
+ static constexpr size_t Alignment(PointerSize pointer_size);
size_t TypesOffset() const {
return types_offset_;
@@ -125,8 +127,6 @@
const size_t call_sites_offset_;
const size_t size_;
- static size_t Alignment(PointerSize pointer_size);
-
static size_t ElementOffset(PointerSize element_size, uint32_t idx);
static size_t ArraySize(PointerSize element_size, uint32_t num_elements);
