Use non volatile lock words in semispace collector.
GSS FormulaEvaluationActions.EvaluateAndApplyChanges:
Before GC time: 9.1s
After GC time: 7.98s
Fixed timing logger errors.
Change-Id: I4193c6ccbbbe7a7220dfaabbf3472a5dcebae616
diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index 05d6693..7c5741b 100644
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -164,7 +164,7 @@
EXPECT_TRUE(reinterpret_cast<byte*>(klass) >= image_end ||
reinterpret_cast<byte*>(klass) < image_begin) << descriptor;
}
- EXPECT_TRUE(Monitor::IsValidLockWord(klass->GetLockWord()));
+ EXPECT_TRUE(Monitor::IsValidLockWord(klass->GetLockWord(false)));
}
}
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index f76587a..3400b01 100644
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -179,7 +179,7 @@
image_bitmap_->Set(obj);
// Before we stomp over the lock word, save the hash code for later.
Monitor::Deflate(Thread::Current(), object);;
- LockWord lw(object->GetLockWord());
+ LockWord lw(object->GetLockWord(false));
switch (lw.GetState()) {
case LockWord::kFatLocked: {
LOG(FATAL) << "Fat locked object " << obj << " found during object copy";
@@ -199,7 +199,7 @@
LOG(FATAL) << "Unreachable.";
break;
}
- object->SetLockWord(LockWord::FromForwardingAddress(offset));
+ object->SetLockWord(LockWord::FromForwardingAddress(offset), false);
DCHECK(IsImageOffsetAssigned(object));
}
@@ -212,13 +212,13 @@
bool ImageWriter::IsImageOffsetAssigned(mirror::Object* object) const {
DCHECK(object != nullptr);
- return object->GetLockWord().GetState() == LockWord::kForwardingAddress;
+ return object->GetLockWord(false).GetState() == LockWord::kForwardingAddress;
}
size_t ImageWriter::GetImageOffset(mirror::Object* object) const {
DCHECK(object != nullptr);
DCHECK(IsImageOffsetAssigned(object));
- LockWord lock_word = object->GetLockWord();
+ LockWord lock_word = object->GetLockWord(false);
size_t offset = lock_word.ForwardingAddress();
DCHECK_LT(offset, image_end_);
return offset;
@@ -555,15 +555,15 @@
heap->VisitObjects(CopyAndFixupObjectsCallback, this);
// Fix up the object previously had hash codes.
for (const std::pair<mirror::Object*, uint32_t>& hash_pair : saved_hashes_) {
- hash_pair.first->SetLockWord(LockWord::FromHashCode(hash_pair.second));
+ hash_pair.first->SetLockWord(LockWord::FromHashCode(hash_pair.second), false);
}
saved_hashes_.clear();
self->EndAssertNoThreadSuspension(old_cause);
}
void ImageWriter::CopyAndFixupObjectsCallback(Object* obj, void* arg) {
- DCHECK(obj != NULL);
- DCHECK(arg != NULL);
+ DCHECK(obj != nullptr);
+ DCHECK(arg != nullptr);
ImageWriter* image_writer = reinterpret_cast<ImageWriter*>(arg);
// see GetLocalAddress for similar computation
size_t offset = image_writer->GetImageOffset(obj);
@@ -575,7 +575,7 @@
Object* copy = reinterpret_cast<Object*>(dst);
// Write in a hash code of objects which have inflated monitors or a hash code in their monitor
// word.
- copy->SetLockWord(LockWord());
+ copy->SetLockWord(LockWord(), false);
image_writer->FixupObject(obj, copy);
}
diff --git a/runtime/gc/collector/semi_space-inl.h b/runtime/gc/collector/semi_space-inl.h
index 8a9611f..d03baf1 100644
--- a/runtime/gc/collector/semi_space-inl.h
+++ b/runtime/gc/collector/semi_space-inl.h
@@ -28,7 +28,7 @@
inline mirror::Object* SemiSpace::GetForwardingAddressInFromSpace(mirror::Object* obj) const {
DCHECK(from_space_->HasAddress(obj));
- LockWord lock_word = obj->GetLockWord();
+ LockWord lock_word = obj->GetLockWord(false);
if (lock_word.GetState() != LockWord::kForwardingAddress) {
return nullptr;
}
@@ -58,8 +58,8 @@
DCHECK(forward_address != nullptr);
// Make sure to only update the forwarding address AFTER you copy the object so that the
// monitor word doesn't get stomped over.
- obj->SetLockWord(LockWord::FromForwardingAddress(
- reinterpret_cast<size_t>(forward_address)));
+ obj->SetLockWord(
+ LockWord::FromForwardingAddress(reinterpret_cast<size_t>(forward_address)), false);
// Push the object onto the mark stack for later processing.
MarkStackPush(forward_address);
}
diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc
index c0e172e..4a1bf18 100644
--- a/runtime/gc/collector/semi_space.cc
+++ b/runtime/gc/collector/semi_space.cc
@@ -117,6 +117,8 @@
immune_region_.Reset();
is_large_object_space_immune_ = false;
saved_bytes_ = 0;
+ bytes_moved_ = 0;
+ objects_moved_ = 0;
self_ = Thread::Current();
// Do any pre GC verification.
timings_.NewSplit("PreGcVerification");
@@ -382,9 +384,9 @@
}
// Record freed memory.
uint64_t from_bytes = from_space_->GetBytesAllocated();
- uint64_t to_bytes = to_space_->GetBytesAllocated();
+ uint64_t to_bytes = bytes_moved_;
uint64_t from_objects = from_space_->GetObjectsAllocated();
- uint64_t to_objects = to_space_->GetObjectsAllocated();
+ uint64_t to_objects = objects_moved_;
CHECK_LE(to_objects, from_objects);
int64_t freed_bytes = from_bytes - to_bytes;
int64_t freed_objects = from_objects - to_objects;
@@ -521,15 +523,13 @@
// If it's allocated before the last GC (older), move
// (pseudo-promote) it to the main free list space (as sort
// of an old generation.)
- size_t bytes_promoted;
space::MallocSpace* promo_dest_space = GetHeap()->GetPrimaryFreeListSpace();
- forward_address = promo_dest_space->Alloc(self_, object_size, &bytes_promoted, nullptr);
- if (forward_address == nullptr) {
+ forward_address = promo_dest_space->Alloc(self_, object_size, &bytes_allocated, nullptr);
+ if (UNLIKELY(forward_address == nullptr)) {
// If out of space, fall back to the to-space.
forward_address = to_space_->Alloc(self_, object_size, &bytes_allocated, nullptr);
} else {
- GetHeap()->num_bytes_allocated_.FetchAndAdd(bytes_promoted);
- bytes_promoted_ += bytes_promoted;
+ bytes_promoted_ += bytes_allocated;
// Dirty the card at the destionation as it may contain
// references (including the class pointer) to the bump pointer
// space.
@@ -573,6 +573,8 @@
// If it's allocated after the last GC (younger), copy it to the to-space.
forward_address = to_space_->Alloc(self_, object_size, &bytes_allocated, nullptr);
}
+ ++objects_moved_;
+ bytes_moved_ += bytes_allocated;
// Copy over the object and add it to the mark stack since we still need to update its
// references.
saved_bytes_ +=
@@ -619,10 +621,9 @@
// Marks all objects in the root set.
void SemiSpace::MarkRoots() {
- timings_.StartSplit("MarkRoots");
+ timings_.NewSplit("MarkRoots");
// TODO: Visit up image roots as well?
Runtime::Current()->VisitRoots(MarkRootCallback, this);
- timings_.EndSplit();
}
mirror::Object* SemiSpace::MarkedForwardingAddressCallback(mirror::Object* object, void* arg) {
@@ -641,7 +642,7 @@
void SemiSpace::Sweep(bool swap_bitmaps) {
DCHECK(mark_stack_->IsEmpty());
- TimingLogger::ScopedSplit("Sweep", &timings_);
+ TimingLogger::ScopedSplit split("Sweep", &timings_);
for (const auto& space : GetHeap()->GetContinuousSpaces()) {
if (space->IsContinuousMemMapAllocSpace()) {
space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace();
@@ -665,13 +666,13 @@
void SemiSpace::SweepLargeObjects(bool swap_bitmaps) {
DCHECK(!is_large_object_space_immune_);
- TimingLogger::ScopedSplit("SweepLargeObjects", &timings_);
+ TimingLogger::ScopedSplit split("SweepLargeObjects", &timings_);
size_t freed_objects = 0;
size_t freed_bytes = 0;
- GetHeap()->GetLargeObjectsSpace()->Sweep(swap_bitmaps, &freed_objects, &freed_bytes);
+ heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps, &freed_objects, &freed_bytes);
freed_large_objects_.FetchAndAdd(freed_objects);
freed_large_object_bytes_.FetchAndAdd(freed_bytes);
- GetHeap()->RecordFree(freed_objects, freed_bytes);
+ heap_->RecordFree(freed_objects, freed_bytes);
}
// Process the "referent" field in a java.lang.ref.Reference. If the referent has not yet been
diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h
index 4169ca9..b6726b2 100644
--- a/runtime/gc/collector/semi_space.h
+++ b/runtime/gc/collector/semi_space.h
@@ -231,6 +231,11 @@
// whole_heap_collection_ once per interval.
int whole_heap_collection_interval_counter_;
+ // How many objects and bytes we moved, used so that we don't need to get the size of the
+ // to_space_ when calculating how many objects and bytes we freed.
+ size_t bytes_moved_;
+ size_t objects_moved_;
+
// How many bytes we avoided dirtying.
size_t saved_bytes_;
diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h
index a6db387..2505855 100644
--- a/runtime/mirror/object-inl.h
+++ b/runtime/mirror/object-inl.h
@@ -51,13 +51,14 @@
OFFSET_OF_OBJECT_MEMBER(Object, klass_), new_klass, false);
}
-inline LockWord Object::GetLockWord() {
- return LockWord(GetField32(OFFSET_OF_OBJECT_MEMBER(Object, monitor_), true));
+inline LockWord Object::GetLockWord(bool is_volatile) {
+ return LockWord(GetField32(OFFSET_OF_OBJECT_MEMBER(Object, monitor_), is_volatile));
}
-inline void Object::SetLockWord(LockWord new_val) {
+inline void Object::SetLockWord(LockWord new_val, bool is_volatile) {
// Force use of non-transactional mode and do not check.
- SetField32<false, false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_), new_val.GetValue(), true);
+ SetField32<false, false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_), new_val.GetValue(),
+ is_volatile);
}
inline bool Object::CasLockWord(LockWord old_val, LockWord new_val) {
diff --git a/runtime/mirror/object.cc b/runtime/mirror/object.cc
index be7e9f2..766bbc9 100644
--- a/runtime/mirror/object.cc
+++ b/runtime/mirror/object.cc
@@ -150,7 +150,7 @@
int32_t Object::IdentityHashCode() const {
mirror::Object* current_this = const_cast<mirror::Object*>(this);
while (true) {
- LockWord lw = current_this->GetLockWord();
+ LockWord lw = current_this->GetLockWord(false);
switch (lw.GetState()) {
case LockWord::kUnlocked: {
// Try to compare and swap in a new hash, if we succeed we will return the hash on the next
diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h
index f652202..fd31dfb 100644
--- a/runtime/mirror/object.h
+++ b/runtime/mirror/object.h
@@ -100,8 +100,10 @@
return OFFSET_OF_OBJECT_MEMBER(Object, monitor_);
}
- LockWord GetLockWord();
- void SetLockWord(LockWord new_val);
+ // As volatile can be false if the mutators are suspended. This is an optimization since it
+ // avoids the barriers.
+ LockWord GetLockWord(bool as_volatile);
+ void SetLockWord(LockWord new_val, bool as_volatile);
bool CasLockWord(LockWord old_val, LockWord new_val) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
uint32_t GetLockOwnerThreadId();
diff --git a/runtime/monitor.cc b/runtime/monitor.cc
index bbc7dd0..38b77d1 100644
--- a/runtime/monitor.cc
+++ b/runtime/monitor.cc
@@ -111,7 +111,7 @@
MutexLock mu(self, monitor_lock_); // Uncontended mutex acquisition as monitor isn't yet public.
CHECK(owner_ == nullptr || owner_ == self || owner_->IsSuspended());
// Propagate the lock state.
- LockWord lw(obj_->GetLockWord());
+ LockWord lw(obj_->GetLockWord(false));
switch (lw.GetState()) {
case LockWord::kThinLocked: {
CHECK_EQ(owner_->GetThreadId(), lw.ThinLockOwner());
@@ -574,7 +574,8 @@
bool Monitor::Deflate(Thread* self, mirror::Object* obj) {
DCHECK(obj != nullptr);
- LockWord lw(obj->GetLockWord());
+ // Don't need volatile since we only deflate with mutators suspended.
+ LockWord lw(obj->GetLockWord(false));
// If the lock isn't an inflated monitor, then we don't need to deflate anything.
if (lw.GetState() == LockWord::kFatLocked) {
Monitor* monitor = lw.FatLockMonitor();
@@ -595,14 +596,15 @@
return false;
}
// Deflate to a thin lock.
- obj->SetLockWord(LockWord::FromThinLockId(owner->GetThreadId(), monitor->lock_count_));
- VLOG(monitor) << "Deflated " << obj << " to thin lock " << owner->GetTid() << " / " << monitor->lock_count_;
+ obj->SetLockWord(LockWord::FromThinLockId(owner->GetThreadId(), monitor->lock_count_), false);
+ VLOG(monitor) << "Deflated " << obj << " to thin lock " << owner->GetTid() << " / "
+ << monitor->lock_count_;
} else if (monitor->HasHashCode()) {
- obj->SetLockWord(LockWord::FromHashCode(monitor->GetHashCode()));
+ obj->SetLockWord(LockWord::FromHashCode(monitor->GetHashCode()), false);
VLOG(monitor) << "Deflated " << obj << " to hash monitor " << monitor->GetHashCode();
} else {
// No lock and no hash, just put an empty lock word inside the object.
- obj->SetLockWord(LockWord());
+ obj->SetLockWord(LockWord(), false);
VLOG(monitor) << "Deflated" << obj << " to empty lock word";
}
// The monitor is deflated, mark the object as nullptr so that we know to delete it during the
@@ -626,7 +628,7 @@
VLOG(monitor) << "monitor: thread " << owner->GetThreadId()
<< " created monitor " << m.get() << " for object " << obj;
Runtime::Current()->GetMonitorList()->Add(m.release());
- CHECK_EQ(obj->GetLockWord().GetState(), LockWord::kFatLocked);
+ CHECK_EQ(obj->GetLockWord(true).GetState(), LockWord::kFatLocked);
}
}
@@ -642,12 +644,12 @@
// Suspend the owner, inflate. First change to blocked and give up mutator_lock_.
ScopedThreadStateChange tsc(self, kBlocked);
self->SetMonitorEnterObject(obj.get());
- if (lock_word == obj->GetLockWord()) { // If lock word hasn't changed.
+ if (lock_word == obj->GetLockWord(true)) { // If lock word hasn't changed.
bool timed_out;
Thread* owner = thread_list->SuspendThreadByThreadId(owner_thread_id, false, &timed_out);
if (owner != nullptr) {
// We succeeded in suspending the thread, check the lock's status didn't change.
- lock_word = obj->GetLockWord();
+ lock_word = obj->GetLockWord(true);
if (lock_word.GetState() == LockWord::kThinLocked &&
lock_word.ThinLockOwner() == owner_thread_id) {
// Go ahead and inflate the lock.
@@ -680,7 +682,7 @@
size_t contention_count = 0;
SirtRef<mirror::Object> sirt_obj(self, obj);
while (true) {
- LockWord lock_word = sirt_obj->GetLockWord();
+ LockWord lock_word = sirt_obj->GetLockWord(true);
switch (lock_word.GetState()) {
case LockWord::kUnlocked: {
LockWord thin_locked(LockWord::FromThinLockId(thread_id, 0));
@@ -697,7 +699,7 @@
uint32_t new_count = lock_word.ThinLockCount() + 1;
if (LIKELY(new_count <= LockWord::kThinLockMaxCount)) {
LockWord thin_locked(LockWord::FromThinLockId(thread_id, new_count));
- sirt_obj->SetLockWord(thin_locked);
+ sirt_obj->SetLockWord(thin_locked, true);
return sirt_obj.get(); // Success!
} else {
// We'd overflow the recursion count, so inflate the monitor.
@@ -737,13 +739,13 @@
DCHECK(self != NULL);
DCHECK(obj != NULL);
obj = FakeUnlock(obj);
- LockWord lock_word = obj->GetLockWord();
+ LockWord lock_word = obj->GetLockWord(true);
SirtRef<mirror::Object> sirt_obj(self, obj);
switch (lock_word.GetState()) {
case LockWord::kHashCode:
// Fall-through.
case LockWord::kUnlocked:
- FailedUnlock(sirt_obj.get(), self, NULL, NULL);
+ FailedUnlock(sirt_obj.get(), self, nullptr, nullptr);
return false; // Failure.
case LockWord::kThinLocked: {
uint32_t thread_id = self->GetThreadId();
@@ -752,16 +754,16 @@
// TODO: there's a race here with the owner dying while we unlock.
Thread* owner =
Runtime::Current()->GetThreadList()->FindThreadByThreadId(lock_word.ThinLockOwner());
- FailedUnlock(sirt_obj.get(), self, owner, NULL);
+ FailedUnlock(sirt_obj.get(), self, owner, nullptr);
return false; // Failure.
} else {
// We own the lock, decrease the recursion count.
if (lock_word.ThinLockCount() != 0) {
uint32_t new_count = lock_word.ThinLockCount() - 1;
LockWord thin_locked(LockWord::FromThinLockId(thread_id, new_count));
- sirt_obj->SetLockWord(thin_locked);
+ sirt_obj->SetLockWord(thin_locked, true);
} else {
- sirt_obj->SetLockWord(LockWord());
+ sirt_obj->SetLockWord(LockWord(), true);
}
return true; // Success!
}
@@ -782,10 +784,9 @@
*/
void Monitor::Wait(Thread* self, mirror::Object *obj, int64_t ms, int32_t ns,
bool interruptShouldThrow, ThreadState why) {
- DCHECK(self != NULL);
- DCHECK(obj != NULL);
-
- LockWord lock_word = obj->GetLockWord();
+ DCHECK(self != nullptr);
+ DCHECK(obj != nullptr);
+ LockWord lock_word = obj->GetLockWord(true);
switch (lock_word.GetState()) {
case LockWord::kHashCode:
// Fall-through.
@@ -801,7 +802,7 @@
} else {
// We own the lock, inflate to enqueue ourself on the Monitor.
Inflate(self, self, obj, 0);
- lock_word = obj->GetLockWord();
+ lock_word = obj->GetLockWord(true);
}
break;
}
@@ -817,10 +818,9 @@
}
void Monitor::DoNotify(Thread* self, mirror::Object* obj, bool notify_all) {
- DCHECK(self != NULL);
- DCHECK(obj != NULL);
-
- LockWord lock_word = obj->GetLockWord();
+ DCHECK(self != nullptr);
+ DCHECK(obj != nullptr);
+ LockWord lock_word = obj->GetLockWord(true);
switch (lock_word.GetState()) {
case LockWord::kHashCode:
// Fall-through.
@@ -855,9 +855,8 @@
}
uint32_t Monitor::GetLockOwnerThreadId(mirror::Object* obj) {
- DCHECK(obj != NULL);
-
- LockWord lock_word = obj->GetLockWord();
+ DCHECK(obj != nullptr);
+ LockWord lock_word = obj->GetLockWord(true);
switch (lock_word.GetState()) {
case LockWord::kHashCode:
// Fall-through.
@@ -902,7 +901,7 @@
if (pretty_object == nullptr) {
os << wait_message << "an unknown object";
} else {
- if ((pretty_object->GetLockWord().GetState() == LockWord::kThinLocked) &&
+ if ((pretty_object->GetLockWord(true).GetState() == LockWord::kThinLocked) &&
Locks::mutator_lock_->IsExclusiveHeld(Thread::Current())) {
// Getting the identity hashcode here would result in lock inflation and suspension of the
// current thread, which isn't safe if this is the only runnable thread.
@@ -1112,7 +1111,7 @@
static mirror::Object* MonitorDeflateCallback(mirror::Object* object, void* arg)
SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
if (Monitor::Deflate(reinterpret_cast<Thread*>(arg), object)) {
- DCHECK_NE(object->GetLockWord().GetState(), LockWord::kFatLocked);
+ DCHECK_NE(object->GetLockWord(true).GetState(), LockWord::kFatLocked);
// If we deflated, return nullptr so that the monitor gets removed from the array.
return nullptr;
}
@@ -1126,9 +1125,8 @@
}
MonitorInfo::MonitorInfo(mirror::Object* obj) : owner_(NULL), entry_count_(0) {
- DCHECK(obj != NULL);
-
- LockWord lock_word = obj->GetLockWord();
+ DCHECK(obj != nullptr);
+ LockWord lock_word = obj->GetLockWord(true);
switch (lock_word.GetState()) {
case LockWord::kUnlocked:
// Fall-through.
diff --git a/runtime/thread.cc b/runtime/thread.cc
index b3d14f0..8691dec 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -876,7 +876,7 @@
if (o == nullptr) {
os << "an unknown object";
} else {
- if ((o->GetLockWord().GetState() == LockWord::kThinLocked) &&
+ if ((o->GetLockWord(false).GetState() == LockWord::kThinLocked) &&
Locks::mutator_lock_->IsExclusiveHeld(Thread::Current())) {
// Getting the identity hashcode here would result in lock inflation and suspension of the
// current thread, which isn't safe if this is the only runnable thread.
diff --git a/runtime/transaction_test.cc b/runtime/transaction_test.cc
index 76b6f27..1dc2da0 100644
--- a/runtime/transaction_test.cc
+++ b/runtime/transaction_test.cc
@@ -55,18 +55,18 @@
// Lock object's monitor outside the transaction.
sirt_obj->MonitorEnter(soa.Self());
- uint32_t old_lock_word = sirt_obj->GetLockWord().GetValue();
+ uint32_t old_lock_word = sirt_obj->GetLockWord(false).GetValue();
Transaction transaction;
Runtime::Current()->EnterTransactionMode(&transaction);
// Unlock object's monitor inside the transaction.
sirt_obj->MonitorExit(soa.Self());
- uint32_t new_lock_word = sirt_obj->GetLockWord().GetValue();
+ uint32_t new_lock_word = sirt_obj->GetLockWord(false).GetValue();
Runtime::Current()->ExitTransactionMode();
// Aborting transaction must not clear the Object::class field.
transaction.Abort();
- uint32_t aborted_lock_word = sirt_obj->GetLockWord().GetValue();
+ uint32_t aborted_lock_word = sirt_obj->GetLockWord(false).GetValue();
EXPECT_NE(old_lock_word, new_lock_word);
EXPECT_EQ(aborted_lock_word, new_lock_word);
}
