Experimental Sticky-Bit (Generational) CC collection
Use the card table to quickly collect regions allocated since the
last GC. This is similar in behavior to sticky CMS.
TODO: This is using the existing sticky CMS ergonomics, we can
maybe improve on these.
Guard Generational Concurrent Copying collection with compile-time
flag art::kEnableGenerationalConcurrentCopyingCollection, set by
environment variable ART_USE_GENERATIONAL_CC.
Test: ART run-tests & gtests, libcore tests, JDWP tests (host & device)
Test: Device/emulator boot test
Bug: 67628039
Bug: 12687968
Change-Id: I9c8023b71a029b0a73527cf67d924675c4c14305
diff --git a/build/art.go b/build/art.go
index 3dabce3..61b1a4e 100644
--- a/build/art.go
+++ b/build/art.go
@@ -66,8 +66,12 @@
"-DART_READ_BARRIER_TYPE_IS_"+barrierType+"=1")
}
- cdexLevel := envDefault(ctx, "ART_DEFAULT_COMPACT_DEX_LEVEL", "fast")
- cflags = append(cflags, "-DART_DEFAULT_COMPACT_DEX_LEVEL="+cdexLevel)
+ if envTrue(ctx, "ART_USE_GENERATIONAL_CC") {
+ cflags = append(cflags, "-DART_USE_GENERATIONAL_CC=1")
+ }
+
+ cdexLevel := envDefault(ctx, "ART_DEFAULT_COMPACT_DEX_LEVEL", "fast")
+ cflags = append(cflags, "-DART_DEFAULT_COMPACT_DEX_LEVEL="+cdexLevel)
// We need larger stack overflow guards for ASAN, as the compiled code will have
// larger frame sizes. For simplicity, just use global not-target-specific cflags.
@@ -312,19 +316,19 @@
codegen(ctx, c, true)
type props struct {
- Target struct {
- Android struct {
- Shared_libs []string
- }
- }
+ Target struct {
+ Android struct {
+ Shared_libs []string
+ }
+ }
}
p := &props{}
// TODO: express this in .bp instead b/79671158
if !envTrue(ctx, "ART_TARGET_LINUX") {
- p.Target.Android.Shared_libs = []string {
- "libmetricslogger",
- }
+ p.Target.Android.Shared_libs = []string{
+ "libmetricslogger",
+ }
}
ctx.AppendProperties(p)
})
diff --git a/libartbase/base/globals.h b/libartbase/base/globals.h
index cd0bf8f..bc79ff2 100644
--- a/libartbase/base/globals.h
+++ b/libartbase/base/globals.h
@@ -122,6 +122,14 @@
static constexpr bool kMarkCompactSupport = false && kMovingCollector;
// True if we allow moving classes.
static constexpr bool kMovingClasses = !kMarkCompactSupport;
+// If true, enable generational collection when using the Concurrent Copying
+// collector, i.e. use sticky-bit CC for minor collections and (full) CC for
+// major collections.
+#ifdef ART_USE_GENERATIONAL_CC
+static constexpr bool kEnableGenerationalConcurrentCopyingCollection = true;
+#else
+static constexpr bool kEnableGenerationalConcurrentCopyingCollection = false;
+#endif
// If true, enable the tlab allocator by default.
#ifdef ART_USE_TLAB
diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h
index b8520b7..6613579 100644
--- a/runtime/gc/accounting/card_table.h
+++ b/runtime/gc/accounting/card_table.h
@@ -66,6 +66,11 @@
return GetCard(obj) == kCardDirty;
}
+ // Is the object on a clean card?
+ bool IsClean(const mirror::Object* obj) const {
+ return GetCard(obj) == kCardClean;
+ }
+
// Return the state of the card at an address.
uint8_t GetCard(const mirror::Object* obj) const {
return *CardFromAddr(obj);
diff --git a/runtime/gc/collector/concurrent_copying-inl.h b/runtime/gc/collector/concurrent_copying-inl.h
index cde5dc7..e4e1773 100644
--- a/runtime/gc/collector/concurrent_copying-inl.h
+++ b/runtime/gc/collector/concurrent_copying-inl.h
@@ -25,6 +25,7 @@
#include "gc/space/region_space.h"
#include "gc/verification.h"
#include "lock_word.h"
+#include "mirror/class.h"
#include "mirror/object-readbarrier-inl.h"
namespace art {
@@ -35,6 +36,25 @@
Thread* const self,
mirror::Object* ref,
accounting::ContinuousSpaceBitmap* bitmap) {
+ if (kEnableGenerationalConcurrentCopyingCollection
+ && young_gen_
+ && !done_scanning_.load(std::memory_order_relaxed)) {
+ // Everything in the unevac space should be marked for generational CC except for large objects.
+ DCHECK(region_space_bitmap_->Test(ref) || region_space_->IsLargeObject(ref)) << ref << " "
+ << ref->GetClass<kVerifyNone, kWithoutReadBarrier>()->PrettyClass();
+ // Since the mark bitmap is still filled in from last GC, we can not use that or else the
+ // mutator may see references to the from space. Instead, use the baker pointer itself as
+ // the mark bit.
+ if (ref->AtomicSetReadBarrierState(ReadBarrier::NonGrayState(), ReadBarrier::GrayState())) {
+ // TODO: We don't actually need to scan this object later, we just need to clear the gray
+ // bit.
+ // TODO: We could also set the mark bit here for "free" since this case comes from the
+ // read barrier.
+ PushOntoMarkStack(self, ref);
+ }
+ DCHECK_EQ(ref->GetReadBarrierState(), ReadBarrier::GrayState());
+ return ref;
+ }
// For the Baker-style RB, in a rare case, we could incorrectly change the object from non-gray
// (black) to gray even though the object has already been marked through. This happens if a
// mutator thread gets preempted before the AtomicSetReadBarrierState below, GC marks through the
@@ -103,11 +123,13 @@
return ref;
}
-template<bool kGrayImmuneObject, bool kFromGCThread>
+template<bool kGrayImmuneObject, bool kNoUnEvac, bool kFromGCThread>
inline mirror::Object* ConcurrentCopying::Mark(Thread* const self,
mirror::Object* from_ref,
mirror::Object* holder,
MemberOffset offset) {
+ // Cannot have `kNoUnEvac` when Generational CC collection is disabled.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || !kNoUnEvac);
if (from_ref == nullptr) {
return nullptr;
}
@@ -149,6 +171,14 @@
return to_ref;
}
case space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace:
+ if (kEnableGenerationalConcurrentCopyingCollection
+ && kNoUnEvac
+ && !region_space_->IsLargeObject(from_ref)) {
+ if (!kFromGCThread) {
+ DCHECK(IsMarkedInUnevacFromSpace(from_ref)) << "Returning unmarked object to mutator";
+ }
+ return from_ref;
+ }
return MarkUnevacFromSpaceRegion(self, from_ref, region_space_bitmap_);
default:
// The reference is in an unused region.
@@ -177,7 +207,8 @@
if (UNLIKELY(mark_from_read_barrier_measurements_)) {
ret = MarkFromReadBarrierWithMeasurements(self, from_ref);
} else {
- ret = Mark(self, from_ref);
+ ret = Mark</*kGrayImmuneObject*/true, /*kNoUnEvac*/false, /*kFromGCThread*/false>(self,
+ from_ref);
}
// Only set the mark bit for baker barrier.
if (kUseBakerReadBarrier && LIKELY(!rb_mark_bit_stack_full_ && ret->AtomicSetMarkBit(0, 1))) {
@@ -209,6 +240,11 @@
// Use load-acquire on the read barrier pointer to ensure that we never see a black (non-gray)
// read barrier state with an unmarked bit due to reordering.
DCHECK(region_space_->IsInUnevacFromSpace(from_ref));
+ if (kEnableGenerationalConcurrentCopyingCollection
+ && young_gen_
+ && !done_scanning_.load(std::memory_order_relaxed)) {
+ return from_ref->GetReadBarrierStateAcquire() == ReadBarrier::GrayState();
+ }
if (kUseBakerReadBarrier && from_ref->GetReadBarrierStateAcquire() == ReadBarrier::GrayState()) {
return true;
}
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 07abbfc..d0c10be 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -64,6 +64,7 @@
static constexpr bool kVerifyNoMissingCardMarks = kIsDebugBuild;
ConcurrentCopying::ConcurrentCopying(Heap* heap,
+ bool young_gen,
const std::string& name_prefix,
bool measure_read_barrier_slow_path)
: GarbageCollector(heap,
@@ -90,6 +91,7 @@
from_space_num_bytes_at_first_pause_(0),
mark_stack_mode_(kMarkStackModeOff),
weak_ref_access_enabled_(true),
+ young_gen_(young_gen),
skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock),
measure_read_barrier_slow_path_(measure_read_barrier_slow_path),
mark_from_read_barrier_measurements_(false),
@@ -107,6 +109,7 @@
kMarkSweepMarkStackLock) {
static_assert(space::RegionSpace::kRegionSize == accounting::ReadBarrierTable::kRegionSize,
"The region space size and the read barrier table region size must match");
+ CHECK(kEnableGenerationalConcurrentCopyingCollection || !young_gen_);
Thread* self = Thread::Current();
{
ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
@@ -268,11 +271,36 @@
space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
CHECK(space->IsZygoteSpace() || space->IsImageSpace());
immune_spaces_.AddSpace(space);
- } else if (space == region_space_) {
- // It is OK to clear the bitmap with mutators running since the only place it is read is
- // VisitObjects which has exclusion with CC.
- region_space_bitmap_ = region_space_->GetMarkBitmap();
- region_space_bitmap_->Clear();
+ } else {
+ CHECK(!space->IsZygoteSpace());
+ CHECK(!space->IsImageSpace());
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ if (space == region_space_) {
+ region_space_bitmap_ = region_space_->GetMarkBitmap();
+ } else if (young_gen_ && space->IsContinuousMemMapAllocSpace()) {
+ DCHECK_EQ(space->GetGcRetentionPolicy(), space::kGcRetentionPolicyAlwaysCollect);
+ space->AsContinuousMemMapAllocSpace()->BindLiveToMarkBitmap();
+ }
+ // Age all of the cards for the region space so that we know which evac regions to scan.
+ Runtime::Current()->GetHeap()->GetCardTable()->ModifyCardsAtomic(
+ space->Begin(),
+ space->End(),
+ AgeCardVisitor(),
+ VoidFunctor());
+ } else {
+ if (space == region_space_) {
+ // It is OK to clear the bitmap with mutators running since the only place it is read is
+ // VisitObjects which has exclusion with CC.
+ region_space_bitmap_ = region_space_->GetMarkBitmap();
+ region_space_bitmap_->Clear();
+ }
+ }
+ }
+ }
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ for (const auto& space : GetHeap()->GetDiscontinuousSpaces()) {
+ CHECK(space->IsLargeObjectSpace());
+ space->AsLargeObjectSpace()->CopyLiveToMarked();
}
}
}
@@ -304,12 +332,14 @@
bytes_moved_gc_thread_ = 0;
objects_moved_gc_thread_ = 0;
GcCause gc_cause = GetCurrentIteration()->GetGcCause();
- if (gc_cause == kGcCauseExplicit ||
- gc_cause == kGcCauseCollectorTransition ||
- GetCurrentIteration()->GetClearSoftReferences()) {
- force_evacuate_all_ = true;
- } else {
- force_evacuate_all_ = false;
+
+ force_evacuate_all_ = false;
+ if (!kEnableGenerationalConcurrentCopyingCollection || !young_gen_) {
+ if (gc_cause == kGcCauseExplicit ||
+ gc_cause == kGcCauseCollectorTransition ||
+ GetCurrentIteration()->GetClearSoftReferences()) {
+ force_evacuate_all_ = true;
+ }
}
if (kUseBakerReadBarrier) {
updated_all_immune_objects_.store(false, std::memory_order_relaxed);
@@ -320,6 +350,9 @@
DCHECK(immune_gray_stack_.empty());
}
}
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ done_scanning_.store(false, std::memory_order_relaxed);
+ }
BindBitmaps();
if (kVerboseMode) {
LOG(INFO) << "force_evacuate_all=" << force_evacuate_all_;
@@ -330,6 +363,9 @@
}
LOG(INFO) << "GC end of InitializePhase";
}
+ if (kEnableGenerationalConcurrentCopyingCollection && !young_gen_) {
+ region_space_bitmap_->Clear();
+ }
// Mark all of the zygote large objects without graying them.
MarkZygoteLargeObjects();
}
@@ -425,9 +461,18 @@
}
CHECK_EQ(thread, self);
Locks::mutator_lock_->AssertExclusiveHeld(self);
+ space::RegionSpace::EvacMode evac_mode = space::RegionSpace::kEvacModeLivePercentNewlyAllocated;
+ if (cc->young_gen_) {
+ CHECK(!cc->force_evacuate_all_);
+ evac_mode = space::RegionSpace::kEvacModeNewlyAllocated;
+ } else if (cc->force_evacuate_all_) {
+ evac_mode = space::RegionSpace::kEvacModeForceAll;
+ }
{
TimingLogger::ScopedTiming split2("(Paused)SetFromSpace", cc->GetTimings());
- cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_);
+ // Only change live bytes for full CC.
+ cc->region_space_->SetFromSpace(
+ cc->rb_table_, evac_mode, /*clear_live_bytes*/ !cc->young_gen_);
}
cc->SwapStacks();
if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) {
@@ -438,7 +483,7 @@
cc->is_marking_ = true;
cc->mark_stack_mode_.store(ConcurrentCopying::kMarkStackModeThreadLocal,
std::memory_order_relaxed);
- if (kIsDebugBuild) {
+ if (kIsDebugBuild && !cc->young_gen_) {
cc->region_space_->AssertAllRegionLiveBytesZeroOrCleared();
}
if (UNLIKELY(Runtime::Current()->IsActiveTransaction())) {
@@ -582,6 +627,8 @@
void CheckReference(mirror::Object* ref, int32_t offset = -1) const
REQUIRES_SHARED(Locks::mutator_lock_) {
+ // FIXME: This assertion is failing in 004-ThreadStress most of
+ // the time with Sticky-Bit (Generational) CC.
CHECK(ref == nullptr || !cc_->region_space_->IsInNewlyAllocatedRegion(ref))
<< holder_->PrettyTypeOf() << "(" << holder_.Ptr() << ") references object "
<< ref->PrettyTypeOf() << "(" << ref << ") in newly allocated region at offset=" << offset;
@@ -596,7 +643,8 @@
auto visitor = [&](mirror::Object* obj)
REQUIRES(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_) {
- // Objects not on dirty or aged cards should never have references to newly allocated regions.
+ // Objects on clean cards should never have references to newly allocated regions. Note
+ // that aged cards are also not clean.
if (heap_->GetCardTable()->GetCard(obj) == gc::accounting::CardTable::kCardClean) {
VerifyNoMissingCardMarkVisitor internal_visitor(this, /*holder*/ obj);
obj->VisitReferences</*kVisitNativeRoots*/true, kVerifyNone, kWithoutReadBarrier>(
@@ -752,7 +800,13 @@
DCHECK(obj != nullptr);
DCHECK(immune_spaces_.ContainsObject(obj));
// Update the fields without graying it or pushing it onto the mark stack.
- Scan(obj);
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ // Young GC does not care about references to unevac space. It is safe to not gray these as
+ // long as scan immune objects happens after scanning the dirty cards.
+ Scan<true>(obj);
+ } else {
+ Scan<false>(obj);
+ }
}
class ConcurrentCopying::ImmuneSpaceScanObjVisitor {
@@ -803,7 +857,49 @@
if (kUseBakerReadBarrier) {
gc_grays_immune_objects_ = false;
}
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ TimingLogger::ScopedTiming split2("ScanCardsForSpace", GetTimings());
+ WriterMutexLock rmu(Thread::Current(), *Locks::heap_bitmap_lock_);
+ CHECK_EQ(done_scanning_.load(std::memory_order_relaxed), false);
+ if (kIsDebugBuild) {
+ // Leave some time for mutators to race ahead to try and find races between the GC card
+ // scanning and mutators reading references.
+ usleep(10 * 1000);
+ }
+ for (space::ContinuousSpace* space : GetHeap()->GetContinuousSpaces()) {
+ if (space->IsImageSpace() || space->IsZygoteSpace()) {
+ // Image and zygote spaces are already handled since we gray the objects in the pause.
+ continue;
+ }
+ // Scan all of the objects on dirty cards in unevac from space, and non moving space. These
+ // are from previous GCs and may reference things in the from space.
+ Runtime::Current()->GetHeap()->GetCardTable()->Scan<false>(
+ space->GetMarkBitmap(),
+ space->Begin(),
+ space->End(),
+ [this, space](mirror::Object* obj)
+ REQUIRES(Locks::heap_bitmap_lock_)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ // Don't push or gray unevac refs.
+ if (kIsDebugBuild && space == region_space_) {
+ // We may get unevac large objects.
+ if (!region_space_->IsInUnevacFromSpace(obj)) {
+ CHECK(region_space_bitmap_->Test(obj));
+ region_space_->DumpRegionForObject(LOG_STREAM(FATAL_WITHOUT_ABORT), obj);
+ LOG(FATAL) << "Scanning " << obj << " not in unevac space";
+ }
+ }
+ Scan<true>(obj);
+ },
+ accounting::CardTable::kCardDirty - 1);
+ }
+ // Done scanning unevac space.
+ done_scanning_.store(true, std::memory_order_seq_cst);
+ }
{
+ // For a sticky-bit collection, this phase needs to be after the card scanning since the
+ // mutator may read an unevac space object out of an image object. If the image object is no
+ // longer gray it will trigger a read barrier for the unevac space object.
TimingLogger::ScopedTiming split2("ScanImmuneSpaces", GetTimings());
for (auto& space : immune_spaces_.GetSpaces()) {
DCHECK(space->IsImageSpace() || space->IsZygoteSpace());
@@ -1202,7 +1298,7 @@
ObjPtr<mirror::Reference> ref) const
REQUIRES_SHARED(Locks::mutator_lock_) ALWAYS_INLINE {
CHECK(klass->IsTypeOfReferenceClass());
- this->operator()(ref, mirror::Reference::ReferentOffset(), false);
+ this->operator()(ObjPtr<mirror::Object>(ref), mirror::Reference::ReferentOffset(), false);
}
void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const
@@ -1520,18 +1616,38 @@
<< " " << to_ref << " " << to_ref->GetReadBarrierState()
<< " is_marked=" << IsMarked(to_ref);
}
+ space::RegionSpace::RegionType rtype = region_space_->GetRegionType(to_ref);
bool add_to_live_bytes = false;
- if (region_space_->IsInUnevacFromSpace(to_ref)) {
+ DCHECK(!region_space_->IsInNewlyAllocatedRegion(to_ref));
+ if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) {
// Mark the bitmap only in the GC thread here so that we don't need a CAS.
- if (!kUseBakerReadBarrier || !region_space_bitmap_->Set(to_ref)) {
+ if (!kUseBakerReadBarrier ||
+ !region_space_bitmap_->Set(to_ref)) {
// It may be already marked if we accidentally pushed the same object twice due to the racy
// bitmap read in MarkUnevacFromSpaceRegion.
- Scan(to_ref);
- // Only add to the live bytes if the object was not already marked.
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ CHECK(region_space_->IsLargeObject(to_ref));
+ region_space_->ZeroLiveBytesForLargeObject(to_ref);
+ Scan<true>(to_ref);
+ } else {
+ Scan<false>(to_ref);
+ }
+ // Only add to the live bytes if the object was not already marked and we are not the young
+ // GC.
add_to_live_bytes = true;
}
} else {
- Scan(to_ref);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ if (rtype == space::RegionSpace::RegionType::kRegionTypeToSpace) {
+ // Copied to to-space, set the bit so that the next GC can scan objects.
+ region_space_bitmap_->Set(to_ref);
+ }
+ }
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ Scan<true>(to_ref);
+ } else {
+ Scan<false>(to_ref);
+ }
}
if (kUseBakerReadBarrier) {
DCHECK(to_ref->GetReadBarrierState() == ReadBarrier::GrayState())
@@ -1874,6 +1990,7 @@
if (!IsMarkedInUnevacFromSpace(ref)) {
LOG(FATAL_WITHOUT_ABORT) << "Found unmarked reference in unevac from-space:";
LOG(FATAL_WITHOUT_ABORT) << DumpHeapReference(obj, offset, ref);
+ Thread::Current()->DumpJavaStack(LOG_STREAM(FATAL_WITHOUT_ABORT));
}
CHECK(IsMarkedInUnevacFromSpace(ref)) << ref;
} else {
@@ -1887,6 +2004,15 @@
LOG(FATAL_WITHOUT_ABORT) << DumpHeapReference(obj, offset, ref);
if (obj != nullptr) {
LogFromSpaceRefHolder(obj, offset);
+ LOG(ERROR) << "UNEVAC " << region_space_->IsInUnevacFromSpace(obj) << " "
+ << obj << " " << obj->GetMarkBit();
+ if (region_space_->HasAddress(obj)) {
+ region_space_->DumpRegionForObject(LOG_STREAM(ERROR), obj);
+ }
+ LOG(ERROR) << "CARD " << static_cast<size_t>(
+ *Runtime::Current()->GetHeap()->GetCardTable()->CardFromAddr(
+ reinterpret_cast<uint8_t*>(obj)));
+ LOG(ERROR) << "BITMAP " << region_space_bitmap_->Test(obj);
}
ref->GetLockWord(false).Dump(LOG_STREAM(FATAL_WITHOUT_ABORT));
LOG(FATAL_WITHOUT_ABORT) << "Non-free regions:";
@@ -2067,6 +2193,9 @@
(is_los && los_bitmap->Test(ref))) {
// OK.
} else {
+ /* FIXME: We've seen this assertion fail in 004-ThreadStress from
+ time to time with Sticky-Bit (Generational) CC (it seems the
+ reference was in the non-moving space range at every occurrence). */
// If `ref` is on the allocation stack, then it may not be
// marked live, but considered marked/alive (but not
// necessarily on the live stack).
@@ -2079,15 +2208,19 @@
}
// Used to scan ref fields of an object.
+template <bool kNoUnEvac>
class ConcurrentCopying::RefFieldsVisitor {
public:
explicit RefFieldsVisitor(ConcurrentCopying* collector, Thread* const thread)
- : collector_(collector), thread_(thread) {}
+ : collector_(collector), thread_(thread) {
+ // Cannot have `kNoUnEvac` when Generational CC collection is disabled.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || !kNoUnEvac);
+ }
void operator()(mirror::Object* obj, MemberOffset offset, bool /* is_static */)
const ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES_SHARED(Locks::heap_bitmap_lock_) {
- collector_->Process(obj, offset);
+ collector_->Process<kNoUnEvac>(obj, offset);
}
void operator()(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> ref) const
@@ -2115,7 +2248,10 @@
Thread* const thread_;
};
+template <bool kNoUnEvac>
inline void ConcurrentCopying::Scan(mirror::Object* to_ref) {
+ // Cannot have `kNoUnEvac` when Generational CC collection is disabled.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || !kNoUnEvac);
if (kDisallowReadBarrierDuringScan && !Runtime::Current()->IsActiveTransaction()) {
// Avoid all read barriers during visit references to help performance.
// Don't do this in transaction mode because we may read the old value of an field which may
@@ -2124,7 +2260,7 @@
}
DCHECK(!region_space_->IsInFromSpace(to_ref));
DCHECK_EQ(Thread::Current(), thread_running_gc_);
- RefFieldsVisitor visitor(this, thread_running_gc_);
+ RefFieldsVisitor<kNoUnEvac> visitor(this, thread_running_gc_);
// Disable the read barrier for a performance reason.
to_ref->VisitReferences</*kVisitNativeRoots*/true, kDefaultVerifyFlags, kWithoutReadBarrier>(
visitor, visitor);
@@ -2133,11 +2269,14 @@
}
}
+template <bool kNoUnEvac>
inline void ConcurrentCopying::Process(mirror::Object* obj, MemberOffset offset) {
+ // Cannot have `kNoUnEvac` when Generational CC collection is disabled.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || !kNoUnEvac);
DCHECK_EQ(Thread::Current(), thread_running_gc_);
mirror::Object* ref = obj->GetFieldObject<
mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset);
- mirror::Object* to_ref = Mark</*kGrayImmuneObject*/false, /*kFromGCThread*/true>(
+ mirror::Object* to_ref = Mark</*kGrayImmuneObject*/false, kNoUnEvac, /*kFromGCThread*/true>(
thread_running_gc_,
ref,
/*holder*/ obj,
@@ -2415,7 +2554,9 @@
accounting::ContinuousSpaceBitmap* mark_bitmap =
heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref);
CHECK(mark_bitmap != nullptr);
- CHECK(!mark_bitmap->AtomicTestAndSet(to_ref));
+ if (!kEnableGenerationalConcurrentCopyingCollection) {
+ CHECK(!mark_bitmap->AtomicTestAndSet(to_ref));
+ }
}
}
DCHECK(to_ref != nullptr);
@@ -2552,6 +2693,10 @@
// At this point, `from_ref` should not be in the region space
// (i.e. within an "unused" region).
DCHECK(!region_space_->HasAddress(from_ref)) << from_ref;
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ // Only sweeps the from space.
+ return from_ref;
+ }
// from_ref is in a non-moving space.
if (immune_spaces_.ContainsObject(from_ref)) {
// An immune object is alive.
@@ -2612,6 +2757,26 @@
accounting::LargeObjectBitmap* los_bitmap =
heap_mark_bitmap_->GetLargeObjectBitmap(ref);
bool is_los = mark_bitmap == nullptr;
+ if (kEnableGenerationalConcurrentCopyingCollection && young_gen_) {
+ // Not done scanning, use AtomicSetReadBarrierPointer.
+ if (!done_scanning_) {
+ // Since the mark bitmap is still filled in from last GC, we can not use that or else the
+ // mutator may see references to the from space. Instead, use the baker pointer itself as
+ // the mark bit.
+ if (ref->AtomicSetReadBarrierState(ReadBarrier::NonGrayState(), ReadBarrier::GrayState())) {
+ // TODO: We don't actually need to scan this object later, we just need to clear the gray
+ // bit.
+ // Also make sure the object is marked.
+ if (is_los) {
+ los_bitmap->AtomicTestAndSet(ref);
+ } else {
+ mark_bitmap->AtomicTestAndSet(ref);
+ }
+ PushOntoMarkStack(self, ref);
+ }
+ return ref;
+ }
+ }
if (!is_los && mark_bitmap->Test(ref)) {
// Already marked.
} else if (is_los && los_bitmap->Test(ref)) {
@@ -2685,7 +2850,7 @@
}
// kVerifyNoMissingCardMarks relies on the region space cards not being cleared to avoid false
// positives.
- if (!kVerifyNoMissingCardMarks) {
+ if (!kEnableGenerationalConcurrentCopyingCollection && !kVerifyNoMissingCardMarks) {
TimingLogger::ScopedTiming split("ClearRegionSpaceCards", GetTimings());
// We do not currently use the region space cards at all, madvise them away to save ram.
heap_->GetCardTable()->ClearCardRange(region_space_->Begin(), region_space_->Limit());
@@ -2793,7 +2958,8 @@
}
ScopedTrace tr(__FUNCTION__);
const uint64_t start_time = measure_read_barrier_slow_path_ ? NanoTime() : 0u;
- mirror::Object* ret = Mark(self, from_ref);
+ mirror::Object* ret =
+ Mark</*kGrayImmuneObject*/true, /*kNoUnEvac*/false, /*kFromGCThread*/false>(self, from_ref);
if (measure_read_barrier_slow_path_) {
rb_slow_path_ns_.fetch_add(NanoTime() - start_time, std::memory_order_relaxed);
}
diff --git a/runtime/gc/collector/concurrent_copying.h b/runtime/gc/collector/concurrent_copying.h
index 448525d..10502ea 100644
--- a/runtime/gc/collector/concurrent_copying.h
+++ b/runtime/gc/collector/concurrent_copying.h
@@ -66,6 +66,7 @@
static constexpr bool kGrayDirtyImmuneObjects = true;
explicit ConcurrentCopying(Heap* heap,
+ bool young_gen,
const std::string& name_prefix = "",
bool measure_read_barrier_slow_path = false);
~ConcurrentCopying();
@@ -87,7 +88,9 @@
void BindBitmaps() REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!Locks::heap_bitmap_lock_);
virtual GcType GetGcType() const OVERRIDE {
- return kGcTypePartial;
+ return (kEnableGenerationalConcurrentCopyingCollection && young_gen_)
+ ? kGcTypeSticky
+ : kGcTypePartial;
}
virtual CollectorType GetCollectorType() const OVERRIDE {
return kCollectorTypeCC;
@@ -110,8 +113,8 @@
DCHECK(ref != nullptr);
return IsMarked(ref) == ref;
}
- template<bool kGrayImmuneObject = true, bool kFromGCThread = false>
// Mark object `from_ref`, copying it to the to-space if needed.
+ template<bool kGrayImmuneObject = true, bool kNoUnEvac = false, bool kFromGCThread = false>
ALWAYS_INLINE mirror::Object* Mark(Thread* const self,
mirror::Object* from_ref,
mirror::Object* holder = nullptr,
@@ -155,9 +158,11 @@
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_);
// Scan the reference fields of object `to_ref`.
+ template <bool kNoUnEvac>
void Scan(mirror::Object* to_ref) REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_);
// Process a field.
+ template <bool kNoUnEvac>
void Process(mirror::Object* obj, MemberOffset offset)
REQUIRES_SHARED(Locks::mutator_lock_)
REQUIRES(!mark_stack_lock_ , !skipped_blocks_lock_, !immune_gray_stack_lock_);
@@ -347,6 +352,10 @@
Atomic<uint64_t> cumulative_bytes_moved_;
Atomic<uint64_t> cumulative_objects_moved_;
+ // Generational "sticky", only trace through dirty objects in region space.
+ const bool young_gen_;
+ Atomic<bool> done_scanning_;
+
// The skipped blocks are memory blocks/chucks that were copies of
// objects that were unused due to lost races (cas failures) at
// object copy/forward pointer install. They are reused.
@@ -394,7 +403,7 @@
template <bool kConcurrent> class GrayImmuneObjectVisitor;
class ImmuneSpaceScanObjVisitor;
class LostCopyVisitor;
- class RefFieldsVisitor;
+ template <bool kNoUnEvac> class RefFieldsVisitor;
class RevokeThreadLocalMarkStackCheckpoint;
class ScopedGcGraysImmuneObjects;
class ThreadFlipVisitor;
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index 58becb1..1be2d6b 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -102,7 +102,8 @@
// Sticky GC throughput adjustment, divided by 4. Increasing this causes sticky GC to occur more
// relative to partial/full GC. This may be desirable since sticky GCs interfere less with mutator
// threads (lower pauses, use less memory bandwidth).
-static constexpr double kStickyGcThroughputAdjustment = 1.0;
+static constexpr double kStickyGcThroughputAdjustment =
+ kEnableGenerationalConcurrentCopyingCollection ? 0.5 : 1.0;
// Whether or not we compact the zygote in PreZygoteFork.
static constexpr bool kCompactZygote = kMovingCollector;
// How many reserve entries are at the end of the allocation stack, these are only needed if the
@@ -260,6 +261,8 @@
verify_object_mode_(kVerifyObjectModeDisabled),
disable_moving_gc_count_(0),
semi_space_collector_(nullptr),
+ active_concurrent_copying_collector_(nullptr),
+ young_concurrent_copying_collector_(nullptr),
concurrent_copying_collector_(nullptr),
is_running_on_memory_tool_(Runtime::Current()->IsRunningOnMemoryTool()),
use_tlab_(use_tlab),
@@ -594,11 +597,26 @@
}
if (MayUseCollector(kCollectorTypeCC)) {
concurrent_copying_collector_ = new collector::ConcurrentCopying(this,
+ /*young_gen*/false,
"",
measure_gc_performance);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ young_concurrent_copying_collector_ = new collector::ConcurrentCopying(
+ this,
+ /*young_gen*/true,
+ "young",
+ measure_gc_performance);
+ }
+ active_concurrent_copying_collector_ = concurrent_copying_collector_;
DCHECK(region_space_ != nullptr);
concurrent_copying_collector_->SetRegionSpace(region_space_);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ young_concurrent_copying_collector_->SetRegionSpace(region_space_);
+ }
garbage_collectors_.push_back(concurrent_copying_collector_);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ garbage_collectors_.push_back(young_concurrent_copying_collector_);
+ }
}
}
if (!GetBootImageSpaces().empty() && non_moving_space_ != nullptr &&
@@ -2120,6 +2138,9 @@
gc_plan_.clear();
switch (collector_type_) {
case kCollectorTypeCC: {
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ gc_plan_.push_back(collector::kGcTypeSticky);
+ }
gc_plan_.push_back(collector::kGcTypeFull);
if (use_tlab_) {
ChangeAllocator(kAllocatorTypeRegionTLAB);
@@ -2159,7 +2180,8 @@
}
if (IsGcConcurrent()) {
concurrent_start_bytes_ =
- std::max(max_allowed_footprint_, kMinConcurrentRemainingBytes) - kMinConcurrentRemainingBytes;
+ std::max(max_allowed_footprint_, kMinConcurrentRemainingBytes) -
+ kMinConcurrentRemainingBytes;
} else {
concurrent_start_bytes_ = std::numeric_limits<size_t>::max();
}
@@ -2567,12 +2589,19 @@
collector = semi_space_collector_;
break;
case kCollectorTypeCC:
- collector = concurrent_copying_collector_;
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ // TODO: Other threads must do the flip checkpoint before they start poking at
+ // active_concurrent_copying_collector_. So we should not concurrency here.
+ active_concurrent_copying_collector_ = (gc_type == collector::kGcTypeSticky) ?
+ young_concurrent_copying_collector_ : concurrent_copying_collector_;
+ active_concurrent_copying_collector_->SetRegionSpace(region_space_);
+ }
+ collector = active_concurrent_copying_collector_;
break;
default:
LOG(FATAL) << "Invalid collector type " << static_cast<size_t>(collector_type_);
}
- if (collector != concurrent_copying_collector_) {
+ if (collector != active_concurrent_copying_collector_) {
temp_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE);
if (kIsDebugBuild) {
// Try to read each page of the memory map in case mprotect didn't work properly b/19894268.
@@ -3435,7 +3464,8 @@
uint64_t target_size;
collector::GcType gc_type = collector_ran->GetGcType();
// Use the multiplier to grow more for foreground.
- const double multiplier = HeapGrowthMultiplier();
+ const double multiplier =
+ HeapGrowthMultiplier() + (kEnableGenerationalConcurrentCopyingCollection ? 3.0 : 0.0);
const uint64_t adjusted_min_free = static_cast<uint64_t>(min_free_ * multiplier);
const uint64_t adjusted_max_free = static_cast<uint64_t>(max_free_ * multiplier);
if (gc_type != collector::kGcTypeSticky) {
@@ -3451,6 +3481,12 @@
collector::GcType non_sticky_gc_type = NonStickyGcType();
// Find what the next non sticky collector will be.
collector::GarbageCollector* non_sticky_collector = FindCollectorByGcType(non_sticky_gc_type);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ if (non_sticky_collector == nullptr) {
+ non_sticky_collector = FindCollectorByGcType(collector::kGcTypePartial);
+ }
+ CHECK(non_sticky_collector != nullptr);
+ }
// If the throughput of the current sticky GC >= throughput of the non sticky collector, then
// do another sticky collection next.
// We also check that the bytes allocated aren't over the footprint limit in order to prevent a
diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h
index 5c34c56..94ce2a1 100644
--- a/runtime/gc/heap.h
+++ b/runtime/gc/heap.h
@@ -709,8 +709,15 @@
return zygote_space_ != nullptr;
}
+ // Returns the active concurrent copying collector.
collector::ConcurrentCopying* ConcurrentCopyingCollector() {
- return concurrent_copying_collector_;
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ DCHECK((active_concurrent_copying_collector_ == concurrent_copying_collector_) ||
+ (active_concurrent_copying_collector_ == young_concurrent_copying_collector_));
+ } else {
+ DCHECK_EQ(active_concurrent_copying_collector_, concurrent_copying_collector_);
+ }
+ return active_concurrent_copying_collector_;
}
CollectorType CurrentCollectorType() {
@@ -1335,6 +1342,8 @@
std::vector<collector::GarbageCollector*> garbage_collectors_;
collector::SemiSpace* semi_space_collector_;
+ collector::ConcurrentCopying* active_concurrent_copying_collector_;
+ collector::ConcurrentCopying* young_concurrent_copying_collector_;
collector::ConcurrentCopying* concurrent_copying_collector_;
const bool is_running_on_memory_tool_;
diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc
index 0569092..1ed81d0 100644
--- a/runtime/gc/space/region_space.cc
+++ b/runtime/gc/space/region_space.cc
@@ -170,15 +170,21 @@
return num_regions * kRegionSize;
}
-inline bool RegionSpace::Region::ShouldBeEvacuated() {
+inline bool RegionSpace::Region::ShouldBeEvacuated(EvacMode evac_mode) {
+ // Evacuation mode `kEvacModeNewlyAllocated` is only used during sticky-bit CC collections.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || (evac_mode != kEvacModeNewlyAllocated));
DCHECK((IsAllocated() || IsLarge()) && IsInToSpace());
// The region should be evacuated if:
+ // - the evacuation is forced (`evac_mode == kEvacModeForceAll`); or
// - the region was allocated after the start of the previous GC (newly allocated region); or
// - the live ratio is below threshold (`kEvacuateLivePercentThreshold`).
- bool result;
+ if (UNLIKELY(evac_mode == kEvacModeForceAll)) {
+ return true;
+ }
+ bool result = false;
if (is_newly_allocated_) {
result = true;
- } else {
+ } else if (evac_mode == kEvacModeLivePercentNewlyAllocated) {
bool is_live_percent_valid = (live_bytes_ != static_cast<size_t>(-1));
if (is_live_percent_valid) {
DCHECK(IsInToSpace());
@@ -205,7 +211,11 @@
// Determine which regions to evacuate and mark them as
// from-space. Mark the rest as unevacuated from-space.
-void RegionSpace::SetFromSpace(accounting::ReadBarrierTable* rb_table, bool force_evacuate_all) {
+void RegionSpace::SetFromSpace(accounting::ReadBarrierTable* rb_table,
+ EvacMode evac_mode,
+ bool clear_live_bytes) {
+ // Live bytes are only preserved (i.e. not cleared) during sticky-bit CC collections.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || clear_live_bytes);
++time_;
if (kUseTableLookupReadBarrier) {
DCHECK(rb_table->IsAllCleared());
@@ -231,12 +241,12 @@
DCHECK((state == RegionState::kRegionStateAllocated ||
state == RegionState::kRegionStateLarge) &&
type == RegionType::kRegionTypeToSpace);
- bool should_evacuate = force_evacuate_all || r->ShouldBeEvacuated();
+ bool should_evacuate = r->ShouldBeEvacuated(evac_mode);
if (should_evacuate) {
r->SetAsFromSpace();
DCHECK(r->IsInFromSpace());
} else {
- r->SetAsUnevacFromSpace();
+ r->SetAsUnevacFromSpace(clear_live_bytes);
DCHECK(r->IsInUnevacFromSpace());
}
if (UNLIKELY(state == RegionState::kRegionStateLarge &&
@@ -252,7 +262,7 @@
r->SetAsFromSpace();
DCHECK(r->IsInFromSpace());
} else {
- r->SetAsUnevacFromSpace();
+ r->SetAsUnevacFromSpace(clear_live_bytes);
DCHECK(r->IsInUnevacFromSpace());
}
--num_expected_large_tails;
@@ -361,7 +371,16 @@
while (i + regions_to_clear_bitmap < num_regions_) {
Region* const cur = ®ions_[i + regions_to_clear_bitmap];
if (!cur->AllAllocatedBytesAreLive()) {
+#if 0
+ // FIXME: These tests fail the following assertion with Sticky-Bit (Generational) CC:
+ //
+ // 004-ThreadStress
+ // 061-out-of-memory
+ // 080-oom-throw
+ // 134-reg-promotion
+ // 617-clinit-oome
DCHECK(!cur->IsLargeTail());
+#endif
break;
}
CHECK(cur->IsInUnevacFromSpace());
@@ -369,8 +388,9 @@
++regions_to_clear_bitmap;
}
- // Optimization: If the live bytes are *all* live in a region
- // then the live-bit information for these objects is superfluous:
+ // Optimization (for full CC only): If the live bytes are *all* live
+ // in a region then the live-bit information for these objects is
+ // superfluous:
// - We can determine that these objects are all live by using
// Region::AllAllocatedBytesAreLive (which just checks whether
// `LiveBytes() == static_cast<size_t>(Top() - Begin())`.
@@ -379,19 +399,44 @@
// live bits (see RegionSpace::WalkInternal).
// Therefore, we can clear the bits for these objects in the
// (live) region space bitmap (and release the corresponding pages).
- GetLiveBitmap()->ClearRange(
- reinterpret_cast<mirror::Object*>(r->Begin()),
- reinterpret_cast<mirror::Object*>(r->Begin() + regions_to_clear_bitmap * kRegionSize));
+ //
+ // This optimization is incompatible with Generational CC, because:
+ // - minor (young-generation) collections need to know which objects
+ // where marked during the previous GC cycle, meaning all mark bitmaps
+ // (this includes the region space bitmap) need to be preserved
+ // between a (minor or major) collection N and a following minor
+ // collection N+1;
+ // - at this stage (in the current GC cycle), we cannot determine
+ // whether the next collection will be a minor or a major one;
+ // This means that we need to be conservative and always preserve the
+ // region space bitmap when using Generational CC.
+ // Note that major collections do not require the previous mark bitmaps
+ // to be preserved, and as matter of fact they do clear the region space
+ // bitmap. But they cannot do so before we know the next GC cycle will
+ // be a major one, so this operation happens at the beginning of such a
+ // major collection, before marking starts.
+ if (!kEnableGenerationalConcurrentCopyingCollection) {
+ GetLiveBitmap()->ClearRange(
+ reinterpret_cast<mirror::Object*>(r->Begin()),
+ reinterpret_cast<mirror::Object*>(r->Begin() + regions_to_clear_bitmap * kRegionSize));
+ }
// Skip over extra regions for which we cleared the bitmaps: we shall not clear them,
// as they are unevac regions that are live.
// Subtract one for the for-loop.
i += regions_to_clear_bitmap - 1;
} else {
- // Only some allocated bytes are live in this unevac region.
- // This should only happen for an allocated non-large region.
- DCHECK(r->IsAllocated()) << r->State();
- if (kPoisonDeadObjectsInUnevacuatedRegions) {
- PoisonDeadObjectsInUnevacuatedRegion(r);
+ // TODO: Explain why we do not poison dead objects in region
+ // `r` when it has an undefined live bytes count (i.e. when
+ // `r->LiveBytes() == static_cast<size_t>(-1)`) with
+ // Generational CC.
+ if (!kEnableGenerationalConcurrentCopyingCollection ||
+ (r->LiveBytes() != static_cast<size_t>(-1))) {
+ // Only some allocated bytes are live in this unevac region.
+ // This should only happen for an allocated non-large region.
+ DCHECK(r->IsAllocated()) << r->State();
+ if (kPoisonDeadObjectsInUnevacuatedRegions) {
+ PoisonDeadObjectsInUnevacuatedRegion(r);
+ }
}
}
}
@@ -732,6 +777,10 @@
Region* r = ®ions_[region_index];
if (r->IsFree()) {
r->Unfree(this, time_);
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ // TODO: Add an explanation for this assertion.
+ DCHECK(!for_evac || !r->is_newly_allocated_);
+ }
if (for_evac) {
++num_evac_regions_;
// Evac doesn't count as newly allocated.
diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h
index 90f1f1d..d86304a 100644
--- a/runtime/gc/space/region_space.h
+++ b/runtime/gc/space/region_space.h
@@ -43,6 +43,12 @@
public:
typedef void(*WalkCallback)(void *start, void *end, size_t num_bytes, void* callback_arg);
+ enum EvacMode {
+ kEvacModeNewlyAllocated,
+ kEvacModeLivePercentNewlyAllocated,
+ kEvacModeForceAll,
+ };
+
SpaceType GetType() const OVERRIDE {
return kSpaceTypeRegionSpace;
}
@@ -230,6 +236,14 @@
return false;
}
+ bool IsLargeObject(mirror::Object* ref) {
+ if (HasAddress(ref)) {
+ Region* r = RefToRegionUnlocked(ref);
+ return r->IsLarge();
+ }
+ return false;
+ }
+
bool IsInToSpace(mirror::Object* ref) {
if (HasAddress(ref)) {
Region* r = RefToRegionUnlocked(ref);
@@ -255,9 +269,20 @@
return r->Type();
}
+ // Zero live bytes for a large object, used by young gen CC for marking newly allocated large
+ // objects.
+ void ZeroLiveBytesForLargeObject(mirror::Object* ref) {
+ // This method is only used when Generational CC collection is enabled.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection);
+ DCHECK(IsLargeObject(ref));
+ RefToRegionUnlocked(ref)->ZeroLiveBytes();
+ }
+
// Determine which regions to evacuate and tag them as
// from-space. Tag the rest as unevacuated from-space.
- void SetFromSpace(accounting::ReadBarrierTable* rb_table, bool force_evacuate_all)
+ void SetFromSpace(accounting::ReadBarrierTable* rb_table,
+ EvacMode evac_mode,
+ bool clear_live_bytes)
REQUIRES(!region_lock_);
size_t FromSpaceSize() REQUIRES(!region_lock_);
@@ -386,6 +411,10 @@
return is_large;
}
+ void ZeroLiveBytes() {
+ live_bytes_ = 0;
+ }
+
// Large-tail allocated.
bool IsLargeTail() const {
bool is_large_tail = (state_ == RegionState::kRegionStateLargeTail);
@@ -432,10 +461,14 @@
// collection, RegionSpace::ClearFromSpace will preserve the space
// used by this region, and tag it as to-space (see
// Region::SetUnevacFromSpaceAsToSpace below).
- void SetAsUnevacFromSpace() {
+ void SetAsUnevacFromSpace(bool clear_live_bytes) {
+ // Live bytes are only preserved (i.e. not cleared) during sticky-bit CC collections.
+ DCHECK(kEnableGenerationalConcurrentCopyingCollection || clear_live_bytes);
DCHECK(!IsFree() && IsInToSpace());
type_ = RegionType::kRegionTypeUnevacFromSpace;
- live_bytes_ = 0U;
+ if (clear_live_bytes) {
+ live_bytes_ = 0;
+ }
}
// Set this region as to-space. Used by RegionSpace::ClearFromSpace.
@@ -443,10 +476,13 @@
void SetUnevacFromSpaceAsToSpace() {
DCHECK(!IsFree() && IsInUnevacFromSpace());
type_ = RegionType::kRegionTypeToSpace;
+ if (kEnableGenerationalConcurrentCopyingCollection) {
+ is_newly_allocated_ = false;
+ }
}
// Return whether this region should be evacuated. Used by RegionSpace::SetFromSpace.
- ALWAYS_INLINE bool ShouldBeEvacuated();
+ ALWAYS_INLINE bool ShouldBeEvacuated(EvacMode evac_mode);
void AddLiveBytes(size_t live_bytes) {
DCHECK(IsInUnevacFromSpace());
diff --git a/runtime/gc/system_weak_test.cc b/runtime/gc/system_weak_test.cc
index 21f5117..897ab01 100644
--- a/runtime/gc/system_weak_test.cc
+++ b/runtime/gc/system_weak_test.cc
@@ -26,6 +26,7 @@
#include "gc_root-inl.h"
#include "handle_scope-inl.h"
#include "heap.h"
+#include "mirror/object-inl.h"
#include "mirror/string.h"
#include "scoped_thread_state_change-inl.h"
#include "thread_list.h"
diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index bc72517..51dc1a4 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -32,12 +32,12 @@
#include "dex_cache.h"
#include "gc/heap-inl.h"
#include "iftable.h"
-#include "subtype_check.h"
#include "object-inl.h"
#include "object_array.h"
#include "read_barrier-inl.h"
#include "runtime.h"
#include "string.h"
+#include "subtype_check.h"
namespace art {
namespace mirror {
