Relocate the always in-lined functions from HeapBitmap.h.
The three essential bitmap access methods have been moved to the new
HeapBitmapInlines.h header file. All other bitmap methods have been
moved into HeapBitmap.c. As an added bonus, source indexing and cross
reference tools should correctly locate all heap bitmap definitions.
Change-Id: I35703403f4f79d121c9c3f411b7d7110d92739d0
diff --git a/vm/alloc/CardTable.c b/vm/alloc/CardTable.c
index 0848653..a745c43 100644
--- a/vm/alloc/CardTable.c
+++ b/vm/alloc/CardTable.c
@@ -17,6 +17,8 @@
#include <sys/mman.h> /* for PROT_* */
#include "Dalvik.h"
+#include "alloc/HeapBitmap.h"
+#include "alloc/HeapBitmapInlines.h"
#include "alloc/HeapSource.h"
#include "alloc/Visit.h"
diff --git a/vm/alloc/HeapBitmap.c b/vm/alloc/HeapBitmap.c
index 41867df..25283b8 100644
--- a/vm/alloc/HeapBitmap.c
+++ b/vm/alloc/HeapBitmap.c
@@ -79,6 +79,88 @@
}
/*
+ * Return true iff <obj> is within the range of pointers that this
+ * bitmap could potentially cover, even if a bit has not been set
+ * for it.
+ */
+bool dvmHeapBitmapCoversAddress(const HeapBitmap *hb, const void *obj)
+{
+ assert(hb != NULL);
+ if (obj != NULL) {
+ const uintptr_t offset = (uintptr_t)obj - hb->base;
+ const size_t index = HB_OFFSET_TO_INDEX(offset);
+ return index < hb->bitsLen / sizeof(*hb->bits);
+ }
+ return false;
+}
+
+/*
+ * Visits set bits in address order. The callback is not permitted to
+ * change the bitmap bits or max during the traversal.
+ */
+void dvmHeapBitmapWalk(const HeapBitmap *bitmap, BitmapCallback *callback,
+ void *arg)
+{
+ uintptr_t end;
+ uintptr_t i;
+
+ assert(bitmap != NULL);
+ assert(bitmap->bits != NULL);
+ assert(callback != NULL);
+ end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
+ for (i = 0; i <= end; ++i) {
+ unsigned long word = bitmap->bits[i];
+ if (UNLIKELY(word != 0)) {
+ unsigned long highBit = 1 << (HB_BITS_PER_WORD - 1);
+ uintptr_t ptrBase = HB_INDEX_TO_OFFSET(i) + bitmap->base;
+ while (word != 0) {
+ const int shift = CLZ(word);
+ void *addr = (void *)(ptrBase + shift * HB_OBJECT_ALIGNMENT);
+ (*callback)(addr, arg);
+ word &= ~(highBit >> shift);
+ }
+ }
+ }
+}
+
+/*
+ * Similar to dvmHeapBitmapWalk but the callback routine is permitted
+ * to change the bitmap bits and max during traversal. Used by the
+ * the root marking scan exclusively.
+ *
+ * The callback is invoked with a finger argument. The finger is a
+ * pointer to an address not yet visited by the traversal. If the
+ * callback sets a bit for an address at or above the finger, this
+ * address will be visited by the traversal. If the callback sets a
+ * bit for an address below the finger, this address will not be
+ * visited.
+ */
+void dvmHeapBitmapScanWalk(HeapBitmap *bitmap,
+ BitmapScanCallback *callback, void *arg)
+{
+ assert(bitmap != NULL);
+ assert(bitmap->bits != NULL);
+ assert(callback != NULL);
+ uintptr_t end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
+ uintptr_t i;
+ for (i = 0; i <= end; ++i) {
+ unsigned long word = bitmap->bits[i];
+ if (UNLIKELY(word != 0)) {
+ unsigned long highBit = 1 << (HB_BITS_PER_WORD - 1);
+ uintptr_t ptrBase = HB_INDEX_TO_OFFSET(i) + bitmap->base;
+ void *finger = (void *)(HB_INDEX_TO_OFFSET(i + 1) + bitmap->base);
+ while (word != 0) {
+ const int shift = CLZ(word);
+ void *addr = (void *)(ptrBase + shift * HB_OBJECT_ALIGNMENT);
+ (*callback)(addr, finger, arg);
+ word &= ~(highBit >> shift);
+ }
+ end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
+ }
+ }
+}
+
+/*
* Walk through the bitmaps in increasing address order, and find the
* object pointers that correspond to garbage objects. Call
* <callback> zero or more times with lists of these object pointers.
diff --git a/vm/alloc/HeapBitmap.h b/vm/alloc/HeapBitmap.h
index 5f7f029..d7936ae 100644
--- a/vm/alloc/HeapBitmap.h
+++ b/vm/alloc/HeapBitmap.h
@@ -18,7 +18,6 @@
#include <limits.h>
#include <stdint.h>
-#include "clz.h"
#define HB_OBJECT_ALIGNMENT 8
#define HB_BITS_PER_WORD (sizeof(unsigned long) * CHAR_BIT)
@@ -42,15 +41,6 @@
(1 << \
(31-(((uintptr_t)(offset_) / HB_OBJECT_ALIGNMENT) % HB_BITS_PER_WORD)))
-/* Return the maximum offset (exclusive) that <hb> can represent.
- */
-#define HB_MAX_OFFSET(hb_) \
- HB_INDEX_TO_OFFSET((hb_)->bitsLen / sizeof(*(hb_)->bits))
-
-#define HB_INLINE_PROTO(p) \
- static inline p __attribute__((always_inline)); \
- static inline p
-
typedef struct {
/* The bitmap data, which points to an mmap()ed area of zeroed
* anonymous memory.
@@ -80,6 +70,9 @@
uintptr_t max;
} HeapBitmap;
+/*
+ * Callback types used by the walking routines.
+ */
typedef void BitmapCallback(void *addr, void *arg);
typedef void BitmapScanCallback(void *addr, void *finger, void *arg);
typedef void BitmapSweepCallback(size_t numPtrs, void **ptrs, void *arg);
@@ -104,74 +97,23 @@
void dvmHeapBitmapZero(HeapBitmap *hb);
/*
- * Visits set bits in address order. The callback is not permitted to
- * change the bitmap bits or max during the traversal.
+ * Returns true if the address range of the bitmap covers the object
+ * address.
*/
-HB_INLINE_PROTO(
- void
- dvmHeapBitmapWalk(const HeapBitmap *bitmap,
- BitmapCallback *callback, void *arg)
-)
-{
- assert(bitmap != NULL);
- assert(bitmap->bits != NULL);
- assert(callback != NULL);
- uintptr_t end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
- uintptr_t i;
- for (i = 0; i <= end; ++i) {
- unsigned long word = bitmap->bits[i];
- if (UNLIKELY(word != 0)) {
- unsigned long highBit = 1 << (HB_BITS_PER_WORD - 1);
- uintptr_t ptrBase = HB_INDEX_TO_OFFSET(i) + bitmap->base;
- while (word != 0) {
- const int shift = CLZ(word);
- void *addr = (void *)(ptrBase + shift * HB_OBJECT_ALIGNMENT);
- (*callback)(addr, arg);
- word &= ~(highBit >> shift);
- }
- }
- }
-}
+bool dvmHeapBitmapCoversAddress(const HeapBitmap *hb, const void *obj);
/*
- * Similar to dvmHeapBitmapWalk but the callback routine is permitted
- * to change the bitmap bits and max during traversal. Used by the
- * the root marking scan exclusively.
- *
- * The callback is invoked with a finger argument. The finger is a
- * pointer to an address not yet visited by the traversal. If the
- * callback sets a bit for an address at or above the finger, this
- * address will be visited by the traversal. If the callback sets a
- * bit for an address below the finger, this address will not be
- * visited.
+ * Applies the callback function to each set address in the bitmap.
*/
-HB_INLINE_PROTO(
- void
- dvmHeapBitmapScanWalk(HeapBitmap *bitmap,
- BitmapScanCallback *callback, void *arg)
-)
-{
- assert(bitmap != NULL);
- assert(bitmap->bits != NULL);
- assert(callback != NULL);
- uintptr_t end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
- uintptr_t i;
- for (i = 0; i <= end; ++i) {
- unsigned long word = bitmap->bits[i];
- if (UNLIKELY(word != 0)) {
- unsigned long highBit = 1 << (HB_BITS_PER_WORD - 1);
- uintptr_t ptrBase = HB_INDEX_TO_OFFSET(i) + bitmap->base;
- void *finger = (void *)(HB_INDEX_TO_OFFSET(i + 1) + bitmap->base);
- while (word != 0) {
- const int shift = CLZ(word);
- void *addr = (void *)(ptrBase + shift * HB_OBJECT_ALIGNMENT);
- (*callback)(addr, finger, arg);
- word &= ~(highBit >> shift);
- }
- end = HB_OFFSET_TO_INDEX(bitmap->max - bitmap->base);
- }
- }
-}
+void dvmHeapBitmapWalk(const HeapBitmap *bitmap,
+ BitmapCallback *callback, void *callbackArg);
+
+/*
+ * Like dvmHeapBitmapWalk but takes a callback function with a finger
+ * address.
+ */
+void dvmHeapBitmapScanWalk(HeapBitmap *bitmap,
+ BitmapScanCallback *callback, void *arg);
/*
* Walk through the bitmaps in increasing address order, and find the
@@ -184,124 +126,4 @@
uintptr_t base, uintptr_t max,
BitmapSweepCallback *callback, void *callbackArg);
-/*
- * Return true iff <obj> is within the range of pointers that this
- * bitmap could potentially cover, even if a bit has not been set
- * for it.
- */
-HB_INLINE_PROTO(
- bool
- dvmHeapBitmapCoversAddress(const HeapBitmap *hb, const void *obj)
-)
-{
- assert(hb != NULL);
-
- if (obj != NULL) {
- const uintptr_t offset = (uintptr_t)obj - hb->base;
- const size_t index = HB_OFFSET_TO_INDEX(offset);
- return index < hb->bitsLen / sizeof(*hb->bits);
- }
- return false;
-}
-
-/*
- * Internal function; do not call directly.
- */
-HB_INLINE_PROTO(
- unsigned long
- _heapBitmapModifyObjectBit(HeapBitmap *hb, const void *obj,
- bool setBit, bool returnOld)
-)
-{
- const uintptr_t offset = (uintptr_t)obj - hb->base;
- const size_t index = HB_OFFSET_TO_INDEX(offset);
- const unsigned long mask = HB_OFFSET_TO_MASK(offset);
-
- assert(hb->bits != NULL);
- assert((uintptr_t)obj >= hb->base);
- assert(index < hb->bitsLen / sizeof(*hb->bits));
-
- if (setBit) {
- if ((uintptr_t)obj > hb->max) {
- hb->max = (uintptr_t)obj;
- }
- if (returnOld) {
- unsigned long *p = hb->bits + index;
- const unsigned long word = *p;
- *p |= mask;
- return word & mask;
- } else {
- hb->bits[index] |= mask;
- }
- } else {
- hb->bits[index] &= ~mask;
- }
- return false;
-}
-
-/*
- * Sets the bit corresponding to <obj>, and returns the previous value
- * of that bit (as zero or non-zero). Does no range checking to see if
- * <obj> is outside of the coverage of the bitmap.
- *
- * NOTE: casting this value to a bool is dangerous, because higher
- * set bits will be lost.
- */
-HB_INLINE_PROTO(
- unsigned long
- dvmHeapBitmapSetAndReturnObjectBit(HeapBitmap *hb, const void *obj)
-)
-{
- return _heapBitmapModifyObjectBit(hb, obj, true, true);
-}
-
-/*
- * Sets the bit corresponding to <obj>, and widens the range of seen
- * pointers if necessary. Does no range checking.
- */
-HB_INLINE_PROTO(
- void
- dvmHeapBitmapSetObjectBit(HeapBitmap *hb, const void *obj)
-)
-{
- (void)_heapBitmapModifyObjectBit(hb, obj, true, false);
-}
-
-/*
- * Clears the bit corresponding to <obj>. Does no range checking.
- */
-HB_INLINE_PROTO(
- void
- dvmHeapBitmapClearObjectBit(HeapBitmap *hb, const void *obj)
-)
-{
- (void)_heapBitmapModifyObjectBit(hb, obj, false, false);
-}
-
-/*
- * Returns the current value of the bit corresponding to <obj>,
- * as zero or non-zero. Does no range checking.
- *
- * NOTE: casting this value to a bool is dangerous, because higher
- * set bits will be lost.
- */
-HB_INLINE_PROTO(
- unsigned long
- dvmHeapBitmapIsObjectBitSet(const HeapBitmap *hb, const void *obj)
-)
-{
- assert(dvmHeapBitmapCoversAddress(hb, obj));
- assert(hb->bits != NULL);
- assert((uintptr_t)obj >= hb->base);
-
- if ((uintptr_t)obj <= hb->max) {
- const uintptr_t offset = (uintptr_t)obj - hb->base;
- return hb->bits[HB_OFFSET_TO_INDEX(offset)] & HB_OFFSET_TO_MASK(offset);
- } else {
- return 0;
- }
-}
-
-#undef HB_INLINE_PROTO
-
-#endif // _DALVIK_HEAP_BITMAP
+#endif /* _DALVIK_HEAP_BITMAP */
diff --git a/vm/alloc/HeapBitmapInlines.h b/vm/alloc/HeapBitmapInlines.h
new file mode 100644
index 0000000..859ffa3
--- /dev/null
+++ b/vm/alloc/HeapBitmapInlines.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _DALVIK_HEAP_BITMAPINLINES
+#define _DALVIK_HEAP_BITMAPINLINES
+
+static unsigned long dvmHeapBitmapSetAndReturnObjectBit(HeapBitmap *hb, const void *obj) __attribute__((used));
+static void dvmHeapBitmapSetObjectBit(HeapBitmap *hb, const void *obj) __attribute__((used));
+static void dvmHeapBitmapClearObjectBit(HeapBitmap *hb, const void *obj) __attribute__((used));
+
+/*
+ * Internal function; do not call directly.
+ */
+static unsigned long _heapBitmapModifyObjectBit(HeapBitmap *hb, const void *obj,
+ bool setBit, bool returnOld)
+{
+ const uintptr_t offset = (uintptr_t)obj - hb->base;
+ const size_t index = HB_OFFSET_TO_INDEX(offset);
+ const unsigned long mask = HB_OFFSET_TO_MASK(offset);
+
+ assert(hb->bits != NULL);
+ assert((uintptr_t)obj >= hb->base);
+ assert(index < hb->bitsLen / sizeof(*hb->bits));
+ if (setBit) {
+ if ((uintptr_t)obj > hb->max) {
+ hb->max = (uintptr_t)obj;
+ }
+ if (returnOld) {
+ unsigned long *p = hb->bits + index;
+ const unsigned long word = *p;
+ *p |= mask;
+ return word & mask;
+ } else {
+ hb->bits[index] |= mask;
+ }
+ } else {
+ hb->bits[index] &= ~mask;
+ }
+ return false;
+}
+
+/*
+ * Sets the bit corresponding to <obj>, and returns the previous value
+ * of that bit (as zero or non-zero). Does no range checking to see if
+ * <obj> is outside of the coverage of the bitmap.
+ *
+ * NOTE: casting this value to a bool is dangerous, because higher
+ * set bits will be lost.
+ */
+static unsigned long dvmHeapBitmapSetAndReturnObjectBit(HeapBitmap *hb,
+ const void *obj)
+{
+ return _heapBitmapModifyObjectBit(hb, obj, true, true);
+}
+
+/*
+ * Sets the bit corresponding to <obj>, and widens the range of seen
+ * pointers if necessary. Does no range checking.
+ */
+static void dvmHeapBitmapSetObjectBit(HeapBitmap *hb, const void *obj)
+{
+ _heapBitmapModifyObjectBit(hb, obj, true, false);
+}
+
+/*
+ * Clears the bit corresponding to <obj>. Does no range checking.
+ */
+static void dvmHeapBitmapClearObjectBit(HeapBitmap *hb, const void *obj)
+{
+ _heapBitmapModifyObjectBit(hb, obj, false, false);
+}
+
+/*
+ * Returns the current value of the bit corresponding to <obj>,
+ * as zero or non-zero. Does no range checking.
+ *
+ * NOTE: casting this value to a bool is dangerous, because higher
+ * set bits will be lost.
+ */
+static unsigned long dvmHeapBitmapIsObjectBitSet(const HeapBitmap *hb,
+ const void *obj)
+{
+ assert(dvmHeapBitmapCoversAddress(hb, obj));
+ assert(hb->bits != NULL);
+ assert((uintptr_t)obj >= hb->base);
+ if ((uintptr_t)obj <= hb->max) {
+ const uintptr_t offset = (uintptr_t)obj - hb->base;
+ return hb->bits[HB_OFFSET_TO_INDEX(offset)] & HB_OFFSET_TO_MASK(offset);
+ } else {
+ return 0;
+ }
+}
+
+#endif /* _DALVIK_HEAP_BITMAPINLINES */
diff --git a/vm/alloc/HeapSource.c b/vm/alloc/HeapSource.c
index 9af7aa0..a3e8893 100644
--- a/vm/alloc/HeapSource.c
+++ b/vm/alloc/HeapSource.c
@@ -24,6 +24,7 @@
#include "alloc/HeapInternal.h"
#include "alloc/HeapSource.h"
#include "alloc/HeapBitmap.h"
+#include "alloc/HeapBitmapInlines.h"
// TODO: find a real header file for these.
extern int dlmalloc_trim(size_t);
diff --git a/vm/alloc/MarkSweep.c b/vm/alloc/MarkSweep.c
index 1a50614..2d073c8 100644
--- a/vm/alloc/MarkSweep.c
+++ b/vm/alloc/MarkSweep.c
@@ -18,6 +18,7 @@
#include "alloc/clz.h"
#include "alloc/CardTable.h"
#include "alloc/HeapBitmap.h"
+#include "alloc/HeapBitmapInlines.h"
#include "alloc/HeapInternal.h"
#include "alloc/HeapSource.h"
#include "alloc/MarkSweep.h"
