| /* |
| * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| package org.graalvm.compiler.hotspot.stubs; |
| |
| import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; |
| import static org.graalvm.compiler.hotspot.nodes.DirectCompareAndSwapNode.compareAndSwap; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_END_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.HEAP_TOP_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.PROTOTYPE_MARK_WORD_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_FAST_REFILL_WASTE_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_NOF_REFILLS_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_REFILL_WASTE_LIMIT_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SIZE_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_SLOW_ALLOCATIONS_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.TLAB_THREAD_ALLOCATED_BYTES_LOCATION; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.getAndClearObjectResult; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.initializeTlab; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.isInstanceKlassFullyInitialized; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadKlassLayoutHelperIntrinsic; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.log2WordSize; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.prototypeMarkWordOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabEnd; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabStart; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readTlabTop; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadAllocatedBytesOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.threadTlabSizeOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabAlignmentReserveInHeapWords; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabFastRefillWasteOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabIntArrayMarkWord; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabNumberOfRefillsOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteIncrement; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabRefillWasteLimitOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabSlowAllocationsOffset; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.tlabStats; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useG1GC; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.useTLAB; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize; |
| import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.writeTlabTop; |
| import static org.graalvm.compiler.hotspot.stubs.StubUtil.handlePendingException; |
| import static org.graalvm.compiler.hotspot.stubs.StubUtil.newDescriptor; |
| import static org.graalvm.compiler.hotspot.stubs.StubUtil.printf; |
| import static org.graalvm.compiler.hotspot.stubs.StubUtil.verifyObject; |
| import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; |
| import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; |
| |
| import org.graalvm.compiler.api.replacements.Fold; |
| import org.graalvm.compiler.api.replacements.Snippet; |
| import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; |
| import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; |
| import org.graalvm.compiler.graph.Node.ConstantNodeParameter; |
| import org.graalvm.compiler.graph.Node.NodeIntrinsic; |
| import org.graalvm.compiler.hotspot.HotSpotForeignCallLinkage; |
| import org.graalvm.compiler.hotspot.meta.HotSpotProviders; |
| import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode; |
| import org.graalvm.compiler.hotspot.nodes.StubForeignCallNode; |
| import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; |
| import org.graalvm.compiler.hotspot.replacements.NewObjectSnippets; |
| import org.graalvm.compiler.hotspot.word.KlassPointer; |
| import org.graalvm.compiler.nodes.ConstantNode; |
| import org.graalvm.compiler.nodes.memory.address.RawAddressNode; |
| import org.graalvm.compiler.word.Word; |
| |
| import jdk.vm.ci.code.Register; |
| import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; |
| import jdk.vm.ci.meta.JavaKind; |
| |
| /** |
| * Stub implementing the fast path for TLAB refill during instance class allocation. This stub is |
| * called from the {@linkplain NewObjectSnippets inline} allocation code when TLAB allocation fails. |
| * If this stub fails to refill the TLAB or allocate the object, it calls out to the HotSpot C++ |
| * runtime for to complete the allocation. |
| */ |
| public class NewInstanceStub extends SnippetStub { |
| |
| public NewInstanceStub(HotSpotProviders providers, HotSpotForeignCallLinkage linkage) { |
| super("newInstance", providers, linkage); |
| } |
| |
| @Override |
| protected Object[] makeConstArgs() { |
| HotSpotResolvedObjectType intArrayType = (HotSpotResolvedObjectType) providers.getMetaAccess().lookupJavaType(int[].class); |
| int count = method.getSignature().getParameterCount(false); |
| Object[] args = new Object[count]; |
| assert checkConstArg(1, "intArrayHub"); |
| assert checkConstArg(2, "threadRegister"); |
| args[1] = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), intArrayType.klass(), null); |
| args[2] = providers.getRegisters().getThreadRegister(); |
| return args; |
| } |
| |
| private static Word allocate(Word thread, int size) { |
| Word top = readTlabTop(thread); |
| Word end = readTlabEnd(thread); |
| Word newTop = top.add(size); |
| /* |
| * this check might lead to problems if the TLAB is within 16GB of the address space end |
| * (checked in c++ code) |
| */ |
| if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { |
| writeTlabTop(thread, newTop); |
| return top; |
| } |
| return Word.zero(); |
| } |
| |
| @Fold |
| static boolean logging() { |
| return StubOptions.TraceNewInstanceStub.getValue(); |
| } |
| |
| /** |
| * Re-attempts allocation after an initial TLAB allocation failed or was skipped (e.g., due to |
| * -XX:-UseTLAB). |
| * |
| * @param hub the hub of the object to be allocated |
| * @param intArrayHub the hub for {@code int[].class} |
| */ |
| @Snippet |
| private static Object newInstance(KlassPointer hub, @ConstantParameter KlassPointer intArrayHub, @ConstantParameter Register threadRegister) { |
| /* |
| * The type is known to be an instance so Klass::_layout_helper is the instance size as a |
| * raw number |
| */ |
| int sizeInBytes = loadKlassLayoutHelperIntrinsic(hub); |
| Word thread = registerAsWord(threadRegister); |
| boolean inlineContiguousAllocationSupported = GraalHotSpotVMConfigNode.inlineContiguousAllocationSupported(); |
| if (!forceSlowPath() && inlineContiguousAllocationSupported) { |
| if (isInstanceKlassFullyInitialized(hub)) { |
| Word memory = refillAllocate(thread, intArrayHub, sizeInBytes, logging()); |
| if (memory.notEqual(0)) { |
| Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(INJECTED_VMCONFIG), PROTOTYPE_MARK_WORD_LOCATION); |
| NewObjectSnippets.formatObjectForStub(hub, sizeInBytes, memory, prototypeMarkWord); |
| return verifyObject(memory.toObject()); |
| } |
| } |
| } |
| |
| if (logging()) { |
| printf("newInstance: calling new_instance_c\n"); |
| } |
| |
| newInstanceC(NEW_INSTANCE_C, thread, hub); |
| handlePendingException(thread, true); |
| return verifyObject(getAndClearObjectResult(thread)); |
| } |
| |
| /** |
| * Attempts to refill the current thread's TLAB and retries the allocation. |
| * |
| * @param intArrayHub the hub for {@code int[].class} |
| * @param sizeInBytes the size of the allocation |
| * @param log specifies if logging is enabled |
| * |
| * @return the newly allocated, uninitialized chunk of memory, or {@link Word#zero()} if the |
| * operation was unsuccessful |
| */ |
| static Word refillAllocate(Word thread, KlassPointer intArrayHub, int sizeInBytes, boolean log) { |
| // If G1 is enabled, the "eden" allocation space is not the same always |
| // and therefore we have to go to slowpath to allocate a new TLAB. |
| if (useG1GC(INJECTED_VMCONFIG)) { |
| return Word.zero(); |
| } |
| if (!useTLAB(INJECTED_VMCONFIG)) { |
| return edenAllocate(Word.unsigned(sizeInBytes), log); |
| } |
| Word intArrayMarkWord = Word.unsigned(tlabIntArrayMarkWord(INJECTED_VMCONFIG)); |
| int alignmentReserveInBytes = tlabAlignmentReserveInHeapWords(INJECTED_VMCONFIG) * wordSize(); |
| |
| Word top = readTlabTop(thread); |
| Word end = readTlabEnd(thread); |
| |
| // calculate amount of free space |
| long tlabFreeSpaceInBytes = end.subtract(top).rawValue(); |
| |
| if (log) { |
| printf("refillTLAB: thread=%p\n", thread.rawValue()); |
| printf("refillTLAB: top=%p\n", top.rawValue()); |
| printf("refillTLAB: end=%p\n", end.rawValue()); |
| printf("refillTLAB: tlabFreeSpaceInBytes=%ld\n", tlabFreeSpaceInBytes); |
| } |
| |
| long tlabFreeSpaceInWords = tlabFreeSpaceInBytes >>> log2WordSize(); |
| |
| // Retain TLAB and allocate object in shared space if |
| // the amount free in the TLAB is too large to discard. |
| Word refillWasteLimit = thread.readWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), TLAB_REFILL_WASTE_LIMIT_LOCATION); |
| if (tlabFreeSpaceInWords <= refillWasteLimit.rawValue()) { |
| if (tlabStats(INJECTED_VMCONFIG)) { |
| // increment number of refills |
| thread.writeInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION) + 1, TLAB_NOF_REFILLS_LOCATION); |
| if (log) { |
| printf("thread: %p -- number_of_refills %d\n", thread.rawValue(), thread.readInt(tlabNumberOfRefillsOffset(INJECTED_VMCONFIG), TLAB_NOF_REFILLS_LOCATION)); |
| } |
| // accumulate wastage |
| int wastage = thread.readInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), TLAB_FAST_REFILL_WASTE_LOCATION) + (int) tlabFreeSpaceInWords; |
| if (log) { |
| printf("thread: %p -- accumulated wastage %d\n", thread.rawValue(), wastage); |
| } |
| thread.writeInt(tlabFastRefillWasteOffset(INJECTED_VMCONFIG), wastage, TLAB_FAST_REFILL_WASTE_LOCATION); |
| } |
| |
| // if TLAB is currently allocated (top or end != null) then |
| // fill [top, end + alignment_reserve) with array object |
| if (top.notEqual(0)) { |
| int headerSize = arrayBaseOffset(JavaKind.Int); |
| // just like the HotSpot assembler stubs, assumes that tlabFreeSpaceInInts fits in |
| // an int |
| int tlabFreeSpaceInInts = (int) tlabFreeSpaceInBytes >>> 2; |
| int length = ((alignmentReserveInBytes - headerSize) >>> 2) + tlabFreeSpaceInInts; |
| NewObjectSnippets.formatArray(intArrayHub, 0, length, headerSize, top, intArrayMarkWord, false, false, false); |
| |
| long allocated = thread.readLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), TLAB_THREAD_ALLOCATED_BYTES_LOCATION); |
| allocated = allocated + top.subtract(readTlabStart(thread)).rawValue(); |
| thread.writeLong(threadAllocatedBytesOffset(INJECTED_VMCONFIG), allocated, TLAB_THREAD_ALLOCATED_BYTES_LOCATION); |
| } |
| |
| // refill the TLAB with an eden allocation |
| Word tlabRefillSizeInWords = thread.readWord(threadTlabSizeOffset(INJECTED_VMCONFIG), TLAB_SIZE_LOCATION); |
| Word tlabRefillSizeInBytes = tlabRefillSizeInWords.multiply(wordSize()); |
| // allocate new TLAB, address returned in top |
| top = edenAllocate(tlabRefillSizeInBytes, log); |
| if (top.notEqual(0)) { |
| end = top.add(tlabRefillSizeInBytes.subtract(alignmentReserveInBytes)); |
| initializeTlab(thread, top, end); |
| |
| return NewInstanceStub.allocate(thread, sizeInBytes); |
| } else { |
| return Word.zero(); |
| } |
| } else { |
| // Retain TLAB |
| Word newRefillWasteLimit = refillWasteLimit.add(tlabRefillWasteIncrement(INJECTED_VMCONFIG)); |
| thread.writeWord(tlabRefillWasteLimitOffset(INJECTED_VMCONFIG), newRefillWasteLimit, TLAB_REFILL_WASTE_LIMIT_LOCATION); |
| if (log) { |
| printf("refillTLAB: retaining TLAB - newRefillWasteLimit=%p\n", newRefillWasteLimit.rawValue()); |
| } |
| |
| if (tlabStats(INJECTED_VMCONFIG)) { |
| thread.writeInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), thread.readInt(tlabSlowAllocationsOffset(INJECTED_VMCONFIG), TLAB_SLOW_ALLOCATIONS_LOCATION) + 1, |
| TLAB_SLOW_ALLOCATIONS_LOCATION); |
| } |
| |
| return edenAllocate(Word.unsigned(sizeInBytes), log); |
| } |
| } |
| |
| /** |
| * Attempts to allocate a chunk of memory from Eden space. |
| * |
| * @param sizeInBytes the size of the chunk to allocate |
| * @param log specifies if logging is enabled |
| * @return the allocated chunk or {@link Word#zero()} if allocation fails |
| */ |
| public static Word edenAllocate(Word sizeInBytes, boolean log) { |
| final long heapTopRawAddress = GraalHotSpotVMConfigNode.heapTopAddress(); |
| final long heapEndRawAddress = GraalHotSpotVMConfigNode.heapEndAddress(); |
| |
| Word heapTopAddress = Word.unsigned(heapTopRawAddress); |
| Word heapEndAddress = Word.unsigned(heapEndRawAddress); |
| |
| while (true) { |
| Word heapTop = heapTopAddress.readWord(0, HEAP_TOP_LOCATION); |
| Word newHeapTop = heapTop.add(sizeInBytes); |
| if (newHeapTop.belowOrEqual(heapTop)) { |
| return Word.zero(); |
| } |
| |
| Word heapEnd = heapEndAddress.readWord(0, HEAP_END_LOCATION); |
| if (newHeapTop.aboveThan(heapEnd)) { |
| return Word.zero(); |
| } |
| |
| if (compareAndSwap(RawAddressNode.address(heapTopAddress), heapTop, newHeapTop, HEAP_TOP_LOCATION).equal(heapTop)) { |
| return heapTop; |
| } |
| } |
| } |
| |
| @Fold |
| static boolean forceSlowPath() { |
| return StubOptions.ForceUseOfNewInstanceStub.getValue(); |
| } |
| |
| public static final ForeignCallDescriptor NEW_INSTANCE_C = newDescriptor(NewInstanceStub.class, "newInstanceC", void.class, Word.class, KlassPointer.class); |
| |
| @NodeIntrinsic(StubForeignCallNode.class) |
| public static native void newInstanceC(@ConstantNodeParameter ForeignCallDescriptor newInstanceC, Word thread, KlassPointer hub); |
| } |
