/*
 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */
package org.graalvm.compiler.hotspot.replacements.arraycopy;

import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;

import java.lang.reflect.Method;
import java.util.EnumMap;
import java.util.Map;

import org.graalvm.compiler.api.directives.GraalDirectives;
import org.graalvm.compiler.api.replacements.Fold;
import org.graalvm.compiler.api.replacements.Snippet;
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
import org.graalvm.compiler.core.common.LocationIdentity;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.graph.Node;
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
import org.graalvm.compiler.hotspot.word.KlassPointer;
import org.graalvm.compiler.nodes.CallTargetNode;
import org.graalvm.compiler.nodes.ConstantNode;
import org.graalvm.compiler.nodes.DeoptimizeNode;
import org.graalvm.compiler.nodes.Invoke;
import org.graalvm.compiler.nodes.InvokeNode;
import org.graalvm.compiler.nodes.NamedLocationIdentity;
import org.graalvm.compiler.nodes.PiNode;
import org.graalvm.compiler.nodes.StructuredGraph;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.extended.UnsafeLoadNode;
import org.graalvm.compiler.nodes.java.ArrayLengthNode;
import org.graalvm.compiler.nodes.spi.LoweringTool;
import org.graalvm.compiler.nodes.type.StampTool;
import org.graalvm.compiler.replacements.SnippetCounter;
import org.graalvm.compiler.replacements.SnippetTemplate;
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
import org.graalvm.compiler.replacements.Snippets;
import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
import org.graalvm.compiler.word.Word;

import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
import jdk.vm.ci.meta.DeoptimizationAction;
import jdk.vm.ci.meta.DeoptimizationReason;
import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.ResolvedJavaMethod;
import jdk.vm.ci.meta.ResolvedJavaType;

public class ArrayCopySnippets implements Snippets {

    private static int checkArrayType(KlassPointer hub) {
        int layoutHelper = readLayoutHelper(hub);
        if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        return layoutHelper;
    }

    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) {
        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
            checkAIOOBECounter.inc();
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
            checkAIOOBECounter.inc();
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
            checkAIOOBECounter.inc();
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
            checkAIOOBECounter.inc();
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
            checkAIOOBECounter.inc();
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        checkSuccessCounter.inc();
    }

    @Snippet
    public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        KlassPointer srcHub = loadHub(nonNullSrc);
        KlassPointer destHub = loadHub(nonNullDest);
        checkArrayType(srcHub);
        checkArrayType(destHub);
        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
        zeroLengthStaticCounter.inc();
    }

    @Snippet
    public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
                    @ConstantParameter SnippetCounter copiedCounter) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
        counter.inc();
        copiedCounter.add(length);
        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
        if (length == 0) {
            zeroLengthDynamicCounter.inc();
        } else {
            nonZeroLengthDynamicCounter.inc();
            nonZeroLengthDynamicCopiedCounter.add(length);
        }
    }

    /**
     * This intrinsic is useful for the case where we know something statically about one of the
     * inputs but not the other.
     */
    @Snippet
    public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        KlassPointer srcHub = loadHub(nonNullSrc);
        KlassPointer destHub = loadHub(nonNullDest);
        if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
        }
        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
        counter.inc();
        copiedCounter.add(length);
        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
        if (length == 0) {
            zeroLengthDynamicCounter.inc();
        } else {
            nonZeroLengthDynamicCounter.inc();
            nonZeroLengthDynamicCopiedCounter.add(length);
        }
    }

    @Snippet
    public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
        if (length > 0) {
            KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
            KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
            if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
                counter.inc();
                copiedCounter.add(length);
                predictedObjectArrayCopyFastPathCounter.inc();
                predictedObjectArrayCopyFastPathCopiedCounter.add(length);
                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
            } else {
                predictedObjectArrayCopySlowPathCounter.inc();
                predictedObjectArrayCopySlowPathCopiedCounter.add(length);
                System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
            }
        }
    }

    /**
     * This is the basic template for the full arraycopy checks, including a check that the
     * underlying type is really an array type.
     */
    @Snippet
    public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
                    @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        KlassPointer srcHub = loadHub(nonNullSrc);
        KlassPointer destHub = loadHub(nonNullDest);
        checkArrayType(srcHub);
        checkArrayType(destHub);
        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
        if (length == 0) {
            zeroLengthDynamicCounter.inc();
        } else {
            nonZeroLengthDynamicCounter.inc();
            nonZeroLengthDynamicCopiedCounter.add(length);
        }
        ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
    }

    /**
     * Snippet for unrolled arraycopy.
     */
    @Snippet
    public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
        if (length == 0) {
            zeroLengthDynamicCounter.inc();
        } else {
            nonZeroLengthDynamicCounter.inc();
            nonZeroLengthDynamicCopiedCounter.add(length);
        }
        ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
    }

    @Snippet
    public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) {
        if (length > 0) {
            KlassPointer destKlass = loadHub(nonNullDest);
            KlassPointer srcKlass = loadHub(nonNullSrc);
            if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
                // no storecheck required.
                objectCheckcastSameTypeCounter.inc();
                objectCheckcastSameTypeCopiedCounter.add(length);
                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
            } else {
                KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
                Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
                objectCheckcastCounter.inc();
                objectCheckcastCopiedCounter.add(length);
                int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
                if (copiedElements != 0) {
                    /*
                     * the checkcast stub doesn't throw the ArrayStoreException, but returns the
                     * number of copied elements (xor'd with -1).
                     */
                    copiedElements ^= -1;
                    System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
                }
            }
        }
    }

    @Snippet
    public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) {
        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
        KlassPointer srcHub = loadHub(nonNullSrc);
        KlassPointer destHub = loadHub(nonNullDest);
        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
            int layoutHelper = checkArrayType(srcHub);
            final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
            if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
                genericObjectExactCallCounter.inc();
                genericObjectExactCallCopiedCounter.add(length);
                ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
            } else {
                genericPrimitiveCallCounter.inc();
                genericPrimitiveCallCopiedCounter.add(length);
                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
            }
        } else {
            SystemArraycopyCounter.inc();
            SystemArraycopyCopiedCounter.add(length);
            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
        }
    }

    @Fold
    static LocationIdentity getArrayLocation(JavaKind kind) {
        return NamedLocationIdentity.getArrayLocation(kind);
    }

    @Snippet
    public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
        final int scale = arrayIndexScale(elementKind);
        int arrayBaseOffset = arrayBaseOffset(elementKind);
        LocationIdentity arrayLocation = getArrayLocation(elementKind);
        if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
            long start = (long) (length - 1) * scale;
            long i = start;
            ExplodeLoopNode.explodeLoop();
            for (int iteration = 0; iteration < length; iteration++) {
                if (i >= 0) {
                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
                    i -= scale;
                }
            }
        } else {
            long end = (long) length * scale;
            long i = 0;
            ExplodeLoopNode.explodeLoop();
            for (int iteration = 0; iteration < length; iteration++) {
                if (i < end) {
                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
                    i += scale;
                }
            }
        }
    }

    private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null;
    private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
    private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");

    private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null;

    private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
    private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
    private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
    private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");

    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);

    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
    private static final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);

    static void createArraycopyCounter(JavaKind kind) {
        arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
        arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));

        arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
        arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
    }

    static {
        createArraycopyCounter(JavaKind.Byte);
        createArraycopyCounter(JavaKind.Boolean);
        createArraycopyCounter(JavaKind.Char);
        createArraycopyCounter(JavaKind.Short);
        createArraycopyCounter(JavaKind.Int);
        createArraycopyCounter(JavaKind.Long);
        createArraycopyCounter(JavaKind.Float);
        createArraycopyCounter(JavaKind.Double);
        createArraycopyCounter(JavaKind.Object);
    }

    private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
    private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
    private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");

    private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null;

    private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
    private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
    private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");

    private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null;

    private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
    private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
    private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
    private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");

    private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
    private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
    private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
                    "used System.arraycopy slow path for predicted Object[] arrays");
    private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");

    public static class Templates extends SnippetTemplate.AbstractTemplates {

        public Templates(HotSpotProviders providers, TargetDescription target) {
            super(providers, providers.getSnippetReflection(), target);
        }

        private ResolvedJavaMethod originalArraycopy() throws GraalError {
            if (originalArraycopy == null) {
                Method method;
                try {
                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
                } catch (NoSuchMethodException | SecurityException e) {
                    throw new GraalError(e);
                }
                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
            }
            return originalArraycopy;
        }

        private ResolvedJavaMethod originalArraycopy;

        private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");

        private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
        private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
        private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
        private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
        private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
        private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");

        private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");

        protected SnippetInfo snippet(String methodName) {
            SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
            info.setOriginalMethod(originalArraycopy());
            return info;
        }

        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
            return selectComponentKind(arraycopy, true);
        }

        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());

            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
                if (!exact) {
                    JavaKind component = getComponentKind(srcType);
                    if (component != null) {
                        return component;
                    }
                    return getComponentKind(destType);
                }
                return null;
            }
            if (exact) {
                if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
                    return null;
                }
                if (!arraycopy.isExact()) {
                    return null;
                }
            }
            return srcType.getComponentType().getJavaKind();
        }

        private static JavaKind getComponentKind(ResolvedJavaType type) {
            if (type != null && type.isArray()) {
                return type.getComponentType().getJavaKind();
            }
            return null;
        }

        private static boolean shouldUnroll(ValueNode length) {
            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
        }

        public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
            JavaKind componentKind = selectComponentKind(arraycopy);
            SnippetInfo snippetInfo = null;
            SnippetInfo slowPathSnippetInfo = null;
            Object slowPathArgument = null;

            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
                snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
            } else if (arraycopy.isExact()) {
                snippetInfo = arraycopyExactIntrinsicSnippet;
                if (shouldUnroll(arraycopy.getLength())) {
                    snippetInfo = arraycopyUnrolledIntrinsicSnippet;
                }
            } else {
                if (componentKind == JavaKind.Object) {
                    ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
                    ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
                    ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
                    ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
                    if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
                        snippetInfo = arraycopySlowPathIntrinsicSnippet;
                        slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
                        slowPathArgument = LocationIdentity.any();
                        /*
                         * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
                         * pretend to kill any() so clear the componentKind.
                         */
                        componentKind = null;
                    }
                }
                if (componentKind == null && snippetInfo == null) {
                    JavaKind predictedKind = selectComponentKind(arraycopy, false);
                    if (predictedKind != null) {
                        /*
                         * At least one array is of a known type requiring no store checks, so
                         * assume the other is of the same type. Generally this is working around
                         * deficiencies in our propagation of type information.
                         */
                        componentKind = predictedKind;
                        if (predictedKind == JavaKind.Object) {
                            snippetInfo = arraycopySlowPathIntrinsicSnippet;
                            slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
                            slowPathArgument = predictedKind;
                            componentKind = null;
                        } else {
                            snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
                        }
                    }
                }
                if (snippetInfo == null) {
                    snippetInfo = arraycopyGenericSnippet;
                }
            }
            Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
            args.add("src", arraycopy.getSource());
            args.add("srcPos", arraycopy.getSourcePosition());
            args.add("dest", arraycopy.getDestination());
            args.add("destPos", arraycopy.getDestinationPosition());
            args.add("length", arraycopy.getLength());
            if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
                args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
            } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
                ValueNode predictedKlass = null;
                if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
                    HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
                } else {
                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
                }
                args.add("predictedKlass", predictedKlass);
                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
                args.addConst("slowPath", slowPathSnippetInfo);
                assert slowPathArgument != null;
                args.addConst("slowPathArgument", slowPathArgument);
            } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
                assert componentKind != null;
                args.addConst("elementKind", componentKind);
                args.addConst("counter", arraycopyCallCounters.get(componentKind));
                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind));
            }
            instantiate(args, arraycopy);
        }

        public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
            StructuredGraph graph = arraycopy.graph();
            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
                // Can't be lowered yet
                return;
            }
            SnippetInfo snippetInfo = arraycopy.getSnippet();
            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
            args.add("nonNullSrc", arraycopy.getSource());
            args.add("srcPos", arraycopy.getSourcePosition());
            args.add("nonNullDest", arraycopy.getDestination());
            args.add("destPos", arraycopy.getDestinationPosition());
            if (snippetInfo == arraycopyUnrolledWorkSnippet) {
                args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
                args.addConst("elementKind", arraycopy.getElementKind());
            } else {
                args.add("length", arraycopy.getLength());
            }
            if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
                args.add("objectArrayKlass", arraycopy.getPredictedKlass());
                args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object));
                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object));
            }
            instantiate(args, arraycopy);
        }

        public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
            StructuredGraph graph = arraycopy.graph();
            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
                // Can't be lowered yet
                return;
            }
            SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
            args.add("nonNullSrc", arraycopy.getSource());
            args.add("srcPos", arraycopy.getSourcePosition());
            args.add("nonNullDest", arraycopy.getDestination());
            args.add("destPos", arraycopy.getDestinationPosition());
            args.addConst("length", arraycopy.getUnrollLength());
            args.addConst("elementKind", arraycopy.getElementKind());
            template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
        }

        /**
         * Instantiate the snippet template and fix up the FrameState of any Invokes of
         * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
         *
         * @param args
         * @param arraycopy
         */
        private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
            StructuredGraph graph = arraycopy.graph();
            SnippetTemplate template = template(args);
            Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
            for (Node originalNode : replacements.keySet()) {
                if (originalNode instanceof Invoke) {
                    Invoke invoke = (Invoke) replacements.get(originalNode);
                    assert invoke.asNode().graph() == graph;
                    CallTargetNode call = invoke.callTarget();

                    if (!call.targetMethod().equals(originalArraycopy)) {
                        throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
                    }
                    // Here we need to fix the bci of the invoke
                    InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
                    if (arraycopy.stateDuring() != null) {
                        newInvoke.setStateDuring(arraycopy.stateDuring());
                    } else {
                        assert arraycopy.stateAfter() != null;
                        newInvoke.setStateAfter(arraycopy.stateAfter());
                    }
                    graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
                } else if (originalNode instanceof ArrayCopySlowPathNode) {
                    ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
                    assert arraycopy.stateAfter() != null;
                    slowPath.setStateAfter(arraycopy.stateAfter());
                    slowPath.setBci(arraycopy.getBci());
                }
            }
        }
    }
}