Refactor SystemArrayCopy intrinsics.
Test: m test-art-host
Test: m test-art-target
Change-Id: I2f9ccdbb831030e670996b97e0c422f505b3abf6
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 28095c4..987a2af 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -41,6 +41,54 @@
using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>;
+#define __ assembler->
+
+// Compute base address for the System.arraycopy intrinsic in `base`.
+static void GenSystemArrayCopyBaseAddress(ArmAssembler* assembler,
+ Primitive::Type type,
+ const Register& array,
+ const Location& pos,
+ const Register& base) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const uint32_t element_size_shift = Primitive::ComponentSizeShift(type);
+ const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
+
+ if (pos.IsConstant()) {
+ int32_t constant = pos.GetConstant()->AsIntConstant()->GetValue();
+ __ AddConstant(base, array, element_size * constant + data_offset);
+ } else {
+ __ add(base, array, ShifterOperand(pos.AsRegister<Register>(), LSL, element_size_shift));
+ __ AddConstant(base, data_offset);
+ }
+}
+
+// Compute end address for the System.arraycopy intrinsic in `end`.
+static void GenSystemArrayCopyEndAddress(ArmAssembler* assembler,
+ Primitive::Type type,
+ const Location& copy_length,
+ const Register& base,
+ const Register& end) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const uint32_t element_size_shift = Primitive::ComponentSizeShift(type);
+
+ if (copy_length.IsConstant()) {
+ int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
+ __ AddConstant(end, base, element_size * constant);
+ } else {
+ __ add(end, base, ShifterOperand(copy_length.AsRegister<Register>(), LSL, element_size_shift));
+ }
+}
+
+#undef __
+
// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT
@@ -55,6 +103,7 @@
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ ArmAssembler* assembler = arm_codegen->GetAssembler();
LocationSummary* locations = instruction_->GetLocations();
DCHECK(locations->CanCall());
DCHECK(instruction_->IsInvokeStaticOrDirect())
@@ -63,9 +112,8 @@
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy);
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
+ Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
Register dest = locations->InAt(2).AsRegister<Register>();
Location dest_pos = locations->InAt(3);
@@ -76,15 +124,7 @@
__ Bind(GetEntryLabel());
// Compute the base destination address in `dst_curr_addr`.
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ AddConstant(dst_curr_addr, dest, element_size * constant + offset);
- } else {
- __ add(dst_curr_addr,
- dest,
- ShifterOperand(dest_pos.AsRegister<Register>(), LSL, element_size_shift));
- __ AddConstant(dst_curr_addr, offset);
- }
+ GenSystemArrayCopyBaseAddress(assembler, type, dest, dest_pos, dst_curr_addr);
Label loop;
__ Bind(&loop);
@@ -108,6 +148,8 @@
DCHECK_NE(src_stop_addr, IP);
DCHECK_NE(tmp, IP);
DCHECK(0 <= tmp && tmp < kNumberOfCoreRegisters) << tmp;
+ // TODO: Load the entrypoint once before the loop, instead of
+ // loading it at every iteration.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp);
// This runtime call does not require a stack map.
@@ -1925,33 +1967,19 @@
__ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel());
}
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
+ const Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
// Compute the base source address in `temp1`.
- if (src_pos.IsConstant()) {
- int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
- __ AddConstant(temp1, src, element_size * constant + offset);
- } else {
- __ add(temp1, src, ShifterOperand(src_pos.AsRegister<Register>(), LSL, element_size_shift));
- __ AddConstant(temp1, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1);
// Compute the end source address in `temp3`.
- if (length.IsConstant()) {
- int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
- __ AddConstant(temp3, temp1, element_size * constant);
- } else {
- __ add(temp3, temp1, ShifterOperand(length.AsRegister<Register>(), LSL, element_size_shift));
- }
+ GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3);
+ // The base destination address is computed later, as `temp2` is
+ // used for intermediate computations.
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// TODO: Also convert this intrinsic to the IsGcMarking strategy?
- // The base destination address is computed later, as `temp2` is
- // used for intermediate computations.
-
// SystemArrayCopy implementation for Baker read barriers (see
// also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier):
//
@@ -1992,6 +2020,8 @@
__ add(src, src, ShifterOperand(temp2, LSR, 32));
// Slow path used to copy array when `src` is gray.
+ // Note that the base destination address is computed in `temp2`
+ // by the slow path code.
SlowPathCode* read_barrier_slow_path =
new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARM(invoke);
codegen_->AddSlowPath(read_barrier_slow_path);
@@ -2006,16 +2036,8 @@
__ b(read_barrier_slow_path->GetEntryLabel(), CS);
// Fast-path copy.
-
// Compute the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ AddConstant(temp2, dest, element_size * constant + offset);
- } else {
- __ add(temp2, dest, ShifterOperand(dest_pos.AsRegister<Register>(), LSL, element_size_shift));
- __ AddConstant(temp2, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
__ Bind(&loop);
@@ -2028,16 +2050,8 @@
__ Bind(&done);
} else {
// Non read barrier code.
-
// Compute the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ AddConstant(temp2, dest, element_size * constant + offset);
- } else {
- __ add(temp2, dest, ShifterOperand(dest_pos.AsRegister<Register>(), LSL, element_size_shift));
- __ AddConstant(temp2, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
Label loop, done;
@@ -2052,11 +2066,7 @@
}
// We only need one card marking on the destination array.
- codegen_->MarkGCCard(temp1,
- temp2,
- dest,
- Register(kNoRegister),
- /* value_can_be_null */ false);
+ codegen_->MarkGCCard(temp1, temp2, dest, Register(kNoRegister), /* value_can_be_null */ false);
__ Bind(intrinsic_slow_path->GetExitLabel());
}
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 807d6cf..b012608 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -198,6 +198,8 @@
DCHECK_NE(LocationFrom(src_stop_addr).reg(), IP0);
DCHECK_NE(tmp_.reg(), IP0);
DCHECK(0 <= tmp_.reg() && tmp_.reg() < kNumberOfWRegisters) << tmp_.reg();
+ // TODO: Load the entrypoint once before the loop, instead of
+ // loading it at every iteration.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(tmp_.reg());
// This runtime call does not require a stack map.
@@ -2191,8 +2193,9 @@
}
}
-// Compute base source address, base destination address, and end source address
-// for System.arraycopy* intrinsics.
+// Compute base source address, base destination address, and end
+// source address for System.arraycopy* intrinsics in `src_base`,
+// `dst_base` and `src_end` respectively.
static void GenSystemArrayCopyAddresses(MacroAssembler* masm,
Primitive::Type type,
const Register& src,
@@ -2203,12 +2206,13 @@
const Register& src_base,
const Register& dst_base,
const Register& src_end) {
+ // This routine is used by the SystemArrayCopy and the SystemArrayCopyChar intrinsics.
DCHECK(type == Primitive::kPrimNot || type == Primitive::kPrimChar)
<< "Unexpected element type: " << type;
const int32_t element_size = Primitive::ComponentSize(type);
const int32_t element_size_shift = Primitive::ComponentSizeShift(type);
+ const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
- uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
if (src_pos.IsConstant()) {
int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
__ Add(src_base, src, element_size * constant + data_offset);
@@ -2712,12 +2716,18 @@
__ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel());
}
+ const Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
+
Register src_curr_addr = temp1.X();
Register dst_curr_addr = temp2.X();
Register src_stop_addr = temp3.X();
+ // Compute base source address, base destination address, and end
+ // source address in `src_curr_addr`, `dst_curr_addr` and
+ // `src_stop_addr` respectively.
GenSystemArrayCopyAddresses(masm,
- Primitive::kPrimNot,
+ type,
src,
src_pos,
dest,
@@ -2727,8 +2737,6 @@
dst_curr_addr,
src_stop_addr);
- const int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
-
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// TODO: Also convert this intrinsic to the IsGcMarking strategy?
@@ -2801,7 +2809,6 @@
__ Bind(&done);
} else {
// Non read barrier code.
-
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
vixl::aarch64::Label loop, done;
@@ -2817,6 +2824,7 @@
__ Bind(&done);
}
}
+
// We only need one card marking on the destination array.
codegen_->MarkGCCard(dest.W(), Register(), /* value_can_be_null */ false);
diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc
index 60bcf2c..0167891 100644
--- a/compiler/optimizing/intrinsics_arm_vixl.cc
+++ b/compiler/optimizing/intrinsics_arm_vixl.cc
@@ -117,6 +117,50 @@
DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARMVIXL);
};
+// Compute base address for the System.arraycopy intrinsic in `base`.
+static void GenSystemArrayCopyBaseAddress(ArmVIXLAssembler* assembler,
+ Primitive::Type type,
+ const vixl32::Register& array,
+ const Location& pos,
+ const vixl32::Register& base) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const uint32_t element_size_shift = Primitive::ComponentSizeShift(type);
+ const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
+
+ if (pos.IsConstant()) {
+ int32_t constant = Int32ConstantFrom(pos);
+ __ Add(base, array, element_size * constant + data_offset);
+ } else {
+ __ Add(base, array, Operand(RegisterFrom(pos), vixl32::LSL, element_size_shift));
+ __ Add(base, base, data_offset);
+ }
+}
+
+// Compute end address for the System.arraycopy intrinsic in `end`.
+static void GenSystemArrayCopyEndAddress(ArmVIXLAssembler* assembler,
+ Primitive::Type type,
+ const Location& copy_length,
+ const vixl32::Register& base,
+ const vixl32::Register& end) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const uint32_t element_size_shift = Primitive::ComponentSizeShift(type);
+
+ if (copy_length.IsConstant()) {
+ int32_t constant = Int32ConstantFrom(copy_length);
+ __ Add(end, base, element_size * constant);
+ } else {
+ __ Add(end, base, Operand(RegisterFrom(copy_length), vixl32::LSL, element_size_shift));
+ }
+}
+
// Slow path implementing the SystemArrayCopy intrinsic copy loop with read barriers.
class ReadBarrierSystemArrayCopySlowPathARMVIXL : public SlowPathCodeARMVIXL {
public:
@@ -137,9 +181,8 @@
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kSystemArrayCopy);
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
+ Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
vixl32::Register dest = InputRegisterAt(instruction_, 2);
Location dest_pos = locations->InAt(3);
@@ -150,15 +193,7 @@
__ Bind(GetEntryLabel());
// Compute the base destination address in `dst_curr_addr`.
- if (dest_pos.IsConstant()) {
- int32_t constant = Int32ConstantFrom(dest_pos);
- __ Add(dst_curr_addr, dest, element_size * constant + offset);
- } else {
- __ Add(dst_curr_addr,
- dest,
- Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift));
- __ Add(dst_curr_addr, dst_curr_addr, offset);
- }
+ GenSystemArrayCopyBaseAddress(assembler, type, dest, dest_pos, dst_curr_addr);
vixl32::Label loop;
__ Bind(&loop);
@@ -182,6 +217,8 @@
DCHECK(!src_stop_addr.Is(ip));
DCHECK(!tmp.Is(ip));
DCHECK(tmp.IsRegister()) << tmp;
+ // TODO: Load the entrypoint once before the loop, instead of
+ // loading it at every iteration.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(tmp.GetCode());
// This runtime call does not require a stack map.
@@ -2243,33 +2280,19 @@
__ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel());
}
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- uint32_t element_size_shift = Primitive::ComponentSizeShift(Primitive::kPrimNot);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
+ const Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
// Compute the base source address in `temp1`.
- if (src_pos.IsConstant()) {
- int32_t constant = Int32ConstantFrom(src_pos);
- __ Add(temp1, src, element_size * constant + offset);
- } else {
- __ Add(temp1, src, Operand(RegisterFrom(src_pos), vixl32::LSL, element_size_shift));
- __ Add(temp1, temp1, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1);
// Compute the end source address in `temp3`.
- if (length.IsConstant()) {
- int32_t constant = Int32ConstantFrom(length);
- __ Add(temp3, temp1, element_size * constant);
- } else {
- __ Add(temp3, temp1, Operand(RegisterFrom(length), vixl32::LSL, element_size_shift));
- }
+ GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3);
+ // The base destination address is computed later, as `temp2` is
+ // used for intermediate computations.
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// TODO: Also convert this intrinsic to the IsGcMarking strategy?
- // The base destination address is computed later, as `temp2` is
- // used for intermediate computations.
-
// SystemArrayCopy implementation for Baker read barriers (see
// also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier):
//
@@ -2310,6 +2333,8 @@
__ Add(src, src, Operand(temp2, vixl32::LSR, 32));
// Slow path used to copy array when `src` is gray.
+ // Note that the base destination address is computed in `temp2`
+ // by the slow path code.
SlowPathCodeARMVIXL* read_barrier_slow_path =
new (GetAllocator()) ReadBarrierSystemArrayCopySlowPathARMVIXL(invoke);
codegen_->AddSlowPath(read_barrier_slow_path);
@@ -2324,28 +2349,17 @@
__ B(cs, read_barrier_slow_path->GetEntryLabel());
// Fast-path copy.
-
// Compute the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = Int32ConstantFrom(dest_pos);
- __ Add(temp2, dest, element_size * constant + offset);
- } else {
- __ Add(temp2, dest, Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift));
- __ Add(temp2, temp2, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
__ Bind(&loop);
-
{
UseScratchRegisterScope temps(assembler->GetVIXLAssembler());
const vixl32::Register temp_reg = temps.Acquire();
-
__ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex));
__ Str(temp_reg, MemOperand(temp2, element_size, PostIndex));
}
-
__ Cmp(temp1, temp3);
__ B(ne, &loop, /* far_target */ false);
@@ -2353,31 +2367,20 @@
__ Bind(&done);
} else {
// Non read barrier code.
-
// Compute the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = Int32ConstantFrom(dest_pos);
- __ Add(temp2, dest, element_size * constant + offset);
- } else {
- __ Add(temp2, dest, Operand(RegisterFrom(dest_pos), vixl32::LSL, element_size_shift));
- __ Add(temp2, temp2, offset);
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
vixl32::Label loop, done;
__ Cmp(temp1, temp3);
__ B(eq, &done, /* far_target */ false);
__ Bind(&loop);
-
{
UseScratchRegisterScope temps(assembler->GetVIXLAssembler());
const vixl32::Register temp_reg = temps.Acquire();
-
__ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex));
__ Str(temp_reg, MemOperand(temp2, element_size, PostIndex));
}
-
__ Cmp(temp1, temp3);
__ B(ne, &loop, /* far_target */ false);
__ Bind(&done);
diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc
index a671788..ecf919b 100644
--- a/compiler/optimizing/intrinsics_x86.cc
+++ b/compiler/optimizing/intrinsics_x86.cc
@@ -2878,6 +2878,49 @@
return instruction->InputAt(input0) == instruction->InputAt(input1);
}
+// Compute base address for the System.arraycopy intrinsic in `base`.
+static void GenSystemArrayCopyBaseAddress(X86Assembler* assembler,
+ Primitive::Type type,
+ const Register& array,
+ const Location& pos,
+ const Register& base) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const ScaleFactor scale_factor = static_cast<ScaleFactor>(Primitive::ComponentSizeShift(type));
+ const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
+
+ if (pos.IsConstant()) {
+ int32_t constant = pos.GetConstant()->AsIntConstant()->GetValue();
+ __ leal(base, Address(array, element_size * constant + data_offset));
+ } else {
+ __ leal(base, Address(array, pos.AsRegister<Register>(), scale_factor, data_offset));
+ }
+}
+
+// Compute end source address for the System.arraycopy intrinsic in `end`.
+static void GenSystemArrayCopyEndAddress(X86Assembler* assembler,
+ Primitive::Type type,
+ const Location& copy_length,
+ const Register& base,
+ const Register& end) {
+ // This routine is only used by the SystemArrayCopy intrinsic at the
+ // moment. We can allow Primitive::kPrimNot as `type` to implement
+ // the SystemArrayCopyChar intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const ScaleFactor scale_factor = static_cast<ScaleFactor>(Primitive::ComponentSizeShift(type));
+
+ if (copy_length.IsConstant()) {
+ int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
+ __ leal(end, Address(base, element_size * constant));
+ } else {
+ __ leal(end, Address(base, copy_length.AsRegister<Register>(), scale_factor, 0));
+ }
+}
+
void IntrinsicLocationsBuilderX86::VisitSystemArrayCopy(HInvoke* invoke) {
// The only read barrier implementation supporting the
// SystemArrayCopy intrinsic is the Baker-style read barriers.
@@ -3182,16 +3225,11 @@
__ j(kNotEqual, intrinsic_slow_path->GetEntryLabel());
}
+ const Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
+
// Compute the base source address in `temp1`.
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- DCHECK_EQ(element_size, 4);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
- if (src_pos.IsConstant()) {
- int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp1, Address(src, element_size * constant + offset));
- } else {
- __ leal(temp1, Address(src, src_pos.AsRegister<Register>(), ScaleFactor::TIMES_4, offset));
- }
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, src, src_pos, temp1);
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// If it is needed (in the case of the fast-path loop), the base
@@ -3199,20 +3237,15 @@
// intermediate computations.
// Compute the end source address in `temp3`.
- if (length.IsConstant()) {
- int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp3, Address(temp1, element_size * constant));
- } else {
- if (length.IsStackSlot()) {
- // Location `length` is again pointing at a stack slot, as
- // register `temp3` (which was containing the length parameter
- // earlier) has been overwritten; restore it now
- DCHECK(length.Equals(length_arg));
- __ movl(temp3, Address(ESP, length.GetStackIndex()));
- length = Location::RegisterLocation(temp3);
- }
- __ leal(temp3, Address(temp1, length.AsRegister<Register>(), ScaleFactor::TIMES_4, 0));
+ if (length.IsStackSlot()) {
+ // Location `length` is again pointing at a stack slot, as
+ // register `temp3` (which was containing the length parameter
+ // earlier) has been overwritten; restore it now
+ DCHECK(length.Equals(length_arg));
+ __ movl(temp3, Address(ESP, length.GetStackIndex()));
+ length = Location::RegisterLocation(temp3);
}
+ GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3);
// SystemArrayCopy implementation for Baker read barriers (see
// also CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier):
@@ -3266,15 +3299,8 @@
__ j(kNotZero, read_barrier_slow_path->GetEntryLabel());
// Fast-path copy.
-
- // Set the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp2, Address(dest, element_size * constant + offset));
- } else {
- __ leal(temp2, Address(dest, dest_pos.AsRegister<Register>(), ScaleFactor::TIMES_4, offset));
- }
-
+ // Compute the base destination address in `temp2`.
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
__ Bind(&loop);
@@ -3291,23 +3317,10 @@
__ Bind(&done);
} else {
// Non read barrier code.
-
// Compute the base destination address in `temp2`.
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp2, Address(dest, element_size * constant + offset));
- } else {
- __ leal(temp2, Address(dest, dest_pos.AsRegister<Register>(), ScaleFactor::TIMES_4, offset));
- }
-
+ GenSystemArrayCopyBaseAddress(GetAssembler(), type, dest, dest_pos, temp2);
// Compute the end source address in `temp3`.
- if (length.IsConstant()) {
- int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp3, Address(temp1, element_size * constant));
- } else {
- __ leal(temp3, Address(temp1, length.AsRegister<Register>(), ScaleFactor::TIMES_4, 0));
- }
-
+ GenSystemArrayCopyEndAddress(GetAssembler(), type, length, temp1, temp3);
// Iterate over the arrays and do a raw copy of the objects. We don't need to
// poison/unpoison.
NearLabel loop, done;
@@ -3326,11 +3339,7 @@
}
// We only need one card marking on the destination array.
- codegen_->MarkGCCard(temp1,
- temp2,
- dest,
- Register(kNoRegister),
- /* value_can_be_null */ false);
+ codegen_->MarkGCCard(temp1, temp2, dest, Register(kNoRegister), /* value_can_be_null */ false);
__ Bind(intrinsic_slow_path->GetExitLabel());
}
diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc
index 9a6dd98..dcea7e5 100644
--- a/compiler/optimizing/intrinsics_x86_64.cc
+++ b/compiler/optimizing/intrinsics_x86_64.cc
@@ -1118,6 +1118,47 @@
CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke);
}
+// Compute base source address, base destination address, and end
+// source address for the System.arraycopy intrinsic in `src_base`,
+// `dst_base` and `src_end` respectively.
+static void GenSystemArrayCopyAddresses(X86_64Assembler* assembler,
+ Primitive::Type type,
+ const CpuRegister& src,
+ const Location& src_pos,
+ const CpuRegister& dst,
+ const Location& dst_pos,
+ const Location& copy_length,
+ const CpuRegister& src_base,
+ const CpuRegister& dst_base,
+ const CpuRegister& src_end) {
+ // This routine is only used by the SystemArrayCopy intrinsic.
+ DCHECK_EQ(type, Primitive::kPrimNot);
+ const int32_t element_size = Primitive::ComponentSize(type);
+ const ScaleFactor scale_factor = static_cast<ScaleFactor>(Primitive::ComponentSizeShift(type));
+ const uint32_t data_offset = mirror::Array::DataOffset(element_size).Uint32Value();
+
+ if (src_pos.IsConstant()) {
+ int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
+ __ leal(src_base, Address(src, element_size * constant + data_offset));
+ } else {
+ __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), scale_factor, data_offset));
+ }
+
+ if (dst_pos.IsConstant()) {
+ int32_t constant = dst_pos.GetConstant()->AsIntConstant()->GetValue();
+ __ leal(dst_base, Address(dst, element_size * constant + data_offset));
+ } else {
+ __ leal(dst_base, Address(dst, dst_pos.AsRegister<CpuRegister>(), scale_factor, data_offset));
+ }
+
+ if (copy_length.IsConstant()) {
+ int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
+ __ leal(src_end, Address(src_base, element_size * constant));
+ } else {
+ __ leal(src_end, Address(src_base, copy_length.AsRegister<CpuRegister>(), scale_factor, 0));
+ }
+}
+
void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) {
// The only read barrier implementation supporting the
// SystemArrayCopy intrinsic is the Baker-style read barriers.
@@ -1366,30 +1407,13 @@
__ j(kNotEqual, intrinsic_slow_path->GetEntryLabel());
}
- // Compute base source address, base destination address, and end source address.
+ const Primitive::Type type = Primitive::kPrimNot;
+ const int32_t element_size = Primitive::ComponentSize(type);
- int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
- uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
- if (src_pos.IsConstant()) {
- int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp1, Address(src, element_size * constant + offset));
- } else {
- __ leal(temp1, Address(src, src_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset));
- }
-
- if (dest_pos.IsConstant()) {
- int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp2, Address(dest, element_size * constant + offset));
- } else {
- __ leal(temp2, Address(dest, dest_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset));
- }
-
- if (length.IsConstant()) {
- int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
- __ leal(temp3, Address(temp1, element_size * constant));
- } else {
- __ leal(temp3, Address(temp1, length.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, 0));
- }
+ // Compute base source address, base destination address, and end
+ // source address in `temp1`, `temp2` and `temp3` respectively.
+ GenSystemArrayCopyAddresses(
+ GetAssembler(), type, src, src_pos, dest, dest_pos, length, temp1, temp2, temp3);
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// SystemArrayCopy implementation for Baker read barriers (see
@@ -1474,11 +1498,7 @@
}
// We only need one card marking on the destination array.
- codegen_->MarkGCCard(temp1,
- temp2,
- dest,
- CpuRegister(kNoRegister),
- /* value_can_be_null */ false);
+ codegen_->MarkGCCard(temp1, temp2, dest, CpuRegister(kNoRegister), /* value_can_be_null */ false);
__ Bind(intrinsic_slow_path->GetExitLabel());
}
