compiler/dex/quick/arm64/fp_arm64.cc - platform/art - Git at Google

 /*
  * Copyright (C) 2011 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *      http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include "codegen_arm64.h"

 #include "arm64_lir.h"
 #include "base/logging.h"
 #include "dex/mir_graph.h"
 #include "dex/quick/mir_to_lir-inl.h"
 #include "utils.h"

 namespace art {

 void Arm64Mir2Lir::GenArithOpFloat(Instruction::Code opcode, RegLocation rl_dest,
                                    RegLocation rl_src1, RegLocation rl_src2) {
   int op = kA64Brk1d;
   RegLocation rl_result;

   switch (opcode) {
     case Instruction::ADD_FLOAT_2ADDR:
     case Instruction::ADD_FLOAT:
       op = kA64Fadd3fff;
       break;
     case Instruction::SUB_FLOAT_2ADDR:
     case Instruction::SUB_FLOAT:
       op = kA64Fsub3fff;
       break;
     case Instruction::DIV_FLOAT_2ADDR:
     case Instruction::DIV_FLOAT:
       op = kA64Fdiv3fff;
       break;
     case Instruction::MUL_FLOAT_2ADDR:
     case Instruction::MUL_FLOAT:
       op = kA64Fmul3fff;
       break;
     case Instruction::REM_FLOAT_2ADDR:
     case Instruction::REM_FLOAT:
       FlushAllRegs();   // Send everything to home location
       CallRuntimeHelperRegLocationRegLocation(kQuickFmodf, rl_src1, rl_src2, false);
       rl_result = GetReturn(kFPReg);
       StoreValue(rl_dest, rl_result);
       return;
     case Instruction::NEG_FLOAT:
       GenNegFloat(rl_dest, rl_src1);
       return;
     default:
       LOG(FATAL) << "Unexpected opcode: " << opcode;
   }
   rl_src1 = LoadValue(rl_src1, kFPReg);
   rl_src2 = LoadValue(rl_src2, kFPReg);
   rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR3(op, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   StoreValue(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenArithOpDouble(Instruction::Code opcode,
                                     RegLocation rl_dest, RegLocation rl_src1, RegLocation rl_src2) {
   int op = kA64Brk1d;
   RegLocation rl_result;

   switch (opcode) {
     case Instruction::ADD_DOUBLE_2ADDR:
     case Instruction::ADD_DOUBLE:
       op = kA64Fadd3fff;
       break;
     case Instruction::SUB_DOUBLE_2ADDR:
     case Instruction::SUB_DOUBLE:
       op = kA64Fsub3fff;
       break;
     case Instruction::DIV_DOUBLE_2ADDR:
     case Instruction::DIV_DOUBLE:
       op = kA64Fdiv3fff;
       break;
     case Instruction::MUL_DOUBLE_2ADDR:
     case Instruction::MUL_DOUBLE:
       op = kA64Fmul3fff;
       break;
     case Instruction::REM_DOUBLE_2ADDR:
     case Instruction::REM_DOUBLE:
       FlushAllRegs();   // Send everything to home location
       {
         RegStorage r_tgt = CallHelperSetup(kQuickFmod);
         LoadValueDirectWideFixed(rl_src1, rs_d0);
         LoadValueDirectWideFixed(rl_src2, rs_d1);
         ClobberCallerSave();
         CallHelper(r_tgt, kQuickFmod, false);
       }
       rl_result = GetReturnWide(kFPReg);
       StoreValueWide(rl_dest, rl_result);
       return;
     case Instruction::NEG_DOUBLE:
       GenNegDouble(rl_dest, rl_src1);
       return;
     default:
       LOG(FATAL) << "Unexpected opcode: " << opcode;
   }

   rl_src1 = LoadValueWide(rl_src1, kFPReg);
   DCHECK(rl_src1.wide);
   rl_src2 = LoadValueWide(rl_src2, kFPReg);
   DCHECK(rl_src2.wide);
   rl_result = EvalLoc(rl_dest, kFPReg, true);
   DCHECK(rl_dest.wide);
   DCHECK(rl_result.wide);
   NewLIR3(WIDE(op), rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenMultiplyByConstantFloat(RegLocation rl_dest, RegLocation rl_src1,
                                               int32_t constant) {
   RegLocation rl_result;
   RegStorage r_tmp = AllocTempSingle();
   LoadConstantNoClobber(r_tmp, constant);
   rl_src1 = LoadValue(rl_src1, kFPReg);
   rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR3(kA64Fmul3fff, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), r_tmp.GetReg());
   StoreValue(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenMultiplyByConstantDouble(RegLocation rl_dest, RegLocation rl_src1,
                                                int64_t constant) {
   RegLocation rl_result;
   RegStorage r_tmp = AllocTempDouble();
   DCHECK(r_tmp.IsDouble());
   LoadConstantWide(r_tmp, constant);
   rl_src1 = LoadValueWide(rl_src1, kFPReg);
   DCHECK(rl_src1.wide);
   rl_result = EvalLocWide(rl_dest, kFPReg, true);
   DCHECK(rl_dest.wide);
   DCHECK(rl_result.wide);
   NewLIR3(WIDE(kA64Fmul3fff), rl_result.reg.GetReg(), rl_src1.reg.GetReg(), r_tmp.GetReg());
   StoreValueWide(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenConversion(Instruction::Code opcode,
                                  RegLocation rl_dest, RegLocation rl_src) {
   int op = kA64Brk1d;
   RegLocation rl_result;
   RegisterClass src_reg_class = kInvalidRegClass;
   RegisterClass dst_reg_class = kInvalidRegClass;

   switch (opcode) {
     case Instruction::INT_TO_FLOAT:
       op = kA64Scvtf2fw;
       src_reg_class = kCoreReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::FLOAT_TO_INT:
       op = kA64Fcvtzs2wf;
       src_reg_class = kFPReg;
       dst_reg_class = kCoreReg;
       break;
     case Instruction::DOUBLE_TO_FLOAT:
       op = kA64Fcvt2sS;
       src_reg_class = kFPReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::FLOAT_TO_DOUBLE:
       op = kA64Fcvt2Ss;
       src_reg_class = kFPReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::INT_TO_DOUBLE:
       op = WIDE(kA64Scvtf2fw);
       src_reg_class = kCoreReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::DOUBLE_TO_INT:
       op = WIDE(kA64Fcvtzs2wf);
       src_reg_class = kFPReg;
       dst_reg_class = kCoreReg;
       break;
     case Instruction::LONG_TO_DOUBLE:
       op = WIDE(kA64Scvtf2fx);
       src_reg_class = kCoreReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::FLOAT_TO_LONG:
       op = kA64Fcvtzs2xf;
       src_reg_class = kFPReg;
       dst_reg_class = kCoreReg;
       break;
     case Instruction::LONG_TO_FLOAT:
       op = kA64Scvtf2fx;
       src_reg_class = kCoreReg;
       dst_reg_class = kFPReg;
       break;
     case Instruction::DOUBLE_TO_LONG:
       op = WIDE(kA64Fcvtzs2xf);
       src_reg_class = kFPReg;
       dst_reg_class = kCoreReg;
       break;
     default:
       LOG(FATAL) << "Unexpected opcode: " << opcode;
   }

   DCHECK_NE(src_reg_class, kInvalidRegClass);
   DCHECK_NE(dst_reg_class, kInvalidRegClass);
   DCHECK_NE(op, kA64Brk1d);

   if (rl_src.wide) {
     rl_src = LoadValueWide(rl_src, src_reg_class);
   } else {
     rl_src = LoadValue(rl_src, src_reg_class);
   }

   rl_result = EvalLoc(rl_dest, dst_reg_class, true);
   NewLIR2(op, rl_result.reg.GetReg(), rl_src.reg.GetReg());

   if (rl_dest.wide) {
     StoreValueWide(rl_dest, rl_result);
   } else {
     StoreValue(rl_dest, rl_result);
   }
 }

 void Arm64Mir2Lir::GenFusedFPCmpBranch(BasicBlock* bb, MIR* mir, bool gt_bias,
                                      bool is_double) {
   LIR* target = &block_label_list_[bb->taken];
   RegLocation rl_src1;
   RegLocation rl_src2;
   if (is_double) {
     rl_src1 = mir_graph_->GetSrcWide(mir, 0);
     rl_src2 = mir_graph_->GetSrcWide(mir, 2);
     rl_src1 = LoadValueWide(rl_src1, kFPReg);
     rl_src2 = LoadValueWide(rl_src2, kFPReg);
     NewLIR2(WIDE(kA64Fcmp2ff), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   } else {
     rl_src1 = mir_graph_->GetSrc(mir, 0);
     rl_src2 = mir_graph_->GetSrc(mir, 1);
     rl_src1 = LoadValue(rl_src1, kFPReg);
     rl_src2 = LoadValue(rl_src2, kFPReg);
     NewLIR2(kA64Fcmp2ff, rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   }
   ConditionCode ccode = mir->meta.ccode;
   switch (ccode) {
     case kCondEq:
     case kCondNe:
       break;
     case kCondLt:
       if (gt_bias) {
         ccode = kCondMi;
       }
       break;
     case kCondLe:
       if (gt_bias) {
         ccode = kCondLs;
       }
       break;
     case kCondGt:
       if (gt_bias) {
         ccode = kCondHi;
       }
       break;
     case kCondGe:
       if (gt_bias) {
         ccode = kCondUge;
       }
       break;
     default:
       LOG(FATAL) << "Unexpected ccode: " << ccode;
   }
   OpCondBranch(ccode, target);
 }


 void Arm64Mir2Lir::GenCmpFP(Instruction::Code opcode, RegLocation rl_dest,
                             RegLocation rl_src1, RegLocation rl_src2) {
   bool is_double = false;
   int default_result = -1;
   RegLocation rl_result;

   switch (opcode) {
     case Instruction::CMPL_FLOAT:
       is_double = false;
       default_result = -1;
       break;
     case Instruction::CMPG_FLOAT:
       is_double = false;
       default_result = 1;
       break;
     case Instruction::CMPL_DOUBLE:
       is_double = true;
       default_result = -1;
       break;
     case Instruction::CMPG_DOUBLE:
       is_double = true;
       default_result = 1;
       break;
     default:
       LOG(FATAL) << "Unexpected opcode: " << opcode;
   }
   if (is_double) {
     rl_src1 = LoadValueWide(rl_src1, kFPReg);
     rl_src2 = LoadValueWide(rl_src2, kFPReg);
     // In case result vreg is also a src vreg, break association to avoid useless copy by EvalLoc()
     ClobberSReg(rl_dest.s_reg_low);
     rl_result = EvalLoc(rl_dest, kCoreReg, true);
     LoadConstant(rl_result.reg, default_result);
     NewLIR2(WIDE(kA64Fcmp2ff), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   } else {
     rl_src1 = LoadValue(rl_src1, kFPReg);
     rl_src2 = LoadValue(rl_src2, kFPReg);
     // In case result vreg is also a srcvreg, break association to avoid useless copy by EvalLoc()
     ClobberSReg(rl_dest.s_reg_low);
     rl_result = EvalLoc(rl_dest, kCoreReg, true);
     LoadConstant(rl_result.reg, default_result);
     NewLIR2(kA64Fcmp2ff, rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   }
   DCHECK(!rl_result.reg.IsFloat());

   // TODO(Arm64): should we rather do this?
   // csinc wD, wzr, wzr, eq
   // csneg wD, wD, wD, le
   // (which requires 2 instructions rather than 3)

   // Rd = if cond then Rd else -Rd.
   NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_result.reg.GetReg(),
           rl_result.reg.GetReg(), (default_result == 1) ? kArmCondPl : kArmCondLe);
   NewLIR4(kA64Csel4rrrc, rl_result.reg.GetReg(), rwzr, rl_result.reg.GetReg(),
           kArmCondEq);
   StoreValue(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenNegFloat(RegLocation rl_dest, RegLocation rl_src) {
   RegLocation rl_result;
   rl_src = LoadValue(rl_src, kFPReg);
   rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(kA64Fneg2ff, rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValue(rl_dest, rl_result);
 }

 void Arm64Mir2Lir::GenNegDouble(RegLocation rl_dest, RegLocation rl_src) {
   RegLocation rl_result;
   rl_src = LoadValueWide(rl_src, kFPReg);
   rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(WIDE(kA64Fneg2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
 }

 static RegisterClass RegClassForAbsFP(RegLocation rl_src, RegLocation rl_dest) {
   // If src is in a core reg or, unlikely, dest has been promoted to a core reg, use core reg.
   if ((rl_src.location == kLocPhysReg && !rl_src.reg.IsFloat()) ||
       (rl_dest.location == kLocPhysReg && !rl_dest.reg.IsFloat())) {
     return kCoreReg;
   }
   // If src is in an fp reg or dest has been promoted to an fp reg, use fp reg.
   if (rl_src.location == kLocPhysReg || rl_dest.location == kLocPhysReg) {
     return kFPReg;
   }
   // With both src and dest in the stack frame we have to perform load+abs+store. Whether this
   // is faster using a core reg or fp reg depends on the particular CPU. For example, on A53
   // it's faster using core reg while on A57 it's faster with fp reg, the difference being
   // bigger on the A53. Without further investigation and testing we prefer core register.
   // (If the result is subsequently used in another fp operation, the dalvik reg will probably
   // get promoted and that should be handled by the cases above.)
   return kCoreReg;
 }

 bool Arm64Mir2Lir::GenInlinedAbsFloat(CallInfo* info) {
   if (info->result.location == kLocInvalid) {
     return true;  // Result is unused: inlining successful, no code generated.
   }
   RegLocation rl_dest = info->result;
   RegLocation rl_src = UpdateLoc(info->args[0]);
   RegisterClass reg_class = RegClassForAbsFP(rl_src, rl_dest);
   rl_src = LoadValue(rl_src, reg_class);
   RegLocation rl_result = EvalLoc(rl_dest, reg_class, true);
   if (reg_class == kFPReg) {
     NewLIR2(kA64Fabs2ff, rl_result.reg.GetReg(), rl_src.reg.GetReg());
   } else {
     // Clear the sign bit in an integer register.
     OpRegRegImm(kOpAnd, rl_result.reg, rl_src.reg, 0x7fffffff);
   }
   StoreValue(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedAbsDouble(CallInfo* info) {
   if (info->result.location == kLocInvalid) {
     return true;  // Result is unused: inlining successful, no code generated.
   }
   RegLocation rl_dest = info->result;
   RegLocation rl_src = UpdateLocWide(info->args[0]);
   RegisterClass reg_class = RegClassForAbsFP(rl_src, rl_dest);
   rl_src = LoadValueWide(rl_src, reg_class);
   RegLocation rl_result = EvalLoc(rl_dest, reg_class, true);
   if (reg_class == kFPReg) {
     NewLIR2(WIDE(kA64Fabs2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   } else {
     // Clear the sign bit in an integer register.
     OpRegRegImm64(kOpAnd, rl_result.reg, rl_src.reg, 0x7fffffffffffffff);
   }
   StoreValueWide(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedSqrt(CallInfo* info) {
   RegLocation rl_src = info->args[0];
   RegLocation rl_dest = InlineTargetWide(info);  // double place for result
   rl_src = LoadValueWide(rl_src, kFPReg);
   RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(WIDE(kA64Fsqrt2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedCeil(CallInfo* info) {
   RegLocation rl_src = info->args[0];
   RegLocation rl_dest = InlineTargetWide(info);
   rl_src = LoadValueWide(rl_src, kFPReg);
   RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(WIDE(kA64Frintp2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedFloor(CallInfo* info) {
   RegLocation rl_src = info->args[0];
   RegLocation rl_dest = InlineTargetWide(info);
   rl_src = LoadValueWide(rl_src, kFPReg);
   RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(WIDE(kA64Frintm2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedRint(CallInfo* info) {
   RegLocation rl_src = info->args[0];
   RegLocation rl_dest = InlineTargetWide(info);
   rl_src = LoadValueWide(rl_src, kFPReg);
   RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR2(WIDE(kA64Frintn2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
   StoreValueWide(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedRound(CallInfo* info, bool is_double) {
   int32_t encoded_imm = EncodeImmSingle(bit_cast<float, uint32_t>(0.5f));
   A64Opcode wide = (is_double) ? WIDE(0) : UNWIDE(0);
   RegLocation rl_src = info->args[0];
   RegLocation rl_dest = (is_double) ? InlineTargetWide(info) : InlineTarget(info);
   rl_src = (is_double) ? LoadValueWide(rl_src, kFPReg) : LoadValue(rl_src, kFPReg);
   RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
   RegStorage r_imm_point5 = (is_double) ? AllocTempDouble() : AllocTempSingle();
   RegStorage r_tmp = (is_double) ? AllocTempDouble() : AllocTempSingle();
   // 0.5f and 0.5d are encoded in the same way.
   NewLIR2(kA64Fmov2fI | wide, r_imm_point5.GetReg(), encoded_imm);
   NewLIR3(kA64Fadd3fff | wide, r_tmp.GetReg(), rl_src.reg.GetReg(), r_imm_point5.GetReg());
   NewLIR2((is_double) ? kA64Fcvtms2xS : kA64Fcvtms2ws, rl_result.reg.GetReg(), r_tmp.GetReg());
   (is_double) ? StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result);
   return true;
 }

 bool Arm64Mir2Lir::GenInlinedMinMaxFP(CallInfo* info, bool is_min, bool is_double) {
   DCHECK_EQ(cu_->instruction_set, kArm64);
   int op = (is_min) ? kA64Fmin3fff : kA64Fmax3fff;
   A64Opcode wide = (is_double) ? WIDE(0) : UNWIDE(0);
   RegLocation rl_src1 = info->args[0];
   RegLocation rl_src2 = (is_double) ? info->args[2] : info->args[1];
   rl_src1 = (is_double) ? LoadValueWide(rl_src1, kFPReg) : LoadValue(rl_src1, kFPReg);
   rl_src2 = (is_double) ? LoadValueWide(rl_src2, kFPReg) : LoadValue(rl_src2, kFPReg);
   RegLocation rl_dest = (is_double) ? InlineTargetWide(info) : InlineTarget(info);
   RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
   NewLIR3(op | wide, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
   (is_double) ?  StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result);
   return true;
 }

 }  // namespace art
	/*
	* Copyright (C) 2011 The Android Open Source Project
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include "codegen_arm64.h"

	#include "arm64_lir.h"
	#include "base/logging.h"
	#include "dex/mir_graph.h"
	#include "dex/quick/mir_to_lir-inl.h"
	#include "utils.h"

	namespace art {

	void Arm64Mir2Lir::GenArithOpFloat(Instruction::Code opcode, RegLocation rl_dest,
	RegLocation rl_src1, RegLocation rl_src2) {
	int op = kA64Brk1d;
	RegLocation rl_result;

	switch (opcode) {
	case Instruction::ADD_FLOAT_2ADDR:
	case Instruction::ADD_FLOAT:
	op = kA64Fadd3fff;
	break;
	case Instruction::SUB_FLOAT_2ADDR:
	case Instruction::SUB_FLOAT:
	op = kA64Fsub3fff;
	break;
	case Instruction::DIV_FLOAT_2ADDR:
	case Instruction::DIV_FLOAT:
	op = kA64Fdiv3fff;
	break;
	case Instruction::MUL_FLOAT_2ADDR:
	case Instruction::MUL_FLOAT:
	op = kA64Fmul3fff;
	break;
	case Instruction::REM_FLOAT_2ADDR:
	case Instruction::REM_FLOAT:
	FlushAllRegs(); // Send everything to home location
	CallRuntimeHelperRegLocationRegLocation(kQuickFmodf, rl_src1, rl_src2, false);
	rl_result = GetReturn(kFPReg);
	StoreValue(rl_dest, rl_result);
	return;
	case Instruction::NEG_FLOAT:
	GenNegFloat(rl_dest, rl_src1);
	return;
	default:
	LOG(FATAL) << "Unexpected opcode: " << opcode;
	}
	rl_src1 = LoadValue(rl_src1, kFPReg);
	rl_src2 = LoadValue(rl_src2, kFPReg);
	rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR3(op, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	StoreValue(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenArithOpDouble(Instruction::Code opcode,
	RegLocation rl_dest, RegLocation rl_src1, RegLocation rl_src2) {
	int op = kA64Brk1d;
	RegLocation rl_result;

	switch (opcode) {
	case Instruction::ADD_DOUBLE_2ADDR:
	case Instruction::ADD_DOUBLE:
	op = kA64Fadd3fff;
	break;
	case Instruction::SUB_DOUBLE_2ADDR:
	case Instruction::SUB_DOUBLE:
	op = kA64Fsub3fff;
	break;
	case Instruction::DIV_DOUBLE_2ADDR:
	case Instruction::DIV_DOUBLE:
	op = kA64Fdiv3fff;
	break;
	case Instruction::MUL_DOUBLE_2ADDR:
	case Instruction::MUL_DOUBLE:
	op = kA64Fmul3fff;
	break;
	case Instruction::REM_DOUBLE_2ADDR:
	case Instruction::REM_DOUBLE:
	FlushAllRegs(); // Send everything to home location
	{
	RegStorage r_tgt = CallHelperSetup(kQuickFmod);
	LoadValueDirectWideFixed(rl_src1, rs_d0);
	LoadValueDirectWideFixed(rl_src2, rs_d1);
	ClobberCallerSave();
	CallHelper(r_tgt, kQuickFmod, false);
	}
	rl_result = GetReturnWide(kFPReg);
	StoreValueWide(rl_dest, rl_result);
	return;
	case Instruction::NEG_DOUBLE:
	GenNegDouble(rl_dest, rl_src1);
	return;
	default:
	LOG(FATAL) << "Unexpected opcode: " << opcode;
	}

	rl_src1 = LoadValueWide(rl_src1, kFPReg);
	DCHECK(rl_src1.wide);
	rl_src2 = LoadValueWide(rl_src2, kFPReg);
	DCHECK(rl_src2.wide);
	rl_result = EvalLoc(rl_dest, kFPReg, true);
	DCHECK(rl_dest.wide);
	DCHECK(rl_result.wide);
	NewLIR3(WIDE(op), rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenMultiplyByConstantFloat(RegLocation rl_dest, RegLocation rl_src1,
	int32_t constant) {
	RegLocation rl_result;
	RegStorage r_tmp = AllocTempSingle();
	LoadConstantNoClobber(r_tmp, constant);
	rl_src1 = LoadValue(rl_src1, kFPReg);
	rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR3(kA64Fmul3fff, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), r_tmp.GetReg());
	StoreValue(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenMultiplyByConstantDouble(RegLocation rl_dest, RegLocation rl_src1,
	int64_t constant) {
	RegLocation rl_result;
	RegStorage r_tmp = AllocTempDouble();
	DCHECK(r_tmp.IsDouble());
	LoadConstantWide(r_tmp, constant);
	rl_src1 = LoadValueWide(rl_src1, kFPReg);
	DCHECK(rl_src1.wide);
	rl_result = EvalLocWide(rl_dest, kFPReg, true);
	DCHECK(rl_dest.wide);
	DCHECK(rl_result.wide);
	NewLIR3(WIDE(kA64Fmul3fff), rl_result.reg.GetReg(), rl_src1.reg.GetReg(), r_tmp.GetReg());
	StoreValueWide(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenConversion(Instruction::Code opcode,
	RegLocation rl_dest, RegLocation rl_src) {
	int op = kA64Brk1d;
	RegLocation rl_result;
	RegisterClass src_reg_class = kInvalidRegClass;
	RegisterClass dst_reg_class = kInvalidRegClass;

	switch (opcode) {
	case Instruction::INT_TO_FLOAT:
	op = kA64Scvtf2fw;
	src_reg_class = kCoreReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::FLOAT_TO_INT:
	op = kA64Fcvtzs2wf;
	src_reg_class = kFPReg;
	dst_reg_class = kCoreReg;
	break;
	case Instruction::DOUBLE_TO_FLOAT:
	op = kA64Fcvt2sS;
	src_reg_class = kFPReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::FLOAT_TO_DOUBLE:
	op = kA64Fcvt2Ss;
	src_reg_class = kFPReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::INT_TO_DOUBLE:
	op = WIDE(kA64Scvtf2fw);
	src_reg_class = kCoreReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::DOUBLE_TO_INT:
	op = WIDE(kA64Fcvtzs2wf);
	src_reg_class = kFPReg;
	dst_reg_class = kCoreReg;
	break;
	case Instruction::LONG_TO_DOUBLE:
	op = WIDE(kA64Scvtf2fx);
	src_reg_class = kCoreReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::FLOAT_TO_LONG:
	op = kA64Fcvtzs2xf;
	src_reg_class = kFPReg;
	dst_reg_class = kCoreReg;
	break;
	case Instruction::LONG_TO_FLOAT:
	op = kA64Scvtf2fx;
	src_reg_class = kCoreReg;
	dst_reg_class = kFPReg;
	break;
	case Instruction::DOUBLE_TO_LONG:
	op = WIDE(kA64Fcvtzs2xf);
	src_reg_class = kFPReg;
	dst_reg_class = kCoreReg;
	break;
	default:
	LOG(FATAL) << "Unexpected opcode: " << opcode;
	}

	DCHECK_NE(src_reg_class, kInvalidRegClass);
	DCHECK_NE(dst_reg_class, kInvalidRegClass);
	DCHECK_NE(op, kA64Brk1d);

	if (rl_src.wide) {
	rl_src = LoadValueWide(rl_src, src_reg_class);
	} else {
	rl_src = LoadValue(rl_src, src_reg_class);
	}

	rl_result = EvalLoc(rl_dest, dst_reg_class, true);
	NewLIR2(op, rl_result.reg.GetReg(), rl_src.reg.GetReg());

	if (rl_dest.wide) {
	StoreValueWide(rl_dest, rl_result);
	} else {
	StoreValue(rl_dest, rl_result);
	}
	}

	void Arm64Mir2Lir::GenFusedFPCmpBranch(BasicBlock* bb, MIR* mir, bool gt_bias,
	bool is_double) {
	LIR* target = &block_label_list_[bb->taken];
	RegLocation rl_src1;
	RegLocation rl_src2;
	if (is_double) {
	rl_src1 = mir_graph_->GetSrcWide(mir, 0);
	rl_src2 = mir_graph_->GetSrcWide(mir, 2);
	rl_src1 = LoadValueWide(rl_src1, kFPReg);
	rl_src2 = LoadValueWide(rl_src2, kFPReg);
	NewLIR2(WIDE(kA64Fcmp2ff), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	} else {
	rl_src1 = mir_graph_->GetSrc(mir, 0);
	rl_src2 = mir_graph_->GetSrc(mir, 1);
	rl_src1 = LoadValue(rl_src1, kFPReg);
	rl_src2 = LoadValue(rl_src2, kFPReg);
	NewLIR2(kA64Fcmp2ff, rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	}
	ConditionCode ccode = mir->meta.ccode;
	switch (ccode) {
	case kCondEq:
	case kCondNe:
	break;
	case kCondLt:
	if (gt_bias) {
	ccode = kCondMi;
	}
	break;
	case kCondLe:
	if (gt_bias) {
	ccode = kCondLs;
	}
	break;
	case kCondGt:
	if (gt_bias) {
	ccode = kCondHi;
	}
	break;
	case kCondGe:
	if (gt_bias) {
	ccode = kCondUge;
	}
	break;
	default:
	LOG(FATAL) << "Unexpected ccode: " << ccode;
	}
	OpCondBranch(ccode, target);
	}


	void Arm64Mir2Lir::GenCmpFP(Instruction::Code opcode, RegLocation rl_dest,
	RegLocation rl_src1, RegLocation rl_src2) {
	bool is_double = false;
	int default_result = -1;
	RegLocation rl_result;

	switch (opcode) {
	case Instruction::CMPL_FLOAT:
	is_double = false;
	default_result = -1;
	break;
	case Instruction::CMPG_FLOAT:
	is_double = false;
	default_result = 1;
	break;
	case Instruction::CMPL_DOUBLE:
	is_double = true;
	default_result = -1;
	break;
	case Instruction::CMPG_DOUBLE:
	is_double = true;
	default_result = 1;
	break;
	default:
	LOG(FATAL) << "Unexpected opcode: " << opcode;
	}
	if (is_double) {
	rl_src1 = LoadValueWide(rl_src1, kFPReg);
	rl_src2 = LoadValueWide(rl_src2, kFPReg);
	// In case result vreg is also a src vreg, break association to avoid useless copy by EvalLoc()
	ClobberSReg(rl_dest.s_reg_low);
	rl_result = EvalLoc(rl_dest, kCoreReg, true);
	LoadConstant(rl_result.reg, default_result);
	NewLIR2(WIDE(kA64Fcmp2ff), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	} else {
	rl_src1 = LoadValue(rl_src1, kFPReg);
	rl_src2 = LoadValue(rl_src2, kFPReg);
	// In case result vreg is also a srcvreg, break association to avoid useless copy by EvalLoc()
	ClobberSReg(rl_dest.s_reg_low);
	rl_result = EvalLoc(rl_dest, kCoreReg, true);
	LoadConstant(rl_result.reg, default_result);
	NewLIR2(kA64Fcmp2ff, rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	}
	DCHECK(!rl_result.reg.IsFloat());

	// TODO(Arm64): should we rather do this?
	// csinc wD, wzr, wzr, eq
	// csneg wD, wD, wD, le
	// (which requires 2 instructions rather than 3)

	// Rd = if cond then Rd else -Rd.
	NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_result.reg.GetReg(),
	rl_result.reg.GetReg(), (default_result == 1) ? kArmCondPl : kArmCondLe);
	NewLIR4(kA64Csel4rrrc, rl_result.reg.GetReg(), rwzr, rl_result.reg.GetReg(),
	kArmCondEq);
	StoreValue(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenNegFloat(RegLocation rl_dest, RegLocation rl_src) {
	RegLocation rl_result;
	rl_src = LoadValue(rl_src, kFPReg);
	rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(kA64Fneg2ff, rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValue(rl_dest, rl_result);
	}

	void Arm64Mir2Lir::GenNegDouble(RegLocation rl_dest, RegLocation rl_src) {
	RegLocation rl_result;
	rl_src = LoadValueWide(rl_src, kFPReg);
	rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(WIDE(kA64Fneg2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	}

	static RegisterClass RegClassForAbsFP(RegLocation rl_src, RegLocation rl_dest) {
	// If src is in a core reg or, unlikely, dest has been promoted to a core reg, use core reg.
	if ((rl_src.location == kLocPhysReg && !rl_src.reg.IsFloat()) \|\|
	(rl_dest.location == kLocPhysReg && !rl_dest.reg.IsFloat())) {
	return kCoreReg;
	}
	// If src is in an fp reg or dest has been promoted to an fp reg, use fp reg.
	if (rl_src.location == kLocPhysReg \|\| rl_dest.location == kLocPhysReg) {
	return kFPReg;
	}
	// With both src and dest in the stack frame we have to perform load+abs+store. Whether this
	// is faster using a core reg or fp reg depends on the particular CPU. For example, on A53
	// it's faster using core reg while on A57 it's faster with fp reg, the difference being
	// bigger on the A53. Without further investigation and testing we prefer core register.
	// (If the result is subsequently used in another fp operation, the dalvik reg will probably
	// get promoted and that should be handled by the cases above.)
	return kCoreReg;
	}

	bool Arm64Mir2Lir::GenInlinedAbsFloat(CallInfo* info) {
	if (info->result.location == kLocInvalid) {
	return true; // Result is unused: inlining successful, no code generated.
	}
	RegLocation rl_dest = info->result;
	RegLocation rl_src = UpdateLoc(info->args[0]);
	RegisterClass reg_class = RegClassForAbsFP(rl_src, rl_dest);
	rl_src = LoadValue(rl_src, reg_class);
	RegLocation rl_result = EvalLoc(rl_dest, reg_class, true);
	if (reg_class == kFPReg) {
	NewLIR2(kA64Fabs2ff, rl_result.reg.GetReg(), rl_src.reg.GetReg());
	} else {
	// Clear the sign bit in an integer register.
	OpRegRegImm(kOpAnd, rl_result.reg, rl_src.reg, 0x7fffffff);
	}
	StoreValue(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedAbsDouble(CallInfo* info) {
	if (info->result.location == kLocInvalid) {
	return true; // Result is unused: inlining successful, no code generated.
	}
	RegLocation rl_dest = info->result;
	RegLocation rl_src = UpdateLocWide(info->args[0]);
	RegisterClass reg_class = RegClassForAbsFP(rl_src, rl_dest);
	rl_src = LoadValueWide(rl_src, reg_class);
	RegLocation rl_result = EvalLoc(rl_dest, reg_class, true);
	if (reg_class == kFPReg) {
	NewLIR2(WIDE(kA64Fabs2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	} else {
	// Clear the sign bit in an integer register.
	OpRegRegImm64(kOpAnd, rl_result.reg, rl_src.reg, 0x7fffffffffffffff);
	}
	StoreValueWide(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedSqrt(CallInfo* info) {
	RegLocation rl_src = info->args[0];
	RegLocation rl_dest = InlineTargetWide(info); // double place for result
	rl_src = LoadValueWide(rl_src, kFPReg);
	RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(WIDE(kA64Fsqrt2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedCeil(CallInfo* info) {
	RegLocation rl_src = info->args[0];
	RegLocation rl_dest = InlineTargetWide(info);
	rl_src = LoadValueWide(rl_src, kFPReg);
	RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(WIDE(kA64Frintp2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedFloor(CallInfo* info) {
	RegLocation rl_src = info->args[0];
	RegLocation rl_dest = InlineTargetWide(info);
	rl_src = LoadValueWide(rl_src, kFPReg);
	RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(WIDE(kA64Frintm2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedRint(CallInfo* info) {
	RegLocation rl_src = info->args[0];
	RegLocation rl_dest = InlineTargetWide(info);
	rl_src = LoadValueWide(rl_src, kFPReg);
	RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR2(WIDE(kA64Frintn2ff), rl_result.reg.GetReg(), rl_src.reg.GetReg());
	StoreValueWide(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedRound(CallInfo* info, bool is_double) {
	int32_t encoded_imm = EncodeImmSingle(bit_cast<float, uint32_t>(0.5f));
	A64Opcode wide = (is_double) ? WIDE(0) : UNWIDE(0);
	RegLocation rl_src = info->args[0];
	RegLocation rl_dest = (is_double) ? InlineTargetWide(info) : InlineTarget(info);
	rl_src = (is_double) ? LoadValueWide(rl_src, kFPReg) : LoadValue(rl_src, kFPReg);
	RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true);
	RegStorage r_imm_point5 = (is_double) ? AllocTempDouble() : AllocTempSingle();
	RegStorage r_tmp = (is_double) ? AllocTempDouble() : AllocTempSingle();
	// 0.5f and 0.5d are encoded in the same way.
	NewLIR2(kA64Fmov2fI \| wide, r_imm_point5.GetReg(), encoded_imm);
	NewLIR3(kA64Fadd3fff \| wide, r_tmp.GetReg(), rl_src.reg.GetReg(), r_imm_point5.GetReg());
	NewLIR2((is_double) ? kA64Fcvtms2xS : kA64Fcvtms2ws, rl_result.reg.GetReg(), r_tmp.GetReg());
	(is_double) ? StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result);
	return true;
	}

	bool Arm64Mir2Lir::GenInlinedMinMaxFP(CallInfo* info, bool is_min, bool is_double) {
	DCHECK_EQ(cu_->instruction_set, kArm64);
	int op = (is_min) ? kA64Fmin3fff : kA64Fmax3fff;
	A64Opcode wide = (is_double) ? WIDE(0) : UNWIDE(0);
	RegLocation rl_src1 = info->args[0];
	RegLocation rl_src2 = (is_double) ? info->args[2] : info->args[1];
	rl_src1 = (is_double) ? LoadValueWide(rl_src1, kFPReg) : LoadValue(rl_src1, kFPReg);
	rl_src2 = (is_double) ? LoadValueWide(rl_src2, kFPReg) : LoadValue(rl_src2, kFPReg);
	RegLocation rl_dest = (is_double) ? InlineTargetWide(info) : InlineTarget(info);
	RegLocation rl_result = EvalLoc(rl_dest, kFPReg, true);
	NewLIR3(op \| wide, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), rl_src2.reg.GetReg());
	(is_double) ? StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result);
	return true;
	}

	} // namespace art