| /* |
| * Copyright (C) 2023 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "code_generator_riscv64.h" |
| |
| #include "android-base/logging.h" |
| #include "android-base/macros.h" |
| #include "arch/riscv64/jni_frame_riscv64.h" |
| #include "arch/riscv64/registers_riscv64.h" |
| #include "base/arena_containers.h" |
| #include "base/macros.h" |
| #include "class_root-inl.h" |
| #include "code_generator_utils.h" |
| #include "dwarf/register.h" |
| #include "gc/heap.h" |
| #include "gc/space/image_space.h" |
| #include "heap_poisoning.h" |
| #include "intrinsics_list.h" |
| #include "intrinsics_riscv64.h" |
| #include "jit/profiling_info.h" |
| #include "linker/linker_patch.h" |
| #include "mirror/class-inl.h" |
| #include "optimizing/nodes.h" |
| #include "optimizing/profiling_info_builder.h" |
| #include "runtime.h" |
| #include "scoped_thread_state_change-inl.h" |
| #include "stack_map_stream.h" |
| #include "trace.h" |
| #include "utils/label.h" |
| #include "utils/riscv64/assembler_riscv64.h" |
| #include "utils/stack_checks.h" |
| |
| namespace art HIDDEN { |
| namespace riscv64 { |
| |
| // Placeholder values embedded in instructions, patched at link time. |
| constexpr uint32_t kLinkTimeOffsetPlaceholderHigh = 0x12345; |
| constexpr uint32_t kLinkTimeOffsetPlaceholderLow = 0x678; |
| |
| // Compare-and-jump packed switch generates approx. 3 + 1.5 * N 32-bit |
| // instructions for N cases. |
| // Table-based packed switch generates approx. 10 32-bit instructions |
| // and N 32-bit data words for N cases. |
| // We switch to the table-based method starting with 6 entries. |
| static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 6; |
| |
| static constexpr XRegister kCoreCalleeSaves[] = { |
| // S1(TR) is excluded as the ART thread register. |
| S0, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, RA |
| }; |
| |
| static constexpr FRegister kFpuCalleeSaves[] = { |
| FS0, FS1, FS2, FS3, FS4, FS5, FS6, FS7, FS8, FS9, FS10, FS11 |
| }; |
| |
| #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kRiscv64PointerSize, x).Int32Value() |
| |
| Location RegisterOrZeroBitPatternLocation(HInstruction* instruction) { |
| DCHECK(!DataType::IsFloatingPointType(instruction->GetType())); |
| return IsZeroBitPattern(instruction) |
| ? Location::ConstantLocation(instruction) |
| : Location::RequiresRegister(); |
| } |
| |
| Location FpuRegisterOrZeroBitPatternLocation(HInstruction* instruction) { |
| DCHECK(DataType::IsFloatingPointType(instruction->GetType())); |
| return IsZeroBitPattern(instruction) |
| ? Location::ConstantLocation(instruction) |
| : Location::RequiresFpuRegister(); |
| } |
| |
| XRegister InputXRegisterOrZero(Location location) { |
| if (location.IsConstant()) { |
| DCHECK(location.GetConstant()->IsZeroBitPattern()); |
| return Zero; |
| } else { |
| return location.AsRegister<XRegister>(); |
| } |
| } |
| |
| Location ValueLocationForStore(HInstruction* value) { |
| if (IsZeroBitPattern(value)) { |
| return Location::ConstantLocation(value); |
| } else if (DataType::IsFloatingPointType(value->GetType())) { |
| return Location::RequiresFpuRegister(); |
| } else { |
| return Location::RequiresRegister(); |
| } |
| } |
| |
| Location Riscv64ReturnLocation(DataType::Type return_type) { |
| switch (return_type) { |
| case DataType::Type::kBool: |
| case DataType::Type::kUint8: |
| case DataType::Type::kInt8: |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| case DataType::Type::kUint32: |
| case DataType::Type::kInt32: |
| case DataType::Type::kReference: |
| case DataType::Type::kUint64: |
| case DataType::Type::kInt64: |
| return Location::RegisterLocation(A0); |
| |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| return Location::FpuRegisterLocation(FA0); |
| |
| case DataType::Type::kVoid: |
| return Location::NoLocation(); |
| } |
| } |
| |
| static RegisterSet OneRegInReferenceOutSaveEverythingCallerSaves() { |
| InvokeRuntimeCallingConvention calling_convention; |
| RegisterSet caller_saves = RegisterSet::Empty(); |
| caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); |
| DCHECK_EQ( |
| calling_convention.GetRegisterAt(0), |
| calling_convention.GetReturnLocation(DataType::Type::kReference).AsRegister<XRegister>()); |
| return caller_saves; |
| } |
| |
| template <ClassStatus kStatus> |
| static constexpr int64_t ShiftedSignExtendedClassStatusValue() { |
| // This is used only for status values that have the highest bit set. |
| static_assert(CLZ(enum_cast<uint32_t>(kStatus)) == kClassStatusLsbPosition); |
| constexpr uint32_t kShiftedStatusValue = enum_cast<uint32_t>(kStatus) << kClassStatusLsbPosition; |
| static_assert(kShiftedStatusValue >= 0x80000000u); |
| return static_cast<int64_t>(kShiftedStatusValue) - (INT64_C(1) << 32); |
| } |
| |
| // Split a 64-bit address used by JIT to the nearest 4KiB-aligned base address and a 12-bit |
| // signed offset. It is usually cheaper to materialize the aligned address than the full address. |
| std::pair<uint64_t, int32_t> SplitJitAddress(uint64_t address) { |
| uint64_t bits0_11 = address & UINT64_C(0xfff); |
| uint64_t bit11 = address & UINT64_C(0x800); |
| // Round the address to nearest 4KiB address because the `imm12` has range [-0x800, 0x800). |
| uint64_t base_address = (address & ~UINT64_C(0xfff)) + (bit11 << 1); |
| int32_t imm12 = dchecked_integral_cast<int32_t>(bits0_11) - |
| dchecked_integral_cast<int32_t>(bit11 << 1); |
| return {base_address, imm12}; |
| } |
| |
| int32_t ReadBarrierMarkEntrypointOffset(Location ref) { |
| DCHECK(ref.IsRegister()); |
| int reg = ref.reg(); |
| DCHECK(T0 <= reg && reg <= T6 && reg != TR) << reg; |
| // Note: Entrypoints for registers X30 (T5) and X31 (T6) are stored in entries |
| // for X0 (Zero) and X1 (RA) because these are not valid registers for marking |
| // and we currently have slots only up to register 29. |
| int entry_point_number = (reg >= 30) ? reg - 30 : reg; |
| return Thread::ReadBarrierMarkEntryPointsOffset<kRiscv64PointerSize>(entry_point_number); |
| } |
| |
| Location InvokeRuntimeCallingConvention::GetReturnLocation(DataType::Type return_type) { |
| return Riscv64ReturnLocation(return_type); |
| } |
| |
| Location InvokeDexCallingConventionVisitorRISCV64::GetReturnLocation(DataType::Type type) const { |
| return Riscv64ReturnLocation(type); |
| } |
| |
| Location InvokeDexCallingConventionVisitorRISCV64::GetMethodLocation() const { |
| return Location::RegisterLocation(kArtMethodRegister); |
| } |
| |
| Location InvokeDexCallingConventionVisitorRISCV64::GetNextLocation(DataType::Type type) { |
| Location next_location; |
| if (type == DataType::Type::kVoid) { |
| LOG(FATAL) << "Unexpected parameter type " << type; |
| } |
| |
| // Note: Unlike the RISC-V C/C++ calling convention, managed ABI does not use |
| // GPRs to pass FP args when we run out of FPRs. |
| if (DataType::IsFloatingPointType(type) && |
| float_index_ < calling_convention.GetNumberOfFpuRegisters()) { |
| next_location = |
| Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); |
| } else if (!DataType::IsFloatingPointType(type) && |
| (gp_index_ < calling_convention.GetNumberOfRegisters())) { |
| next_location = Location::RegisterLocation(calling_convention.GetRegisterAt(gp_index_++)); |
| } else { |
| size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_); |
| next_location = DataType::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset) : |
| Location::StackSlot(stack_offset); |
| } |
| |
| // Space on the stack is reserved for all arguments. |
| stack_index_ += DataType::Is64BitType(type) ? 2 : 1; |
| |
| return next_location; |
| } |
| |
| Location CriticalNativeCallingConventionVisitorRiscv64::GetNextLocation(DataType::Type type) { |
| DCHECK_NE(type, DataType::Type::kReference); |
| |
| Location location = Location::NoLocation(); |
| if (DataType::IsFloatingPointType(type)) { |
| if (fpr_index_ < kParameterFpuRegistersLength) { |
| location = Location::FpuRegisterLocation(kParameterFpuRegisters[fpr_index_]); |
| ++fpr_index_; |
| } else { |
| // Native ABI allows passing excessive FP args in GPRs. This is facilitated by |
| // inserting fake conversion intrinsic calls (`Double.doubleToRawLongBits()` |
| // or `Float.floatToRawIntBits()`) by `CriticalNativeAbiFixupRiscv64`. |
| // Remaining FP args shall be passed on the stack. |
| CHECK_EQ(gpr_index_, kRuntimeParameterCoreRegistersLength); |
| } |
| } else { |
| // Native ABI uses the same core registers as a runtime call. |
| if (gpr_index_ < kRuntimeParameterCoreRegistersLength) { |
| location = Location::RegisterLocation(kRuntimeParameterCoreRegisters[gpr_index_]); |
| ++gpr_index_; |
| } |
| } |
| if (location.IsInvalid()) { |
| // Only a `float` gets a single slot. Integral args need to be sign-extended to 64 bits. |
| if (type == DataType::Type::kFloat32) { |
| location = Location::StackSlot(stack_offset_); |
| } else { |
| location = Location::DoubleStackSlot(stack_offset_); |
| } |
| stack_offset_ += kFramePointerSize; |
| |
| if (for_register_allocation_) { |
| location = Location::Any(); |
| } |
| } |
| return location; |
| } |
| |
| Location CriticalNativeCallingConventionVisitorRiscv64::GetReturnLocation( |
| DataType::Type type) const { |
| // The result is returned the same way in native ABI and managed ABI. No result conversion is |
| // needed, see comments in `Riscv64JniCallingConvention::RequiresSmallResultTypeExtension()`. |
| InvokeDexCallingConventionVisitorRISCV64 dex_calling_convention; |
| return dex_calling_convention.GetReturnLocation(type); |
| } |
| |
| Location CriticalNativeCallingConventionVisitorRiscv64::GetMethodLocation() const { |
| // Pass the method in the hidden argument T0. |
| return Location::RegisterLocation(T0); |
| } |
| |
| #define __ down_cast<CodeGeneratorRISCV64*>(codegen)->GetAssembler()-> // NOLINT |
| |
| void LocationsBuilderRISCV64::HandleInvoke(HInvoke* instruction) { |
| InvokeDexCallingConventionVisitorRISCV64 calling_convention_visitor; |
| CodeGenerator::CreateCommonInvokeLocationSummary(instruction, &calling_convention_visitor); |
| } |
| |
| class CompileOptimizedSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| CompileOptimizedSlowPathRISCV64(HSuspendCheck* suspend_check, XRegister base, int32_t imm12) |
| : SlowPathCodeRISCV64(suspend_check), |
| base_(base), |
| imm12_(imm12) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| uint32_t entrypoint_offset = |
| GetThreadOffset<kRiscv64PointerSize>(kQuickCompileOptimized).Int32Value(); |
| __ Bind(GetEntryLabel()); |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| riscv64::ScratchRegisterScope srs(riscv64_codegen->GetAssembler()); |
| XRegister counter = srs.AllocateXRegister(); |
| __ LoadConst32(counter, ProfilingInfo::GetOptimizeThreshold()); |
| __ Sh(counter, base_, imm12_); |
| if (instruction_ != nullptr) { |
| // Only saves live vector regs for SIMD. |
| SaveLiveRegisters(codegen, instruction_->GetLocations()); |
| } |
| __ Loadd(RA, TR, entrypoint_offset); |
| // Note: we don't record the call here (and therefore don't generate a stack |
| // map), as the entrypoint should never be suspended. |
| __ Jalr(RA); |
| if (instruction_ != nullptr) { |
| // Only restores live vector regs for SIMD. |
| RestoreLiveRegisters(codegen, instruction_->GetLocations()); |
| } |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { return "CompileOptimizedSlowPath"; } |
| |
| private: |
| XRegister base_; |
| const int32_t imm12_; |
| |
| DISALLOW_COPY_AND_ASSIGN(CompileOptimizedSlowPathRISCV64); |
| }; |
| |
| class SuspendCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| SuspendCheckSlowPathRISCV64(HSuspendCheck* instruction, HBasicBlock* successor) |
| : SlowPathCodeRISCV64(instruction), successor_(successor) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); // Only saves live vector registers for SIMD. |
| riscv64_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickTestSuspend, void, void>(); |
| RestoreLiveRegisters(codegen, locations); // Only restores live vector registers for SIMD. |
| if (successor_ == nullptr) { |
| __ J(GetReturnLabel()); |
| } else { |
| __ J(riscv64_codegen->GetLabelOf(successor_)); |
| } |
| } |
| |
| Riscv64Label* GetReturnLabel() { |
| DCHECK(successor_ == nullptr); |
| return &return_label_; |
| } |
| |
| const char* GetDescription() const override { return "SuspendCheckSlowPathRISCV64"; } |
| |
| HBasicBlock* GetSuccessor() const { return successor_; } |
| |
| private: |
| // If not null, the block to branch to after the suspend check. |
| HBasicBlock* const successor_; |
| |
| // If `successor_` is null, the label to branch to after the suspend check. |
| Riscv64Label return_label_; |
| |
| DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathRISCV64); |
| }; |
| |
| class NullCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit NullCheckSlowPathRISCV64(HNullCheck* instr) : SlowPathCodeRISCV64(instr) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| if (instruction_->CanThrowIntoCatchBlock()) { |
| // Live registers will be restored in the catch block if caught. |
| SaveLiveRegisters(codegen, instruction_->GetLocations()); |
| } |
| riscv64_codegen->InvokeRuntime( |
| kQuickThrowNullPointer, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); |
| } |
| |
| bool IsFatal() const override { return true; } |
| |
| const char* GetDescription() const override { return "NullCheckSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathRISCV64); |
| }; |
| |
| class BoundsCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit BoundsCheckSlowPathRISCV64(HBoundsCheck* instruction) |
| : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| if (instruction_->CanThrowIntoCatchBlock()) { |
| // Live registers will be restored in the catch block if caught. |
| SaveLiveRegisters(codegen, instruction_->GetLocations()); |
| } |
| // We're moving two locations to locations that could overlap, so we need a parallel |
| // move resolver. |
| InvokeRuntimeCallingConvention calling_convention; |
| codegen->EmitParallelMoves(locations->InAt(0), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(0)), |
| DataType::Type::kInt32, |
| locations->InAt(1), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(1)), |
| DataType::Type::kInt32); |
| QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() ? |
| kQuickThrowStringBounds : |
| kQuickThrowArrayBounds; |
| riscv64_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); |
| CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); |
| } |
| |
| bool IsFatal() const override { return true; } |
| |
| const char* GetDescription() const override { return "BoundsCheckSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathRISCV64); |
| }; |
| |
| class LoadClassSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| LoadClassSlowPathRISCV64(HLoadClass* cls, HInstruction* at) : SlowPathCodeRISCV64(at), cls_(cls) { |
| DCHECK(at->IsLoadClass() || at->IsClinitCheck()); |
| DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); |
| } |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| Location out = locations->Out(); |
| const uint32_t dex_pc = instruction_->GetDexPc(); |
| bool must_resolve_type = instruction_->IsLoadClass() && cls_->MustResolveTypeOnSlowPath(); |
| bool must_do_clinit = instruction_->IsClinitCheck() || cls_->MustGenerateClinitCheck(); |
| |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); |
| |
| InvokeRuntimeCallingConvention calling_convention; |
| if (must_resolve_type) { |
| DCHECK(IsSameDexFile(cls_->GetDexFile(), riscv64_codegen->GetGraph()->GetDexFile()) || |
| riscv64_codegen->GetCompilerOptions().WithinOatFile(&cls_->GetDexFile()) || |
| ContainsElement(Runtime::Current()->GetClassLinker()->GetBootClassPath(), |
| &cls_->GetDexFile())); |
| dex::TypeIndex type_index = cls_->GetTypeIndex(); |
| __ LoadConst32(calling_convention.GetRegisterAt(0), type_index.index_); |
| if (cls_->NeedsAccessCheck()) { |
| CheckEntrypointTypes<kQuickResolveTypeAndVerifyAccess, void*, uint32_t>(); |
| riscv64_codegen->InvokeRuntime( |
| kQuickResolveTypeAndVerifyAccess, instruction_, dex_pc, this); |
| } else { |
| CheckEntrypointTypes<kQuickResolveType, void*, uint32_t>(); |
| riscv64_codegen->InvokeRuntime(kQuickResolveType, instruction_, dex_pc, this); |
| } |
| // If we also must_do_clinit, the resolved type is now in the correct register. |
| } else { |
| DCHECK(must_do_clinit); |
| Location source = instruction_->IsLoadClass() ? out : locations->InAt(0); |
| riscv64_codegen->MoveLocation( |
| Location::RegisterLocation(calling_convention.GetRegisterAt(0)), source, cls_->GetType()); |
| } |
| if (must_do_clinit) { |
| riscv64_codegen->InvokeRuntime(kQuickInitializeStaticStorage, instruction_, dex_pc, this); |
| CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, mirror::Class*>(); |
| } |
| |
| // Move the class to the desired location. |
| if (out.IsValid()) { |
| DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); |
| DataType::Type type = DataType::Type::kReference; |
| DCHECK_EQ(type, instruction_->GetType()); |
| riscv64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type); |
| } |
| RestoreLiveRegisters(codegen, locations); |
| |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { return "LoadClassSlowPathRISCV64"; } |
| |
| private: |
| // The class this slow path will load. |
| HLoadClass* const cls_; |
| |
| DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathRISCV64); |
| }; |
| |
| class DeoptimizationSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit DeoptimizationSlowPathRISCV64(HDeoptimize* instruction) |
| : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| LocationSummary* locations = instruction_->GetLocations(); |
| SaveLiveRegisters(codegen, locations); |
| InvokeRuntimeCallingConvention calling_convention; |
| __ LoadConst32(calling_convention.GetRegisterAt(0), |
| static_cast<uint32_t>(instruction_->AsDeoptimize()->GetDeoptimizationKind())); |
| riscv64_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickDeoptimize, void, DeoptimizationKind>(); |
| } |
| |
| const char* GetDescription() const override { return "DeoptimizationSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathRISCV64); |
| }; |
| |
| // Slow path generating a read barrier for a GC root. |
| class ReadBarrierForRootSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| ReadBarrierForRootSlowPathRISCV64(HInstruction* instruction, Location out, Location root) |
| : SlowPathCodeRISCV64(instruction), out_(out), root_(root) { |
| } |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| DCHECK(codegen->EmitReadBarrier()); |
| LocationSummary* locations = instruction_->GetLocations(); |
| DataType::Type type = DataType::Type::kReference; |
| XRegister reg_out = out_.AsRegister<XRegister>(); |
| DCHECK(locations->CanCall()); |
| DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); |
| DCHECK(instruction_->IsLoadClass() || |
| instruction_->IsLoadString() || |
| (instruction_->IsInvoke() && instruction_->GetLocations()->Intrinsified())) |
| << "Unexpected instruction in read barrier for GC root slow path: " |
| << instruction_->DebugName(); |
| |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); |
| |
| InvokeRuntimeCallingConvention calling_convention; |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| riscv64_codegen->MoveLocation(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), |
| root_, |
| DataType::Type::kReference); |
| riscv64_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, |
| instruction_, |
| instruction_->GetDexPc(), |
| this); |
| CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); |
| riscv64_codegen->MoveLocation(out_, calling_convention.GetReturnLocation(type), type); |
| |
| RestoreLiveRegisters(codegen, locations); |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { return "ReadBarrierForRootSlowPathRISCV64"; } |
| |
| private: |
| const Location out_; |
| const Location root_; |
| |
| DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathRISCV64); |
| }; |
| |
| class MethodEntryExitHooksSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit MethodEntryExitHooksSlowPathRISCV64(HInstruction* instruction) |
| : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| QuickEntrypointEnum entry_point = |
| (instruction_->IsMethodEntryHook()) ? kQuickMethodEntryHook : kQuickMethodExitHook; |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); |
| if (instruction_->IsMethodExitHook()) { |
| __ Li(A4, riscv64_codegen->GetFrameSize()); |
| } |
| riscv64_codegen->InvokeRuntime(entry_point, instruction_, instruction_->GetDexPc(), this); |
| RestoreLiveRegisters(codegen, locations); |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { |
| return "MethodEntryExitHooksSlowPathRISCV"; |
| } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(MethodEntryExitHooksSlowPathRISCV64); |
| }; |
| |
| class ArraySetSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit ArraySetSlowPathRISCV64(HInstruction* instruction) : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); |
| |
| InvokeRuntimeCallingConvention calling_convention; |
| HParallelMove parallel_move(codegen->GetGraph()->GetAllocator()); |
| parallel_move.AddMove( |
| locations->InAt(0), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(0)), |
| DataType::Type::kReference, |
| nullptr); |
| parallel_move.AddMove( |
| locations->InAt(1), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(1)), |
| DataType::Type::kInt32, |
| nullptr); |
| parallel_move.AddMove( |
| locations->InAt(2), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(2)), |
| DataType::Type::kReference, |
| nullptr); |
| codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); |
| |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| riscv64_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); |
| RestoreLiveRegisters(codegen, locations); |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { return "ArraySetSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathRISCV64); |
| }; |
| |
| class TypeCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit TypeCheckSlowPathRISCV64(HInstruction* instruction, bool is_fatal) |
| : SlowPathCodeRISCV64(instruction), is_fatal_(is_fatal) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| LocationSummary* locations = instruction_->GetLocations(); |
| |
| uint32_t dex_pc = instruction_->GetDexPc(); |
| DCHECK(instruction_->IsCheckCast() |
| || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| |
| __ Bind(GetEntryLabel()); |
| if (!is_fatal_ || instruction_->CanThrowIntoCatchBlock()) { |
| SaveLiveRegisters(codegen, locations); |
| } |
| |
| // We're moving two locations to locations that could overlap, so we need a parallel |
| // move resolver. |
| InvokeRuntimeCallingConvention calling_convention; |
| codegen->EmitParallelMoves(locations->InAt(0), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(0)), |
| DataType::Type::kReference, |
| locations->InAt(1), |
| Location::RegisterLocation(calling_convention.GetRegisterAt(1)), |
| DataType::Type::kReference); |
| if (instruction_->IsInstanceOf()) { |
| riscv64_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, instruction_, dex_pc, this); |
| CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); |
| DataType::Type ret_type = instruction_->GetType(); |
| Location ret_loc = calling_convention.GetReturnLocation(ret_type); |
| riscv64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type); |
| } else { |
| DCHECK(instruction_->IsCheckCast()); |
| riscv64_codegen->InvokeRuntime(kQuickCheckInstanceOf, instruction_, dex_pc, this); |
| CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); |
| } |
| |
| if (!is_fatal_) { |
| RestoreLiveRegisters(codegen, locations); |
| __ J(GetExitLabel()); |
| } |
| } |
| |
| const char* GetDescription() const override { return "TypeCheckSlowPathRISCV64"; } |
| |
| bool IsFatal() const override { return is_fatal_; } |
| |
| private: |
| const bool is_fatal_; |
| |
| DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathRISCV64); |
| }; |
| |
| class DivZeroCheckSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit DivZeroCheckSlowPathRISCV64(HDivZeroCheck* instruction) |
| : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| __ Bind(GetEntryLabel()); |
| riscv64_codegen->InvokeRuntime( |
| kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); |
| } |
| |
| bool IsFatal() const override { return true; } |
| |
| const char* GetDescription() const override { return "DivZeroCheckSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathRISCV64); |
| }; |
| |
| class ReadBarrierMarkSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| ReadBarrierMarkSlowPathRISCV64(HInstruction* instruction, Location ref, Location entrypoint) |
| : SlowPathCodeRISCV64(instruction), ref_(ref), entrypoint_(entrypoint) { |
| DCHECK(entrypoint.IsRegister()); |
| } |
| |
| const char* GetDescription() const override { return "ReadBarrierMarkSlowPathRISCV64"; } |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| DCHECK(codegen->EmitReadBarrier()); |
| LocationSummary* locations = instruction_->GetLocations(); |
| XRegister ref_reg = ref_.AsRegister<XRegister>(); |
| DCHECK(locations->CanCall()); |
| DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; |
| DCHECK(instruction_->IsInstanceFieldGet() || |
| instruction_->IsStaticFieldGet() || |
| instruction_->IsArrayGet() || |
| instruction_->IsArraySet() || |
| instruction_->IsLoadClass() || |
| instruction_->IsLoadString() || |
| instruction_->IsInstanceOf() || |
| instruction_->IsCheckCast() || |
| (instruction_->IsInvoke() && instruction_->GetLocations()->Intrinsified())) |
| << "Unexpected instruction in read barrier marking slow path: " |
| << instruction_->DebugName(); |
| |
| __ Bind(GetEntryLabel()); |
| // No need to save live registers; it's taken care of by the |
| // entrypoint. Also, there is no need to update the stack mask, |
| // as this runtime call will not trigger a garbage collection. |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| DCHECK(ref_reg >= T0 && ref_reg != TR); |
| |
| // "Compact" slow path, saving two moves. |
| // |
| // Instead of using the standard runtime calling convention (input |
| // and output in A0 and V0 respectively): |
| // |
| // A0 <- ref |
| // V0 <- ReadBarrierMark(A0) |
| // ref <- V0 |
| // |
| // we just use rX (the register containing `ref`) as input and output |
| // of a dedicated entrypoint: |
| // |
| // rX <- ReadBarrierMarkRegX(rX) |
| // |
| riscv64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); |
| DCHECK_NE(entrypoint_.AsRegister<XRegister>(), TMP); // A taken branch can clobber `TMP`. |
| __ Jalr(entrypoint_.AsRegister<XRegister>()); // Clobbers `RA` (used as the `entrypoint_`). |
| __ J(GetExitLabel()); |
| } |
| |
| private: |
| // The location (register) of the marked object reference. |
| const Location ref_; |
| |
| // The location of the already loaded entrypoint. |
| const Location entrypoint_; |
| |
| DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathRISCV64); |
| }; |
| |
| class LoadStringSlowPathRISCV64 : public SlowPathCodeRISCV64 { |
| public: |
| explicit LoadStringSlowPathRISCV64(HLoadString* instruction) |
| : SlowPathCodeRISCV64(instruction) {} |
| |
| void EmitNativeCode(CodeGenerator* codegen) override { |
| DCHECK(instruction_->IsLoadString()); |
| DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); |
| LocationSummary* locations = instruction_->GetLocations(); |
| DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); |
| const dex::StringIndex string_index = instruction_->AsLoadString()->GetStringIndex(); |
| CodeGeneratorRISCV64* riscv64_codegen = down_cast<CodeGeneratorRISCV64*>(codegen); |
| InvokeRuntimeCallingConvention calling_convention; |
| __ Bind(GetEntryLabel()); |
| SaveLiveRegisters(codegen, locations); |
| |
| __ LoadConst32(calling_convention.GetRegisterAt(0), string_index.index_); |
| riscv64_codegen->InvokeRuntime( |
| kQuickResolveString, instruction_, instruction_->GetDexPc(), this); |
| CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); |
| |
| DataType::Type type = DataType::Type::kReference; |
| DCHECK_EQ(type, instruction_->GetType()); |
| riscv64_codegen->MoveLocation( |
| locations->Out(), calling_convention.GetReturnLocation(type), type); |
| RestoreLiveRegisters(codegen, locations); |
| |
| __ J(GetExitLabel()); |
| } |
| |
| const char* GetDescription() const override { return "LoadStringSlowPathRISCV64"; } |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathRISCV64); |
| }; |
| |
| #undef __ |
| #define __ down_cast<Riscv64Assembler*>(GetAssembler())-> // NOLINT |
| |
| template <typename Reg, |
| void (Riscv64Assembler::*opS)(Reg, FRegister, FRegister), |
| void (Riscv64Assembler::*opD)(Reg, FRegister, FRegister)> |
| inline void InstructionCodeGeneratorRISCV64::FpBinOp( |
| Reg rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| Riscv64Assembler* assembler = down_cast<CodeGeneratorRISCV64*>(codegen_)->GetAssembler(); |
| if (type == DataType::Type::kFloat32) { |
| (assembler->*opS)(rd, rs1, rs2); |
| } else { |
| DCHECK_EQ(type, DataType::Type::kFloat64); |
| (assembler->*opD)(rd, rs1, rs2); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::FAdd( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FAddS, &Riscv64Assembler::FAddD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FSub( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FSubS, &Riscv64Assembler::FSubD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FDiv( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FDivS, &Riscv64Assembler::FDivD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FMul( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FMulS, &Riscv64Assembler::FMulD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FMin( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FMinS, &Riscv64Assembler::FMinD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FMax( |
| FRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<FRegister, &Riscv64Assembler::FMaxS, &Riscv64Assembler::FMaxD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FEq( |
| XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<XRegister, &Riscv64Assembler::FEqS, &Riscv64Assembler::FEqD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FLt( |
| XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<XRegister, &Riscv64Assembler::FLtS, &Riscv64Assembler::FLtD>(rd, rs1, rs2, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FLe( |
| XRegister rd, FRegister rs1, FRegister rs2, DataType::Type type) { |
| FpBinOp<XRegister, &Riscv64Assembler::FLeS, &Riscv64Assembler::FLeD>(rd, rs1, rs2, type); |
| } |
| |
| template <typename Reg, |
| void (Riscv64Assembler::*opS)(Reg, FRegister), |
| void (Riscv64Assembler::*opD)(Reg, FRegister)> |
| inline void InstructionCodeGeneratorRISCV64::FpUnOp( |
| Reg rd, FRegister rs1, DataType::Type type) { |
| Riscv64Assembler* assembler = down_cast<CodeGeneratorRISCV64*>(codegen_)->GetAssembler(); |
| if (type == DataType::Type::kFloat32) { |
| (assembler->*opS)(rd, rs1); |
| } else { |
| DCHECK_EQ(type, DataType::Type::kFloat64); |
| (assembler->*opD)(rd, rs1); |
| } |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FAbs( |
| FRegister rd, FRegister rs1, DataType::Type type) { |
| FpUnOp<FRegister, &Riscv64Assembler::FAbsS, &Riscv64Assembler::FAbsD>(rd, rs1, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FNeg( |
| FRegister rd, FRegister rs1, DataType::Type type) { |
| FpUnOp<FRegister, &Riscv64Assembler::FNegS, &Riscv64Assembler::FNegD>(rd, rs1, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FMv( |
| FRegister rd, FRegister rs1, DataType::Type type) { |
| FpUnOp<FRegister, &Riscv64Assembler::FMvS, &Riscv64Assembler::FMvD>(rd, rs1, type); |
| } |
| |
| inline void InstructionCodeGeneratorRISCV64::FMvX( |
| XRegister rd, FRegister rs1, DataType::Type type) { |
| FpUnOp<XRegister, &Riscv64Assembler::FMvXW, &Riscv64Assembler::FMvXD>(rd, rs1, type); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::FClass( |
| XRegister rd, FRegister rs1, DataType::Type type) { |
| FpUnOp<XRegister, &Riscv64Assembler::FClassS, &Riscv64Assembler::FClassD>(rd, rs1, type); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::Load( |
| Location out, XRegister rs1, int32_t offset, DataType::Type type) { |
| switch (type) { |
| case DataType::Type::kBool: |
| case DataType::Type::kUint8: |
| __ Loadbu(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kInt8: |
| __ Loadb(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kUint16: |
| __ Loadhu(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kInt16: |
| __ Loadh(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kInt32: |
| __ Loadw(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kInt64: |
| __ Loadd(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kReference: |
| __ Loadwu(out.AsRegister<XRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kFloat32: |
| __ FLoadw(out.AsFpuRegister<FRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kFloat64: |
| __ FLoadd(out.AsFpuRegister<FRegister>(), rs1, offset); |
| break; |
| case DataType::Type::kUint32: |
| case DataType::Type::kUint64: |
| case DataType::Type::kVoid: |
| LOG(FATAL) << "Unreachable type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::Store( |
| Location value, XRegister rs1, int32_t offset, DataType::Type type) { |
| DCHECK_IMPLIES(value.IsConstant(), IsZeroBitPattern(value.GetConstant())); |
| if (kPoisonHeapReferences && type == DataType::Type::kReference && !value.IsConstant()) { |
| riscv64::ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| __ Mv(tmp, value.AsRegister<XRegister>()); |
| codegen_->PoisonHeapReference(tmp); |
| __ Storew(tmp, rs1, offset); |
| return; |
| } |
| switch (type) { |
| case DataType::Type::kBool: |
| case DataType::Type::kUint8: |
| case DataType::Type::kInt8: |
| __ Storeb(InputXRegisterOrZero(value), rs1, offset); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| __ Storeh(InputXRegisterOrZero(value), rs1, offset); |
| break; |
| case DataType::Type::kFloat32: |
| if (!value.IsConstant()) { |
| __ FStorew(value.AsFpuRegister<FRegister>(), rs1, offset); |
| break; |
| } |
| FALLTHROUGH_INTENDED; |
| case DataType::Type::kInt32: |
| case DataType::Type::kReference: |
| __ Storew(InputXRegisterOrZero(value), rs1, offset); |
| break; |
| case DataType::Type::kFloat64: |
| if (!value.IsConstant()) { |
| __ FStored(value.AsFpuRegister<FRegister>(), rs1, offset); |
| break; |
| } |
| FALLTHROUGH_INTENDED; |
| case DataType::Type::kInt64: |
| __ Stored(InputXRegisterOrZero(value), rs1, offset); |
| break; |
| case DataType::Type::kUint32: |
| case DataType::Type::kUint64: |
| case DataType::Type::kVoid: |
| LOG(FATAL) << "Unreachable type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::StoreSeqCst(Location value, |
| XRegister rs1, |
| int32_t offset, |
| DataType::Type type, |
| HInstruction* instruction) { |
| if (DataType::Size(type) >= 4u) { |
| // Use AMOSWAP for 32-bit and 64-bit data types. |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister swap_src = kNoXRegister; |
| if (kPoisonHeapReferences && type == DataType::Type::kReference && !value.IsConstant()) { |
| swap_src = srs.AllocateXRegister(); |
| __ Mv(swap_src, value.AsRegister<XRegister>()); |
| codegen_->PoisonHeapReference(swap_src); |
| } else if (DataType::IsFloatingPointType(type) && !value.IsConstant()) { |
| swap_src = srs.AllocateXRegister(); |
| FMvX(swap_src, value.AsFpuRegister<FRegister>(), type); |
| } else { |
| swap_src = InputXRegisterOrZero(value); |
| } |
| XRegister addr = rs1; |
| if (offset != 0) { |
| addr = srs.AllocateXRegister(); |
| __ AddConst64(addr, rs1, offset); |
| } |
| if (DataType::Is64BitType(type)) { |
| __ AmoSwapD(Zero, swap_src, addr, AqRl::kRelease); |
| } else { |
| __ AmoSwapW(Zero, swap_src, addr, AqRl::kRelease); |
| } |
| if (instruction != nullptr) { |
| codegen_->MaybeRecordImplicitNullCheck(instruction); |
| } |
| } else { |
| // Use fences for smaller data types. |
| codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); |
| Store(value, rs1, offset, type); |
| if (instruction != nullptr) { |
| codegen_->MaybeRecordImplicitNullCheck(instruction); |
| } |
| codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::ShNAdd( |
| XRegister rd, XRegister rs1, XRegister rs2, DataType::Type type) { |
| switch (type) { |
| case DataType::Type::kBool: |
| case DataType::Type::kUint8: |
| case DataType::Type::kInt8: |
| DCHECK_EQ(DataType::SizeShift(type), 0u); |
| __ Add(rd, rs1, rs2); |
| break; |
| case DataType::Type::kUint16: |
| case DataType::Type::kInt16: |
| DCHECK_EQ(DataType::SizeShift(type), 1u); |
| __ Sh1Add(rd, rs1, rs2); |
| break; |
| case DataType::Type::kInt32: |
| case DataType::Type::kReference: |
| case DataType::Type::kFloat32: |
| DCHECK_EQ(DataType::SizeShift(type), 2u); |
| __ Sh2Add(rd, rs1, rs2); |
| break; |
| case DataType::Type::kInt64: |
| case DataType::Type::kFloat64: |
| DCHECK_EQ(DataType::SizeShift(type), 3u); |
| __ Sh3Add(rd, rs1, rs2); |
| break; |
| case DataType::Type::kUint32: |
| case DataType::Type::kUint64: |
| case DataType::Type::kVoid: |
| LOG(FATAL) << "Unreachable type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| Riscv64Assembler* ParallelMoveResolverRISCV64::GetAssembler() const { |
| return codegen_->GetAssembler(); |
| } |
| |
| void ParallelMoveResolverRISCV64::EmitMove(size_t index) { |
| MoveOperands* move = moves_[index]; |
| codegen_->MoveLocation(move->GetDestination(), move->GetSource(), move->GetType()); |
| } |
| |
| void ParallelMoveResolverRISCV64::EmitSwap(size_t index) { |
| MoveOperands* move = moves_[index]; |
| codegen_->SwapLocations(move->GetDestination(), move->GetSource(), move->GetType()); |
| } |
| |
| void ParallelMoveResolverRISCV64::SpillScratch([[maybe_unused]] int reg) { |
| LOG(FATAL) << "Unimplemented"; |
| UNREACHABLE(); |
| } |
| |
| void ParallelMoveResolverRISCV64::RestoreScratch([[maybe_unused]] int reg) { |
| LOG(FATAL) << "Unimplemented"; |
| UNREACHABLE(); |
| } |
| |
| void ParallelMoveResolverRISCV64::Exchange(int index1, int index2, bool double_slot) { |
| // We have 2 scratch X registers and 1 scratch F register that we can use. We prefer |
| // to use X registers for the swap but if both offsets are too big, we need to reserve |
| // one of the X registers for address adjustment and use an F register. |
| bool use_fp_tmp2 = false; |
| if (!IsInt<12>(index2)) { |
| if (!IsInt<12>(index1)) { |
| use_fp_tmp2 = true; |
| } else { |
| std::swap(index1, index2); |
| } |
| } |
| DCHECK_IMPLIES(!IsInt<12>(index2), use_fp_tmp2); |
| |
| Location loc1(double_slot ? Location::DoubleStackSlot(index1) : Location::StackSlot(index1)); |
| Location loc2(double_slot ? Location::DoubleStackSlot(index2) : Location::StackSlot(index2)); |
| riscv64::ScratchRegisterScope srs(GetAssembler()); |
| Location tmp = Location::RegisterLocation(srs.AllocateXRegister()); |
| DataType::Type tmp_type = double_slot ? DataType::Type::kInt64 : DataType::Type::kInt32; |
| Location tmp2 = use_fp_tmp2 |
| ? Location::FpuRegisterLocation(srs.AllocateFRegister()) |
| : Location::RegisterLocation(srs.AllocateXRegister()); |
| DataType::Type tmp2_type = use_fp_tmp2 |
| ? (double_slot ? DataType::Type::kFloat64 : DataType::Type::kFloat32) |
| : tmp_type; |
| |
| codegen_->MoveLocation(tmp, loc1, tmp_type); |
| codegen_->MoveLocation(tmp2, loc2, tmp2_type); |
| if (use_fp_tmp2) { |
| codegen_->MoveLocation(loc2, tmp, tmp_type); |
| } else { |
| // We cannot use `Stored()` or `Storew()` via `MoveLocation()` because we have |
| // no more scratch registers available. Use `Sd()` or `Sw()` explicitly. |
| DCHECK(IsInt<12>(index2)); |
| if (double_slot) { |
| __ Sd(tmp.AsRegister<XRegister>(), SP, index2); |
| } else { |
| __ Sw(tmp.AsRegister<XRegister>(), SP, index2); |
| } |
| srs.FreeXRegister(tmp.AsRegister<XRegister>()); // Free a temporary for `MoveLocation()`. |
| } |
| codegen_->MoveLocation(loc1, tmp2, tmp2_type); |
| } |
| |
| InstructionCodeGeneratorRISCV64::InstructionCodeGeneratorRISCV64(HGraph* graph, |
| CodeGeneratorRISCV64* codegen) |
| : InstructionCodeGenerator(graph, codegen), |
| assembler_(codegen->GetAssembler()), |
| codegen_(codegen) {} |
| |
| void InstructionCodeGeneratorRISCV64::GenerateClassInitializationCheck( |
| SlowPathCodeRISCV64* slow_path, XRegister class_reg) { |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| XRegister tmp2 = srs.AllocateXRegister(); |
| |
| // We shall load the full 32-bit status word with sign-extension and compare as unsigned |
| // to a sign-extended shifted status value. This yields the same comparison as loading and |
| // materializing unsigned but the constant is materialized with a single LUI instruction. |
| __ Loadw(tmp, class_reg, mirror::Class::StatusOffset().SizeValue()); // Sign-extended. |
| __ Li(tmp2, ShiftedSignExtendedClassStatusValue<ClassStatus::kVisiblyInitialized>()); |
| __ Bltu(tmp, tmp2, slow_path->GetEntryLabel()); |
| __ Bind(slow_path->GetExitLabel()); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateBitstringTypeCheckCompare( |
| HTypeCheckInstruction* instruction, XRegister temp) { |
| UNUSED(instruction); |
| UNUSED(temp); |
| LOG(FATAL) << "Unimplemented"; |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateSuspendCheck(HSuspendCheck* instruction, |
| HBasicBlock* successor) { |
| if (instruction->IsNoOp()) { |
| if (successor != nullptr) { |
| __ J(codegen_->GetLabelOf(successor)); |
| } |
| return; |
| } |
| |
| if (codegen_->CanUseImplicitSuspendCheck()) { |
| LOG(FATAL) << "Unimplemented ImplicitSuspendCheck"; |
| return; |
| } |
| |
| SuspendCheckSlowPathRISCV64* slow_path = |
| down_cast<SuspendCheckSlowPathRISCV64*>(instruction->GetSlowPath()); |
| |
| if (slow_path == nullptr) { |
| slow_path = |
| new (codegen_->GetScopedAllocator()) SuspendCheckSlowPathRISCV64(instruction, successor); |
| instruction->SetSlowPath(slow_path); |
| codegen_->AddSlowPath(slow_path); |
| if (successor != nullptr) { |
| DCHECK(successor->IsLoopHeader()); |
| } |
| } else { |
| DCHECK_EQ(slow_path->GetSuccessor(), successor); |
| } |
| |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| __ Loadw(tmp, TR, Thread::ThreadFlagsOffset<kRiscv64PointerSize>().Int32Value()); |
| static_assert(Thread::SuspendOrCheckpointRequestFlags() != std::numeric_limits<uint32_t>::max()); |
| static_assert(IsPowerOfTwo(Thread::SuspendOrCheckpointRequestFlags() + 1u)); |
| // Shift out other bits. Use an instruction that can be 16-bit with the "C" Standard Extension. |
| __ Slli(tmp, tmp, CLZ(static_cast<uint64_t>(Thread::SuspendOrCheckpointRequestFlags()))); |
| if (successor == nullptr) { |
| __ Bnez(tmp, slow_path->GetEntryLabel()); |
| __ Bind(slow_path->GetReturnLabel()); |
| } else { |
| __ Beqz(tmp, codegen_->GetLabelOf(successor)); |
| __ J(slow_path->GetEntryLabel()); |
| // slow_path will return to GetLabelOf(successor). |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateReferenceLoadOneRegister( |
| HInstruction* instruction, |
| Location out, |
| uint32_t offset, |
| Location maybe_temp, |
| ReadBarrierOption read_barrier_option) { |
| XRegister out_reg = out.AsRegister<XRegister>(); |
| if (read_barrier_option == kWithReadBarrier) { |
| DCHECK(codegen_->EmitReadBarrier()); |
| if (kUseBakerReadBarrier) { |
| // Load with fast path based Baker's read barrier. |
| // /* HeapReference<Object> */ out = *(out + offset) |
| codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, |
| out, |
| out_reg, |
| offset, |
| maybe_temp, |
| /* needs_null_check= */ false); |
| } else { |
| // Load with slow path based read barrier. |
| // Save the value of `out` into `maybe_temp` before overwriting it |
| // in the following move operation, as we will need it for the |
| // read barrier below. |
| __ Mv(maybe_temp.AsRegister<XRegister>(), out_reg); |
| // /* HeapReference<Object> */ out = *(out + offset) |
| __ Loadwu(out_reg, out_reg, offset); |
| codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); |
| } |
| } else { |
| // Plain load with no read barrier. |
| // /* HeapReference<Object> */ out = *(out + offset) |
| __ Loadwu(out_reg, out_reg, offset); |
| codegen_->MaybeUnpoisonHeapReference(out_reg); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateReferenceLoadTwoRegisters( |
| HInstruction* instruction, |
| Location out, |
| Location obj, |
| uint32_t offset, |
| Location maybe_temp, |
| ReadBarrierOption read_barrier_option) { |
| XRegister out_reg = out.AsRegister<XRegister>(); |
| XRegister obj_reg = obj.AsRegister<XRegister>(); |
| if (read_barrier_option == kWithReadBarrier) { |
| DCHECK(codegen_->EmitReadBarrier()); |
| if (kUseBakerReadBarrier) { |
| // Load with fast path based Baker's read barrier. |
| // /* HeapReference<Object> */ out = *(obj + offset) |
| codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, |
| out, |
| obj_reg, |
| offset, |
| maybe_temp, |
| /* needs_null_check= */ false); |
| } else { |
| // Load with slow path based read barrier. |
| // /* HeapReference<Object> */ out = *(obj + offset) |
| __ Loadwu(out_reg, obj_reg, offset); |
| codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); |
| } |
| } else { |
| // Plain load with no read barrier. |
| // /* HeapReference<Object> */ out = *(obj + offset) |
| __ Loadwu(out_reg, obj_reg, offset); |
| codegen_->MaybeUnpoisonHeapReference(out_reg); |
| } |
| } |
| |
| SlowPathCodeRISCV64* CodeGeneratorRISCV64::AddGcRootBakerBarrierBarrierSlowPath( |
| HInstruction* instruction, Location root, Location temp) { |
| SlowPathCodeRISCV64* slow_path = |
| new (GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64(instruction, root, temp); |
| AddSlowPath(slow_path); |
| return slow_path; |
| } |
| |
| void CodeGeneratorRISCV64::EmitBakerReadBarierMarkingCheck( |
| SlowPathCodeRISCV64* slow_path, Location root, Location temp) { |
| const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(root); |
| // Loading the entrypoint does not require a load acquire since it is only changed when |
| // threads are suspended or running a checkpoint. |
| __ Loadd(temp.AsRegister<XRegister>(), TR, entry_point_offset); |
| __ Bnez(temp.AsRegister<XRegister>(), slow_path->GetEntryLabel()); |
| __ Bind(slow_path->GetExitLabel()); |
| } |
| |
| void CodeGeneratorRISCV64::GenerateGcRootFieldLoad(HInstruction* instruction, |
| Location root, |
| XRegister obj, |
| uint32_t offset, |
| ReadBarrierOption read_barrier_option, |
| Riscv64Label* label_low) { |
| DCHECK_IMPLIES(label_low != nullptr, offset == kLinkTimeOffsetPlaceholderLow) << offset; |
| XRegister root_reg = root.AsRegister<XRegister>(); |
| if (read_barrier_option == kWithReadBarrier) { |
| DCHECK(EmitReadBarrier()); |
| if (kUseBakerReadBarrier) { |
| // Note that we do not actually check the value of `GetIsGcMarking()` |
| // to decide whether to mark the loaded GC root or not. Instead, we |
| // load into `temp` (T6) the read barrier mark entry point corresponding |
| // to register `root`. If `temp` is null, it means that `GetIsGcMarking()` |
| // is false, and vice versa. |
| // |
| // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. |
| // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() |
| // if (temp != null) { |
| // root = temp(root) |
| // } |
| // |
| // TODO(riscv64): Introduce a "marking register" that holds the pointer to one of the |
| // register marking entrypoints if marking (null if not marking) and make sure that |
| // marking entrypoints for other registers are at known offsets, so that we can call |
| // them using the "marking register" plus the offset embedded in the JALR instruction. |
| |
| if (label_low != nullptr) { |
| __ Bind(label_low); |
| } |
| // /* GcRoot<mirror::Object> */ root = *(obj + offset) |
| __ Loadwu(root_reg, obj, offset); |
| static_assert( |
| sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), |
| "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " |
| "have different sizes."); |
| static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), |
| "art::mirror::CompressedReference<mirror::Object> and int32_t " |
| "have different sizes."); |
| |
| // Use RA as temp. It is clobbered in the slow path anyway. |
| Location temp = Location::RegisterLocation(RA); |
| SlowPathCodeRISCV64* slow_path = |
| AddGcRootBakerBarrierBarrierSlowPath(instruction, root, temp); |
| EmitBakerReadBarierMarkingCheck(slow_path, root, temp); |
| } else { |
| // GC root loaded through a slow path for read barriers other |
| // than Baker's. |
| // /* GcRoot<mirror::Object>* */ root = obj + offset |
| if (label_low != nullptr) { |
| __ Bind(label_low); |
| } |
| __ AddConst32(root_reg, obj, offset); |
| // /* mirror::Object* */ root = root->Read() |
| GenerateReadBarrierForRootSlow(instruction, root, root); |
| } |
| } else { |
| // Plain GC root load with no read barrier. |
| // /* GcRoot<mirror::Object> */ root = *(obj + offset) |
| if (label_low != nullptr) { |
| __ Bind(label_low); |
| } |
| __ Loadwu(root_reg, obj, offset); |
| // Note that GC roots are not affected by heap poisoning, thus we |
| // do not have to unpoison `root_reg` here. |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateTestAndBranch(HInstruction* instruction, |
| size_t condition_input_index, |
| Riscv64Label* true_target, |
| Riscv64Label* false_target) { |
| HInstruction* cond = instruction->InputAt(condition_input_index); |
| |
| if (true_target == nullptr && false_target == nullptr) { |
| // Nothing to do. The code always falls through. |
| return; |
| } else if (cond->IsIntConstant()) { |
| // Constant condition, statically compared against "true" (integer value 1). |
| if (cond->AsIntConstant()->IsTrue()) { |
| if (true_target != nullptr) { |
| __ J(true_target); |
| } |
| } else { |
| DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); |
| if (false_target != nullptr) { |
| __ J(false_target); |
| } |
| } |
| return; |
| } |
| |
| // The following code generates these patterns: |
| // (1) true_target == nullptr && false_target != nullptr |
| // - opposite condition true => branch to false_target |
| // (2) true_target != nullptr && false_target == nullptr |
| // - condition true => branch to true_target |
| // (3) true_target != nullptr && false_target != nullptr |
| // - condition true => branch to true_target |
| // - branch to false_target |
| if (IsBooleanValueOrMaterializedCondition(cond)) { |
| // The condition instruction has been materialized, compare the output to 0. |
| Location cond_val = instruction->GetLocations()->InAt(condition_input_index); |
| DCHECK(cond_val.IsRegister()); |
| if (true_target == nullptr) { |
| __ Beqz(cond_val.AsRegister<XRegister>(), false_target); |
| } else { |
| __ Bnez(cond_val.AsRegister<XRegister>(), true_target); |
| } |
| } else { |
| // The condition instruction has not been materialized, use its inputs as |
| // the comparison and its condition as the branch condition. |
| HCondition* condition = cond->AsCondition(); |
| DataType::Type type = condition->InputAt(0)->GetType(); |
| LocationSummary* locations = condition->GetLocations(); |
| IfCondition if_cond = condition->GetCondition(); |
| Riscv64Label* branch_target = true_target; |
| |
| if (true_target == nullptr) { |
| if_cond = condition->GetOppositeCondition(); |
| branch_target = false_target; |
| } |
| |
| switch (type) { |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| GenerateFpCondition(if_cond, condition->IsGtBias(), type, locations, branch_target); |
| break; |
| default: |
| // Integral types and reference equality. |
| GenerateIntLongCompareAndBranch(if_cond, locations, branch_target); |
| break; |
| } |
| } |
| |
| // If neither branch falls through (case 3), the conditional branch to `true_target` |
| // was already emitted (case 2) and we need to emit a jump to `false_target`. |
| if (true_target != nullptr && false_target != nullptr) { |
| __ J(false_target); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::DivRemOneOrMinusOne(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsDiv() || instruction->IsRem()); |
| DataType::Type type = instruction->GetResultType(); |
| |
| LocationSummary* locations = instruction->GetLocations(); |
| Location second = locations->InAt(1); |
| DCHECK(second.IsConstant()); |
| |
| XRegister out = locations->Out().AsRegister<XRegister>(); |
| XRegister dividend = locations->InAt(0).AsRegister<XRegister>(); |
| int64_t imm = Int64FromConstant(second.GetConstant()); |
| DCHECK(imm == 1 || imm == -1); |
| |
| if (instruction->IsRem()) { |
| __ Mv(out, Zero); |
| } else { |
| if (imm == -1) { |
| if (type == DataType::Type::kInt32) { |
| __ Subw(out, Zero, dividend); |
| } else { |
| DCHECK_EQ(type, DataType::Type::kInt64); |
| __ Sub(out, Zero, dividend); |
| } |
| } else if (out != dividend) { |
| __ Mv(out, dividend); |
| } |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::DivRemByPowerOfTwo(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsDiv() || instruction->IsRem()); |
| DataType::Type type = instruction->GetResultType(); |
| DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64) << type; |
| |
| LocationSummary* locations = instruction->GetLocations(); |
| Location second = locations->InAt(1); |
| DCHECK(second.IsConstant()); |
| |
| XRegister out = locations->Out().AsRegister<XRegister>(); |
| XRegister dividend = locations->InAt(0).AsRegister<XRegister>(); |
| int64_t imm = Int64FromConstant(second.GetConstant()); |
| int64_t abs_imm = static_cast<uint64_t>(AbsOrMin(imm)); |
| int ctz_imm = CTZ(abs_imm); |
| DCHECK_GE(ctz_imm, 1); // Division by +/-1 is handled by `DivRemOneOrMinusOne()`. |
| |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| // Calculate the negative dividend adjustment `tmp = dividend < 0 ? abs_imm - 1 : 0`. |
| // This adjustment is needed for rounding the division result towards zero. |
| if (type == DataType::Type::kInt32 || ctz_imm == 1) { |
| // A 32-bit dividend is sign-extended to 64-bit, so we can use the upper bits. |
| // And for a 64-bit division by +/-2, we need just the sign bit. |
| DCHECK_IMPLIES(type == DataType::Type::kInt32, ctz_imm < 32); |
| __ Srli(tmp, dividend, 64 - ctz_imm); |
| } else { |
| // For other 64-bit divisions, we need to replicate the sign bit. |
| __ Srai(tmp, dividend, 63); |
| __ Srli(tmp, tmp, 64 - ctz_imm); |
| } |
| // The rest of the calculation can use 64-bit operations even for 32-bit div/rem. |
| __ Add(tmp, tmp, dividend); |
| if (instruction->IsDiv()) { |
| __ Srai(out, tmp, ctz_imm); |
| if (imm < 0) { |
| __ Neg(out, out); |
| } |
| } else { |
| if (ctz_imm <= 11) { |
| __ Andi(tmp, tmp, -abs_imm); |
| } else { |
| ScratchRegisterScope srs2(GetAssembler()); |
| XRegister tmp2 = srs2.AllocateXRegister(); |
| __ Li(tmp2, -abs_imm); |
| __ And(tmp, tmp, tmp2); |
| } |
| __ Sub(out, dividend, tmp); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsDiv() || instruction->IsRem()); |
| LocationSummary* locations = instruction->GetLocations(); |
| XRegister dividend = locations->InAt(0).AsRegister<XRegister>(); |
| XRegister out = locations->Out().AsRegister<XRegister>(); |
| Location second = locations->InAt(1); |
| int64_t imm = Int64FromConstant(second.GetConstant()); |
| DataType::Type type = instruction->GetResultType(); |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| |
| // TODO: optimize with constant. |
| __ LoadConst64(tmp, imm); |
| if (instruction->IsDiv()) { |
| if (type == DataType::Type::kInt32) { |
| __ Divw(out, dividend, tmp); |
| } else { |
| __ Div(out, dividend, tmp); |
| } |
| } else { |
| if (type == DataType::Type::kInt32) { |
| __ Remw(out, dividend, tmp); |
| } else { |
| __ Rem(out, dividend, tmp); |
| } |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateDivRemIntegral(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsDiv() || instruction->IsRem()); |
| DataType::Type type = instruction->GetResultType(); |
| DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64) << type; |
| |
| LocationSummary* locations = instruction->GetLocations(); |
| XRegister out = locations->Out().AsRegister<XRegister>(); |
| Location second = locations->InAt(1); |
| |
| if (second.IsConstant()) { |
| int64_t imm = Int64FromConstant(second.GetConstant()); |
| if (imm == 0) { |
| // Do not generate anything. DivZeroCheck would prevent any code to be executed. |
| } else if (imm == 1 || imm == -1) { |
| DivRemOneOrMinusOne(instruction); |
| } else if (IsPowerOfTwo(AbsOrMin(imm))) { |
| DivRemByPowerOfTwo(instruction); |
| } else { |
| DCHECK(imm <= -2 || imm >= 2); |
| GenerateDivRemWithAnyConstant(instruction); |
| } |
| } else { |
| XRegister dividend = locations->InAt(0).AsRegister<XRegister>(); |
| XRegister divisor = second.AsRegister<XRegister>(); |
| if (instruction->IsDiv()) { |
| if (type == DataType::Type::kInt32) { |
| __ Divw(out, dividend, divisor); |
| } else { |
| __ Div(out, dividend, divisor); |
| } |
| } else { |
| if (type == DataType::Type::kInt32) { |
| __ Remw(out, dividend, divisor); |
| } else { |
| __ Rem(out, dividend, divisor); |
| } |
| } |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateIntLongCondition(IfCondition cond, |
| LocationSummary* locations) { |
| XRegister rd = locations->Out().AsRegister<XRegister>(); |
| GenerateIntLongCondition(cond, locations, rd, /*to_all_bits=*/ false); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateIntLongCondition(IfCondition cond, |
| LocationSummary* locations, |
| XRegister rd, |
| bool to_all_bits) { |
| XRegister rs1 = locations->InAt(0).AsRegister<XRegister>(); |
| Location rs2_location = locations->InAt(1); |
| bool use_imm = rs2_location.IsConstant(); |
| int64_t imm = use_imm ? CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant()) : 0; |
| XRegister rs2 = use_imm ? kNoXRegister : rs2_location.AsRegister<XRegister>(); |
| bool reverse_condition = false; |
| switch (cond) { |
| case kCondEQ: |
| case kCondNE: |
| if (!use_imm) { |
| __ Sub(rd, rs1, rs2); // SUB is OK here even for 32-bit comparison. |
| } else if (imm != 0) { |
| DCHECK(IsInt<12>(-imm)); |
| __ Addi(rd, rs1, -imm); // ADDI is OK here even for 32-bit comparison. |
| } // else test `rs1` directly without subtraction for `use_imm && imm == 0`. |
| if (cond == kCondEQ) { |
| __ Seqz(rd, (use_imm && imm == 0) ? rs1 : rd); |
| } else { |
| __ Snez(rd, (use_imm && imm == 0) ? rs1 : rd); |
| } |
| break; |
| |
| case kCondLT: |
| case kCondGE: |
| if (use_imm) { |
| DCHECK(IsInt<12>(imm)); |
| __ Slti(rd, rs1, imm); |
| } else { |
| __ Slt(rd, rs1, rs2); |
| } |
| // Calculate `rs1 >= rhs` as `!(rs1 < rhs)` since there's only the SLT but no SGE. |
| reverse_condition = (cond == kCondGE); |
| break; |
| |
| case kCondLE: |
| case kCondGT: |
| if (use_imm) { |
| // Calculate `rs1 <= imm` as `rs1 < imm + 1`. |
| DCHECK(IsInt<12>(imm + 1)); // The value that overflows would fail this check. |
| __ Slti(rd, rs1, imm + 1); |
| } else { |
| __ Slt(rd, rs2, rs1); |
| } |
| // Calculate `rs1 > imm` as `!(rs1 < imm + 1)` and calculate |
| // `rs1 <= rs2` as `!(rs2 < rs1)` since there's only the SLT but no SGE. |
| reverse_condition = ((cond == kCondGT) == use_imm); |
| break; |
| |
| case kCondB: |
| case kCondAE: |
| if (use_imm) { |
| // Sltiu sign-extends its 12-bit immediate operand before the comparison |
| // and thus lets us compare directly with unsigned values in the ranges |
| // [0, 0x7ff] and [0x[ffffffff]fffff800, 0x[ffffffff]ffffffff]. |
| DCHECK(IsInt<12>(imm)); |
| __ Sltiu(rd, rs1, imm); |
| } else { |
| __ Sltu(rd, rs1, rs2); |
| } |
| // Calculate `rs1 AE rhs` as `!(rs1 B rhs)` since there's only the SLTU but no SGEU. |
| reverse_condition = (cond == kCondAE); |
| break; |
| |
| case kCondBE: |
| case kCondA: |
| if (use_imm) { |
| // Calculate `rs1 BE imm` as `rs1 B imm + 1`. |
| // Sltiu sign-extends its 12-bit immediate operand before the comparison |
| // and thus lets us compare directly with unsigned values in the ranges |
| // [0, 0x7ff] and [0x[ffffffff]fffff800, 0x[ffffffff]ffffffff]. |
| DCHECK(IsInt<12>(imm + 1)); // The value that overflows would fail this check. |
| __ Sltiu(rd, rs1, imm + 1); |
| } else { |
| __ Sltu(rd, rs2, rs1); |
| } |
| // Calculate `rs1 A imm` as `!(rs1 B imm + 1)` and calculate |
| // `rs1 BE rs2` as `!(rs2 B rs1)` since there's only the SLTU but no SGEU. |
| reverse_condition = ((cond == kCondA) == use_imm); |
| break; |
| } |
| if (to_all_bits) { |
| // Store the result to all bits; in other words, "true" is represented by -1. |
| if (reverse_condition) { |
| __ Addi(rd, rd, -1); // 0 -> -1, 1 -> 0 |
| } else { |
| __ Neg(rd, rd); // 0 -> 0, 1 -> -1 |
| } |
| } else { |
| if (reverse_condition) { |
| __ Xori(rd, rd, 1); |
| } |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateIntLongCompareAndBranch(IfCondition cond, |
| LocationSummary* locations, |
| Riscv64Label* label) { |
| XRegister left = locations->InAt(0).AsRegister<XRegister>(); |
| Location right_location = locations->InAt(1); |
| if (right_location.IsConstant()) { |
| DCHECK_EQ(CodeGenerator::GetInt64ValueOf(right_location.GetConstant()), 0); |
| switch (cond) { |
| case kCondEQ: |
| case kCondBE: // <= 0 if zero |
| __ Beqz(left, label); |
| break; |
| case kCondNE: |
| case kCondA: // > 0 if non-zero |
| __ Bnez(left, label); |
| break; |
| case kCondLT: |
| __ Bltz(left, label); |
| break; |
| case kCondGE: |
| __ Bgez(left, label); |
| break; |
| case kCondLE: |
| __ Blez(left, label); |
| break; |
| case kCondGT: |
| __ Bgtz(left, label); |
| break; |
| case kCondB: // always false |
| break; |
| case kCondAE: // always true |
| __ J(label); |
| break; |
| } |
| } else { |
| XRegister right_reg = right_location.AsRegister<XRegister>(); |
| switch (cond) { |
| case kCondEQ: |
| __ Beq(left, right_reg, label); |
| break; |
| case kCondNE: |
| __ Bne(left, right_reg, label); |
| break; |
| case kCondLT: |
| __ Blt(left, right_reg, label); |
| break; |
| case kCondGE: |
| __ Bge(left, right_reg, label); |
| break; |
| case kCondLE: |
| __ Ble(left, right_reg, label); |
| break; |
| case kCondGT: |
| __ Bgt(left, right_reg, label); |
| break; |
| case kCondB: |
| __ Bltu(left, right_reg, label); |
| break; |
| case kCondAE: |
| __ Bgeu(left, right_reg, label); |
| break; |
| case kCondBE: |
| __ Bleu(left, right_reg, label); |
| break; |
| case kCondA: |
| __ Bgtu(left, right_reg, label); |
| break; |
| } |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateFpCondition(IfCondition cond, |
| bool gt_bias, |
| DataType::Type type, |
| LocationSummary* locations, |
| Riscv64Label* label) { |
| DCHECK_EQ(label != nullptr, locations->Out().IsInvalid()); |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister rd = |
| (label != nullptr) ? srs.AllocateXRegister() : locations->Out().AsRegister<XRegister>(); |
| GenerateFpCondition(cond, gt_bias, type, locations, label, rd, /*to_all_bits=*/ false); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateFpCondition(IfCondition cond, |
| bool gt_bias, |
| DataType::Type type, |
| LocationSummary* locations, |
| Riscv64Label* label, |
| XRegister rd, |
| bool to_all_bits) { |
| // RISCV-V FP compare instructions yield the following values: |
| // l<r l=r l>r Unordered |
| // FEQ l,r 0 1 0 0 |
| // FLT l,r 1 0 0 0 |
| // FLT r,l 0 0 1 0 |
| // FLE l,r 1 1 0 0 |
| // FLE r,l 0 1 1 0 |
| // |
| // We can calculate the `Compare` results using the following formulas: |
| // l<r l=r l>r Unordered |
| // Compare/gt_bias -1 0 1 1 = ((FLE l,r) ^ 1) - (FLT l,r) |
| // Compare/lt_bias -1 0 1 -1 = ((FLE r,l) - 1) + (FLT r,l) |
| // These are emitted in `VisitCompare()`. |
| // |
| // This function emits a fused `Condition(Compare(., .), 0)`. If we compare the |
| // `Compare` results above with 0, we get the following values and formulas: |
| // l<r l=r l>r Unordered |
| // CondEQ/- 0 1 0 0 = (FEQ l, r) |
| // CondNE/- 1 0 1 1 = (FEQ l, r) ^ 1 |
| // CondLT/gt_bias 1 0 0 0 = (FLT l,r) |
| // CondLT/lt_bias 1 0 0 1 = (FLE r,l) ^ 1 |
| // CondLE/gt_bias 1 1 0 0 = (FLE l,r) |
| // CondLE/lt_bias 1 1 0 1 = (FLT r,l) ^ 1 |
| // CondGT/gt_bias 0 0 1 1 = (FLE l,r) ^ 1 |
| // CondGT/lt_bias 0 0 1 0 = (FLT r,l) |
| // CondGE/gt_bias 0 1 1 1 = (FLT l,r) ^ 1 |
| // CondGE/lt_bias 0 1 1 0 = (FLE r,l) |
| // (CondEQ/CondNE comparison with zero yields the same result with gt_bias and lt_bias.) |
| // |
| // If the condition is not materialized, the `^ 1` is not emitted, |
| // instead the condition is reversed by emitting BEQZ instead of BNEZ. |
| |
| FRegister rs1 = locations->InAt(0).AsFpuRegister<FRegister>(); |
| FRegister rs2 = locations->InAt(1).AsFpuRegister<FRegister>(); |
| |
| bool reverse_condition = false; |
| switch (cond) { |
| case kCondEQ: |
| FEq(rd, rs1, rs2, type); |
| break; |
| case kCondNE: |
| FEq(rd, rs1, rs2, type); |
| reverse_condition = true; |
| break; |
| case kCondLT: |
| if (gt_bias) { |
| FLt(rd, rs1, rs2, type); |
| } else { |
| FLe(rd, rs2, rs1, type); |
| reverse_condition = true; |
| } |
| break; |
| case kCondLE: |
| if (gt_bias) { |
| FLe(rd, rs1, rs2, type); |
| } else { |
| FLt(rd, rs2, rs1, type); |
| reverse_condition = true; |
| } |
| break; |
| case kCondGT: |
| if (gt_bias) { |
| FLe(rd, rs1, rs2, type); |
| reverse_condition = true; |
| } else { |
| FLt(rd, rs2, rs1, type); |
| } |
| break; |
| case kCondGE: |
| if (gt_bias) { |
| FLt(rd, rs1, rs2, type); |
| reverse_condition = true; |
| } else { |
| FLe(rd, rs2, rs1, type); |
| } |
| break; |
| default: |
| LOG(FATAL) << "Unexpected floating-point condition " << cond; |
| UNREACHABLE(); |
| } |
| |
| if (label != nullptr) { |
| if (reverse_condition) { |
| __ Beqz(rd, label); |
| } else { |
| __ Bnez(rd, label); |
| } |
| } else if (to_all_bits) { |
| // Store the result to all bits; in other words, "true" is represented by -1. |
| if (reverse_condition) { |
| __ Addi(rd, rd, -1); // 0 -> -1, 1 -> 0 |
| } else { |
| __ Neg(rd, rd); // 0 -> 0, 1 -> -1 |
| } |
| } else { |
| if (reverse_condition) { |
| __ Xori(rd, rd, 1); |
| } |
| } |
| } |
| |
| void CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, |
| Location ref, |
| XRegister obj, |
| uint32_t offset, |
| Location temp, |
| bool needs_null_check) { |
| GenerateReferenceLoadWithBakerReadBarrier( |
| instruction, ref, obj, offset, /*index=*/ Location::NoLocation(), temp, needs_null_check); |
| } |
| |
| void CodeGeneratorRISCV64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, |
| Location ref, |
| XRegister obj, |
| uint32_t data_offset, |
| Location index, |
| Location temp, |
| bool needs_null_check) { |
| GenerateReferenceLoadWithBakerReadBarrier( |
| instruction, ref, obj, data_offset, index, temp, needs_null_check); |
| } |
| |
| void CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, |
| Location ref, |
| XRegister obj, |
| uint32_t offset, |
| Location index, |
| Location temp, |
| bool needs_null_check) { |
| // For now, use the same approach as for GC roots plus unpoison the reference if needed. |
| // TODO(riscv64): Implement checking if the holder is black. |
| UNUSED(temp); |
| |
| DCHECK(EmitBakerReadBarrier()); |
| XRegister reg = ref.AsRegister<XRegister>(); |
| if (index.IsValid()) { |
| DCHECK(!needs_null_check); |
| DCHECK(index.IsRegister()); |
| DataType::Type type = DataType::Type::kReference; |
| DCHECK_EQ(type, instruction->GetType()); |
| if (instruction->IsArrayGet()) { |
| // /* HeapReference<Object> */ ref = *(obj + index * element_size + offset) |
| instruction_visitor_.ShNAdd(reg, index.AsRegister<XRegister>(), obj, type); |
| } else { |
| // /* HeapReference<Object> */ ref = *(obj + index + offset) |
| DCHECK(instruction->IsInvoke()); |
| DCHECK(instruction->GetLocations()->Intrinsified()); |
| __ Add(reg, index.AsRegister<XRegister>(), obj); |
| } |
| __ Loadwu(reg, reg, offset); |
| } else { |
| // /* HeapReference<Object> */ ref = *(obj + offset) |
| __ Loadwu(reg, obj, offset); |
| if (needs_null_check) { |
| MaybeRecordImplicitNullCheck(instruction); |
| } |
| } |
| MaybeUnpoisonHeapReference(reg); |
| |
| // Slow path marking the reference. |
| XRegister tmp = RA; // Use RA as temp. It is clobbered in the slow path anyway. |
| SlowPathCodeRISCV64* slow_path = new (GetScopedAllocator()) ReadBarrierMarkSlowPathRISCV64( |
| instruction, ref, Location::RegisterLocation(tmp)); |
| AddSlowPath(slow_path); |
| |
| const int32_t entry_point_offset = ReadBarrierMarkEntrypointOffset(ref); |
| // Loading the entrypoint does not require a load acquire since it is only changed when |
| // threads are suspended or running a checkpoint. |
| __ Loadd(tmp, TR, entry_point_offset); |
| __ Bnez(tmp, slow_path->GetEntryLabel()); |
| __ Bind(slow_path->GetExitLabel()); |
| } |
| |
| SlowPathCodeRISCV64* CodeGeneratorRISCV64::AddReadBarrierSlowPath(HInstruction* instruction, |
| Location out, |
| Location ref, |
| Location obj, |
| uint32_t offset, |
| Location index) { |
| UNUSED(instruction); |
| UNUSED(out); |
| UNUSED(ref); |
| UNUSED(obj); |
| UNUSED(offset); |
| UNUSED(index); |
| LOG(FATAL) << "Unimplemented"; |
| UNREACHABLE(); |
| } |
| |
| void CodeGeneratorRISCV64::GenerateReadBarrierSlow(HInstruction* instruction, |
| Location out, |
| Location ref, |
| Location obj, |
| uint32_t offset, |
| Location index) { |
| UNUSED(instruction); |
| UNUSED(out); |
| UNUSED(ref); |
| UNUSED(obj); |
| UNUSED(offset); |
| UNUSED(index); |
| LOG(FATAL) << "Unimplemented"; |
| } |
| |
| void CodeGeneratorRISCV64::MaybeGenerateReadBarrierSlow(HInstruction* instruction, |
| Location out, |
| Location ref, |
| Location obj, |
| uint32_t offset, |
| Location index) { |
| if (EmitReadBarrier()) { |
| // Baker's read barriers shall be handled by the fast path |
| // (CodeGeneratorRISCV64::GenerateReferenceLoadWithBakerReadBarrier). |
| DCHECK(!kUseBakerReadBarrier); |
| // If heap poisoning is enabled, unpoisoning will be taken care of |
| // by the runtime within the slow path. |
| GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); |
| } else if (kPoisonHeapReferences) { |
| UnpoisonHeapReference(out.AsRegister<XRegister>()); |
| } |
| } |
| |
| void CodeGeneratorRISCV64::GenerateReadBarrierForRootSlow(HInstruction* instruction, |
| Location out, |
| Location root) { |
| DCHECK(EmitReadBarrier()); |
| |
| // Insert a slow path based read barrier *after* the GC root load. |
| // |
| // Note that GC roots are not affected by heap poisoning, so we do |
| // not need to do anything special for this here. |
| SlowPathCodeRISCV64* slow_path = |
| new (GetScopedAllocator()) ReadBarrierForRootSlowPathRISCV64(instruction, out, root); |
| AddSlowPath(slow_path); |
| |
| __ J(slow_path->GetEntryLabel()); |
| __ Bind(slow_path->GetExitLabel()); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleGoto(HInstruction* instruction, |
| HBasicBlock* successor) { |
| if (successor->IsExitBlock()) { |
| DCHECK(instruction->GetPrevious()->AlwaysThrows()); |
| return; // no code needed |
| } |
| |
| HBasicBlock* block = instruction->GetBlock(); |
| HInstruction* previous = instruction->GetPrevious(); |
| HLoopInformation* info = block->GetLoopInformation(); |
| |
| if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { |
| codegen_->MaybeIncrementHotness(info->GetSuspendCheck(), /*is_frame_entry=*/ false); |
| GenerateSuspendCheck(info->GetSuspendCheck(), successor); |
| return; // `GenerateSuspendCheck()` emitted the jump. |
| } |
| if (block->IsEntryBlock() && previous != nullptr && previous->IsSuspendCheck()) { |
| GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); |
| } |
| if (!codegen_->GoesToNextBlock(block, successor)) { |
| __ J(codegen_->GetLabelOf(successor)); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenPackedSwitchWithCompares(XRegister adjusted, |
| XRegister temp, |
| uint32_t num_entries, |
| HBasicBlock* switch_block) { |
| // Note: The `adjusted` register holds `value - lower_bound`. If the `lower_bound` is 0, |
| // `adjusted` is the original `value` register and we must not clobber it. Otherwise, |
| // `adjusted` is the `temp`. The caller already emitted the `adjusted < num_entries` check. |
| |
| // Create a set of compare/jumps. |
| ArrayRef<HBasicBlock* const> successors(switch_block->GetSuccessors()); |
| uint32_t index = 0; |
| for (; num_entries - index >= 2u; index += 2u) { |
| // Jump to `successors[index]` if `value == lower_bound + index`. |
| // Note that `adjusted` holds `value - lower_bound - index`. |
| __ Beqz(adjusted, codegen_->GetLabelOf(successors[index])); |
| if (num_entries - index == 2u) { |
| break; // The last entry shall match, so the branch shall be unconditional. |
| } |
| // Jump to `successors[index + 1]` if `value == lower_bound + index + 1`. |
| // Modify `adjusted` to hold `value - lower_bound - index - 2` for this comparison. |
| __ Addi(temp, adjusted, -2); |
| adjusted = temp; |
| __ Bltz(adjusted, codegen_->GetLabelOf(successors[index + 1])); |
| } |
| // For the last entry, unconditionally jump to `successors[num_entries - 1]`. |
| __ J(codegen_->GetLabelOf(successors[num_entries - 1u])); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenTableBasedPackedSwitch(XRegister adjusted, |
| XRegister temp, |
| uint32_t num_entries, |
| HBasicBlock* switch_block) { |
| // Note: The `adjusted` register holds `value - lower_bound`. If the `lower_bound` is 0, |
| // `adjusted` is the original `value` register and we must not clobber it. Otherwise, |
| // `adjusted` is the `temp`. The caller already emitted the `adjusted < num_entries` check. |
| |
| // Create a jump table. |
| ArenaVector<Riscv64Label*> labels(num_entries, |
| __ GetAllocator()->Adapter(kArenaAllocSwitchTable)); |
| const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors(); |
| for (uint32_t i = 0; i < num_entries; i++) { |
| labels[i] = codegen_->GetLabelOf(successors[i]); |
| } |
| JumpTable* table = __ CreateJumpTable(std::move(labels)); |
| |
| // Load the address of the jump table. |
| // Note: The `LoadLabelAddress()` emits AUIPC+ADD. It is possible to avoid the ADD and |
| // instead embed that offset in the LW below as well as all jump table entries but |
| // that would need some invasive changes in the jump table handling in the assembler. |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister table_base = srs.AllocateXRegister(); |
| __ LoadLabelAddress(table_base, table->GetLabel()); |
| |
| // Load the PC difference from the jump table. |
| // TODO(riscv64): Use SH2ADD from the Zba extension. |
| __ Slli(temp, adjusted, 2); |
| __ Add(temp, temp, table_base); |
| __ Lw(temp, temp, 0); |
| |
| // Compute the absolute target address by adding the table start address |
| // (the table contains offsets to targets relative to its start). |
| __ Add(temp, temp, table_base); |
| // And jump. |
| __ Jr(temp); |
| } |
| |
| int32_t InstructionCodeGeneratorRISCV64::VecAddress(LocationSummary* locations, |
| size_t size, |
| /*out*/ XRegister* adjusted_base) { |
| UNUSED(locations); |
| UNUSED(size); |
| UNUSED(adjusted_base); |
| LOG(FATAL) << "Unimplemented"; |
| UNREACHABLE(); |
| } |
| |
| void LocationsBuilderRISCV64::HandleBinaryOp(HBinaryOperation* instruction) { |
| DCHECK_EQ(instruction->InputCount(), 2u); |
| LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
| DataType::Type type = instruction->GetResultType(); |
| switch (type) { |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: { |
| locations->SetInAt(0, Location::RequiresRegister()); |
| HInstruction* right = instruction->InputAt(1); |
| bool can_use_imm = false; |
| if (instruction->IsMin() || instruction->IsMax()) { |
| can_use_imm = IsZeroBitPattern(instruction); |
| } else if (right->IsConstant()) { |
| int64_t imm = CodeGenerator::GetInt64ValueOf(right->AsConstant()); |
| can_use_imm = IsInt<12>(instruction->IsSub() ? -imm : imm); |
| } |
| if (can_use_imm) { |
| locations->SetInAt(1, Location::ConstantLocation(right)); |
| } else { |
| locations->SetInAt(1, Location::RequiresRegister()); |
| } |
| locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); |
| break; |
| } |
| |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetInAt(1, Location::RequiresFpuRegister()); |
| if (instruction->IsMin() || instruction->IsMax()) { |
| locations->SetOut(Location::RequiresFpuRegister(), Location::kOutputOverlap); |
| } else { |
| locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); |
| } |
| break; |
| |
| default: |
| LOG(FATAL) << "Unexpected " << instruction->DebugName() << " type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleBinaryOp(HBinaryOperation* instruction) { |
| DataType::Type type = instruction->GetType(); |
| LocationSummary* locations = instruction->GetLocations(); |
| |
| switch (type) { |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: { |
| XRegister rd = locations->Out().AsRegister<XRegister>(); |
| XRegister rs1 = locations->InAt(0).AsRegister<XRegister>(); |
| Location rs2_location = locations->InAt(1); |
| |
| bool use_imm = rs2_location.IsConstant(); |
| XRegister rs2 = use_imm ? kNoXRegister : rs2_location.AsRegister<XRegister>(); |
| int64_t imm = use_imm ? CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant()) : 0; |
| |
| if (instruction->IsAnd()) { |
| if (use_imm) { |
| __ Andi(rd, rs1, imm); |
| } else { |
| __ And(rd, rs1, rs2); |
| } |
| } else if (instruction->IsOr()) { |
| if (use_imm) { |
| __ Ori(rd, rs1, imm); |
| } else { |
| __ Or(rd, rs1, rs2); |
| } |
| } else if (instruction->IsXor()) { |
| if (use_imm) { |
| __ Xori(rd, rs1, imm); |
| } else { |
| __ Xor(rd, rs1, rs2); |
| } |
| } else if (instruction->IsAdd() || instruction->IsSub()) { |
| if (type == DataType::Type::kInt32) { |
| if (use_imm) { |
| __ Addiw(rd, rs1, instruction->IsSub() ? -imm : imm); |
| } else if (instruction->IsAdd()) { |
| __ Addw(rd, rs1, rs2); |
| } else { |
| DCHECK(instruction->IsSub()); |
| __ Subw(rd, rs1, rs2); |
| } |
| } else { |
| if (use_imm) { |
| __ Addi(rd, rs1, instruction->IsSub() ? -imm : imm); |
| } else if (instruction->IsAdd()) { |
| __ Add(rd, rs1, rs2); |
| } else { |
| DCHECK(instruction->IsSub()); |
| __ Sub(rd, rs1, rs2); |
| } |
| } |
| } else if (instruction->IsMin()) { |
| DCHECK_IMPLIES(use_imm, imm == 0); |
| __ Min(rd, rs1, use_imm ? Zero : rs2); |
| } else { |
| DCHECK(instruction->IsMax()); |
| DCHECK_IMPLIES(use_imm, imm == 0); |
| __ Max(rd, rs1, use_imm ? Zero : rs2); |
| } |
| break; |
| } |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: { |
| FRegister rd = locations->Out().AsFpuRegister<FRegister>(); |
| FRegister rs1 = locations->InAt(0).AsFpuRegister<FRegister>(); |
| FRegister rs2 = locations->InAt(1).AsFpuRegister<FRegister>(); |
| if (instruction->IsAdd()) { |
| FAdd(rd, rs1, rs2, type); |
| } else if (instruction->IsSub()) { |
| FSub(rd, rs1, rs2, type); |
| } else { |
| DCHECK(instruction->IsMin() || instruction->IsMax()); |
| // If one of the operands is NaN and the other is not, riscv64 instructions FMIN/FMAX |
| // return the other operand while we want to return the NaN operand. |
| DCHECK_NE(rd, rs1); // Requested `Location::kOutputOverlap`. |
| DCHECK_NE(rd, rs2); // Requested `Location::kOutputOverlap`. |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister tmp = srs.AllocateXRegister(); |
| XRegister tmp2 = srs.AllocateXRegister(); |
| Riscv64Label done; |
| // Return `rs1` if it's NaN. |
| FClass(tmp, rs1, type); |
| __ Li(tmp2, kFClassNaNMinValue); |
| FMv(rd, rs1, type); |
| __ Bgeu(tmp, tmp2, &done); |
| // Return `rs2` if it's NaN. |
| FClass(tmp, rs2, type); |
| FMv(rd, rs2, type); |
| __ Bgeu(tmp, tmp2, &done); |
| // Calculate Min/Max for non-NaN arguments. |
| if (instruction->IsMin()) { |
| FMin(rd, rs1, rs2, type); |
| } else { |
| FMax(rd, rs1, rs2, type); |
| } |
| __ Bind(&done); |
| } |
| break; |
| } |
| default: |
| LOG(FATAL) << "Unexpected binary operation type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| void LocationsBuilderRISCV64::HandleCondition(HCondition* instruction) { |
| LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
| switch (instruction->InputAt(0)->GetType()) { |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| locations->SetInAt(0, Location::RequiresFpuRegister()); |
| locations->SetInAt(1, Location::RequiresFpuRegister()); |
| break; |
| |
| default: { |
| locations->SetInAt(0, Location::RequiresRegister()); |
| HInstruction* rhs = instruction->InputAt(1); |
| bool use_imm = false; |
| if (rhs->IsConstant()) { |
| int64_t imm = CodeGenerator::GetInt64ValueOf(rhs->AsConstant()); |
| if (instruction->IsEmittedAtUseSite()) { |
| // For `HIf`, materialize all non-zero constants with an `HParallelMove`. |
| // Note: For certain constants and conditions, the code could be improved. |
| // For example, 2048 takes two instructions to materialize but the negative |
| // -2048 could be embedded in ADDI for EQ/NE comparison. |
| use_imm = (imm == 0); |
| } else { |
| // Constants that cannot be embedded in an instruction's 12-bit immediate shall be |
| // materialized with an `HParallelMove`. This simplifies the code and avoids cases |
| // with arithmetic overflow. Adjust the `imm` if needed for a particular instruction. |
| switch (instruction->GetCondition()) { |
| case kCondEQ: |
| case kCondNE: |
| imm = -imm; // ADDI with negative immediate (there is no SUBI). |
| break; |
| case kCondLE: |
| case kCondGT: |
| case kCondBE: |
| case kCondA: |
| imm += 1; // SLTI/SLTIU with adjusted immediate (there is no SLEI/SLEIU). |
| break; |
| default: |
| break; |
| } |
| use_imm = IsInt<12>(imm); |
| } |
| } |
| if (use_imm) { |
| locations->SetInAt(1, Location::ConstantLocation(rhs)); |
| } else { |
| locations->SetInAt(1, Location::RequiresRegister()); |
| } |
| break; |
| } |
| } |
| if (!instruction->IsEmittedAtUseSite()) { |
| locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleCondition(HCondition* instruction) { |
| if (instruction->IsEmittedAtUseSite()) { |
| return; |
| } |
| |
| DataType::Type type = instruction->InputAt(0)->GetType(); |
| LocationSummary* locations = instruction->GetLocations(); |
| switch (type) { |
| case DataType::Type::kFloat32: |
| case DataType::Type::kFloat64: |
| GenerateFpCondition(instruction->GetCondition(), instruction->IsGtBias(), type, locations); |
| return; |
| default: |
| // Integral types and reference equality. |
| GenerateIntLongCondition(instruction->GetCondition(), locations); |
| return; |
| } |
| } |
| |
| void LocationsBuilderRISCV64::HandleShift(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsShl() || |
| instruction->IsShr() || |
| instruction->IsUShr() || |
| instruction->IsRol() || |
| instruction->IsRor()); |
| |
| LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction); |
| DataType::Type type = instruction->GetResultType(); |
| switch (type) { |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: { |
| locations->SetInAt(0, Location::RequiresRegister()); |
| locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); |
| locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); |
| break; |
| } |
| default: |
| LOG(FATAL) << "Unexpected shift type " << type; |
| UNREACHABLE(); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleShift(HBinaryOperation* instruction) { |
| DCHECK(instruction->IsShl() || |
| instruction->IsShr() || |
| instruction->IsUShr() || |
| instruction->IsRol() || |
| instruction->IsRor()); |
| |
| LocationSummary* locations = instruction->GetLocations(); |
| DataType::Type type = instruction->GetType(); |
| |
| switch (type) { |
| case DataType::Type::kInt32: |
| case DataType::Type::kInt64: { |
| XRegister rd = locations->Out().AsRegister<XRegister>(); |
| XRegister rs1 = locations->InAt(0).AsRegister<XRegister>(); |
| Location rs2_location = locations->InAt(1); |
| |
| if (rs2_location.IsConstant()) { |
| int64_t imm = CodeGenerator::GetInt64ValueOf(rs2_location.GetConstant()); |
| if (instruction->IsRol()) { |
| imm = -imm; |
| } |
| uint32_t shamt = |
| imm & (type == DataType::Type::kInt32 ? kMaxIntShiftDistance : kMaxLongShiftDistance); |
| |
| if (shamt == 0) { |
| if (rd != rs1) { |
| __ Mv(rd, rs1); |
| } |
| } else if (type == DataType::Type::kInt32) { |
| if (instruction->IsShl()) { |
| __ Slliw(rd, rs1, shamt); |
| } else if (instruction->IsShr()) { |
| __ Sraiw(rd, rs1, shamt); |
| } else if (instruction->IsUShr()) { |
| __ Srliw(rd, rs1, shamt); |
| } else if (instruction->IsRol()) { |
| __ Roriw(rd, rs1, shamt); |
| } else { |
| DCHECK(instruction->IsRor()); |
| __ Roriw(rd, rs1, shamt); |
| } |
| } else { |
| if (instruction->IsShl()) { |
| __ Slli(rd, rs1, shamt); |
| } else if (instruction->IsShr()) { |
| __ Srai(rd, rs1, shamt); |
| } else if (instruction->IsUShr()) { |
| __ Srli(rd, rs1, shamt); |
| } else if (instruction->IsRol()) { |
| __ Rori(rd, rs1, shamt); |
| } else { |
| DCHECK(instruction->IsRor()); |
| __ Rori(rd, rs1, shamt); |
| } |
| } |
| } else { |
| XRegister rs2 = rs2_location.AsRegister<XRegister>(); |
| if (type == DataType::Type::kInt32) { |
| if (instruction->IsShl()) { |
| __ Sllw(rd, rs1, rs2); |
| } else if (instruction->IsShr()) { |
| __ Sraw(rd, rs1, rs2); |
| } else if (instruction->IsUShr()) { |
| __ Srlw(rd, rs1, rs2); |
| } else if (instruction->IsRol()) { |
| __ Rolw(rd, rs1, rs2); |
| } else { |
| DCHECK(instruction->IsRor()); |
| __ Rorw(rd, rs1, rs2); |
| } |
| } else { |
| if (instruction->IsShl()) { |
| __ Sll(rd, rs1, rs2); |
| } else if (instruction->IsShr()) { |
| __ Sra(rd, rs1, rs2); |
| } else if (instruction->IsUShr()) { |
| __ Srl(rd, rs1, rs2); |
| } else if (instruction->IsRol()) { |
| __ Rol(rd, rs1, rs2); |
| } else { |
| DCHECK(instruction->IsRor()); |
| __ Ror(rd, rs1, rs2); |
| } |
| } |
| } |
| break; |
| } |
| default: |
| LOG(FATAL) << "Unexpected shift operation type " << type; |
| } |
| } |
| |
| void CodeGeneratorRISCV64::MaybeMarkGCCard(XRegister object, |
| XRegister value, |
| bool value_can_be_null) { |
| Riscv64Label done; |
| if (value_can_be_null) { |
| __ Beqz(value, &done); |
| } |
| MarkGCCard(object); |
| __ Bind(&done); |
| } |
| |
| void CodeGeneratorRISCV64::MarkGCCard(XRegister object) { |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister card = srs.AllocateXRegister(); |
| XRegister temp = srs.AllocateXRegister(); |
| // Load the address of the card table into `card`. |
| __ Loadd(card, TR, Thread::CardTableOffset<kRiscv64PointerSize>().Int32Value()); |
| |
| // Calculate the address of the card corresponding to `object`. |
| __ Srli(temp, object, gc::accounting::CardTable::kCardShift); |
| __ Add(temp, card, temp); |
| // Write the `art::gc::accounting::CardTable::kCardDirty` value into the |
| // `object`'s card. |
| // |
| // Register `card` contains the address of the card table. Note that the card |
| // table's base is biased during its creation so that it always starts at an |
| // address whose least-significant byte is equal to `kCardDirty` (see |
| // art::gc::accounting::CardTable::Create). Therefore the SB instruction |
| // below writes the `kCardDirty` (byte) value into the `object`'s card |
| // (located at `card + object >> kCardShift`). |
| // |
| // This dual use of the value in register `card` (1. to calculate the location |
| // of the card to mark; and 2. to load the `kCardDirty` value) saves a load |
| // (no need to explicitly load `kCardDirty` as an immediate value). |
| __ Sb(card, temp, 0); // No scratch register left for `Storeb()`. |
| } |
| |
| void CodeGeneratorRISCV64::CheckGCCardIsValid(XRegister object) { |
| Riscv64Label done; |
| ScratchRegisterScope srs(GetAssembler()); |
| XRegister card = srs.AllocateXRegister(); |
| XRegister temp = srs.AllocateXRegister(); |
| // Load the address of the card table into `card`. |
| __ Loadd(card, TR, Thread::CardTableOffset<kRiscv64PointerSize>().Int32Value()); |
| |
| // Calculate the address of the card corresponding to `object`. |
| __ Srli(temp, object, gc::accounting::CardTable::kCardShift); |
| __ Add(temp, card, temp); |
| // assert (!clean || !self->is_gc_marking) |
| __ Lb(temp, temp, 0); |
| static_assert(gc::accounting::CardTable::kCardClean == 0); |
| __ Bnez(temp, &done); |
| __ Loadw(temp, TR, Thread::IsGcMarkingOffset<kRiscv64PointerSize>().Int32Value()); |
| __ Beqz(temp, &done); |
| __ Unimp(); |
| __ Bind(&done); |
| } |
| |
| void LocationsBuilderRISCV64::HandleFieldSet(HInstruction* instruction) { |
| LocationSummary* locations = |
| new (GetGraph()->GetAllocator()) LocationSummary(instruction, LocationSummary::kNoCall); |
| locations->SetInAt(0, Location::RequiresRegister()); |
| locations->SetInAt(1, ValueLocationForStore(instruction->InputAt(1))); |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleFieldSet(HInstruction* instruction, |
| const FieldInfo& field_info, |
| bool value_can_be_null, |
| WriteBarrierKind write_barrier_kind) { |
| DataType::Type type = field_info.GetFieldType(); |
| LocationSummary* locations = instruction->GetLocations(); |
| XRegister obj = locations->InAt(0).AsRegister<XRegister>(); |
| Location value = locations->InAt(1); |
| DCHECK_IMPLIES(value.IsConstant(), IsZeroBitPattern(value.GetConstant())); |
| bool is_volatile = field_info.IsVolatile(); |
| uint32_t offset = field_info.GetFieldOffset().Uint32Value(); |
| |
| if (is_volatile) { |
| StoreSeqCst(value, obj, offset, type, instruction); |
| } else { |
| Store(value, obj, offset, type); |
| codegen_->MaybeRecordImplicitNullCheck(instruction); |
| } |
| |
| bool needs_write_barrier = |
| codegen_->StoreNeedsWriteBarrier(type, instruction->InputAt(1), write_barrier_kind); |
| if (needs_write_barrier) { |
| if (value.IsConstant()) { |
| DCHECK_EQ(write_barrier_kind, WriteBarrierKind::kEmitBeingReliedOn); |
| codegen_->MarkGCCard(obj); |
| } else { |
| codegen_->MaybeMarkGCCard( |
| obj, |
| value.AsRegister<XRegister>(), |
| value_can_be_null && write_barrier_kind == WriteBarrierKind::kEmitNotBeingReliedOn); |
| } |
| } else if (codegen_->ShouldCheckGCCard(type, instruction->InputAt(1), write_barrier_kind)) { |
| codegen_->CheckGCCardIsValid(obj); |
| } |
| } |
| |
| void LocationsBuilderRISCV64::HandleFieldGet(HInstruction* instruction) { |
| DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); |
| |
| bool object_field_get_with_read_barrier = |
| (instruction->GetType() == DataType::Type::kReference) && codegen_->EmitReadBarrier(); |
| LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary( |
| instruction, |
| object_field_get_with_read_barrier |
| ? LocationSummary::kCallOnSlowPath |
| : LocationSummary::kNoCall); |
| |
| // Input for object receiver. |
| locations->SetInAt(0, Location::RequiresRegister()); |
| |
| if (DataType::IsFloatingPointType(instruction->GetType())) { |
| locations->SetOut(Location::RequiresFpuRegister()); |
| } else { |
| // The output overlaps for an object field get when read barriers |
| // are enabled: we do not want the load to overwrite the object's |
| // location, as we need it to emit the read barrier. |
| locations->SetOut( |
| Location::RequiresRegister(), |
| object_field_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); |
| } |
| |
| if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { |
| locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. |
| // We need a temporary register for the read barrier marking slow |
| // path in CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier. |
| locations->AddTemp(Location::RequiresRegister()); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::HandleFieldGet(HInstruction* instruction, |
| const FieldInfo& field_info) { |
| DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); |
| DCHECK_EQ(DataType::Size(field_info.GetFieldType()), DataType::Size(instruction->GetType())); |
| DataType::Type type = instruction->GetType(); |
| LocationSummary* locations = instruction->GetLocations(); |
| Location obj_loc = locations->InAt(0); |
| XRegister obj = obj_loc.AsRegister<XRegister>(); |
| Location dst_loc = locations->Out(); |
| bool is_volatile = field_info.IsVolatile(); |
| uint32_t offset = field_info.GetFieldOffset().Uint32Value(); |
| |
| if (is_volatile) { |
| codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); |
| } |
| |
| if (type == DataType::Type::kReference && codegen_->EmitBakerReadBarrier()) { |
| // /* HeapReference<Object> */ dst = *(obj + offset) |
| Location temp_loc = locations->GetTemp(0); |
| // Note that a potential implicit null check is handled in this |
| // CodeGeneratorRISCV64::GenerateFieldLoadWithBakerReadBarrier call. |
| codegen_->GenerateFieldLoadWithBakerReadBarrier(instruction, |
| dst_loc, |
| obj, |
| offset, |
| temp_loc, |
| /* needs_null_check= */ true); |
| } else { |
| Load(dst_loc, obj, offset, type); |
| codegen_->MaybeRecordImplicitNullCheck(instruction); |
| } |
| |
| if (is_volatile) { |
| codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); |
| } |
| |
| if (type == DataType::Type::kReference && !codegen_->EmitBakerReadBarrier()) { |
| // If read barriers are enabled, emit read barriers other than |
| // Baker's using a slow path (and also unpoison the loaded |
| // reference, if heap poisoning is enabled). |
| codegen_->MaybeGenerateReadBarrierSlow(instruction, dst_loc, dst_loc, obj_loc, offset); |
| } |
| } |
| |
| void InstructionCodeGeneratorRISCV64::GenerateMethodEntryExitHook(HInstruction* instruction) { |
| SlowPathCodeRISCV64* slow_path = |
| new (codegen_->GetScopedAllocator()) MethodEntryExitHooksSlowPathRISCV64(instruction); |
| codegen_->AddSlowPath(slow_path); |
| |
| ScratchRegisterScope temps(GetAssembler()); |
| XRegister tmp = temps.AllocateXRegister(); |
| |
| if (instruction->IsMethodExitHook()) { |
| // Check if we are required to check if the caller needs a deoptimization. Strictly speaking it |
| // would be sufficient to check if CheckCallerForDeopt bit is set. Though it is faster to check |
| // if it is just non-zero. kCHA bit isn't used in debuggable runtimes as cha optimization is |
| // disabled in debuggable runtime. The other bit is used when this method itself requires a |
| // deoptimization due to redefinition. So it is safe to just check for non-zero value here. |
| __ Loadwu(tmp, SP, codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); |
| __ Bnez(tmp, slow_path->GetEntryLabel()); |
| } |
| |
| uint64_t hook_offset = instruction->IsMethodExitHook() ? |
| instrumentation::Instrumentation::HaveMethodExitListenersOffset().SizeValue() : |
| instrumentation::Instrumentation::HaveMethodEntryListenersOffset().SizeValue(); |
| auto [base_hook_address, hook_imm12] = SplitJitAddress( |
| reinterpret_cast64<uint64_t>(Runtime::Current()->GetInstrumentation()) + hook_offset); |
| __ LoadConst64(tmp, base_hook_address); |
| __ Lbu(tmp, tmp, hook_imm12); |
| // Check if there are any method entry / exit listeners. If no, continue. |
| __ Beqz(tmp, slow_path->GetExitLabel()); |
| // Check if there are any slow (jvmti / trace with thread cpu time) method entry / exit listeners. |
| // If yes, just take the slow path. |
| static_assert(instrumentation::Instrumentation::kFastTraceListeners == 1u); |
| __ Addi(tmp, tmp, -1); |
| __ Bnez(tmp, slow_path->GetEntryLabel()); |
| |
| // Allocate second core scratch register. We can no longer use `Stored()` |
| // and similar macro instructions because there is no core scratch register left. |
| XRegister tmp2 = temps.AllocateXRegister(); |
| |
| // Check if there is place in the buffer to store a new entry, if no, take the slow path. |
| int32_t trace_buffer_curr_entry_offset = |
| Thread::TraceBufferCurrPtrOffset<kRiscv64PointerSize>().Int32Value(); |
| __ Loadd(tmp, TR, trace_buffer_curr_entry_offset); |
| __ Loadd(tmp2, TR, Thread::TraceBufferPtrOffset<kRiscv64PointerSize>().SizeValue()); |
| __ Addi(tmp, tmp, -dchecked_integral_cast<int32_t>(kNumEntriesForWallClock * sizeof(void*))); |
| __ Blt(tmp, tmp2, slow_path->GetEntryLabel()); |
| |
| // Update the index in the `Thread`. |
| __ Sd(tmp, TR, trace_buffer_curr_entry_offset); |
| |
| // Record method pointer and trace action. |
| __ Ld(tmp2, SP, 0); |
| // Use last two bits to encode trace method action. For MethodEntry it is 0 |
| // so no need to set the bits since they are 0 already. |
| DCHECK_GE(ArtMethod::Alignment(kRuntimePointerSize), static_cast<size_t>(4)); |
| static_assert(enum_cast<int32_t>(TraceAction::kTraceMethodEnter) == 0); |
| static_assert(enum_cast |