| // Copyright 2019, VIXL authors |
| // All rights reserved. |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // * Redistributions of source code must retain the above copyright notice, |
| // this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above copyright notice, |
| // this list of conditions and the following disclaimer in the documentation |
| // and/or other materials provided with the distribution. |
| // * Neither the name of ARM Limited nor the names of its contributors may be |
| // used to endorse or promote products derived from this software without |
| // specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND |
| // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE |
| // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include "assembler-aarch64.h" |
| |
| namespace vixl { |
| namespace aarch64 { |
| |
| void Assembler::ResolveSVEImm8Shift(int* imm8, int* shift) { |
| if (*shift < 0) { |
| VIXL_ASSERT(*shift == -1); |
| // Derive the shift amount from the immediate. |
| if (IsInt8(*imm8)) { |
| *shift = 0; |
| } else if ((*imm8 % 256) == 0) { |
| *imm8 /= 256; |
| *shift = 8; |
| } |
| } |
| |
| VIXL_ASSERT(IsInt8(*imm8)); |
| VIXL_ASSERT((*shift == 0) || (*shift == 8)); |
| } |
| |
| // SVEAddressGeneration. |
| |
| void Assembler::adr(const ZRegister& zd, const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsVectorPlusVector()); |
| VIXL_ASSERT( |
| AreSameLaneSize(zd, addr.GetVectorBase(), addr.GetVectorOffset())); |
| |
| int lane_size = zd.GetLaneSizeInBits(); |
| VIXL_ASSERT((lane_size == kSRegSize) || (lane_size == kDRegSize)); |
| |
| int shift_amount = addr.GetShiftAmount(); |
| VIXL_ASSERT((shift_amount >= 0) && (shift_amount <= 3)); |
| |
| Instr op = 0xffffffff; |
| Instr msz = shift_amount << 10; |
| SVEOffsetModifier mod = addr.GetOffsetModifier(); |
| switch (mod) { |
| case SVE_UXTW: |
| VIXL_ASSERT(lane_size == kDRegSize); |
| op = ADR_z_az_d_u32_scaled; |
| break; |
| case SVE_SXTW: |
| VIXL_ASSERT(lane_size == kDRegSize); |
| op = ADR_z_az_d_s32_scaled; |
| break; |
| case SVE_LSL: |
| case NO_SVE_OFFSET_MODIFIER: |
| op = (lane_size == kSRegSize) ? ADR_z_az_s_same_scaled |
| : ADR_z_az_d_same_scaled; |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| } |
| Emit(op | msz | Rd(zd) | Rn(addr.GetVectorBase()) | |
| Rm(addr.GetVectorOffset())); |
| } |
| |
| void Assembler::SVELogicalImmediate(const ZRegister& zdn, |
| uint64_t imm, |
| Instr op) { |
| unsigned bit_n, imm_s, imm_r; |
| unsigned lane_size = zdn.GetLaneSizeInBits(); |
| // Check that the immediate can be encoded in the instruction. |
| if (IsImmLogical(imm, lane_size, &bit_n, &imm_s, &imm_r)) { |
| Emit(op | Rd(zdn) | SVEBitN(bit_n) | SVEImmRotate(imm_r, lane_size) | |
| SVEImmSetBits(imm_s, lane_size)); |
| } else { |
| VIXL_UNREACHABLE(); |
| } |
| } |
| |
| void Assembler::and_(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| SVELogicalImmediate(zd, imm, AND_z_zi); |
| } |
| |
| void Assembler::dupm(const ZRegister& zd, uint64_t imm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| // DUPM_z_i is an SVEBroadcastBitmaskImmOp, but its encoding and constraints |
| // are similar enough to SVEBitwiseLogicalWithImm_UnpredicatedOp, that we can |
| // use the logical immediate encoder to get the correct behaviour. |
| SVELogicalImmediate(zd, imm, DUPM_z_i); |
| } |
| |
| void Assembler::eor(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| SVELogicalImmediate(zd, imm, EOR_z_zi); |
| } |
| |
| void Assembler::orr(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| SVELogicalImmediate(zd, imm, ORR_z_zi); |
| } |
| |
| // SVEBitwiseLogicalUnpredicated. |
| void Assembler::and_(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| Emit(AND_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::bic(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| Emit(BIC_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::eor(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| Emit(EOR_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::orr(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| Emit(ORR_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEBitwiseShiftPredicated. |
| |
| void Assembler::SVEBitwiseShiftImmediatePred(const ZRegister& zdn, |
| const PRegisterM& pg, |
| Instr encoded_imm_and_tsz, |
| Instr op) { |
| Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz) |
| << 5; |
| Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22; |
| Emit(op | tszh | tszl_and_imm | PgLow8(pg) | Rd(zdn)); |
| } |
| |
| void Assembler::asr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0000 100. .... .... .... |
| // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 0 | Pg<12:10> | |
| // tszl<9:8> | imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASR_z_p_zi); |
| } |
| |
| void Assembler::asr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D |
| // 0000 0100 ..01 1000 100. .... .... .... |
| // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || |
| ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && |
| (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); |
| Instr op = ASR_z_p_zw; |
| if (AreSameLaneSize(zd, zn, zm)) { |
| op = ASR_z_p_zz; |
| } |
| Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::asrd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // ASRD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0100 100. .... .... .... |
| // tszh<23:22> | opc<19:18> = 01 | L<17> = 0 | U<16> = 0 | Pg<12:10> | |
| // tszl<9:8> | imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASRD_z_p_zi); |
| } |
| |
| void Assembler::asrr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ASRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0100 100. .... .... .... |
| // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(ASRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::lsl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0011 100. .... .... .... |
| // tszh<23:22> | opc<19:18> = 00 | L<17> = 1 | U<16> = 1 | Pg<12:10> | |
| // tszl<9:8> | imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSL_z_p_zi); |
| } |
| |
| void Assembler::lsl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D |
| // 0000 0100 ..01 1011 100. .... .... .... |
| // size<23:22> | R<18> = 0 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || |
| ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && |
| (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); |
| Instr op = LSL_z_p_zw; |
| if (AreSameLaneSize(zd, zn, zm)) { |
| op = LSL_z_p_zz; |
| } |
| Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::lslr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // LSLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0111 100. .... .... .... |
| // size<23:22> | R<18> = 1 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(LSLR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::lsr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0001 100. .... .... .... |
| // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 1 | Pg<12:10> | |
| // tszl<9:8> | imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSR_z_p_zi); |
| } |
| |
| void Assembler::lsr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D |
| // 0000 0100 ..01 1001 100. .... .... .... |
| // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || |
| ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && |
| (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); |
| Instr op = LSR_z_p_zw; |
| if (AreSameLaneSize(zd, zn, zm)) { |
| op = LSR_z_p_zz; |
| } |
| Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::lsrr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // LSRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0101 100. .... .... .... |
| // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(LSRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // SVEBitwiseShiftUnpredicated. |
| |
| Instr Assembler::EncodeSVEShiftLeftImmediate(int shift, int lane_size_in_bits) { |
| VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits)); |
| return lane_size_in_bits + shift; |
| } |
| |
| Instr Assembler::EncodeSVEShiftRightImmediate(int shift, |
| int lane_size_in_bits) { |
| VIXL_ASSERT((shift > 0) && (shift <= lane_size_in_bits)); |
| return (2 * lane_size_in_bits) - shift; |
| } |
| |
| void Assembler::SVEBitwiseShiftImmediate(const ZRegister& zd, |
| const ZRegister& zn, |
| Instr encoded_imm_and_tsz, |
| Instr op) { |
| Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz) |
| << 16; |
| Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22; |
| Emit(op | tszh | tszl_and_imm | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::asr(const ZRegister& zd, const ZRegister& zn, int shift) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, ASR_z_zi); |
| } |
| |
| void Assembler::asr(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); |
| |
| Emit(ASR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::lsl(const ZRegister& zd, const ZRegister& zn, int shift) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSL_z_zi); |
| } |
| |
| void Assembler::lsl(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); |
| |
| Emit(LSL_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::lsr(const ZRegister& zd, const ZRegister& zn, int shift) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSR_z_zi); |
| } |
| |
| void Assembler::lsr(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); |
| |
| Emit(LSR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEElementCount. |
| |
| #define VIXL_SVE_INC_DEC_LIST(V) \ |
| V(cntb, CNTB_r_s) \ |
| V(cnth, CNTH_r_s) \ |
| V(cntw, CNTW_r_s) \ |
| V(cntd, CNTD_r_s) \ |
| V(decb, DECB_r_rs) \ |
| V(dech, DECH_r_rs) \ |
| V(decw, DECW_r_rs) \ |
| V(decd, DECD_r_rs) \ |
| V(incb, INCB_r_rs) \ |
| V(inch, INCH_r_rs) \ |
| V(incw, INCW_r_rs) \ |
| V(incd, INCD_r_rs) \ |
| V(sqdecb, SQDECB_r_rs_x) \ |
| V(sqdech, SQDECH_r_rs_x) \ |
| V(sqdecw, SQDECW_r_rs_x) \ |
| V(sqdecd, SQDECD_r_rs_x) \ |
| V(sqincb, SQINCB_r_rs_x) \ |
| V(sqinch, SQINCH_r_rs_x) \ |
| V(sqincw, SQINCW_r_rs_x) \ |
| V(sqincd, SQINCD_r_rs_x) |
| |
| #define VIXL_DEFINE_ASM_FUNC(FN, OP) \ |
| void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(rdn.IsX()); \ |
| Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \ |
| ImmUnsignedField<19, 16>(multiplier - 1)); \ |
| } |
| VIXL_SVE_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC) |
| #undef VIXL_DEFINE_ASM_FUNC |
| |
| #define VIXL_SVE_UQINC_UQDEC_LIST(V) \ |
| V(uqdecb, (rdn.IsX() ? UQDECB_r_rs_x : UQDECB_r_rs_uw)) \ |
| V(uqdech, (rdn.IsX() ? UQDECH_r_rs_x : UQDECH_r_rs_uw)) \ |
| V(uqdecw, (rdn.IsX() ? UQDECW_r_rs_x : UQDECW_r_rs_uw)) \ |
| V(uqdecd, (rdn.IsX() ? UQDECD_r_rs_x : UQDECD_r_rs_uw)) \ |
| V(uqincb, (rdn.IsX() ? UQINCB_r_rs_x : UQINCB_r_rs_uw)) \ |
| V(uqinch, (rdn.IsX() ? UQINCH_r_rs_x : UQINCH_r_rs_uw)) \ |
| V(uqincw, (rdn.IsX() ? UQINCW_r_rs_x : UQINCW_r_rs_uw)) \ |
| V(uqincd, (rdn.IsX() ? UQINCD_r_rs_x : UQINCD_r_rs_uw)) |
| |
| #define VIXL_DEFINE_ASM_FUNC(FN, OP) \ |
| void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \ |
| ImmUnsignedField<19, 16>(multiplier - 1)); \ |
| } |
| VIXL_SVE_UQINC_UQDEC_LIST(VIXL_DEFINE_ASM_FUNC) |
| #undef VIXL_DEFINE_ASM_FUNC |
| |
| #define VIXL_SVE_SQX_INC_DEC_LIST(V) \ |
| V(sqdecb, SQDECB) \ |
| V(sqdech, SQDECH) \ |
| V(sqdecw, SQDECW) \ |
| V(sqdecd, SQDECD) \ |
| V(sqincb, SQINCB) \ |
| V(sqinch, SQINCH) \ |
| V(sqincw, SQINCW) \ |
| V(sqincd, SQINCD) |
| |
| #define VIXL_DEFINE_ASM_FUNC(FN, OP) \ |
| void Assembler::FN(const Register& xd, \ |
| const Register& wn, \ |
| int pattern, \ |
| int multiplier) { \ |
| USE(wn); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(wn.IsW() && xd.Is(wn.X())); \ |
| Emit(OP##_r_rs_sx | Rd(xd) | ImmSVEPredicateConstraint(pattern) | \ |
| ImmUnsignedField<19, 16>(multiplier - 1)); \ |
| } |
| VIXL_SVE_SQX_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC) |
| #undef VIXL_DEFINE_ASM_FUNC |
| |
| #define VIXL_SVE_INC_DEC_VEC_LIST(V) \ |
| V(dech, DEC, H) \ |
| V(decw, DEC, W) \ |
| V(decd, DEC, D) \ |
| V(inch, INC, H) \ |
| V(incw, INC, W) \ |
| V(incd, INC, D) \ |
| V(sqdech, SQDEC, H) \ |
| V(sqdecw, SQDEC, W) \ |
| V(sqdecd, SQDEC, D) \ |
| V(sqinch, SQINC, H) \ |
| V(sqincw, SQINC, W) \ |
| V(sqincd, SQINC, D) \ |
| V(uqdech, UQDEC, H) \ |
| V(uqdecw, UQDEC, W) \ |
| V(uqdecd, UQDEC, D) \ |
| V(uqinch, UQINC, H) \ |
| V(uqincw, UQINC, W) \ |
| V(uqincd, UQINC, D) |
| |
| #define VIXL_DEFINE_ASM_FUNC(FN, OP, T) \ |
| void Assembler::FN(const ZRegister& zdn, int pattern, int multiplier) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() == k##T##RegSizeInBytes); \ |
| Emit(OP##T##_z_zs | Rd(zdn) | ImmSVEPredicateConstraint(pattern) | \ |
| ImmUnsignedField<19, 16>(multiplier - 1)); \ |
| } |
| VIXL_SVE_INC_DEC_VEC_LIST(VIXL_DEFINE_ASM_FUNC) |
| #undef VIXL_DEFINE_ASM_FUNC |
| |
| // SVEFPAccumulatingReduction. |
| |
| void Assembler::fadda(const VRegister& vd, |
| const PRegister& pg, |
| const VRegister& vn, |
| const ZRegister& zm) { |
| // FADDA <V><dn>, <Pg>, <V><dn>, <Zm>.<T> |
| // 0110 0101 ..01 1000 001. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Vdn<4:0> |
| |
| USE(vn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.Is(vn)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zm.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zm, vd)); |
| |
| Emit(FADDA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // SVEFPArithmetic_Predicated. |
| |
| void Assembler::fabd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 1000 100. .... .... .... |
| // size<23:22> | opc<19:16> = 1000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1000 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FADD_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0000 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fdiv(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 1101 100. .... .... .... |
| // size<23:22> | opc<19:16> = 1101 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fdivr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 1100 100. .... .... .... |
| // size<23:22> | opc<19:16> = 1100 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmax(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1110 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 110 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FMAX_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fmax(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0110 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0110 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmaxnm(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1100 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 100 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FMAXNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fmaxnm(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0100 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0100 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMAXNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmin(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1111 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 111 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FMIN_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fmin(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0111 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0111 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fminnm(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1101 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 101 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FMINNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fminnm(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0101 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0101 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMINNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmul(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1010 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 010 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((imm == 0.5) || (imm == 2.0)); |
| |
| Instr i1 = (imm == 2.0) ? (1 << 5) : 0; |
| Emit(FMUL_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fmul(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0010 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmulx(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMULX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 1010 100. .... .... .... |
| // size<23:22> | opc<19:16> = 1010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMULX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fscale(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FSCALE <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 1001 100. .... .... .... |
| // size<23:22> | opc<19:16> = 1001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FSCALE_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1001 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 001 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FSUB_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0001 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FSUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| double imm) { |
| // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> |
| // 0110 0101 ..01 1011 100. ..00 00.. .... |
| // size<23:22> | opc<18:16> = 011 | Pg<12:10> | i1<5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); |
| |
| Instr i1 = (imm == 1.0) ? (1 << 5) : 0; |
| Emit(FSUBR_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); |
| } |
| |
| void Assembler::fsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0101 ..00 0011 100. .... .... .... |
| // size<23:22> | opc<19:16> = 0011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FSUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::ftmad(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int imm3) { |
| // FTMAD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<imm> |
| // 0110 0101 ..01 0... 1000 00.. .... .... |
| // size<23:22> | imm3<18:16> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FTMAD_z_zzi | SVESize(zd) | Rd(zd) | Rn(zm) | |
| ImmUnsignedField<18, 16>(imm3)); |
| } |
| |
| // SVEFPArithmeticUnpredicated. |
| |
| void Assembler::fadd(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0000 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fmul(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0000 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::frecps(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FRECPS <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0001 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRECPS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::frsqrts(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FRSQRTS <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0001 11.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRSQRTS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fsub(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0000 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ftsmul(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FTSMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 0000 11.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FTSMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEFPCompareVectors. |
| |
| void Assembler::facge(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FACGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 110. .... ...1 .... |
| // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FACGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::facgt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FACGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 111. .... ...1 .... |
| // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FACGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fcmeq(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 011. .... ...0 .... |
| // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FCMEQ_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fcmge(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 010. .... ...0 .... |
| // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FCMGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fcmgt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 010. .... ...1 .... |
| // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FCMGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fcmne(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 011. .... ...1 .... |
| // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FCMNE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fcmuo(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FCMUO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..0. .... 110. .... ...0 .... |
| // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | |
| // o3<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FCMUO_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEFPCompareWithZero. |
| |
| void Assembler::fcmeq(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0010 001. .... ...0 .... |
| // size<23:22> | eq<17> = 1 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMEQ_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcmge(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0000 001. .... ...0 .... |
| // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMGE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcmgt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0000 001. .... ...1 .... |
| // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMGT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcmle(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0001 001. .... ...1 .... |
| // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMLE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcmlt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0001 001. .... ...0 .... |
| // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMLT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcmne(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| double zero) { |
| // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 |
| // 0110 0101 ..01 0011 001. .... ...0 .... |
| // size<23:22> | eq<17> = 1 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(zero == 0.0); |
| USE(zero); |
| |
| Emit(FCMNE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // SVEFPComplexAddition. |
| |
| void Assembler::fcadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int rot) { |
| // FCADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>, <const> |
| // 0110 0100 ..00 000. 100. .... .... .... |
| // size<23:22> | rot<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((rot == 90) || (rot == 270)); |
| |
| Instr rotate_bit = (rot == 90) ? 0 : (1 << 16); |
| Emit(FCADD_z_p_zz | rotate_bit | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // SVEFPComplexMulAdd. |
| |
| void Assembler::fcmla(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int rot) { |
| // FCMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>, <const> |
| // 0110 0100 ..0. .... 0... .... .... .... |
| // size<23:22> | Zm<20:16> | rot<14:13> | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); |
| |
| Instr rotate_bit = (rot / 90) << 13; |
| Emit(FCMLA_z_p_zzz | rotate_bit | SVESize(zda) | Rd(zda) | PgLow8(pg) | |
| Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEFPComplexMulAddIndex. |
| |
| void Assembler::fcmla(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index, |
| int rot) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); |
| VIXL_ASSERT(index >= 0); |
| |
| int lane_size = zda.GetLaneSizeInBytes(); |
| |
| Instr zm_and_idx = 0; |
| Instr op = FCMLA_z_zzzi_h; |
| if (lane_size == kHRegSizeInBytes) { |
| // Zm<18:16> | i2<20:19> |
| VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 3)); |
| zm_and_idx = (index << 19) | Rx<18, 16>(zm); |
| } else { |
| // Zm<19:16> | i1<20> |
| VIXL_ASSERT(lane_size == kSRegSizeInBytes); |
| VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 1)); |
| zm_and_idx = (index << 20) | Rx<19, 16>(zm); |
| op = FCMLA_z_zzzi_s; |
| } |
| |
| Instr rotate_bit = (rot / 90) << 10; |
| Emit(op | zm_and_idx | rotate_bit | Rd(zda) | Rn(zn)); |
| } |
| |
| // SVEFPFastReduction. |
| |
| void Assembler::faddv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // FADDV <V><d>, <Pg>, <Zn>.<T> |
| // 0110 0101 ..00 0000 001. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zn, vd)); |
| |
| Emit(FADDV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fmaxnmv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // FMAXNMV <V><d>, <Pg>, <Zn>.<T> |
| // 0110 0101 ..00 0100 001. .... .... .... |
| // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zn, vd)); |
| |
| Emit(FMAXNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fmaxv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // FMAXV <V><d>, <Pg>, <Zn>.<T> |
| // 0110 0101 ..00 0110 001. .... .... .... |
| // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zn, vd)); |
| |
| Emit(FMAXV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fminnmv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // FMINNMV <V><d>, <Pg>, <Zn>.<T> |
| // 0110 0101 ..00 0101 001. .... .... .... |
| // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zn, vd)); |
| |
| Emit(FMINNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fminv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // FMINV <V><d>, <Pg>, <Zn>.<T> |
| // 0110 0101 ..00 0111 001. .... .... .... |
| // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(AreSameLaneSize(zn, vd)); |
| |
| Emit(FMINV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // SVEFPMulAdd. |
| |
| void Assembler::fmad(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // FMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0110 0101 ..1. .... 100. .... .... .... |
| // size<23:22> | Za<20:16> | opc<14:13> = 00 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); |
| } |
| |
| void Assembler::fmla(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..1. .... 000. .... .... .... |
| // size<23:22> | Zm<20:16> | opc<14:13> = 00 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fmls(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..1. .... 001. .... .... .... |
| // size<23:22> | Zm<20:16> | opc<14:13> = 01 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fmsb(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // FMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0110 0101 ..1. .... 101. .... .... .... |
| // size<23:22> | Za<20:16> | opc<14:13> = 01 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); |
| } |
| |
| void Assembler::fnmad(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // FNMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0110 0101 ..1. .... 110. .... .... .... |
| // size<23:22> | Za<20:16> | opc<14:13> = 10 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FNMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); |
| } |
| |
| void Assembler::fnmla(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FNMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..1. .... 010. .... .... .... |
| // size<23:22> | Zm<20:16> | opc<14:13> = 10 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FNMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fnmls(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FNMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0110 0101 ..1. .... 011. .... .... .... |
| // size<23:22> | Zm<20:16> | opc<14:13> = 11 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FNMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::fnmsb(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // FNMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0110 0101 ..1. .... 111. .... .... .... |
| // size<23:22> | Za<20:16> | opc<14:13> = 11 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FNMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); |
| } |
| |
| Instr Assembler::SVEFPMulIndexHelper(unsigned lane_size_in_bytes_log2, |
| const ZRegister& zm, |
| int index, |
| Instr op_h, |
| Instr op_s, |
| Instr op_d) { |
| Instr size = lane_size_in_bytes_log2 << SVESize_offset; |
| Instr zm_with_index = Rm(zm); |
| Instr op = 0xffffffff; |
| // Allowable register number and lane index depends on the lane size. |
| switch (lane_size_in_bytes_log2) { |
| case kHRegSizeInBytesLog2: |
| VIXL_ASSERT(zm.GetCode() <= 7); |
| VIXL_ASSERT(IsUint3(index)); |
| // For H-sized lanes, size is encoded as 0b0x, where x is used as the top |
| // bit of the index. So, if index is less than four, the top bit of index |
| // is zero, and therefore size is 0b00. Otherwise, it's 0b01, the usual |
| // encoding for H-sized lanes. |
| if (index < 4) size = 0; |
| // Top two bits of "zm" encode the index. |
| zm_with_index |= (index & 3) << (Rm_offset + 3); |
| op = op_h; |
| break; |
| case kSRegSizeInBytesLog2: |
| VIXL_ASSERT(zm.GetCode() <= 7); |
| VIXL_ASSERT(IsUint2(index)); |
| // Top two bits of "zm" encode the index. |
| zm_with_index |= (index & 3) << (Rm_offset + 3); |
| op = op_s; |
| break; |
| case kDRegSizeInBytesLog2: |
| VIXL_ASSERT(zm.GetCode() <= 15); |
| VIXL_ASSERT(IsUint1(index)); |
| // Top bit of "zm" encodes the index. |
| zm_with_index |= (index & 1) << (Rm_offset + 4); |
| op = op_d; |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| } |
| return op | zm_with_index | size; |
| } |
| |
| // SVEFPMulAddIndex. |
| |
| void Assembler::fmla(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| // The encoding of opcode, index, Zm, and size are synthesized in this |
| // variable. |
| Instr synthesized_op = SVEFPMulIndexHelper(zda.GetLaneSizeInBytesLog2(), |
| zm, |
| index, |
| FMLA_z_zzzi_h, |
| FMLA_z_zzzi_s, |
| FMLA_z_zzzi_d); |
| |
| Emit(synthesized_op | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::fmls(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| // The encoding of opcode, index, Zm, and size are synthesized in this |
| // variable. |
| Instr synthesized_op = SVEFPMulIndexHelper(zda.GetLaneSizeInBytesLog2(), |
| zm, |
| index, |
| FMLS_z_zzzi_h, |
| FMLS_z_zzzi_s, |
| FMLS_z_zzzi_d); |
| |
| Emit(synthesized_op | Rd(zda) | Rn(zn)); |
| } |
| |
| // SVEFPMulIndex. |
| |
| // This prototype maps to 3 instruction encodings: |
| void Assembler::fmul(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| unsigned index) { |
| // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>[<imm>] |
| // 0110 0100 ..1. .... 0010 00.. .... .... |
| // size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| // The encoding of opcode, index, Zm, and size are synthesized in this |
| // variable. |
| Instr synthesized_op = SVEFPMulIndexHelper(zd.GetLaneSizeInBytesLog2(), |
| zm, |
| index, |
| FMUL_z_zzi_h, |
| FMUL_z_zzi_s, |
| FMUL_z_zzi_d); |
| |
| Emit(synthesized_op | Rd(zd) | Rn(zn)); |
| } |
| |
| // SVEFPUnaryOpPredicated. |
| |
| void Assembler::fcvt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Instr op = 0xffffffff; |
| switch (zn.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kSRegSizeInBytes: |
| op = FCVT_z_p_z_h2s; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVT_z_p_z_h2d; |
| break; |
| } |
| break; |
| case kSRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = FCVT_z_p_z_s2h; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVT_z_p_z_s2d; |
| break; |
| } |
| break; |
| case kDRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = FCVT_z_p_z_d2h; |
| break; |
| case kSRegSizeInBytes: |
| op = FCVT_z_p_z_d2s; |
| break; |
| } |
| break; |
| } |
| VIXL_ASSERT(op != 0xffffffff); |
| |
| Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcvtzs(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr op = 0xffffffff; |
| switch (zn.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = FCVTZS_z_p_z_fp162h; |
| break; |
| case kSRegSizeInBytes: |
| op = FCVTZS_z_p_z_fp162w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZS_z_p_z_fp162x; |
| break; |
| } |
| break; |
| case kSRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kSRegSizeInBytes: |
| op = FCVTZS_z_p_z_s2w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZS_z_p_z_s2x; |
| break; |
| } |
| break; |
| case kDRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kSRegSizeInBytes: |
| op = FCVTZS_z_p_z_d2w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZS_z_p_z_d2x; |
| break; |
| } |
| break; |
| } |
| VIXL_ASSERT(op != 0xffffffff); |
| |
| Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcvtzu(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr op = 0xffffffff; |
| switch (zn.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = FCVTZU_z_p_z_fp162h; |
| break; |
| case kSRegSizeInBytes: |
| op = FCVTZU_z_p_z_fp162w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZU_z_p_z_fp162x; |
| break; |
| } |
| break; |
| case kSRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kSRegSizeInBytes: |
| op = FCVTZU_z_p_z_s2w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZU_z_p_z_s2x; |
| break; |
| } |
| break; |
| case kDRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kSRegSizeInBytes: |
| op = FCVTZU_z_p_z_d2w; |
| break; |
| case kDRegSizeInBytes: |
| op = FCVTZU_z_p_z_d2x; |
| break; |
| } |
| break; |
| } |
| VIXL_ASSERT(op != 0xffffffff); |
| |
| Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frecpx(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FRECPX <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0110 0101 ..00 1100 101. .... .... .... |
| // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRECPX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frinta(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTA_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frinti(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTI_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frintm(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTM_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frintn(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTN_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frintp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTP_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frintx(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::frintz(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRINTZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fsqrt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FSQRT <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0110 0101 ..00 1101 101. .... .... .... |
| // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FSQRT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::scvtf(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr op = 0xffffffff; |
| switch (zn.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = SCVTF_z_p_z_h2fp16; |
| break; |
| } |
| break; |
| case kSRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = SCVTF_z_p_z_w2fp16; |
| break; |
| case kSRegSizeInBytes: |
| op = SCVTF_z_p_z_w2s; |
| break; |
| case kDRegSizeInBytes: |
| op = SCVTF_z_p_z_w2d; |
| break; |
| } |
| break; |
| case kDRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = SCVTF_z_p_z_x2fp16; |
| break; |
| case kSRegSizeInBytes: |
| op = SCVTF_z_p_z_x2s; |
| break; |
| case kDRegSizeInBytes: |
| op = SCVTF_z_p_z_x2d; |
| break; |
| } |
| break; |
| } |
| VIXL_ASSERT(op != 0xffffffff); |
| |
| Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::ucvtf(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Instr op = 0xffffffff; |
| switch (zn.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = UCVTF_z_p_z_h2fp16; |
| break; |
| } |
| break; |
| case kSRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = UCVTF_z_p_z_w2fp16; |
| break; |
| case kSRegSizeInBytes: |
| op = UCVTF_z_p_z_w2s; |
| break; |
| case kDRegSizeInBytes: |
| op = UCVTF_z_p_z_w2d; |
| break; |
| } |
| break; |
| case kDRegSizeInBytes: |
| switch (zd.GetLaneSizeInBytes()) { |
| case kHRegSizeInBytes: |
| op = UCVTF_z_p_z_x2fp16; |
| break; |
| case kSRegSizeInBytes: |
| op = UCVTF_z_p_z_x2s; |
| break; |
| case kDRegSizeInBytes: |
| op = UCVTF_z_p_z_x2d; |
| break; |
| } |
| break; |
| } |
| VIXL_ASSERT(op != 0xffffffff); |
| |
| Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // SVEFPUnaryOpUnpredicated. |
| |
| void Assembler::frecpe(const ZRegister& zd, const ZRegister& zn) { |
| // FRECPE <Zd>.<T>, <Zn>.<T> |
| // 0110 0101 ..00 1110 0011 00.. .... .... |
| // size<23:22> | opc<18:16> = 110 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRECPE_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::frsqrte(const ZRegister& zd, const ZRegister& zn) { |
| // FRSQRTE <Zd>.<T>, <Zn>.<T> |
| // 0110 0101 ..00 1111 0011 00.. .... .... |
| // size<23:22> | opc<18:16> = 111 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FRSQRTE_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| // SVEIncDecByPredicateCount. |
| |
| void Assembler::decp(const Register& rdn, const PRegisterWithLaneSize& pg) { |
| // DECP <Xdn>, <Pg>.<T> |
| // 0010 0101 ..10 1101 1000 100. .... .... |
| // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rdn.IsX()); |
| |
| Emit(DECP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::decp(const ZRegister& zdn, const PRegister& pg) { |
| // DECP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1101 1000 000. .... .... |
| // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> | |
| // Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(DECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::incp(const Register& rdn, const PRegisterWithLaneSize& pg) { |
| // INCP <Xdn>, <Pg>.<T> |
| // 0010 0101 ..10 1100 1000 100. .... .... |
| // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rdn.IsX()); |
| |
| Emit(INCP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::incp(const ZRegister& zdn, const PRegister& pg) { |
| // INCP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1100 1000 000. .... .... |
| // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> | |
| // Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(INCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::sqdecp(const Register& xd, |
| const PRegisterWithLaneSize& pg, |
| const Register& wn) { |
| // SQDECP <Xdn>, <Pg>.<T>, <Wdn> |
| // 0010 0101 ..10 1010 1000 100. .... .... |
| // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| USE(wn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn)); |
| |
| Emit(SQDECP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::sqdecp(const Register& xdn, const PRegisterWithLaneSize& pg) { |
| // SQDECP <Xdn>, <Pg>.<T> |
| // 0010 0101 ..10 1010 1000 110. .... .... |
| // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xdn.IsX()); |
| |
| Emit(SQDECP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::sqdecp(const ZRegister& zdn, const PRegister& pg) { |
| // SQDECP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1010 1000 000. .... .... |
| // size<23:22> | D<17> = 1 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(SQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::sqincp(const Register& xd, |
| const PRegisterWithLaneSize& pg, |
| const Register& wn) { |
| // SQINCP <Xdn>, <Pg>.<T>, <Wdn> |
| // 0010 0101 ..10 1000 1000 100. .... .... |
| // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| USE(wn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn)); |
| |
| Emit(SQINCP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::sqincp(const Register& xdn, const PRegisterWithLaneSize& pg) { |
| // SQINCP <Xdn>, <Pg>.<T> |
| // 0010 0101 ..10 1000 1000 110. .... .... |
| // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xdn.IsX()); |
| |
| Emit(SQINCP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::sqincp(const ZRegister& zdn, const PRegister& pg) { |
| // SQINCP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1000 1000 000. .... .... |
| // size<23:22> | D<17> = 0 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(SQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::uqdecp(const Register& rdn, const PRegisterWithLaneSize& pg) { |
| // UQDECP <Wdn>, <Pg>.<T> |
| // UQDECP <Xdn>, <Pg>.<T> |
| // 0010 0101 ..10 1011 1000 10.. .... .... |
| // size<23:22> | D<17> = 1 | U<16> = 1 | sf<10> | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Instr op = rdn.IsX() ? UQDECP_r_p_r_x : UQDECP_r_p_r_uw; |
| Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::uqdecp(const ZRegister& zdn, const PRegister& pg) { |
| // UQDECP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1011 1000 000. .... .... |
| // size<23:22> | D<17> = 1 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(UQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::uqincp(const Register& rdn, const PRegisterWithLaneSize& pg) { |
| // UQINCP <Wdn>, <Pg>.<T> |
| // 0010 0101 ..10 1001 1000 100. .... .... |
| // size<23:22> | D<17> = 0 | U<16> = 1 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | |
| // Rdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Instr op = rdn.IsX() ? UQINCP_r_p_r_x : UQINCP_r_p_r_uw; |
| Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); |
| } |
| |
| void Assembler::uqincp(const ZRegister& zdn, const PRegister& pg) { |
| // UQINCP <Zdn>.<T>, <Pg> |
| // 0010 0101 ..10 1001 1000 000. .... .... |
| // size<23:22> | D<17> = 0 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| |
| Emit(UQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); |
| } |
| |
| // SVEIndexGeneration. |
| |
| void Assembler::index(const ZRegister& zd, int start, int step) { |
| // INDEX <Zd>.<T>, #<imm1>, #<imm2> |
| // 0000 0100 ..1. .... 0100 00.. .... .... |
| // size<23:22> | step<20:16> | start<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(INDEX_z_ii | SVESize(zd) | ImmField<20, 16>(step) | |
| ImmField<9, 5>(start) | Rd(zd)); |
| } |
| |
| void Assembler::index(const ZRegister& zd, |
| const Register& rn, |
| const Register& rm) { |
| // INDEX <Zd>.<T>, <R><n>, <R><m> |
| // 0000 0100 ..1. .... 0100 11.. .... .... |
| // size<23:22> | Rm<20:16> | Rn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= |
| zd.GetLaneSizeInBits()); |
| VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >= |
| zd.GetLaneSizeInBits()); |
| |
| Emit(INDEX_z_rr | SVESize(zd) | Rd(zd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::index(const ZRegister& zd, const Register& rn, int imm5) { |
| // INDEX <Zd>.<T>, <R><n>, #<imm> |
| // 0000 0100 ..1. .... 0100 01.. .... .... |
| // size<23:22> | imm5<20:16> | Rn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= |
| zd.GetLaneSizeInBits()); |
| |
| Emit(INDEX_z_ri | SVESize(zd) | Rd(zd) | Rn(rn) | ImmField<20, 16>(imm5)); |
| } |
| |
| void Assembler::index(const ZRegister& zd, int imm5, const Register& rm) { |
| // INDEX <Zd>.<T>, #<imm>, <R><m> |
| // 0000 0100 ..1. .... 0100 10.. .... .... |
| // size<23:22> | Rm<20:16> | imm5<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >= |
| zd.GetLaneSizeInBits()); |
| |
| Emit(INDEX_z_ir | SVESize(zd) | Rd(zd) | ImmField<9, 5>(imm5) | Rm(rm)); |
| } |
| |
| // SVEIntArithmeticUnpredicated. |
| |
| void Assembler::add(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0000 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(ADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqadd(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0001 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(SQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqsub(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0001 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(SQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sub(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0000 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(SUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uqadd(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0001 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(UQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uqsub(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0001 11.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(UQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEIntBinaryArithmeticPredicated. |
| |
| void Assembler::add(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 0000 000. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(ADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::and_(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // AND <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 1010 000. .... .... .... |
| // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(AND_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::bic(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // BIC <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 1011 000. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(BIC_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::eor(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // EOR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 1001 000. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(EOR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::mul(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // MUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0000 000. .... .... .... |
| // size<23:22> | H<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(MUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::orr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ORR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 1000 000. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(ORR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sabd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1100 000. .... .... .... |
| // size<23:22> | opc<18:17> = 10 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sdiv(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0100 000. .... .... .... |
| // size<23:22> | R<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(SDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sdivr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0110 000. .... .... .... |
| // size<23:22> | R<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(SDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::smax(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1000 000. .... .... .... |
| // size<23:22> | opc<18:17> = 00 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::smin(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1010 000. .... .... .... |
| // size<23:22> | opc<18:17> = 01 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::smulh(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0010 000. .... .... .... |
| // size<23:22> | H<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 0001 000. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::subr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 0011 000. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(SUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uabd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1101 000. .... .... .... |
| // size<23:22> | opc<18:17> = 10 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(UABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::udiv(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0101 000. .... .... .... |
| // size<23:22> | R<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(UDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::udivr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0111 000. .... .... .... |
| // size<23:22> | R<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(UDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::umax(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1001 000. .... .... .... |
| // size<23:22> | opc<18:17> = 00 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(UMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::umin(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..00 1011 000. .... .... .... |
| // size<23:22> | opc<18:17> = 01 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(UMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::umulh(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0100 ..01 0011 000. .... .... .... |
| // size<23:22> | H<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(UMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // SVEIntCompareScalars. |
| |
| void Assembler::ctermeq(const Register& rn, const Register& rm) { |
| // CTERMEQ <R><n>, <R><m> |
| // 0010 0101 1.1. .... 0010 00.. ...0 0000 |
| // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 0 |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000; |
| |
| Emit(CTERMEQ_rr | sz | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::ctermne(const Register& rn, const Register& rm) { |
| // CTERMNE <R><n>, <R><m> |
| // 0010 0101 1.1. .... 0010 00.. ...1 0000 |
| // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 1 |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000; |
| |
| Emit(CTERMNE_rr | sz | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilele(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILELE <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 01.. ...1 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> | |
| // eq<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; |
| |
| Emit(WHILELE_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilelo(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILELO <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 11.. ...0 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> | |
| // eq<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; |
| |
| Emit(WHILELO_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilels(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILELS <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 11.. ...1 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> | |
| // eq<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; |
| |
| Emit(WHILELS_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilelt(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILELT <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 01.. ...0 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> | |
| // eq<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameSizeAndType(rn, rm)); |
| const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; |
| |
| Emit(WHILELT_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| SVEIntCompareVectorsOp op) { |
| Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm, |
| SVEIntCompareSignedImmOp op) { |
| Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm)); |
| } |
| |
| void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| unsigned imm, |
| SVEIntCompareUnsignedImmOp op) { |
| Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | |
| ImmUnsignedField<20, 14>(imm)); |
| } |
| |
| void Assembler::cmp(Condition cond, |
| const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| switch (cond) { |
| case eq: |
| cmpeq(pd, pg, zn, zm); |
| break; |
| case ge: |
| cmpge(pd, pg, zn, zm); |
| break; |
| case gt: |
| cmpgt(pd, pg, zn, zm); |
| break; |
| case le: |
| cmple(pd, pg, zn, zm); |
| break; |
| case lt: |
| cmplt(pd, pg, zn, zm); |
| break; |
| case ne: |
| cmpne(pd, pg, zn, zm); |
| break; |
| case hi: |
| cmphi(pd, pg, zn, zm); |
| break; |
| case hs: |
| cmphs(pd, pg, zn, zm); |
| break; |
| case lo: |
| cmplo(pd, pg, zn, zm); |
| break; |
| case ls: |
| cmpls(pd, pg, zn, zm); |
| break; |
| default: |
| VIXL_UNREACHABLE(); |
| } |
| } |
| |
| // SVEIntCompareSignedImm. |
| |
| void Assembler::cmpeq(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 100. .... ...0 .... |
| // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPEQ_p_p_zi); |
| } |
| |
| void Assembler::cmpge(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 000. .... ...0 .... |
| // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPGE_p_p_zi); |
| } |
| |
| void Assembler::cmpgt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 000. .... ...1 .... |
| // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPGT_p_p_zi); |
| } |
| |
| void Assembler::cmple(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 001. .... ...1 .... |
| // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPLE_p_p_zi); |
| } |
| |
| void Assembler::cmplt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 001. .... ...0 .... |
| // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPLT_p_p_zi); |
| } |
| |
| void Assembler::cmpne(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // CMPNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0101 ..0. .... 100. .... ...1 .... |
| // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> |
| // | ne<4> = 1 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm5, CMPNE_p_p_zi); |
| } |
| |
| // SVEIntCompareUnsignedImm. |
| |
| void Assembler::cmphi(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| unsigned imm7) { |
| // CMPHI <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0100 ..1. .... ..0. .... ...1 .... |
| // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm7, CMPHI_p_p_zi); |
| } |
| |
| void Assembler::cmphs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| unsigned imm7) { |
| // CMPHS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0100 ..1. .... ..0. .... ...0 .... |
| // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm7, CMPHS_p_p_zi); |
| } |
| |
| void Assembler::cmplo(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| unsigned imm7) { |
| // CMPLO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0100 ..1. .... ..1. .... ...0 .... |
| // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm7, CMPLO_p_p_zi); |
| } |
| |
| void Assembler::cmpls(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| unsigned imm7) { |
| // CMPLS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> |
| // 0010 0100 ..1. .... ..1. .... ...1 .... |
| // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| |
| CompareVectors(pd, pg, zn, imm7, CMPLS_p_p_zi); |
| } |
| |
| // SVEIntCompareVectors. |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPEQ_p_p_zw |
| // CMPEQ_p_p_zz |
| void Assembler::cmpeq(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPEQ_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPEQ_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPGE_p_p_zw |
| // CMPGE_p_p_zz |
| void Assembler::cmpge(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPGE_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPGE_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPGT_p_p_zw |
| // CMPGT_p_p_zz |
| void Assembler::cmpgt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPGT_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPGT_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPHI_p_p_zw |
| // CMPHI_p_p_zz |
| void Assembler::cmphi(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPHI_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPHI_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPHS_p_p_zw |
| // CMPHS_p_p_zz |
| void Assembler::cmphs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPHS_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPHS_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| void Assembler::cmple(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| if (AreSameLaneSize(zn, zm)) { |
| cmpge(pd, pg, zm, zn); |
| return; |
| } |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(!zn.IsLaneSizeD()); |
| |
| CompareVectors(pd, pg, zn, zm, CMPLE_p_p_zw); |
| } |
| |
| void Assembler::cmplo(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| if (AreSameLaneSize(zn, zm)) { |
| cmphi(pd, pg, zm, zn); |
| return; |
| } |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(!zn.IsLaneSizeD()); |
| |
| CompareVectors(pd, pg, zn, zm, CMPLO_p_p_zw); |
| } |
| |
| void Assembler::cmpls(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| if (AreSameLaneSize(zn, zm)) { |
| cmphs(pd, pg, zm, zn); |
| return; |
| } |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(!zn.IsLaneSizeD()); |
| |
| CompareVectors(pd, pg, zn, zm, CMPLS_p_p_zw); |
| } |
| |
| void Assembler::cmplt(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| if (AreSameLaneSize(zn, zm)) { |
| cmpgt(pd, pg, zm, zn); |
| return; |
| } |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(!zn.IsLaneSizeD()); |
| |
| CompareVectors(pd, pg, zn, zm, CMPLT_p_p_zw); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // CMPNE_p_p_zw |
| // CMPNE_p_p_zz |
| void Assembler::cmpne(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn)); |
| SVEIntCompareVectorsOp op = CMPNE_p_p_zz; |
| if (!AreSameLaneSize(zn, zm)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| op = CMPNE_p_p_zw; |
| } |
| CompareVectors(pd, pg, zn, zm, op); |
| } |
| |
| // SVEIntMiscUnpredicated. |
| |
| void Assembler::fexpa(const ZRegister& zd, const ZRegister& zn) { |
| // FEXPA <Zd>.<T>, <Zn>.<T> |
| // 0000 0100 ..10 0000 1011 10.. .... .... |
| // size<23:22> | opc<20:16> = 00000 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FEXPA_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::ftssel(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FTSSEL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 1011 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<10> = 0 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FTSSEL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::movprfx(const ZRegister& zd, const ZRegister& zn) { |
| // MOVPRFX <Zd>, <Zn> |
| // 0000 0100 0010 0000 1011 11.. .... .... |
| // opc<23:22> = 00 | opc2<20:16> = 00000 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(MOVPRFX_z_z | Rd(zd) | Rn(zn)); |
| } |
| |
| // SVEIntMulAddPredicated. |
| |
| void Assembler::mad(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // MAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0000 0100 ..0. .... 110. .... .... .... |
| // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Za<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| |
| Emit(MAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za)); |
| } |
| |
| void Assembler::mla(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // MLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..0. .... 010. .... .... .... |
| // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(MLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::mls(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // MLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..0. .... 011. .... .... .... |
| // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(MLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::msb(const ZRegister& zdn, |
| const PRegisterM& pg, |
| const ZRegister& zm, |
| const ZRegister& za) { |
| // MSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> |
| // 0000 0100 ..0. .... 111. .... .... .... |
| // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Za<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); |
| |
| Emit(MSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za)); |
| } |
| |
| // SVEIntMulAddUnpredicated. |
| |
| void Assembler::sdot(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); |
| VIXL_ASSERT(AreSameLaneSize(zm, zn)); |
| |
| Emit(SDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::udot(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); |
| VIXL_ASSERT(AreSameLaneSize(zm, zn)); |
| |
| Emit(UDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEIntReduction. |
| |
| void Assembler::andv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(ANDV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::eorv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(EORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::movprfx(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // MOVPRFX <Zd>.<T>, <Pg>/<ZM>, <Zn>.<T> |
| // 0000 0100 ..01 000. 001. .... .... .... |
| // size<23:22> | opc<18:17> = 00 | M<16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); |
| VIXL_ASSERT(!pg.HasLaneSize()); |
| |
| Instr m = pg.IsMerging() ? 0x00010000 : 0x00000000; |
| Emit(MOVPRFX_z_p_z | SVESize(zd) | m | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::orv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(ORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::saddv(const VRegister& dd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kDRegSizeInBytes); |
| |
| Emit(SADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::smaxv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(SMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::sminv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(SMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uaddv(const VRegister& dd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(UADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::umaxv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(UMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uminv(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(UMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // SVEIntUnaryArithmeticPredicated. |
| |
| void Assembler::abs(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // ABS <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0110 101. .... .... .... |
| // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(ABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::cls(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // CLS <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1000 101. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(CLS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::clz(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // CLZ <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1001 101. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(CLZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::cnot(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // CNOT <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1011 101. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(CNOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::cnt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // CNT <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1010 101. .... .... .... |
| // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(CNT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fabs(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FABS <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1100 101. .... .... .... |
| // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fneg(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FNEG <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1101 101. .... .... .... |
| // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(FNEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::neg(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // NEG <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0111 101. .... .... .... |
| // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(NEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::not_(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // NOT <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 1110 101. .... .... .... |
| // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(NOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::sxtb(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SXTB <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0000 101. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes); |
| |
| Emit(SXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::sxth(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SXTH <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0010 101. .... .... .... |
| // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes); |
| |
| Emit(SXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::sxtw(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SXTW <Zd>.D, <Pg>/M, <Zn>.D |
| // 0000 0100 ..01 0100 101. .... .... .... |
| // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes); |
| |
| Emit(SXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uxtb(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // UXTB <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0001 101. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes); |
| |
| Emit(UXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uxth(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // UXTH <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0100 ..01 0011 101. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes); |
| |
| Emit(UXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uxtw(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // UXTW <Zd>.D, <Pg>/M, <Zn>.D |
| // 0000 0100 ..01 0101 101. .... .... .... |
| // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes); |
| |
| Emit(UXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // SVEIntWideImmPredicated. |
| |
| void Assembler::cpy(const ZRegister& zd, |
| const PRegister& pg, |
| int imm8, |
| int shift) { |
| // CPY <Zd>.<T>, <Pg>/<ZM>, #<imm>{, <shift>} |
| // 0000 0101 ..01 .... 0... .... .... .... |
| // size<23:22> | Pg<19:16> | M<14> | sh<13> | imm8<12:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); |
| |
| ResolveSVEImm8Shift(&imm8, &shift); |
| |
| Instr sh = (shift > 0) ? (1 << 13) : 0; |
| Instr m = pg.IsMerging() ? (1 << 14) : 0; |
| Emit(CPY_z_p_i | m | sh | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | |
| ImmField<12, 5>(imm8)); |
| } |
| |
| void Assembler::fcpy(const ZRegister& zd, const PRegisterM& pg, double imm) { |
| // FCPY <Zd>.<T>, <Pg>/M, #<const> |
| // 0000 0101 ..01 .... 110. .... .... .... |
| // size<23:22> | Pg<19:16> | imm8<12:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Instr imm_field = ImmUnsignedField<12, 5>(FP64ToImm8(imm)); |
| Emit(FCPY_z_p_i | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | imm_field); |
| } |
| |
| // SVEIntAddSubtractImmUnpredicated. |
| |
| void Assembler::SVEIntAddSubtractImmUnpredicatedHelper( |
| SVEIntAddSubtractImm_UnpredicatedOp op, |
| const ZRegister& zd, |
| int imm8, |
| int shift) { |
| if (shift < 0) { |
| VIXL_ASSERT(shift == -1); |
| // Derive the shift amount from the immediate. |
| if (IsUint8(imm8)) { |
| shift = 0; |
| } else if (IsUint16(imm8) && ((imm8 % 256) == 0)) { |
| imm8 /= 256; |
| shift = 8; |
| } |
| } |
| |
| VIXL_ASSERT(IsUint8(imm8)); |
| VIXL_ASSERT((shift == 0) || (shift == 8)); |
| |
| Instr shift_bit = (shift > 0) ? (1 << 13) : 0; |
| Emit(op | SVESize(zd) | Rd(zd) | shift_bit | ImmUnsignedField<12, 5>(imm8)); |
| } |
| |
| void Assembler::add(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // ADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0000 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(ADD_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::dup(const ZRegister& zd, int imm8, int shift) { |
| // DUP <Zd>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..11 1000 11.. .... .... .... |
| // size<23:22> | opc<18:17> = 00 | sh<13> | imm8<12:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| ResolveSVEImm8Shift(&imm8, &shift); |
| VIXL_ASSERT((shift < 8) || !zd.IsLaneSizeB()); |
| |
| Instr shift_bit = (shift > 0) ? (1 << 13) : 0; |
| Emit(DUP_z_i | SVESize(zd) | Rd(zd) | shift_bit | ImmField<12, 5>(imm8)); |
| } |
| |
| void Assembler::fdup(const ZRegister& zd, double imm) { |
| // FDUP <Zd>.<T>, #<const> |
| // 0010 0101 ..11 1001 110. .... .... .... |
| // size<23:22> | opc<18:17> = 00 | o2<13> = 0 | imm8<12:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Instr encoded_imm = FP64ToImm8(imm) << 5; |
| Emit(FDUP_z_i | SVESize(zd) | encoded_imm | Rd(zd)); |
| } |
| |
| void Assembler::mul(const ZRegister& zd, const ZRegister& zn, int imm8) { |
| // MUL <Zdn>.<T>, <Zdn>.<T>, #<imm> |
| // 0010 0101 ..11 0000 110. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(MUL_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); |
| } |
| |
| void Assembler::smax(const ZRegister& zd, const ZRegister& zn, int imm8) { |
| // SMAX <Zdn>.<T>, <Zdn>.<T>, #<imm> |
| // 0010 0101 ..10 1000 110. .... .... .... |
| // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(SMAX_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); |
| } |
| |
| void Assembler::smin(const ZRegister& zd, const ZRegister& zn, int imm8) { |
| // SMIN <Zdn>.<T>, <Zdn>.<T>, #<imm> |
| // 0010 0101 ..10 1010 110. .... .... .... |
| // size<23:22> | opc<18:16> = 010 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(SMIN_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); |
| } |
| |
| void Assembler::sqadd(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // SQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0100 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 100 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(SQADD_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::sqsub(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // SQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0110 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 110 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(SQSUB_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::sub(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // SUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0001 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(SUB_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::subr(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // SUBR <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0011 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(SUBR_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::umax(const ZRegister& zd, const ZRegister& zn, int imm8) { |
| // UMAX <Zdn>.<T>, <Zdn>.<T>, #<imm> |
| // 0010 0101 ..10 1001 110. .... .... .... |
| // size<23:22> | opc<18:16> = 001 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(UMAX_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8)); |
| } |
| |
| void Assembler::umin(const ZRegister& zd, const ZRegister& zn, int imm8) { |
| // UMIN <Zdn>.<T>, <Zdn>.<T>, #<imm> |
| // 0010 0101 ..10 1011 110. .... .... .... |
| // size<23:22> | opc<18:16> = 011 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(UMIN_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8)); |
| } |
| |
| void Assembler::uqadd(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // UQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0101 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 101 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(UQADD_z_zi, zd, imm8, shift); |
| } |
| |
| void Assembler::uqsub(const ZRegister& zd, |
| const ZRegister& zn, |
| int imm8, |
| int shift) { |
| // UQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} |
| // 0010 0101 ..10 0111 11.. .... .... .... |
| // size<23:22> | opc<18:16> = 111 | sh<13> | imm8<12:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| SVEIntAddSubtractImmUnpredicatedHelper(UQSUB_z_zi, zd, imm8, shift); |
| } |
| |
| // SVEMemLoad. |
| |
| void Assembler::SVELdSt1Helper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| bool is_signed, |
| Instr op) { |
| VIXL_ASSERT(addr.IsContiguous()); |
| |
| Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr); |
| Instr dtype = |
| SVEDtype(msize_in_bytes_log2, zt.GetLaneSizeInBytesLog2(), is_signed); |
| Emit(op | mem_op | dtype | Rt(zt) | PgLow8(pg)); |
| } |
| |
| void Assembler::SVELdSt234Helper(int num_regs, |
| const ZRegister& zt1, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| Instr op) { |
| VIXL_ASSERT((num_regs >= 2) && (num_regs <= 4)); |
| |
| unsigned msize_in_bytes_log2 = zt1.GetLaneSizeInBytesLog2(); |
| Instr num = (num_regs - 1) << 21; |
| Instr msz = msize_in_bytes_log2 << 23; |
| Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, num_regs, addr); |
| Emit(op | mem_op | msz | num | Rt(zt1) | PgLow8(pg)); |
| } |
| |
| void Assembler::SVELd1Helper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr, |
| bool is_signed) { |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); |
| if (is_signed) { |
| // Sign-extension is only possible when the vector elements are larger than |
| // the elements in memory. |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); |
| } |
| |
| if (addr.IsScatterGather()) { |
| bool is_load = true; |
| bool is_ff = false; |
| SVEScatterGatherHelper(msize_in_bytes_log2, |
| zt, |
| pg, |
| addr, |
| is_load, |
| is_signed, |
| is_ff); |
| return; |
| } |
| |
| Instr op = 0xffffffff; |
| if (addr.IsScalarPlusImmediate()) { |
| op = SVEContiguousLoad_ScalarPlusImmFixed; |
| } else if (addr.IsScalarPlusScalar()) { |
| // Rm must not be xzr. |
| VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); |
| op = SVEContiguousLoad_ScalarPlusScalarFixed; |
| } else { |
| VIXL_UNIMPLEMENTED(); |
| } |
| SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op); |
| } |
| |
| void Assembler::SVELdff1Helper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr, |
| bool is_signed) { |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); |
| if (is_signed) { |
| // Sign-extension is only possible when the vector elements are larger than |
| // the elements in memory. |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); |
| } |
| |
| if (addr.IsScatterGather()) { |
| bool is_load = true; |
| bool is_ff = true; |
| SVEScatterGatherHelper(msize_in_bytes_log2, |
| zt, |
| pg, |
| addr, |
| is_load, |
| is_signed, |
| is_ff); |
| return; |
| } |
| |
| if (addr.IsPlainScalar()) { |
| // SVEMemOperand(x0) is treated as a scalar-plus-immediate form ([x0, #0]). |
| // In these instructions, we want to treat it as [x0, xzr]. |
| SVEMemOperand addr_scalar_plus_scalar(addr.GetScalarBase(), xzr); |
| // Guard against infinite recursion. |
| VIXL_ASSERT(!addr_scalar_plus_scalar.IsPlainScalar()); |
| SVELdff1Helper(msize_in_bytes_log2, |
| zt, |
| pg, |
| addr_scalar_plus_scalar, |
| is_signed); |
| return; |
| } |
| |
| Instr op = 0xffffffff; |
| if (addr.IsScalarPlusScalar()) { |
| op = SVEContiguousFirstFaultLoad_ScalarPlusScalarFixed; |
| } else { |
| VIXL_UNIMPLEMENTED(); |
| } |
| SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op); |
| } |
| |
| void Assembler::SVEScatterGatherHelper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| bool is_load, |
| bool is_signed, |
| bool is_first_fault) { |
| VIXL_ASSERT(addr.IsScatterGather()); |
| VIXL_ASSERT(zt.IsLaneSizeS() || zt.IsLaneSizeD()); |
| VIXL_ASSERT(is_load || !is_first_fault); |
| VIXL_ASSERT(is_load || !is_signed); |
| |
| Instr op = 0xffffffff; |
| if (addr.IsVectorPlusImmediate()) { |
| VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorBase())); |
| if (is_load) { |
| if (zt.IsLaneSizeS()) { |
| op = SVE32BitGatherLoad_VectorPlusImmFixed; |
| } else { |
| op = SVE64BitGatherLoad_VectorPlusImmFixed; |
| } |
| } else { |
| if (zt.IsLaneSizeS()) { |
| op = SVE32BitScatterStore_VectorPlusImmFixed; |
| } else { |
| op = SVE64BitScatterStore_VectorPlusImmFixed; |
| } |
| } |
| } else { |
| VIXL_ASSERT(addr.IsScalarPlusVector()); |
| VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorOffset())); |
| SVEOffsetModifier mod = addr.GetOffsetModifier(); |
| if (zt.IsLaneSizeS()) { |
| VIXL_ASSERT((mod == SVE_UXTW) || (mod == SVE_SXTW)); |
| unsigned shift_amount = addr.GetShiftAmount(); |
| if (shift_amount == 0) { |
| if (is_load) { |
| op = SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsetsFixed; |
| } else { |
| op = SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsetsFixed; |
| } |
| } else if (shift_amount == 1) { |
| VIXL_ASSERT(msize_in_bytes_log2 == kHRegSizeInBytesLog2); |
| if (is_load) { |
| op = SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsetsFixed; |
| } else { |
| op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed; |
| } |
| } else { |
| VIXL_ASSERT(shift_amount == 2); |
| VIXL_ASSERT(msize_in_bytes_log2 == kSRegSizeInBytesLog2); |
| if (is_load) { |
| op = SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsetsFixed; |
| } else { |
| op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed; |
| } |
| } |
| } else if (zt.IsLaneSizeD()) { |
| switch (mod) { |
| case NO_SVE_OFFSET_MODIFIER: |
| if (is_load) { |
| op = SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsetsFixed; |
| } else { |
| op = SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsetsFixed; |
| } |
| break; |
| case SVE_LSL: |
| if (is_load) { |
| op = SVE64BitGatherLoad_ScalarPlus64BitScaledOffsetsFixed; |
| } else { |
| op = SVE64BitScatterStore_ScalarPlus64BitScaledOffsetsFixed; |
| } |
| break; |
| case SVE_UXTW: |
| case SVE_SXTW: { |
| unsigned shift_amount = addr.GetShiftAmount(); |
| if (shift_amount == 0) { |
| if (is_load) { |
| op = |
| SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsetsFixed; |
| } else { |
| op = |
| SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsetsFixed; |
| } |
| } else { |
| VIXL_ASSERT(shift_amount == msize_in_bytes_log2); |
| if (is_load) { |
| op = SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsetsFixed; |
| } else { |
| op = |
| SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsetsFixed; |
| } |
| } |
| break; |
| } |
| default: |
| VIXL_UNIMPLEMENTED(); |
| } |
| } |
| } |
| |
| Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr, is_load); |
| Instr msz = ImmUnsignedField<24, 23>(msize_in_bytes_log2); |
| Instr u = (!is_load || is_signed) ? 0 : (1 << 14); |
| Instr ff = is_first_fault ? (1 << 13) : 0; |
| Emit(op | mem_op | msz | u | ff | Rt(zt) | PgLow8(pg)); |
| } |
| |
| void Assembler::SVELd234Helper(int num_regs, |
| const ZRegister& zt1, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| if (addr.IsScalarPlusScalar()) { |
| // Rm must not be xzr. |
| VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); |
| } |
| |
| Instr op; |
| if (addr.IsScalarPlusImmediate()) { |
| op = SVELoadMultipleStructures_ScalarPlusImmFixed; |
| } else if (addr.IsScalarPlusScalar()) { |
| op = SVELoadMultipleStructures_ScalarPlusScalarFixed; |
| } else { |
| // These instructions don't support any other addressing modes. |
| VIXL_ABORT(); |
| } |
| SVELdSt234Helper(num_regs, zt1, pg, addr, op); |
| } |
| |
| // SVEMemContiguousLoad. |
| |
| #define VIXL_DEFINE_LD1(MSZ, LANE_SIZE) \ |
| void Assembler::ld1##MSZ(const ZRegister& zt, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \ |
| } |
| #define VIXL_DEFINE_LD2(MSZ, LANE_SIZE) \ |
| void Assembler::ld2##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVELd234Helper(2, zt1, pg, addr); \ |
| } |
| #define VIXL_DEFINE_LD3(MSZ, LANE_SIZE) \ |
| void Assembler::ld3##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const ZRegister& zt3, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2, zt3); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVELd234Helper(3, zt1, pg, addr); \ |
| } |
| #define VIXL_DEFINE_LD4(MSZ, LANE_SIZE) \ |
| void Assembler::ld4##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const ZRegister& zt3, \ |
| const ZRegister& zt4, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2, zt3, zt4); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVELd234Helper(4, zt1, pg, addr); \ |
| } |
| |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD1) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD2) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD3) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD4) |
| |
| #define VIXL_DEFINE_LD1S(MSZ, LANE_SIZE) \ |
| void Assembler::ld1s##MSZ(const ZRegister& zt, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \ |
| } |
| VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LD1S) |
| |
| // SVEMem32BitGatherAndUnsizedContiguous. |
| |
| void Assembler::SVELd1BroadcastHelper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr, |
| bool is_signed) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate()); |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); |
| if (is_signed) { |
| // Sign-extension is only possible when the vector elements are larger than |
| // the elements in memory. |
| VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); |
| } |
| |
| int64_t imm = addr.GetImmediateOffset(); |
| int divisor = 1 << msize_in_bytes_log2; |
| VIXL_ASSERT(imm % divisor == 0); |
| Instr dtype = SVEDtypeSplit(msize_in_bytes_log2, |
| zt.GetLaneSizeInBytesLog2(), |
| is_signed); |
| |
| Emit(SVELoadAndBroadcastElementFixed | dtype | RnSP(addr.GetScalarBase()) | |
| ImmUnsignedField<21, 16>(imm / divisor) | Rt(zt) | PgLow8(pg)); |
| } |
| |
| // This prototype maps to 4 instruction encodings: |
| // LD1RB_z_p_bi_u16 |
| // LD1RB_z_p_bi_u32 |
| // LD1RB_z_p_bi_u64 |
| // LD1RB_z_p_bi_u8 |
| void Assembler::ld1rb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, false); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // LD1RH_z_p_bi_u16 |
| // LD1RH_z_p_bi_u32 |
| // LD1RH_z_p_bi_u64 |
| void Assembler::ld1rh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, false); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LD1RW_z_p_bi_u32 |
| // LD1RW_z_p_bi_u64 |
| void Assembler::ld1rw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kSRegSizeInBytesLog2, zt, pg, addr, false); |
| } |
| |
| void Assembler::ld1rd(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kDRegSizeInBytesLog2, zt, pg, addr, false); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // LD1RSB_z_p_bi_s16 |
| // LD1RSB_z_p_bi_s32 |
| // LD1RSB_z_p_bi_s64 |
| void Assembler::ld1rsb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, true); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LD1RSH_z_p_bi_s32 |
| // LD1RSH_z_p_bi_s64 |
| void Assembler::ld1rsh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, true); |
| } |
| |
| void Assembler::ld1rsw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| SVELd1BroadcastHelper(kWRegSizeInBytesLog2, zt, pg, addr, true); |
| } |
| |
| void Assembler::ldr(const CPURegister& rt, const SVEMemOperand& addr) { |
| // LDR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}] |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister()); |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| int64_t imm9 = addr.GetImmediateOffset(); |
| VIXL_ASSERT(IsInt9(imm9)); |
| Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10; |
| Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16; |
| |
| Instr op = LDR_z_bi; |
| if (rt.IsPRegister()) { |
| op = LDR_p_bi; |
| } |
| Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l); |
| } |
| |
| // SVEMem64BitGather. |
| |
| // This prototype maps to 3 instruction encodings: |
| // LDFF1B_z_p_bz_d_64_unscaled |
| // LDFF1B_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1b(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1B { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D] |
| // 1100 0100 010. .... 111. .... .... .... |
| // msz<24:23> = 00 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1B_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LDFF1B_z_p_ai_d |
| // LDFF1B_z_p_ai_s |
| void Assembler::ldff1b(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1B { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0100 001. .... 111. .... .... .... |
| // msz<24:23> = 00 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1B_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 4 instruction encodings: |
| // LDFF1D_z_p_bz_d_64_scaled |
| // LDFF1D_z_p_bz_d_64_unscaled |
| // LDFF1D_z_p_bz_d_x32_scaled |
| // LDFF1D_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1d(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1D { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #3] |
| // 1100 0101 111. .... 111. .... .... .... |
| // msz<24:23> = 11 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1D_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| void Assembler::ldff1d(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1D { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0101 101. .... 111. .... .... .... |
| // msz<24:23> = 11 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1D_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 6 instruction encodings: |
| // LDFF1H_z_p_bz_d_64_scaled |
| // LDFF1H_z_p_bz_d_64_unscaled |
| // LDFF1H_z_p_bz_d_x32_scaled |
| // LDFF1H_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1h(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1H { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1] |
| // 1100 0100 111. .... 111. .... .... .... |
| // msz<24:23> = 01 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1H_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LDFF1H_z_p_ai_d |
| // LDFF1H_z_p_ai_s |
| void Assembler::ldff1h(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1H { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0100 101. .... 111. .... .... .... |
| // msz<24:23> = 01 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1H_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // LDFF1SB_z_p_bz_d_64_unscaled |
| // LDFF1SB_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1sb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D] |
| // 1100 0100 010. .... 101. .... .... .... |
| // msz<24:23> = 00 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SB_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LDFF1SB_z_p_ai_d |
| // LDFF1SB_z_p_ai_s |
| void Assembler::ldff1sb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0100 001. .... 101. .... .... .... |
| // msz<24:23> = 00 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SB_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | |
| ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 6 instruction encodings: |
| // LDFF1SH_z_p_bz_d_64_scaled |
| // LDFF1SH_z_p_bz_d_64_unscaled |
| // LDFF1SH_z_p_bz_d_x32_scaled |
| // LDFF1SH_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1sh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1] |
| // 1100 0100 111. .... 101. .... .... .... |
| // msz<24:23> = 01 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SH_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LDFF1SH_z_p_ai_d |
| // LDFF1SH_z_p_ai_s |
| void Assembler::ldff1sh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0100 101. .... 101. .... .... .... |
| // msz<24:23> = 01 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SH_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | |
| ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 4 instruction encodings: |
| // LDFF1SW_z_p_bz_d_64_scaled |
| // LDFF1SW_z_p_bz_d_64_unscaled |
| // LDFF1SW_z_p_bz_d_x32_scaled |
| // LDFF1SW_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1sw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2] |
| // 1100 0101 011. .... 101. .... .... .... |
| // msz<24:23> = 10 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SW_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| void Assembler::ldff1sw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0101 001. .... 101. .... .... .... |
| // msz<24:23> = 10 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1SW_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | |
| ImmField<20, 16>(imm5)); |
| } |
| |
| // This prototype maps to 6 instruction encodings: |
| // LDFF1W_z_p_bz_d_64_scaled |
| // LDFF1W_z_p_bz_d_64_unscaled |
| // LDFF1W_z_p_bz_d_x32_scaled |
| // LDFF1W_z_p_bz_d_x32_unscaled |
| void Assembler::ldff1w(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const Register& xn, |
| const ZRegister& zm) { |
| // LDFF1W { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2] |
| // 1100 0101 011. .... 111. .... .... .... |
| // msz<24:23> = 10 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> |
| // | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1W_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // LDFF1W_z_p_ai_d |
| // LDFF1W_z_p_ai_s |
| void Assembler::ldff1w(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| int imm5) { |
| // LDFF1W { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] |
| // 1100 0101 001. .... 111. .... .... .... |
| // msz<24:23> = 10 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | |
| // Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LDFF1W_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); |
| } |
| |
| void Assembler::SVEGatherPrefetchVectorPlusImmediateHelper( |
| PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| int prefetch_size) { |
| VIXL_ASSERT(addr.IsVectorPlusImmediate()); |
| ZRegister zn = addr.GetVectorBase(); |
| VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); |
| |
| Instr op = 0xffffffff; |
| switch (prefetch_size) { |
| case kBRegSize: |
| op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_ai_s) |
| : static_cast<Instr>(PRFB_i_p_ai_d); |
| break; |
| case kHRegSize: |
| op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_ai_s) |
| : static_cast<Instr>(PRFH_i_p_ai_d); |
| break; |
| case kSRegSize: |
| op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_ai_s) |
| : static_cast<Instr>(PRFW_i_p_ai_d); |
| break; |
| case kDRegSize: |
| op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_ai_s) |
| : static_cast<Instr>(PRFD_i_p_ai_d); |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| int64_t imm5 = addr.GetImmediateOffset(); |
| Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | Rn(zn) | |
| ImmUnsignedField<20, 16>(imm5)); |
| } |
| |
| void Assembler::SVEGatherPrefetchScalarPlusImmediateHelper( |
| PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| int prefetch_size) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate()); |
| int64_t imm6 = addr.GetImmediateOffset(); |
| |
| Instr op = 0xffffffff; |
| switch (prefetch_size) { |
| case kBRegSize: |
| op = PRFB_i_p_bi_s; |
| break; |
| case kHRegSize: |
| op = PRFH_i_p_bi_s; |
| break; |
| case kSRegSize: |
| op = PRFW_i_p_bi_s; |
| break; |
| case kDRegSize: |
| op = PRFD_i_p_bi_s; |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | |
| RnSP(addr.GetScalarBase()) | ImmField<21, 16>(imm6)); |
| } |
| |
| void Assembler::SVEContiguousPrefetchScalarPlusScalarHelper( |
| PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| int prefetch_size) { |
| VIXL_ASSERT(addr.IsScalarPlusScalar()); |
| Instr op = 0xffffffff; |
| |
| switch (prefetch_size) { |
| case kBRegSize: |
| VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER); |
| op = PRFB_i_p_br_s; |
| break; |
| case kHRegSize: |
| VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); |
| VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2); |
| op = PRFH_i_p_br_s; |
| break; |
| case kSRegSize: |
| VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); |
| VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2); |
| op = PRFW_i_p_br_s; |
| break; |
| case kDRegSize: |
| VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); |
| VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2); |
| op = PRFD_i_p_br_s; |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); |
| Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | |
| RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset())); |
| } |
| |
| void Assembler::SVEContiguousPrefetchScalarPlusVectorHelper( |
| PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| int prefetch_size) { |
| VIXL_ASSERT(addr.IsScalarPlusVector()); |
| ZRegister zm = addr.GetVectorOffset(); |
| SVEOffsetModifier mod = addr.GetOffsetModifier(); |
| |
| // All prefetch scalar-plus-vector addressing modes use a shift corresponding |
| // to the element size. |
| switch (prefetch_size) { |
| case kBRegSize: |
| VIXL_ASSERT(addr.GetShiftAmount() == kBRegSizeInBytesLog2); |
| break; |
| case kHRegSize: |
| VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2); |
| break; |
| case kSRegSize: |
| VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2); |
| break; |
| case kDRegSize: |
| VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2); |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| Instr sx = 0; |
| Instr op = 0xffffffff; |
| if ((mod == NO_SVE_OFFSET_MODIFIER) || (mod == SVE_LSL)) { |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| |
| switch (prefetch_size) { |
| case kBRegSize: |
| VIXL_ASSERT(mod == NO_SVE_OFFSET_MODIFIER); |
| op = PRFB_i_p_bz_d_64_scaled; |
| break; |
| case kHRegSize: |
| VIXL_ASSERT(mod == SVE_LSL); |
| op = PRFH_i_p_bz_d_64_scaled; |
| break; |
| case kSRegSize: |
| VIXL_ASSERT(mod == SVE_LSL); |
| op = PRFW_i_p_bz_d_64_scaled; |
| break; |
| case kDRegSize: |
| VIXL_ASSERT(mod == SVE_LSL); |
| op = PRFD_i_p_bz_d_64_scaled; |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| } else { |
| VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW)); |
| VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD()); |
| |
| switch (prefetch_size) { |
| case kBRegSize: |
| op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_bz_s_x32_scaled) |
| : static_cast<Instr>(PRFB_i_p_bz_d_x32_scaled); |
| break; |
| case kHRegSize: |
| op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_bz_s_x32_scaled) |
| : static_cast<Instr>(PRFH_i_p_bz_d_x32_scaled); |
| break; |
| case kSRegSize: |
| op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_bz_s_x32_scaled) |
| : static_cast<Instr>(PRFW_i_p_bz_d_x32_scaled); |
| break; |
| case kDRegSize: |
| op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_bz_s_x32_scaled) |
| : static_cast<Instr>(PRFD_i_p_bz_d_x32_scaled); |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| if (mod == SVE_SXTW) { |
| sx = 1 << 22; |
| } |
| } |
| |
| Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | sx | |
| RnSP(addr.GetScalarBase()) | Rm(zm)); |
| } |
| |
| void Assembler::SVEPrefetchHelper(PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| int prefetch_size) { |
| if (addr.IsVectorPlusImmediate()) { |
| // For example: |
| // [z0.s, #0] |
| SVEGatherPrefetchVectorPlusImmediateHelper(prfop, pg, addr, prefetch_size); |
| |
| } else if (addr.IsScalarPlusImmediate()) { |
| // For example: |
| // [x0, #42, mul vl] |
| SVEGatherPrefetchScalarPlusImmediateHelper(prfop, pg, addr, prefetch_size); |
| |
| } else if (addr.IsScalarPlusVector()) { |
| // For example: |
| // [x0, z0.s, sxtw] |
| SVEContiguousPrefetchScalarPlusVectorHelper(prfop, pg, addr, prefetch_size); |
| |
| } else if (addr.IsScalarPlusScalar()) { |
| // For example: |
| // [x0, x1] |
| SVEContiguousPrefetchScalarPlusScalarHelper(prfop, pg, addr, prefetch_size); |
| |
| } else { |
| VIXL_UNIMPLEMENTED(); |
| } |
| } |
| |
| void Assembler::prfb(PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| SVEPrefetchHelper(prfop, pg, addr, kBRegSize); |
| } |
| |
| void Assembler::prfd(PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| SVEPrefetchHelper(prfop, pg, addr, kDRegSize); |
| } |
| |
| void Assembler::prfh(PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| SVEPrefetchHelper(prfop, pg, addr, kHRegSize); |
| } |
| |
| void Assembler::prfw(PrefetchOperation prfop, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| SVEPrefetchHelper(prfop, pg, addr, kSRegSize); |
| } |
| |
| void Assembler::SVELd1St1ScaImmHelper(const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr, |
| Instr regoffset_op, |
| Instr immoffset_op, |
| int imm_divisor) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsScalarPlusScalar() || addr.IsScalarPlusImmediate()); |
| |
| Instr op; |
| if (addr.IsScalarPlusScalar()) { |
| op = regoffset_op | Rm(addr.GetScalarOffset()); |
| } else { |
| int64_t imm = addr.GetImmediateOffset(); |
| VIXL_ASSERT(((imm % imm_divisor) == 0) && IsInt4(imm / imm_divisor)); |
| op = immoffset_op | ImmField<19, 16>(imm / imm_divisor); |
| } |
| Emit(op | Rt(zt) | PgLow8(pg) | RnSP(addr.GetScalarBase())); |
| } |
| |
| void Assembler::ld1rqb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(0)); |
| VIXL_ASSERT(zt.IsLaneSizeB()); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LD1RQB_z_p_br_contiguous, |
| LD1RQB_z_p_bi_u8, |
| 16); |
| } |
| |
| void Assembler::ld1rqd(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(3)); |
| VIXL_ASSERT(zt.IsLaneSizeD()); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LD1RQD_z_p_br_contiguous, |
| LD1RQD_z_p_bi_u64, |
| 16); |
| } |
| |
| void Assembler::ld1rqh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(1)); |
| VIXL_ASSERT(zt.IsLaneSizeH()); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LD1RQH_z_p_br_contiguous, |
| LD1RQH_z_p_bi_u16, |
| 16); |
| } |
| |
| void Assembler::ld1rqw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(2)); |
| VIXL_ASSERT(zt.IsLaneSizeS()); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LD1RQW_z_p_br_contiguous, |
| LD1RQW_z_p_bi_u32, |
| 16); |
| } |
| |
| #define VIXL_DEFINE_LDFF1(MSZ, LANE_SIZE) \ |
| void Assembler::ldff1##MSZ(const ZRegister& zt, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \ |
| } |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LDFF1) |
| |
| #define VIXL_DEFINE_LDFF1S(MSZ, LANE_SIZE) \ |
| void Assembler::ldff1s##MSZ(const ZRegister& zt, \ |
| const PRegisterZ& pg, \ |
| const SVEMemOperand& addr) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \ |
| } |
| VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LDFF1S) |
| |
| void Assembler::ldnf1b(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(0, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ false, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1d(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(3, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ false, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1h(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(1, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ false, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1sb(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(0, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ true, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1sh(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(1, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ true, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1sw(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(2, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ true, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnf1w(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(addr.IsPlainRegister() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| |
| SVELdSt1Helper(2, |
| zt, |
| pg, |
| addr, |
| /* is_signed = */ false, |
| SVEContiguousNonFaultLoad_ScalarPlusImmFixed); |
| } |
| |
| void Assembler::ldnt1b(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LDNT1B_z_p_br_contiguous, |
| LDNT1B_z_p_bi_contiguous); |
| } |
| |
| void Assembler::ldnt1d(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LDNT1D_z_p_br_contiguous, |
| LDNT1D_z_p_bi_contiguous); |
| } |
| |
| void Assembler::ldnt1h(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LDNT1H_z_p_br_contiguous, |
| LDNT1H_z_p_bi_contiguous); |
| } |
| |
| void Assembler::ldnt1w(const ZRegister& zt, |
| const PRegisterZ& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| LDNT1W_z_p_br_contiguous, |
| LDNT1W_z_p_bi_contiguous); |
| } |
| |
| Instr Assembler::SVEMemOperandHelper(unsigned msize_in_bytes_log2, |
| int num_regs, |
| const SVEMemOperand& addr, |
| bool is_load) { |
| VIXL_ASSERT((num_regs >= 1) && (num_regs <= 4)); |
| |
| Instr op = 0xfffffff; |
| if (addr.IsScalarPlusImmediate()) { |
| VIXL_ASSERT((addr.GetImmediateOffset() == 0) || addr.IsMulVl()); |
| int64_t imm = addr.GetImmediateOffset(); |
| VIXL_ASSERT((imm % num_regs) == 0); |
| op = RnSP(addr.GetScalarBase()) | ImmField<19, 16>(imm / num_regs); |
| |
| } else if (addr.IsScalarPlusScalar()) { |
| VIXL_ASSERT(addr.GetScalarOffset().IsZero() || |
| addr.IsEquivalentToLSL(msize_in_bytes_log2)); |
| op = RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset()); |
| |
| } else if (addr.IsVectorPlusImmediate()) { |
| ZRegister zn = addr.GetVectorBase(); |
| uint64_t imm = addr.GetImmediateOffset(); |
| VIXL_ASSERT(num_regs == 1); |
| VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); |
| VIXL_ASSERT(IsMultiple(imm, (1 << msize_in_bytes_log2))); |
| op = Rn(zn) | ImmUnsignedField<20, 16>(imm >> msize_in_bytes_log2); |
| |
| } else if (addr.IsScalarPlusVector()) { |
| // We have to support several different addressing modes. Some instructions |
| // support a subset of these, but the SVEMemOperand encoding is consistent. |
| Register xn = addr.GetScalarBase(); |
| ZRegister zm = addr.GetVectorOffset(); |
| SVEOffsetModifier mod = addr.GetOffsetModifier(); |
| Instr modifier_bit = 1 << (is_load ? 22 : 14); |
| Instr xs = (mod == SVE_SXTW) ? modifier_bit : 0; |
| VIXL_ASSERT(num_regs == 1); |
| |
| if (mod == SVE_LSL) { |
| // 64-bit scaled offset: [<Xn|SP>, <Zm>.D, LSL #<shift>] |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(addr.GetShiftAmount() == msize_in_bytes_log2); |
| } else if (mod == NO_SVE_OFFSET_MODIFIER) { |
| // 64-bit unscaled offset: [<Xn|SP>, <Zm>.D] |
| VIXL_ASSERT(zm.IsLaneSizeD()); |
| VIXL_ASSERT(addr.GetShiftAmount() == 0); |
| } else { |
| // 32-bit scaled offset: [<Xn|SP>, <Zm>.S, <mod> #<shift>] |
| // 32-bit unscaled offset: [<Xn|SP>, <Zm>.S, <mod>] |
| // 32-bit unpacked scaled offset: [<Xn|SP>, <Zm>.D, <mod> #<shift>] |
| // 32-bit unpacked unscaled offset: [<Xn|SP>, <Zm>.D, <mod>] |
| VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD()); |
| VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW)); |
| VIXL_ASSERT((addr.GetShiftAmount() == 0) || |
| (addr.GetShiftAmount() == msize_in_bytes_log2)); |
| } |
| |
| // The form itself is encoded in the instruction opcode. |
| op = RnSP(xn) | Rm(zm) | xs; |
| } else { |
| VIXL_UNIMPLEMENTED(); |
| } |
| |
| return op; |
| } |
| |
| // SVEMemStore. |
| |
| void Assembler::SVESt1Helper(unsigned msize_in_bytes_log2, |
| const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| if (addr.IsScalarPlusScalar()) { |
| // Rm must not be xzr. |
| VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); |
| } |
| |
| if (addr.IsScatterGather()) { |
| bool is_load = false; |
| bool is_signed = false; |
| bool is_ff = false; |
| SVEScatterGatherHelper(msize_in_bytes_log2, |
| zt, |
| pg, |
| addr, |
| is_load, |
| is_signed, |
| is_ff); |
| return; |
| } |
| |
| Instr op; |
| if (addr.IsScalarPlusImmediate()) { |
| op = SVEContiguousStore_ScalarPlusImmFixed; |
| } else if (addr.IsScalarPlusScalar()) { |
| op = SVEContiguousStore_ScalarPlusScalarFixed; |
| } else { |
| VIXL_UNIMPLEMENTED(); |
| op = 0xffffffff; |
| } |
| SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, false, op); |
| } |
| |
| void Assembler::SVESt234Helper(int num_regs, |
| const ZRegister& zt1, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| if (addr.IsScalarPlusScalar()) { |
| // Rm must not be xzr. |
| VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); |
| } |
| |
| Instr op; |
| if (addr.IsScalarPlusImmediate()) { |
| op = SVEStoreMultipleStructures_ScalarPlusImmFixed; |
| } else if (addr.IsScalarPlusScalar()) { |
| op = SVEStoreMultipleStructures_ScalarPlusScalarFixed; |
| } else { |
| // These instructions don't support any other addressing modes. |
| VIXL_ABORT(); |
| } |
| SVELdSt234Helper(num_regs, zt1, pg, addr, op); |
| } |
| |
| #define VIXL_DEFINE_ST1(MSZ, LANE_SIZE) \ |
| void Assembler::st1##MSZ(const ZRegister& zt, \ |
| const PRegister& pg, \ |
| const SVEMemOperand& addr) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| SVESt1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr); \ |
| } |
| #define VIXL_DEFINE_ST2(MSZ, LANE_SIZE) \ |
| void Assembler::st2##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const PRegister& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVESt234Helper(2, zt1, pg, addr); \ |
| } |
| #define VIXL_DEFINE_ST3(MSZ, LANE_SIZE) \ |
| void Assembler::st3##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const ZRegister& zt3, \ |
| const PRegister& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2, zt3); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVESt234Helper(3, zt1, pg, addr); \ |
| } |
| #define VIXL_DEFINE_ST4(MSZ, LANE_SIZE) \ |
| void Assembler::st4##MSZ(const ZRegister& zt1, \ |
| const ZRegister& zt2, \ |
| const ZRegister& zt3, \ |
| const ZRegister& zt4, \ |
| const PRegister& pg, \ |
| const SVEMemOperand& addr) { \ |
| USE(zt2, zt3, zt4); \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ |
| VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \ |
| VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \ |
| VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ |
| SVESt234Helper(4, zt1, pg, addr); \ |
| } |
| |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST1) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST2) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST3) |
| VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST4) |
| |
| void Assembler::stnt1b(const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| STNT1B_z_p_br_contiguous, |
| STNT1B_z_p_bi_contiguous); |
| } |
| |
| void Assembler::stnt1d(const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| STNT1D_z_p_br_contiguous, |
| STNT1D_z_p_bi_contiguous); |
| } |
| |
| void Assembler::stnt1h(const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| STNT1H_z_p_br_contiguous, |
| STNT1H_z_p_bi_contiguous); |
| } |
| |
| void Assembler::stnt1w(const ZRegister& zt, |
| const PRegister& pg, |
| const SVEMemOperand& addr) { |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || |
| (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2))); |
| SVELd1St1ScaImmHelper(zt, |
| pg, |
| addr, |
| STNT1W_z_p_br_contiguous, |
| STNT1W_z_p_bi_contiguous); |
| } |
| |
| void Assembler::str(const CPURegister& rt, const SVEMemOperand& addr) { |
| // STR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}] |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister()); |
| VIXL_ASSERT(addr.IsPlainScalar() || |
| (addr.IsScalarPlusImmediate() && |
| (addr.GetOffsetModifier() == SVE_MUL_VL))); |
| int64_t imm9 = addr.GetImmediateOffset(); |
| VIXL_ASSERT(IsInt9(imm9)); |
| Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10; |
| Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16; |
| |
| Instr op = STR_z_bi; |
| if (rt.IsPRegister()) { |
| op = STR_p_bi; |
| } |
| Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l); |
| } |
| |
| // SVEMulIndex. |
| |
| void Assembler::sdot(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| |
| Instr op = 0xffffffff; |
| switch (zda.GetLaneSizeInBits()) { |
| case kSRegSize: |
| VIXL_ASSERT(IsUint2(index)); |
| op = SDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn); |
| break; |
| case kDRegSize: |
| VIXL_ASSERT(IsUint1(index)); |
| op = SDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn); |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| Emit(op); |
| } |
| |
| void Assembler::udot(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| |
| Instr op = 0xffffffff; |
| switch (zda.GetLaneSizeInBits()) { |
| case kSRegSize: |
| VIXL_ASSERT(IsUint2(index)); |
| op = UDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn); |
| break; |
| case kDRegSize: |
| VIXL_ASSERT(IsUint1(index)); |
| op = UDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn); |
| break; |
| default: |
| VIXL_UNIMPLEMENTED(); |
| break; |
| } |
| |
| Emit(op); |
| } |
| |
| // SVEPartitionBreak. |
| |
| void Assembler::brka(const PRegisterWithLaneSize& pd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| |
| Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000; |
| Emit(BRKA_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn)); |
| } |
| |
| void Assembler::brkas(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| |
| Emit(BRKAS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); |
| } |
| |
| void Assembler::brkb(const PRegisterWithLaneSize& pd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| |
| Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000; |
| Emit(BRKB_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn)); |
| } |
| |
| void Assembler::brkbs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| |
| Emit(BRKBS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); |
| } |
| |
| void Assembler::brkn(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| USE(pm); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| VIXL_ASSERT(pd.Is(pm)); |
| |
| Emit(BRKN_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); |
| } |
| |
| void Assembler::brkns(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| USE(pm); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); |
| VIXL_ASSERT(pd.Is(pm)); |
| |
| Emit(BRKNS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); |
| } |
| |
| // SVEPermutePredicate. |
| |
| void Assembler::punpkhi(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn) { |
| // PUNPKHI <Pd>.H, <Pn>.B |
| // 0000 0101 0011 0001 0100 000. ...0 .... |
| // H<16> = 1 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeH()); |
| VIXL_ASSERT(pn.IsLaneSizeB()); |
| |
| Emit(PUNPKHI_p_p | Pd(pd) | Pn(pn)); |
| } |
| |
| void Assembler::punpklo(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn) { |
| // PUNPKLO <Pd>.H, <Pn>.B |
| // 0000 0101 0011 0000 0100 000. ...0 .... |
| // H<16> = 0 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.IsLaneSizeH()); |
| VIXL_ASSERT(pn.IsLaneSizeB()); |
| |
| Emit(PUNPKLO_p_p | Pd(pd) | Pn(pn)); |
| } |
| |
| void Assembler::rev(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn) { |
| // REV <Pd>.<T>, <Pn>.<T> |
| // 0000 0101 ..11 0100 0100 000. ...0 .... |
| // size<23:22> | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn)); |
| |
| Emit(REV_p_p | SVESize(pd) | Pd(pd) | Rx<8, 5>(pn)); |
| } |
| |
| void Assembler::trn1(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // TRN1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0101 000. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 0 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(TRN1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::trn2(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // TRN2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0101 010. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 1 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(TRN2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::uzp1(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // UZP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0100 100. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 0 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(UZP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::uzp2(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // UZP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0100 110. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 1 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(UZP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::zip1(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // ZIP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0100 000. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 0 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(ZIP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::zip2(const PRegisterWithLaneSize& pd, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // ZIP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> |
| // 0000 0101 ..10 .... 0100 010. ...0 .... |
| // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 1 | Pn<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); |
| |
| Emit(ZIP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); |
| } |
| |
| // SVEPermuteVectorExtract. |
| |
| void Assembler::ext(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| unsigned offset) { |
| // EXT <Zdn>.B, <Zdn>.B, <Zm>.B, #<imm> |
| // 0000 0101 001. .... 000. .... .... .... |
| // imm8h<20:16> | imm8l<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(IsUint8(offset)); |
| |
| int imm8h = ExtractUnsignedBitfield32(7, 3, offset); |
| int imm8l = ExtractUnsignedBitfield32(2, 0, offset); |
| Emit(EXT_z_zi_des | Rd(zd) | Rn(zm) | ImmUnsignedField<20, 16>(imm8h) | |
| ImmUnsignedField<12, 10>(imm8l)); |
| } |
| |
| // SVEPermuteVectorInterleaving. |
| |
| void Assembler::trn1(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // TRN1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0111 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(TRN1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::trn2(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // TRN2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0111 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(TRN2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uzp1(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UZP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0110 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(UZP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uzp2(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UZP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0110 11.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(UZP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::zip1(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ZIP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0110 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(ZIP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::zip2(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ZIP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0110 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(ZIP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEPermuteVectorPredicated. |
| |
| void Assembler::clasta(const Register& rd, |
| const PRegister& pg, |
| const Register& rn, |
| const ZRegister& zm) { |
| // CLASTA <R><dn>, <Pg>, <R><dn>, <Zm>.<T> |
| // 0000 0101 ..11 0000 101. .... .... .... |
| // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Rdn<4:0> |
| |
| USE(rn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rd.Is(rn)); |
| |
| Emit(CLASTA_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::clasta(const VRegister& vd, |
| const PRegister& pg, |
| const VRegister& vn, |
| const ZRegister& zm) { |
| // CLASTA <V><dn>, <Pg>, <V><dn>, <Zm>.<T> |
| // 0000 0101 ..10 1010 100. .... .... .... |
| // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Vdn<4:0> |
| |
| USE(vn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.Is(vn)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(AreSameLaneSize(vd, zm)); |
| |
| Emit(CLASTA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::clasta(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // CLASTA <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0101 ..10 1000 100. .... .... .... |
| // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(CLASTA_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::clastb(const Register& rd, |
| const PRegister& pg, |
| const Register& rn, |
| const ZRegister& zm) { |
| // CLASTB <R><dn>, <Pg>, <R><dn>, <Zm>.<T> |
| // 0000 0101 ..11 0001 101. .... .... .... |
| // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Rdn<4:0> |
| |
| USE(rn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(rd.Is(rn)); |
| |
| Emit(CLASTB_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::clastb(const VRegister& vd, |
| const PRegister& pg, |
| const VRegister& vn, |
| const ZRegister& zm) { |
| // CLASTB <V><dn>, <Pg>, <V><dn>, <Zm>.<T> |
| // 0000 0101 ..10 1011 100. .... .... .... |
| // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Vdn<4:0> |
| |
| USE(vn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.Is(vn)); |
| VIXL_ASSERT(vd.IsScalar()); |
| VIXL_ASSERT(AreSameLaneSize(vd, zm)); |
| |
| Emit(CLASTB_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::clastb(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // CLASTB <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0101 ..10 1001 100. .... .... .... |
| // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(CLASTB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::compact(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // COMPACT <Zd>.<T>, <Pg>, <Zn>.<T> |
| // 0000 0101 1.10 0001 100. .... .... .... |
| // sz<22> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT((zd.GetLaneSizeInBits() == kSRegSize) || |
| (zd.GetLaneSizeInBits() == kDRegSize)); |
| |
| Instr sz = (zd.GetLaneSizeInBits() == kDRegSize) ? (1 << 22) : 0; |
| Emit(COMPACT_z_p_z | sz | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::cpy(const ZRegister& zd, |
| const PRegisterM& pg, |
| const Register& rn) { |
| // CPY <Zd>.<T>, <Pg>/M, <R><n|SP> |
| // 0000 0101 ..10 1000 101. .... .... .... |
| // size<23:22> | Pg<12:10> | Rn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= |
| zd.GetLaneSizeInBits()); |
| |
| Emit(CPY_z_p_r | SVESize(zd) | Rd(zd) | PgLow8(pg) | RnSP(rn)); |
| } |
| |
| void Assembler::cpy(const ZRegister& zd, |
| const PRegisterM& pg, |
| const VRegister& vn) { |
| // CPY <Zd>.<T>, <Pg>/M, <V><n> |
| // 0000 0101 ..10 0000 100. .... .... .... |
| // size<23:22> | Pg<12:10> | Vn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vn.IsScalar()); |
| VIXL_ASSERT(static_cast<unsigned>(vn.GetSizeInBits()) == |
| zd.GetLaneSizeInBits()); |
| |
| Emit(CPY_z_p_v | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(vn)); |
| } |
| |
| void Assembler::lasta(const Register& rd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // LASTA <R><d>, <Pg>, <Zn>.<T> |
| // 0000 0101 ..10 0000 101. .... .... .... |
| // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LASTA_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::lasta(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // LASTA <V><d>, <Pg>, <Zn>.<T> |
| // 0000 0101 ..10 0010 100. .... .... .... |
| // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(LASTA_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::lastb(const Register& rd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // LASTB <R><d>, <Pg>, <Zn>.<T> |
| // 0000 0101 ..10 0001 101. .... .... .... |
| // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(LASTB_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::lastb(const VRegister& vd, |
| const PRegister& pg, |
| const ZRegister& zn) { |
| // LASTB <V><d>, <Pg>, <Zn>.<T> |
| // 0000 0101 ..10 0011 100. .... .... .... |
| // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Vd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vd.IsScalar()); |
| |
| Emit(LASTB_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::rbit(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // RBIT <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0101 ..10 0111 100. .... .... .... |
| // size<23:22> | opc<17:16> = 11 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(RBIT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::revb(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // REVB <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0101 ..10 0100 100. .... .... .... |
| // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.IsLaneSizeH() || zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(REVB_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::revh(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // REVH <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0000 0101 ..10 0101 100. .... .... .... |
| // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(REVH_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::revw(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // REVW <Zd>.D, <Pg>/M, <Zn>.D |
| // 0000 0101 ..10 0110 100. .... .... .... |
| // size<23:22> | opc<17:16> = 10 | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| |
| Emit(REVW_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::splice(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SPLICE <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> |
| // 0000 0101 ..10 1100 100. .... .... .... |
| // size<23:22> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(SPLICE_z_p_zz_des | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // SVEPermuteVectorUnpredicated. |
| |
| void Assembler::dup(const ZRegister& zd, const Register& xn) { |
| // DUP <Zd>.<T>, <R><n|SP> |
| // 0000 0101 ..10 0000 0011 10.. .... .... |
| // size<23:22> | Rn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(DUP_z_r | SVESize(zd) | Rd(zd) | RnSP(xn)); |
| } |
| |
| void Assembler::dup(const ZRegister& zd, const ZRegister& zn, unsigned index) { |
| // DUP <Zd>.<T>, <Zn>.<T>[<imm>] |
| // 0000 0101 ..1. .... 0010 00.. .... .... |
| // imm2<23:22> | tsz<20:16> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(zd, zn)); |
| VIXL_ASSERT((index * zd.GetLaneSizeInBits()) < 512); |
| int n = zd.GetLaneSizeInBytesLog2(); |
| unsigned imm_7 = (index << (n + 1)) | (1 << n); |
| VIXL_ASSERT(IsUint7(imm_7)); |
| unsigned imm_2 = ExtractUnsignedBitfield32(6, 5, imm_7); |
| unsigned tsz_5 = ExtractUnsignedBitfield32(4, 0, imm_7); |
| |
| Emit(DUP_z_zi | ImmUnsignedField<23, 22>(imm_2) | |
| ImmUnsignedField<20, 16>(tsz_5) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::insr(const ZRegister& zdn, const Register& rm) { |
| // INSR <Zdn>.<T>, <R><m> |
| // 0000 0101 ..10 0100 0011 10.. .... .... |
| // size<23:22> | Rm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(INSR_z_r | SVESize(zdn) | Rd(zdn) | Rn(rm)); |
| } |
| |
| void Assembler::insr(const ZRegister& zdn, const VRegister& vm) { |
| // INSR <Zdn>.<T>, <V><m> |
| // 0000 0101 ..11 0100 0011 10.. .... .... |
| // size<23:22> | Vm<9:5> | Zdn<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(vm.IsScalar()); |
| |
| Emit(INSR_z_v | SVESize(zdn) | Rd(zdn) | Rn(vm)); |
| } |
| |
| void Assembler::rev(const ZRegister& zd, const ZRegister& zn) { |
| // REV <Zd>.<T>, <Zn>.<T> |
| // 0000 0101 ..11 1000 0011 10.. .... .... |
| // size<23:22> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(zd, zn)); |
| |
| Emit(REV_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::sunpkhi(const ZRegister& zd, const ZRegister& zn) { |
| // SUNPKHI <Zd>.<T>, <Zn>.<Tb> |
| // 0000 0101 ..11 0001 0011 10.. .... .... |
| // size<23:22> | U<17> = 0 | H<16> = 1 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| |
| Emit(SUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::sunpklo(const ZRegister& zd, const ZRegister& zn) { |
| // SUNPKLO <Zd>.<T>, <Zn>.<Tb> |
| // 0000 0101 ..11 0000 0011 10.. .... .... |
| // size<23:22> | U<17> = 0 | H<16> = 0 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| |
| Emit(SUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::tbl(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // TBL <Zd>.<T>, { <Zn>.<T> }, <Zm>.<T> |
| // 0000 0101 ..1. .... 0011 00.. .... .... |
| // size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(zd, zn, zm)); |
| |
| Emit(TBL_z_zz_1 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uunpkhi(const ZRegister& zd, const ZRegister& zn) { |
| // UUNPKHI <Zd>.<T>, <Zn>.<Tb> |
| // 0000 0101 ..11 0011 0011 10.. .... .... |
| // size<23:22> | U<17> = 1 | H<16> = 1 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| |
| Emit(UUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::uunpklo(const ZRegister& zd, const ZRegister& zn) { |
| // UUNPKLO <Zd>.<T>, <Zn>.<Tb> |
| // 0000 0101 ..11 0010 0011 10.. .... .... |
| // size<23:22> | U<17> = 1 | H<16> = 0 | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| |
| Emit(UUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); |
| } |
| |
| // SVEPredicateCount. |
| |
| void Assembler::cntp(const Register& xd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn) { |
| // CNTP <Xd>, <Pg>, <Pn>.<T> |
| // 0010 0101 ..10 0000 10.. ..0. .... .... |
| // size<23:22> | opc<18:16> = 000 | Pg<13:10> | o2<9> = 0 | Pn<8:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX()); |
| VIXL_ASSERT(pg.IsUnqualified()); |
| if (pg.HasLaneSize()) VIXL_ASSERT(AreSameFormat(pg, pn)); |
| |
| Emit(CNTP_r_p_p | SVESize(pn) | Rd(xd) | Pg<13, 10>(pg) | Pn(pn)); |
| } |
| |
| // SVEPredicateLogicalOp. |
| void Assembler::and_(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(AND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::ands(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(ANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::bic(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(BIC_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::bics(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(BICS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::eor(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(EOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::eors(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(EORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::nand(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(NAND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::nands(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(NANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::nor(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(NOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::nors(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(NORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::orn(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(ORN_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::orns(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(ORNS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::orr(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(ORR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::orrs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameFormat(pd, pn, pm)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| Emit(ORRS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::sel(const PRegisterWithLaneSize& pd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| Emit(SEL_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| // SVEPredicateMisc. |
| |
| void Assembler::pfalse(const PRegisterWithLaneSize& pd) { |
| // PFALSE <Pd>.B |
| // 0010 0101 0001 1000 1110 0100 0000 .... |
| // op<23> = 0 | S<22> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| // Ignore the lane size, since it makes no difference to the operation. |
| |
| Emit(PFALSE_p | Pd(pd)); |
| } |
| |
| void Assembler::pfirst(const PRegisterWithLaneSize& pd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn) { |
| // PFIRST <Pdn>.B, <Pg>, <Pdn>.B |
| // 0010 0101 0101 1000 1100 000. ...0 .... |
| // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pdn<3:0> |
| |
| USE(pn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.Is(pn)); |
| VIXL_ASSERT(pd.IsLaneSizeB()); |
| |
| Emit(PFIRST_p_p_p | Pd(pd) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::pnext(const PRegisterWithLaneSize& pd, |
| const PRegister& pg, |
| const PRegisterWithLaneSize& pn) { |
| // PNEXT <Pdn>.<T>, <Pg>, <Pdn>.<T> |
| // 0010 0101 ..01 1001 1100 010. ...0 .... |
| // size<23:22> | Pg<8:5> | Pdn<3:0> |
| |
| USE(pn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pd.Is(pn)); |
| |
| Emit(PNEXT_p_p_p | SVESize(pd) | Pd(pd) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::ptest(const PRegister& pg, const PRegisterWithLaneSize& pn) { |
| // PTEST <Pg>, <Pn>.B |
| // 0010 0101 0101 0000 11.. ..0. ...0 0000 |
| // op<23> = 0 | S<22> = 1 | Pg<13:10> | Pn<8:5> | opc2<3:0> = 0000 |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(pn.IsLaneSizeB()); |
| |
| Emit(PTEST_p_p | Pg<13, 10>(pg) | Rx<8, 5>(pn)); |
| } |
| |
| void Assembler::ptrue(const PRegisterWithLaneSize& pd, int pattern) { |
| // PTRUE <Pd>.<T>{, <pattern>} |
| // 0010 0101 ..01 1000 1110 00.. ...0 .... |
| // size<23:22> | S<16> = 0 | pattern<9:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(PTRUE_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern)); |
| } |
| |
| void Assembler::ptrues(const PRegisterWithLaneSize& pd, int pattern) { |
| // PTRUES <Pd>.<T>{, <pattern>} |
| // 0010 0101 ..01 1001 1110 00.. ...0 .... |
| // size<23:22> | S<16> = 1 | pattern<9:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(PTRUES_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern)); |
| } |
| |
| void Assembler::rdffr(const PRegisterWithLaneSize& pd) { |
| // RDFFR <Pd>.B |
| // 0010 0101 0001 1001 1111 0000 0000 .... |
| // op<23> = 0 | S<22> = 0 | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(RDFFR_p_f | Pd(pd)); |
| } |
| |
| void Assembler::rdffr(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) { |
| // RDFFR <Pd>.B, <Pg>/Z |
| // 0010 0101 0001 1000 1111 000. ...0 .... |
| // op<23> = 0 | S<22> = 0 | Pg<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(RDFFR_p_p_f | Pd(pd) | Pg<8, 5>(pg)); |
| } |
| |
| void Assembler::rdffrs(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) { |
| // RDFFRS <Pd>.B, <Pg>/Z |
| // 0010 0101 0101 1000 1111 000. ...0 .... |
| // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(RDFFRS_p_p_f | Pd(pd) | Pg<8, 5>(pg)); |
| } |
| |
| // SVEPropagateBreak. |
| |
| void Assembler::brkpa(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // BRKPA <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B |
| // 0010 0101 0000 .... 11.. ..0. ...0 .... |
| // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(BRKPA_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::brkpas(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // BRKPAS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B |
| // 0010 0101 0100 .... 11.. ..0. ...0 .... |
| // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(BRKPAS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::brkpb(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // BRKPB <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B |
| // 0010 0101 0000 .... 11.. ..0. ...1 .... |
| // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(BRKPB_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| void Assembler::brkpbs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn, |
| const PRegisterWithLaneSize& pm) { |
| // BRKPBS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B |
| // 0010 0101 0100 .... 11.. ..0. ...1 .... |
| // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(BRKPBS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); |
| } |
| |
| // SVEStackFrameAdjustment. |
| |
| void Assembler::addpl(const Register& xd, const Register& xn, int imm6) { |
| // ADDPL <Xd|SP>, <Xn|SP>, #<imm> |
| // 0000 0100 011. .... 0101 0... .... .... |
| // op<22> = 1 | Rn<20:16> | imm6<10:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX()); |
| VIXL_ASSERT(xn.IsX()); |
| |
| Emit(ADDPL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6)); |
| } |
| |
| void Assembler::addvl(const Register& xd, const Register& xn, int imm6) { |
| // ADDVL <Xd|SP>, <Xn|SP>, #<imm> |
| // 0000 0100 001. .... 0101 0... .... .... |
| // op<22> = 0 | Rn<20:16> | imm6<10:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX()); |
| VIXL_ASSERT(xn.IsX()); |
| |
| Emit(ADDVL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6)); |
| } |
| |
| // SVEStackFrameSize. |
| |
| void Assembler::rdvl(const Register& xd, int imm6) { |
| // RDVL <Xd>, #<imm> |
| // 0000 0100 1011 1111 0101 0... .... .... |
| // op<22> = 0 | opc2<20:16> = 11111 | imm6<10:5> | Rd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(xd.IsX()); |
| |
| Emit(RDVL_r_i | Rd(xd) | ImmField<10, 5>(imm6)); |
| } |
| |
| // SVEVectorSelect. |
| |
| void Assembler::sel(const ZRegister& zd, |
| const PRegister& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(SEL_z_p_zz | SVESize(zd) | Rd(zd) | Pg<13, 10>(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| // SVEWriteFFR. |
| |
| void Assembler::setffr() { |
| // SETFFR |
| // 0010 0101 0010 1100 1001 0000 0000 0000 |
| // opc<23:22> = 00 |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(SETFFR_f); |
| } |
| |
| void Assembler::wrffr(const PRegisterWithLaneSize& pn) { |
| // WRFFR <Pn>.B |
| // 0010 0101 0010 1000 1001 000. ...0 0000 |
| // opc<23:22> = 00 | Pn<8:5> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| |
| Emit(WRFFR_f_p | Rx<8, 5>(pn)); |
| } |
| |
| // Aliases. |
| |
| void Assembler::bic(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| and_(zd, zn, ~imm); |
| } |
| |
| void Assembler::eon(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| eor(zd, zn, ~imm); |
| } |
| |
| void Assembler::orn(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { |
| orr(zd, zn, ~imm); |
| } |
| |
| |
| void Assembler::fmov(const ZRegister& zd, const PRegisterM& pg, double imm) { |
| if (IsPositiveZero(imm)) { |
| cpy(zd, pg, 0); |
| } else { |
| fcpy(zd, pg, imm); |
| } |
| } |
| |
| void Assembler::fmov(const ZRegister& zd, double imm) { |
| if (IsPositiveZero(imm)) { |
| dup(zd, imm); |
| } else { |
| fdup(zd, imm); |
| } |
| } |
| |
| void Assembler::mov(const PRegister& pd, const PRegister& pn) { |
| // If the inputs carry a lane size, they must match. |
| VIXL_ASSERT((!pd.HasLaneSize() && !pn.HasLaneSize()) || |
| AreSameLaneSize(pd, pn)); |
| orr(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB()); |
| } |
| |
| void Assembler::mov(const PRegisterWithLaneSize& pd, |
| const PRegisterM& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| sel(pd, pg, pn, pd); |
| } |
| |
| void Assembler::mov(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| and_(pd, pg, pn, pn); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, |
| const PRegister& pg, |
| int imm8, |
| int shift) { |
| VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); |
| cpy(zd, pg, imm8, shift); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, const Register& xn) { dup(zd, xn); } |
| |
| void Assembler::mov(const ZRegister& zd, const VRegister& vn) { |
| VIXL_ASSERT(vn.IsScalar()); |
| VIXL_ASSERT(AreSameLaneSize(zd, vn)); |
| dup(zd, vn.Z().WithSameLaneSizeAs(vn), 0); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, const ZRegister& zn) { |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| orr(zd.VnD(), zn.VnD(), zn.VnD()); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, const ZRegister& zn, unsigned index) { |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| dup(zd, zn, index); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, |
| const PRegisterM& pg, |
| const Register& rn) { |
| cpy(zd, pg, rn); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, |
| const PRegisterM& pg, |
| const VRegister& vn) { |
| VIXL_ASSERT(vn.IsScalar()); |
| VIXL_ASSERT(AreSameLaneSize(zd, vn)); |
| cpy(zd, pg, vn); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| sel(zd, pg, zn, zd); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, uint64_t imm) { |
| // Mov is an alias of dupm for certain values of imm. Whilst this matters in |
| // the disassembler, for the assembler, we don't distinguish between the |
| // two mnemonics, and simply call dupm. |
| dupm(zd, imm); |
| } |
| |
| void Assembler::mov(const ZRegister& zd, int imm8, int shift) { |
| dup(zd, imm8, shift); |
| } |
| |
| void Assembler::movs(const PRegister& pd, const PRegister& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| orrs(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB()); |
| } |
| |
| void Assembler::movs(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| ands(pd, pg, pn, pn); |
| } |
| |
| void Assembler::not_(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| eor(pd, pg, pn, pg.VnB()); |
| } |
| |
| void Assembler::nots(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const PRegisterWithLaneSize& pn) { |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); |
| eors(pd, pg, pn, pg.VnB()); |
| } |
| |
| // SVE2 |
| |
| void Assembler::adclb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 0.0. .... 1101 00.. .... .... |
| // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| |
| Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; |
| Emit(0x4500d000 | sz | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::adclt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 0.0. .... 1101 01.. .... .... |
| // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| |
| Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; |
| Emit(0x4500d400 | sz | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::addhnb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0110 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45206000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::addhnt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0110 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45206400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::addp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // ADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0001 101. .... .... .... |
| // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4411a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::bcax(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // BCAX <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 011. .... 0011 10.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04603800 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::bdep(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // BDEP <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1011 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4500b400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::bext(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // BEXT <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1011 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4500b000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::bgrp(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // BGRP <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1011 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4500b800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::bsl(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // BSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 001. .... 0011 11.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04203c00 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::bsl1n(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // BSL1N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 011. .... 0011 11.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04603c00 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::bsl2n(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // BSL2N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 101. .... 0011 11.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04a03c00 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::cadd(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int rot) { |
| // CADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const> |
| // 0100 0101 ..00 0000 1101 1... .... .... |
| // size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT((rot == 90) || (rot == 270)); |
| |
| Instr rotate_bit = (rot == 90) ? 0 : (1 << 10); |
| Emit(0x4500d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // cdot_z_zzzi_d |
| // cdot_z_zzzi_s |
| void Assembler::cdot(const ZRegister& zda, const ZRegister& zn) { |
| // CDOT <Zda>.D, <Zn>.H, <Zm>.H[<imm>], <const> |
| // 0100 0100 111. .... 0100 .... .... .... |
| // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e04000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::cdot(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // CDOT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>, <const> |
| // 0100 0100 ..0. .... 0001 .... .... .... |
| // size<23:22> | Zm<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44001000 | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // cmla_z_zzzi_h |
| // cmla_z_zzzi_s |
| void Assembler::cmla(const ZRegister& zda, const ZRegister& zn) { |
| // CMLA <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const> |
| // 0100 0100 101. .... 0110 .... .... .... |
| // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44a06000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::cmla(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // CMLA <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const> |
| // 0100 0100 ..0. .... 0010 .... .... .... |
| // size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x44002000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::eor3(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // EOR3 <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 001. .... 0011 10.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04203800 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::eorbt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // EORBT <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1001 00.. .... .... |
| // size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x45009000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::eortb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // EORTB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1001 01.. .... .... |
| // size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x45009400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ext(const ZRegister& zd, |
| const ZRegister& zn1, |
| const ZRegister& zn2) { |
| // EXT <Zd>.B, { <Zn1>.B, <Zn2>.B }, #<imm> |
| // 0000 0101 011. .... 000. .... .... .... |
| // imm8h<20:16> | imm8l<12:10> | Zn<9:5> | Zd<4:0> |
| |
| USE(zn2); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreConsecutive(zn1, zn2)); |
| |
| Emit(0x05600000 | Rd(zd) | Rn(zn1) | Rn(zn2)); |
| } |
| |
| void Assembler::faddp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0100 ..01 0000 100. .... .... .... |
| // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x64108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fcvtlt_z_p_z_h2s |
| // fcvtlt_z_p_z_s2d |
| void Assembler::fcvtlt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FCVTLT <Zd>.S, <Pg>/M, <Zn>.H |
| // 0110 0100 1000 1001 101. .... .... .... |
| // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x6489a000 | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fcvtnt_z_p_z_d2s |
| // fcvtnt_z_p_z_s2h |
| void Assembler::fcvtnt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FCVTNT <Zd>.S, <Pg>/M, <Zn>.D |
| // 0110 0100 1100 1010 101. .... .... .... |
| // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x64caa000 | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcvtx(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FCVTX <Zd>.S, <Pg>/M, <Zn>.D |
| // 0110 0101 0000 1010 101. .... .... .... |
| // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x650aa000 | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fcvtxnt(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FCVTXNT <Zd>.S, <Pg>/M, <Zn>.D |
| // 0110 0100 0000 1010 101. .... .... .... |
| // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x640aa000 | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::flogb(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // FLOGB <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0110 0101 0001 1..0 101. .... .... .... |
| // opc<23:22> | opc2<18:17> | U<16> | Pg<12:10> | Zn<9:5> | Zd<4:0> | size<> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x6518a000 | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::fmaxnmp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMAXNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0100 ..01 0100 100. .... .... .... |
| // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x64148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fmaxp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0100 ..01 0110 100. .... .... .... |
| // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x64168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fminnmp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMINNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0100 ..01 0101 100. .... .... .... |
| // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x64158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::fminp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0110 0100 ..01 0111 100. .... .... .... |
| // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x64178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fmlalb_z_zzz |
| // fmlalb_z_zzzi_s |
| void Assembler::fmlalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLALB <Zda>.S, <Zn>.H, <Zm>.H |
| // 0110 0100 101. .... 1000 00.. .... .... |
| // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x64a08000 | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fmlalt_z_zzz |
| // fmlalt_z_zzzi_s |
| void Assembler::fmlalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLALT <Zda>.S, <Zn>.H, <Zm>.H |
| // 0110 0100 101. .... 1000 01.. .... .... |
| // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x64a08400 | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fmlslb_z_zzz |
| // fmlslb_z_zzzi_s |
| void Assembler::fmlslb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLSLB <Zda>.S, <Zn>.H, <Zm>.H |
| // 0110 0100 101. .... 1010 00.. .... .... |
| // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x64a0a000 | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // fmlslt_z_zzz |
| // fmlslt_z_zzzi_s |
| void Assembler::fmlslt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // FMLSLT <Zda>.S, <Zn>.H, <Zm>.H |
| // 0110 0100 101. .... 1010 01.. .... .... |
| // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x64a0a400 | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::histcnt(const ZRegister& zd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // HISTCNT <Zd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..1. .... 110. .... .... .... |
| // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); |
| |
| Emit(0x4520c000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::histseg(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // HISTSEG <Zd>.B, <Zn>.B, <Zm>.B |
| // 0100 0101 ..1. .... 1010 00.. .... .... |
| // size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x4520a000 | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| #if 0 |
| |
| // This prototype maps to 2 instruction encodings: |
| // ldnt1b_z_p_ar_d_64_unscaled |
| // ldnt1b_z_p_ar_s_x32_unscaled |
| void Assembler::ldnt1b(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1B { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0100 000. .... 110. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc400c000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| void Assembler::ldnt1d(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1D { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0101 100. .... 110. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc580c000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // ldnt1h_z_p_ar_d_64_unscaled |
| // ldnt1h_z_p_ar_s_x32_unscaled |
| void Assembler::ldnt1h(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1H { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0100 100. .... 110. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc480c000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // ldnt1sb_z_p_ar_d_64_unscaled |
| // ldnt1sb_z_p_ar_s_x32_unscaled |
| void Assembler::ldnt1sb(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1SB { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0100 000. .... 100. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc4008000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // ldnt1sh_z_p_ar_d_64_unscaled |
| // ldnt1sh_z_p_ar_s_x32_unscaled |
| void Assembler::ldnt1sh(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1SH { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0100 100. .... 100. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc4808000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| void Assembler::ldnt1sw(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1SW { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0101 000. .... 100. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc5008000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // ldnt1w_z_p_ar_d_64_unscaled |
| // ldnt1w_z_p_ar_s_x32_unscaled |
| void Assembler::ldnt1w(const ZRegister& zt, const PRegisterZ& pg, const ZRegister& zn, const Register& rm) { |
| // LDNT1W { <Zt>.D }, <Pg>/Z, [<Zn>.D{, <Xm>}] |
| // 1100 0101 000. .... 110. .... .... .... |
| // msz<24:23> | Rm<20:16> | U<14> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xc500c000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| #endif |
| |
| void Assembler::match(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // MATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..1. .... 100. .... ...0 .... |
| // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn, zm)); |
| VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH()); |
| |
| Emit(0x45208000 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // mla_z_zzzi_d |
| // mla_z_zzzi_h |
| // mla_z_zzzi_s |
| void Assembler::mla(const ZRegister& zda, const ZRegister& zn) { |
| // MLA <Zda>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 0000 10.. .... .... |
| // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e00800 | Rd(zda) | Rn(zn)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // mls_z_zzzi_d |
| // mls_z_zzzi_h |
| // mls_z_zzzi_s |
| void Assembler::mls(const ZRegister& zda, const ZRegister& zn) { |
| // MLS <Zda>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 0000 11.. .... .... |
| // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e00c00 | Rd(zda) | Rn(zn)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // mul_z_zzi_d |
| // mul_z_zzi_h |
| // mul_z_zzi_s |
| void Assembler::mul(const ZRegister& zd, const ZRegister& zn) { |
| // MUL <Zd>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 1111 10.. .... .... |
| // size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0f800 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::mul(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // MUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0110 00.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x04206000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::nbsl(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int Zk) { |
| // NBSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D |
| // 0000 0100 111. .... 0011 11.. .... .... |
| // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Emit(0x04e03c00 | Rd(zd) | Rm(zm) | ImmField<9, 5>(Zk)); |
| } |
| |
| void Assembler::nmatch(const PRegisterWithLaneSize& pd, |
| const PRegisterZ& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // NMATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..1. .... 100. .... ...1 .... |
| // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(pd, zn, zm)); |
| VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH()); |
| |
| Emit(0x45208010 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::pmul(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // PMUL <Zd>.B, <Zn>.B, <Zm>.B |
| // 0000 0100 001. .... 0110 01.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x04206400 | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::pmullb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // PMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0110 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| // SVEPmull128 is not supported |
| VIXL_ASSERT(!zd.IsLaneSizeQ()); |
| |
| Emit(0x45006800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::pmullt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // PMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0110 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| // SVEPmull128 is not supported |
| VIXL_ASSERT(!zd.IsLaneSizeQ()); |
| |
| Emit(0x45006c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::raddhnb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // RADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0110 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45206800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::raddhnt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // RADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0110 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45206c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| #define VIXL_SVE_SHR_LIST(V) \ |
| V(rshrnb, 0x45201800) \ |
| V(rshrnt, 0x45201c00) \ |
| V(shrnb, 0x45201000) \ |
| V(shrnt, 0x45201400) \ |
| V(sqrshrnb, 0x45202800) \ |
| V(sqrshrnt, 0x45202c00) \ |
| V(sqrshrunb, 0x45200800) \ |
| V(sqrshrunt, 0x45200c00) \ |
| V(sqshrnb, 0x45202000) \ |
| V(sqshrnt, 0x45202400) \ |
| V(sqshrunb, 0x45200000) \ |
| V(sqshrunt, 0x45200400) \ |
| V(uqrshrnb, 0x45203800) \ |
| V(uqrshrnt, 0x45203c00) \ |
| V(uqshrnb, 0x45203000) \ |
| V(uqshrnt, 0x45203400) |
| |
| #define VIXL_DEFINE_ASM_FUNC(MNE, X) \ |
| void Assembler::MNE(const ZRegister& zd, const ZRegister& zn, int shift) { \ |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); \ |
| VIXL_ASSERT(!zd.IsLaneSizeD() && !zd.IsLaneSizeQ()); \ |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); \ |
| Instr encoded_imm = \ |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); \ |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, X); \ |
| } |
| VIXL_SVE_SHR_LIST(VIXL_DEFINE_ASM_FUNC) |
| #undef VIXL_DEFINE_ASM_FUNC |
| |
| void Assembler::rsubhnb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // RSUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0111 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45207800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::rsubhnt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // RSUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0111 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45207c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::saba(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1111 10.. .... .... |
| // size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x4500f800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sabalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1100 00.. .... .... |
| // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4500c000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sabalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1100 01.. .... .... |
| // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4500c400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sabdlb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0011 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45003000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sabdlt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0011 01.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45003400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sadalp(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb> |
| // 0100 0100 ..00 0100 101. .... .... .... |
| // size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4404a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::saddlb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0000 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45000000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::saddlbt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SADDLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1000 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); |
| |
| Emit(0x45008000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::saddlt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0000 01.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45000400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::saddwb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0100 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45004000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::saddwt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0100 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45004400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sbclb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SBCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 1.0. .... 1101 00.. .... .... |
| // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| |
| Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; |
| Emit(0x4580d000 | sz | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sbclt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SBCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 1.0. .... 1101 01.. .... .... |
| // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); |
| |
| Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; |
| Emit(0x4580d400 | sz | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::shadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0000 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::shsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0010 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44128000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::shsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0110 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sli(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // SLI <Zd>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1111 01.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f400); |
| } |
| |
| void Assembler::smaxp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0100 101. .... .... .... |
| // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4414a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sminp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0110 101. .... .... .... |
| // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4416a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smlalb_z_zzzi_d |
| // smlalb_z_zzzi_s |
| void Assembler::smlalb(const ZRegister& zda, const ZRegister& zn) { |
| // SMLALB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1000 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e08000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::smlalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0100 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44004000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smlalt_z_zzzi_d |
| // smlalt_z_zzzi_s |
| void Assembler::smlalt(const ZRegister& zda, const ZRegister& zn) { |
| // SMLALT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1000 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e08400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::smlalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0100 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44004400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smlslb_z_zzzi_d |
| // smlslb_z_zzzi_s |
| void Assembler::smlslb(const ZRegister& zda, const ZRegister& zn) { |
| // SMLSLB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1010 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0a000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::smlslb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0101 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44005000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smlslt_z_zzzi_d |
| // smlslt_z_zzzi_s |
| void Assembler::smlslt(const ZRegister& zda, const ZRegister& zn) { |
| // SMLSLT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1010 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0a400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::smlslt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0101 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44005400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::smulh(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0110 10.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x04206800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smullb_z_zzi_d |
| // smullb_z_zzi_s |
| void Assembler::smullb(const ZRegister& zd, const ZRegister& zn) { |
| // SMULLB <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1100 .0.. .... .... |
| // size<23:22> | opc<20:16> | U<12> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0c000 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::smullb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0111 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45007000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // smullt_z_zzi_d |
| // smullt_z_zzi_s |
| void Assembler::smullt(const ZRegister& zd, const ZRegister& zn) { |
| // SMULLT <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1100 .1.. .... .... |
| // size<23:22> | opc<20:16> | U<12> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0c400 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::smullt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0111 01.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45007400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // TODO: merge SVE2 constructive splice into SVE1 destructive splice. |
| // void Assembler::splice(const ZRegister& zd, const PRegister& pg, const |
| // ZRegister& zn1, const ZRegister& zn2) { |
| // // SPLICE <Zd>.<T>, <Pg>, { <Zn1>.<T>, <Zn2>.<T> } |
| // // 0000 0101 ..10 1101 100. .... .... .... |
| // // size<23:22> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| // |
| // USE(zn2); |
| // VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| // VIXL_ASSERT(AreConsecutive(zn1, zn2)); |
| // VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2)); |
| // |
| // Emit(0x052d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn1) | Rn(zn2)); |
| //} |
| |
| void Assembler::sqabs(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SQABS <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0100 0100 ..00 1000 101. .... .... .... |
| // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(0x4408a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::sqadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1000 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44188000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqcadd(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int rot) { |
| // SQCADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const> |
| // 0100 0101 ..00 0001 1101 1... .... .... |
| // size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| VIXL_ASSERT((rot == 90) || (rot == 270)); |
| |
| Instr rotate_bit = (rot == 90) ? 0 : (1 << 10); |
| Emit(0x4501d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqdmlalb_z_zzzi_d |
| // sqdmlalb_z_zzzi_s |
| void Assembler::sqdmlalb(const ZRegister& zda, const ZRegister& zn) { |
| // SQDMLALB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 0010 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e02000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqdmlalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0110 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44006000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqdmlalbt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLALBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0000 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44000800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqdmlalt_z_zzzi_d |
| // sqdmlalt_z_zzzi_s |
| void Assembler::sqdmlalt(const ZRegister& zda, const ZRegister& zn) { |
| // SQDMLALT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 0010 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e02400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqdmlalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0110 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44006400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqdmlslb_z_zzzi_d |
| // sqdmlslb_z_zzzi_s |
| void Assembler::sqdmlslb(const ZRegister& zda, const ZRegister& zn) { |
| // SQDMLSLB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 0011 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e03000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqdmlslb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0110 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44006800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqdmlslbt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLSLBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0000 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44000c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqdmlslt_z_zzzi_d |
| // sqdmlslt_z_zzzi_s |
| void Assembler::sqdmlslt(const ZRegister& zda, const ZRegister& zn) { |
| // SQDMLSLT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 0011 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e03400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqdmlslt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0110 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44006c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // sqdmulh_z_zzi_d |
| // sqdmulh_z_zzi_h |
| // sqdmulh_z_zzi_s |
| void Assembler::sqdmulh(const ZRegister& zd, const ZRegister& zn) { |
| // SQDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 1111 00.. .... .... |
| // size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0f000 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::sqdmulh(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0111 00.. .... .... |
| // size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x04207000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqdmullb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| // SQDMULLB <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1110 .0.. .... .... |
| // size<23:22> | opc<20:16> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeS()); |
| |
| Instr imm_field; |
| Instr zm_id; |
| if (zd.IsLaneSizeS()) { |
| VIXL_ASSERT(IsUint7(index)); |
| imm_field = ExtractUnsignedBitfield32(2, 1, index) << 19; |
| zm_id = Rx<18, 16>(zm); |
| } else { |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(IsUint3(index)); |
| imm_field = ExtractBit(index, 1) << 20; |
| zm_id = Rx<19, 16>(zm); |
| } |
| |
| // Synthesize the low part of immediate encoding. |
| imm_field |= ExtractBit(index, 0) << 11; |
| |
| Emit(0x44a0e000 | SVESize(zd) | Rd(zd) | Rn(zn) | zm_id | imm_field); |
| } |
| |
| void Assembler::sqdmullb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0110 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45006000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqdmullt_z_zzi_d |
| // sqdmullt_z_zzi_s |
| void Assembler::sqdmullt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int index) { |
| // SQDMULLT <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1110 .1.. .... .... |
| // size<23:22> | opc<20:16> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Instr imm_field; |
| Instr zm_id; |
| if (zd.IsLaneSizeS()) { |
| VIXL_ASSERT(IsUint7(index)); |
| imm_field = ExtractUnsignedBitfield32(2, 1, index) << 19; |
| zm_id = Rx<18, 16>(zm); |
| } else { |
| VIXL_ASSERT(zd.IsLaneSizeD()); |
| VIXL_ASSERT(IsUint3(index)); |
| imm_field = ExtractBit(index, 1) << 20; |
| zm_id = Rx<19, 16>(zm); |
| } |
| |
| // Synthesize the low part of immediate encoding. |
| imm_field |= ExtractBit(index, 0) << 11; |
| |
| Emit(0x44a0e400 | SVESize(zd) | Rd(zd) | Rn(zn) | zm_id | imm_field); |
| } |
| |
| void Assembler::sqdmullt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQDMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0110 01.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45006400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqneg(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // SQNEG <Zd>.<T>, <Pg>/M, <Zn>.<T> |
| // 0100 0100 ..00 1001 101. .... .... .... |
| // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| |
| Emit(0x4409a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // sqrdcmlah_z_zzzi_h |
| // sqrdcmlah_z_zzzi_s |
| void Assembler::sqrdcmlah(const ZRegister& zda, const ZRegister& zn) { |
| // SQRDCMLAH <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const> |
| // 0100 0100 101. .... 0111 .... .... .... |
| // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44a07000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqrdcmlah(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRDCMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const> |
| // 0100 0100 ..0. .... 0011 .... .... .... |
| // size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x44003000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // sqrdmlah_z_zzzi_d |
| // sqrdmlah_z_zzzi_h |
| // sqrdmlah_z_zzzi_s |
| void Assembler::sqrdmlah(const ZRegister& zda, const ZRegister& zn) { |
| // SQRDMLAH <Zda>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 0001 00.. .... .... |
| // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e01000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqrdmlah(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRDMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0100 ..0. .... 0111 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x44007000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // sqrdmlsh_z_zzzi_d |
| // sqrdmlsh_z_zzzi_h |
| // sqrdmlsh_z_zzzi_s |
| void Assembler::sqrdmlsh(const ZRegister& zda, const ZRegister& zn) { |
| // SQRDMLSH <Zda>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 0001 01.. .... .... |
| // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e01400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::sqrdmlsh(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRDMLSH <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0100 ..0. .... 0111 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x44007400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 3 instruction encodings: |
| // sqrdmulh_z_zzi_d |
| // sqrdmulh_z_zzi_h |
| // sqrdmulh_z_zzi_s |
| void Assembler::sqrdmulh(const ZRegister& zd, const ZRegister& zn) { |
| // SQRDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>] |
| // 0100 0100 111. .... 1111 01.. .... .... |
| // size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0f400 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::sqrdmulh(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0111 01.. .... .... |
| // size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x04207400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::sqrshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1010 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqrshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1110 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0110 100. .... .... .... |
| // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | |
| // imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04068000); |
| } |
| |
| void Assembler::sqshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1000 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44088000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1100 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqshlu(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // SQSHLU <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 1111 100. .... .... .... |
| // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | |
| // imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040f8000); |
| } |
| |
| void Assembler::sqsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1010 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1110 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sqxtnb(const ZRegister& zd, const ZRegister& zn) { |
| // SQXTNB <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0100 00.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45204000); |
| } |
| |
| void Assembler::sqxtnt(const ZRegister& zd, const ZRegister& zn) { |
| // SQXTNT <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0100 01.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45204400); |
| } |
| |
| void Assembler::sqxtunb(const ZRegister& zd, const ZRegister& zn) { |
| // SQXTUNB <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0101 00.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45205000); |
| } |
| |
| void Assembler::sqxtunt(const ZRegister& zd, const ZRegister& zn) { |
| // SQXTUNT <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0101 01.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45205400); |
| } |
| |
| void Assembler::srhadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SRHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0100 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::sri(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // SRI <Zd>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1111 00.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f000); |
| } |
| |
| void Assembler::srshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 0010 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44028000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::srshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 0110 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44068000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::srshr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // SRSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 1100 100. .... .... .... |
| // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | |
| // imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040c8000); |
| } |
| |
| void Assembler::srsra(const ZRegister& zda, const ZRegister& zn, int shift) { |
| // SRSRA <Zda>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1110 10.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e800); |
| } |
| |
| void Assembler::sshllb(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // SSHLLB <Zd>.<T>, <Zn>.<Tb>, #<const> |
| // 0100 0101 0.0. .... 1010 00.. .... .... |
| // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a000); |
| } |
| |
| void Assembler::sshllt(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // SSHLLT <Zd>.<T>, <Zn>.<Tb>, #<const> |
| // 0100 0101 0.0. .... 1010 01.. .... .... |
| // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a400); |
| } |
| |
| void Assembler::ssra(const ZRegister& zda, const ZRegister& zn, int shift) { |
| // SSRA <Zda>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1110 00.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e000); |
| } |
| |
| void Assembler::ssublb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0001 00.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45001000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ssublbt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1000 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); |
| |
| Emit(0x45008800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ssublt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0001 01.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45001400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ssubltb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBLTB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1000 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); |
| |
| Emit(0x45008c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ssubwb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0101 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45005000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::ssubwt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SSUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0101 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45005400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| #if 0 |
| // This prototype maps to 2 instruction encodings: |
| // stnt1b_z_p_ar_d_64_unscaled |
| // stnt1b_z_p_ar_s_x32_unscaled |
| void Assembler::stnt1b(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { |
| // STNT1B { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] |
| // 1110 0100 000. .... 001. .... .... .... |
| // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xe4002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| void Assembler::stnt1d(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { |
| // STNT1D { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] |
| // 1110 0101 100. .... 001. .... .... .... |
| // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xe5802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // stnt1h_z_p_ar_d_64_unscaled |
| // stnt1h_z_p_ar_s_x32_unscaled |
| void Assembler::stnt1h(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { |
| // STNT1H { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] |
| // 1110 0100 100. .... 001. .... .... .... |
| // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xe4802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // stnt1w_z_p_ar_d_64_unscaled |
| // stnt1w_z_p_ar_s_x32_unscaled |
| void Assembler::stnt1w(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { |
| // STNT1W { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] |
| // 1110 0101 000. .... 001. .... .... .... |
| // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0xe5002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); |
| } |
| #endif |
| |
| void Assembler::subhnb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0111 00.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45207000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::subhnt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..1. .... 0111 01.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45207400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::suqadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // SUQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1100 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::tbl(const ZRegister& zd, |
| const ZRegister& zn1, |
| const ZRegister& zn2, |
| const ZRegister& zm) { |
| // TBL <Zd>.<T>, { <Zn1>.<T>, <Zn2>.<T> }, <Zm>.<T> |
| // 0000 0101 ..1. .... 0010 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0> |
| |
| USE(zn2); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreConsecutive(zn1, zn2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2, zm)); |
| |
| Emit(0x05202800 | SVESize(zd) | Rd(zd) | Rn(zn1) | Rn(zn2) | Rm(zm)); |
| } |
| |
| void Assembler::tbx(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // TBX <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0101 ..1. .... 0010 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x05202c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uaba(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0100 0101 ..0. .... 1111 11.. .... .... |
| // size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| |
| Emit(0x4500fc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uabalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1100 10.. .... .... |
| // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4500c800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uabalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 1100 11.. .... .... |
| // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4500cc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uabdlb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0011 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45003800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uabdlt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0011 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45003c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uadalp(const ZRegister& zda, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // UADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb> |
| // 0100 0100 ..00 0101 101. .... .... .... |
| // size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x4405a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::uaddlb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0000 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45000800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uaddlt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0000 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45000c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uaddwb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0100 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45004800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uaddwt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0100 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45004c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uhadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0001 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44118000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uhsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0011 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44138000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uhsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0111 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::umaxp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0101 101. .... .... .... |
| // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4415a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uminp(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0111 101. .... .... .... |
| // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x4417a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umlalb_z_zzzi_d |
| // umlalb_z_zzzi_s |
| void Assembler::umlalb(const ZRegister& zda, const ZRegister& zn) { |
| // UMLALB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1001 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e09000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::umlalb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0100 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44004800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umlalt_z_zzzi_d |
| // umlalt_z_zzzi_s |
| void Assembler::umlalt(const ZRegister& zda, const ZRegister& zn) { |
| // UMLALT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1001 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e09400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::umlalt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0100 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44004c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umlslb_z_zzzi_d |
| // umlslb_z_zzzi_s |
| void Assembler::umlslb(const ZRegister& zda, const ZRegister& zn) { |
| // UMLSLB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1011 .0.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0b000 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::umlslb(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0101 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44005800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umlslt_z_zzzi_d |
| // umlslt_z_zzzi_s |
| void Assembler::umlslt(const ZRegister& zda, const ZRegister& zn) { |
| // UMLSLT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1011 .1.. .... .... |
| // size<23:22> | opc<20:16> | S<13> | U<12> | il<11> | T<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0b400 | Rd(zda) | Rn(zn)); |
| } |
| |
| void Assembler::umlslt(const ZRegister& zda, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0100 ..0. .... 0101 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); |
| VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x44005c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::umulh(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> |
| // 0000 0100 ..1. .... 0110 11.. .... .... |
| // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x04206c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umullb_z_zzi_d |
| // umullb_z_zzi_s |
| void Assembler::umullb(const ZRegister& zd, const ZRegister& zn) { |
| // UMULLB <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1101 .0.. .... .... |
| // size<23:22> | opc<20:16> | U<12> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0d000 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::umullb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0111 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45007800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| // This prototype maps to 2 instruction encodings: |
| // umullt_z_zzi_d |
| // umullt_z_zzi_s |
| void Assembler::umullt(const ZRegister& zd, const ZRegister& zn) { |
| // UMULLT <Zd>.D, <Zn>.S, <Zm>.S[<imm>] |
| // 0100 0100 111. .... 1101 .1.. .... .... |
| // size<23:22> | opc<20:16> | U<12> | il<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x44e0d400 | Rd(zd) | Rn(zn)); |
| } |
| |
| void Assembler::umullt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0111 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); |
| |
| Emit(0x45007c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::uqadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1001 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44198000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqrshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1011 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqrshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1111 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 0111 100. .... .... .... |
| // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | |
| // imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04078000); |
| } |
| |
| void Assembler::uqshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1001 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44098000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 1101 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x440d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqsub(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1011 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqsubr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // UQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1111 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::uqxtnb(const ZRegister& zd, const ZRegister& zn) { |
| // UQXTNB <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0100 10.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45204800); |
| } |
| |
| void Assembler::uqxtnt(const ZRegister& zd, const ZRegister& zn) { |
| // UQXTNT <Zd>.<T>, <Zn>.<Tb> |
| // 0100 0101 0.1. .000 0100 11.. .... .... |
| // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); |
| |
| // XTN instructions look like immediate shifts with zero shift distance. |
| Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, size, 0x45204c00); |
| } |
| |
| void Assembler::urecpe(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // URECPE <Zd>.S, <Pg>/M, <Zn>.S |
| // 0100 0100 ..00 0000 101. .... .... .... |
| // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS()); |
| |
| Emit(0x4400a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::urhadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // URHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 0101 100. .... .... .... |
| // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::urshl(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // URSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 0011 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44038000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::urshlr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // URSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..00 0111 100. .... .... .... |
| // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | |
| // Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x44078000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::urshr(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| int shift) { |
| // URSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> |
| // 0000 0100 ..00 1101 100. .... .... .... |
| // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | |
| // imm3<7:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040d8000); |
| } |
| |
| void Assembler::ursqrte(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn) { |
| // URSQRTE <Zd>.S, <Pg>/M, <Zn>.S |
| // 0100 0100 ..00 0001 101. .... .... .... |
| // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS()); |
| |
| Emit(0x4401a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); |
| } |
| |
| void Assembler::ursra(const ZRegister& zda, const ZRegister& zn, int shift) { |
| // URSRA <Zda>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1110 11.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500ec00); |
| } |
| |
| void Assembler::ushllb(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // USHLLB <Zd>.<T>, <Zn>.<Tb>, #<const> |
| // 0100 0101 0.0. .... 1010 10.. .... .... |
| // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a800); |
| } |
| |
| void Assembler::ushllt(const ZRegister& zd, const ZRegister& zn, int shift) { |
| // USHLLT <Zd>.<T>, <Zn>.<Tb>, #<const> |
| // 0100 0101 0.0. .... 1010 11.. .... .... |
| // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(!zd.IsLaneSizeB()); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500ac00); |
| } |
| |
| void Assembler::usqadd(const ZRegister& zd, |
| const PRegisterM& pg, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // USQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> |
| // 0100 0100 ..01 1101 100. .... .... .... |
| // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); |
| |
| Emit(0x441d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); |
| } |
| |
| void Assembler::usra(const ZRegister& zda, const ZRegister& zn, int shift) { |
| // USRA <Zda>.<T>, <Zn>.<T>, #<const> |
| // 0100 0101 ..0. .... 1110 01.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | |
| // Zda<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zda, zn)); |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); |
| |
| SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e400); |
| } |
| |
| void Assembler::usublb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // USUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0001 10.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45001800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::usublt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // USUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0001 11.. .... .... |
| // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | |
| // Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zn, zm)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45001c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::usubwb(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // USUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0101 10.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45005800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::usubwt(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm) { |
| // USUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> |
| // 0100 0101 ..0. .... 0101 11.. .... .... |
| // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zn)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); |
| VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); |
| |
| Emit(0x45005c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); |
| } |
| |
| void Assembler::whilege(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILEGE <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 00.. ...0 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25200000 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilegt(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILEGT <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 00.. ...1 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25200010 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilehi(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILEHI <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 10.. ...1 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25200810 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilehs(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILEHS <Pd>.<T>, <R><n>, <R><m> |
| // 0010 0101 ..1. .... 000. 10.. ...0 .... |
| // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | |
| // Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25200800 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilerw(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILERW <Pd>.<T>, <Xn>, <Xm> |
| // 0010 0101 ..1. .... 0011 00.. ...1 .... |
| // size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25203010 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::whilewr(const PRegisterWithLaneSize& pd, |
| const Register& rn, |
| const Register& rm) { |
| // WHILEWR <Pd>.<T>, <Xn>, <Xm> |
| // 0010 0101 ..1. .... 0011 00.. ...0 .... |
| // size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0> |
| |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| |
| Emit(0x25203000 | Pd(pd) | Rn(rn) | Rm(rm)); |
| } |
| |
| void Assembler::xar(const ZRegister& zd, |
| const ZRegister& zn, |
| const ZRegister& zm, |
| int shift) { |
| // XAR <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<const> |
| // 0000 0100 ..1. .... 0011 01.. .... .... |
| // tszh<23:22> | tszl<20:19> | imm3<18:16> | Zm<9:5> | Zdn<4:0> |
| |
| USE(zn); |
| VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); |
| VIXL_ASSERT(zd.Is(zn)); |
| VIXL_ASSERT(AreSameLaneSize(zd, zm)); |
| |
| Instr encoded_imm = |
| EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); |
| SVEBitwiseShiftImmediate(zd, zm, encoded_imm, 0x04203400); |
| } |
| |
| } // namespace aarch64 |
| } // namespace vixl |