| // SPDX-License-Identifier: GPL-2.0-only |
| /****************************************************************************** |
| * emulate.c |
| * |
| * Generic x86 (32-bit and 64-bit) instruction decoder and emulator. |
| * |
| * Copyright (c) 2005 Keir Fraser |
| * |
| * Linux coding style, mod r/m decoder, segment base fixes, real-mode |
| * privileged instructions: |
| * |
| * Copyright (C) 2006 Qumranet |
| * Copyright 2010 Red Hat, Inc. and/or its affiliates. |
| * |
| * Avi Kivity <avi@qumranet.com> |
| * Yaniv Kamay <yaniv@qumranet.com> |
| * |
| * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 |
| */ |
| |
| #include <linux/kvm_host.h> |
| #include "kvm_cache_regs.h" |
| #include "kvm_emulate.h" |
| #include <linux/stringify.h> |
| #include <asm/fpu/api.h> |
| #include <asm/debugreg.h> |
| #include <asm/nospec-branch.h> |
| |
| #include "x86.h" |
| #include "tss.h" |
| #include "mmu.h" |
| #include "pmu.h" |
| |
| /* |
| * Operand types |
| */ |
| #define OpNone 0ull |
| #define OpImplicit 1ull /* No generic decode */ |
| #define OpReg 2ull /* Register */ |
| #define OpMem 3ull /* Memory */ |
| #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */ |
| #define OpDI 5ull /* ES:DI/EDI/RDI */ |
| #define OpMem64 6ull /* Memory, 64-bit */ |
| #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */ |
| #define OpDX 8ull /* DX register */ |
| #define OpCL 9ull /* CL register (for shifts) */ |
| #define OpImmByte 10ull /* 8-bit sign extended immediate */ |
| #define OpOne 11ull /* Implied 1 */ |
| #define OpImm 12ull /* Sign extended up to 32-bit immediate */ |
| #define OpMem16 13ull /* Memory operand (16-bit). */ |
| #define OpMem32 14ull /* Memory operand (32-bit). */ |
| #define OpImmU 15ull /* Immediate operand, zero extended */ |
| #define OpSI 16ull /* SI/ESI/RSI */ |
| #define OpImmFAddr 17ull /* Immediate far address */ |
| #define OpMemFAddr 18ull /* Far address in memory */ |
| #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */ |
| #define OpES 20ull /* ES */ |
| #define OpCS 21ull /* CS */ |
| #define OpSS 22ull /* SS */ |
| #define OpDS 23ull /* DS */ |
| #define OpFS 24ull /* FS */ |
| #define OpGS 25ull /* GS */ |
| #define OpMem8 26ull /* 8-bit zero extended memory operand */ |
| #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */ |
| #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */ |
| #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */ |
| #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */ |
| |
| #define OpBits 5 /* Width of operand field */ |
| #define OpMask ((1ull << OpBits) - 1) |
| |
| /* |
| * Opcode effective-address decode tables. |
| * Note that we only emulate instructions that have at least one memory |
| * operand (excluding implicit stack references). We assume that stack |
| * references and instruction fetches will never occur in special memory |
| * areas that require emulation. So, for example, 'mov <imm>,<reg>' need |
| * not be handled. |
| */ |
| |
| /* Operand sizes: 8-bit operands or specified/overridden size. */ |
| #define ByteOp (1<<0) /* 8-bit operands. */ |
| /* Destination operand type. */ |
| #define DstShift 1 |
| #define ImplicitOps (OpImplicit << DstShift) |
| #define DstReg (OpReg << DstShift) |
| #define DstMem (OpMem << DstShift) |
| #define DstAcc (OpAcc << DstShift) |
| #define DstDI (OpDI << DstShift) |
| #define DstMem64 (OpMem64 << DstShift) |
| #define DstMem16 (OpMem16 << DstShift) |
| #define DstImmUByte (OpImmUByte << DstShift) |
| #define DstDX (OpDX << DstShift) |
| #define DstAccLo (OpAccLo << DstShift) |
| #define DstMask (OpMask << DstShift) |
| /* Source operand type. */ |
| #define SrcShift 6 |
| #define SrcNone (OpNone << SrcShift) |
| #define SrcReg (OpReg << SrcShift) |
| #define SrcMem (OpMem << SrcShift) |
| #define SrcMem16 (OpMem16 << SrcShift) |
| #define SrcMem32 (OpMem32 << SrcShift) |
| #define SrcImm (OpImm << SrcShift) |
| #define SrcImmByte (OpImmByte << SrcShift) |
| #define SrcOne (OpOne << SrcShift) |
| #define SrcImmUByte (OpImmUByte << SrcShift) |
| #define SrcImmU (OpImmU << SrcShift) |
| #define SrcSI (OpSI << SrcShift) |
| #define SrcXLat (OpXLat << SrcShift) |
| #define SrcImmFAddr (OpImmFAddr << SrcShift) |
| #define SrcMemFAddr (OpMemFAddr << SrcShift) |
| #define SrcAcc (OpAcc << SrcShift) |
| #define SrcImmU16 (OpImmU16 << SrcShift) |
| #define SrcImm64 (OpImm64 << SrcShift) |
| #define SrcDX (OpDX << SrcShift) |
| #define SrcMem8 (OpMem8 << SrcShift) |
| #define SrcAccHi (OpAccHi << SrcShift) |
| #define SrcMask (OpMask << SrcShift) |
| #define BitOp (1<<11) |
| #define MemAbs (1<<12) /* Memory operand is absolute displacement */ |
| #define String (1<<13) /* String instruction (rep capable) */ |
| #define Stack (1<<14) /* Stack instruction (push/pop) */ |
| #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */ |
| #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */ |
| #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */ |
| #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */ |
| #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */ |
| #define Escape (5<<15) /* Escape to coprocessor instruction */ |
| #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */ |
| #define ModeDual (7<<15) /* Different instruction for 32/64 bit */ |
| #define Sse (1<<18) /* SSE Vector instruction */ |
| /* Generic ModRM decode. */ |
| #define ModRM (1<<19) |
| /* Destination is only written; never read. */ |
| #define Mov (1<<20) |
| /* Misc flags */ |
| #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */ |
| #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */ |
| #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */ |
| #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */ |
| #define Undefined (1<<25) /* No Such Instruction */ |
| #define Lock (1<<26) /* lock prefix is allowed for the instruction */ |
| #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */ |
| #define No64 (1<<28) |
| #define PageTable (1 << 29) /* instruction used to write page table */ |
| #define NotImpl (1 << 30) /* instruction is not implemented */ |
| /* Source 2 operand type */ |
| #define Src2Shift (31) |
| #define Src2None (OpNone << Src2Shift) |
| #define Src2Mem (OpMem << Src2Shift) |
| #define Src2CL (OpCL << Src2Shift) |
| #define Src2ImmByte (OpImmByte << Src2Shift) |
| #define Src2One (OpOne << Src2Shift) |
| #define Src2Imm (OpImm << Src2Shift) |
| #define Src2ES (OpES << Src2Shift) |
| #define Src2CS (OpCS << Src2Shift) |
| #define Src2SS (OpSS << Src2Shift) |
| #define Src2DS (OpDS << Src2Shift) |
| #define Src2FS (OpFS << Src2Shift) |
| #define Src2GS (OpGS << Src2Shift) |
| #define Src2Mask (OpMask << Src2Shift) |
| #define Mmx ((u64)1 << 40) /* MMX Vector instruction */ |
| #define AlignMask ((u64)7 << 41) |
| #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */ |
| #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */ |
| #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */ |
| #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */ |
| #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */ |
| #define NoWrite ((u64)1 << 45) /* No writeback */ |
| #define SrcWrite ((u64)1 << 46) /* Write back src operand */ |
| #define NoMod ((u64)1 << 47) /* Mod field is ignored */ |
| #define Intercept ((u64)1 << 48) /* Has valid intercept field */ |
| #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */ |
| #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */ |
| #define NearBranch ((u64)1 << 52) /* Near branches */ |
| #define No16 ((u64)1 << 53) /* No 16 bit operand */ |
| #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */ |
| #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */ |
| |
| #define DstXacc (DstAccLo | SrcAccHi | SrcWrite) |
| |
| #define X2(x...) x, x |
| #define X3(x...) X2(x), x |
| #define X4(x...) X2(x), X2(x) |
| #define X5(x...) X4(x), x |
| #define X6(x...) X4(x), X2(x) |
| #define X7(x...) X4(x), X3(x) |
| #define X8(x...) X4(x), X4(x) |
| #define X16(x...) X8(x), X8(x) |
| |
| #define NR_FASTOP (ilog2(sizeof(ulong)) + 1) |
| #define FASTOP_SIZE 8 |
| |
| struct opcode { |
| u64 flags : 56; |
| u64 intercept : 8; |
| union { |
| int (*execute)(struct x86_emulate_ctxt *ctxt); |
| const struct opcode *group; |
| const struct group_dual *gdual; |
| const struct gprefix *gprefix; |
| const struct escape *esc; |
| const struct instr_dual *idual; |
| const struct mode_dual *mdual; |
| void (*fastop)(struct fastop *fake); |
| } u; |
| int (*check_perm)(struct x86_emulate_ctxt *ctxt); |
| }; |
| |
| struct group_dual { |
| struct opcode mod012[8]; |
| struct opcode mod3[8]; |
| }; |
| |
| struct gprefix { |
| struct opcode pfx_no; |
| struct opcode pfx_66; |
| struct opcode pfx_f2; |
| struct opcode pfx_f3; |
| }; |
| |
| struct escape { |
| struct opcode op[8]; |
| struct opcode high[64]; |
| }; |
| |
| struct instr_dual { |
| struct opcode mod012; |
| struct opcode mod3; |
| }; |
| |
| struct mode_dual { |
| struct opcode mode32; |
| struct opcode mode64; |
| }; |
| |
| #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a |
| |
| enum x86_transfer_type { |
| X86_TRANSFER_NONE, |
| X86_TRANSFER_CALL_JMP, |
| X86_TRANSFER_RET, |
| X86_TRANSFER_TASK_SWITCH, |
| }; |
| |
| static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr) |
| { |
| if (!(ctxt->regs_valid & (1 << nr))) { |
| ctxt->regs_valid |= 1 << nr; |
| ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr); |
| } |
| return ctxt->_regs[nr]; |
| } |
| |
| static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr) |
| { |
| ctxt->regs_valid |= 1 << nr; |
| ctxt->regs_dirty |= 1 << nr; |
| return &ctxt->_regs[nr]; |
| } |
| |
| static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr) |
| { |
| reg_read(ctxt, nr); |
| return reg_write(ctxt, nr); |
| } |
| |
| static void writeback_registers(struct x86_emulate_ctxt *ctxt) |
| { |
| unsigned reg; |
| |
| for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16) |
| ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]); |
| } |
| |
| static void invalidate_registers(struct x86_emulate_ctxt *ctxt) |
| { |
| ctxt->regs_dirty = 0; |
| ctxt->regs_valid = 0; |
| } |
| |
| /* |
| * These EFLAGS bits are restored from saved value during emulation, and |
| * any changes are written back to the saved value after emulation. |
| */ |
| #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\ |
| X86_EFLAGS_PF|X86_EFLAGS_CF) |
| |
| #ifdef CONFIG_X86_64 |
| #define ON64(x) x |
| #else |
| #define ON64(x) |
| #endif |
| |
| /* |
| * fastop functions have a special calling convention: |
| * |
| * dst: rax (in/out) |
| * src: rdx (in/out) |
| * src2: rcx (in) |
| * flags: rflags (in/out) |
| * ex: rsi (in:fastop pointer, out:zero if exception) |
| * |
| * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for |
| * different operand sizes can be reached by calculation, rather than a jump |
| * table (which would be bigger than the code). |
| */ |
| static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); |
| |
| #define __FOP_FUNC(name) \ |
| ".align " __stringify(FASTOP_SIZE) " \n\t" \ |
| ".type " name ", @function \n\t" \ |
| name ":\n\t" |
| |
| #define FOP_FUNC(name) \ |
| __FOP_FUNC(#name) |
| |
| #define __FOP_RET(name) \ |
| "ret \n\t" \ |
| ".size " name ", .-" name "\n\t" |
| |
| #define FOP_RET(name) \ |
| __FOP_RET(#name) |
| |
| #define FOP_START(op) \ |
| extern void em_##op(struct fastop *fake); \ |
| asm(".pushsection .text, \"ax\" \n\t" \ |
| ".global em_" #op " \n\t" \ |
| ".align " __stringify(FASTOP_SIZE) " \n\t" \ |
| "em_" #op ":\n\t" |
| |
| #define FOP_END \ |
| ".popsection") |
| |
| #define __FOPNOP(name) \ |
| __FOP_FUNC(name) \ |
| __FOP_RET(name) |
| |
| #define FOPNOP() \ |
| __FOPNOP(__stringify(__UNIQUE_ID(nop))) |
| |
| #define FOP1E(op, dst) \ |
| __FOP_FUNC(#op "_" #dst) \ |
| "10: " #op " %" #dst " \n\t" \ |
| __FOP_RET(#op "_" #dst) |
| |
| #define FOP1EEX(op, dst) \ |
| FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception) |
| |
| #define FASTOP1(op) \ |
| FOP_START(op) \ |
| FOP1E(op##b, al) \ |
| FOP1E(op##w, ax) \ |
| FOP1E(op##l, eax) \ |
| ON64(FOP1E(op##q, rax)) \ |
| FOP_END |
| |
| /* 1-operand, using src2 (for MUL/DIV r/m) */ |
| #define FASTOP1SRC2(op, name) \ |
| FOP_START(name) \ |
| FOP1E(op, cl) \ |
| FOP1E(op, cx) \ |
| FOP1E(op, ecx) \ |
| ON64(FOP1E(op, rcx)) \ |
| FOP_END |
| |
| /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */ |
| #define FASTOP1SRC2EX(op, name) \ |
| FOP_START(name) \ |
| FOP1EEX(op, cl) \ |
| FOP1EEX(op, cx) \ |
| FOP1EEX(op, ecx) \ |
| ON64(FOP1EEX(op, rcx)) \ |
| FOP_END |
| |
| #define FOP2E(op, dst, src) \ |
| __FOP_FUNC(#op "_" #dst "_" #src) \ |
| #op " %" #src ", %" #dst " \n\t" \ |
| __FOP_RET(#op "_" #dst "_" #src) |
| |
| #define FASTOP2(op) \ |
| FOP_START(op) \ |
| FOP2E(op##b, al, dl) \ |
| FOP2E(op##w, ax, dx) \ |
| FOP2E(op##l, eax, edx) \ |
| ON64(FOP2E(op##q, rax, rdx)) \ |
| FOP_END |
| |
| /* 2 operand, word only */ |
| #define FASTOP2W(op) \ |
| FOP_START(op) \ |
| FOPNOP() \ |
| FOP2E(op##w, ax, dx) \ |
| FOP2E(op##l, eax, edx) \ |
| ON64(FOP2E(op##q, rax, rdx)) \ |
| FOP_END |
| |
| /* 2 operand, src is CL */ |
| #define FASTOP2CL(op) \ |
| FOP_START(op) \ |
| FOP2E(op##b, al, cl) \ |
| FOP2E(op##w, ax, cl) \ |
| FOP2E(op##l, eax, cl) \ |
| ON64(FOP2E(op##q, rax, cl)) \ |
| FOP_END |
| |
| /* 2 operand, src and dest are reversed */ |
| #define FASTOP2R(op, name) \ |
| FOP_START(name) \ |
| FOP2E(op##b, dl, al) \ |
| FOP2E(op##w, dx, ax) \ |
| FOP2E(op##l, edx, eax) \ |
| ON64(FOP2E(op##q, rdx, rax)) \ |
| FOP_END |
| |
| #define FOP3E(op, dst, src, src2) \ |
| __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \ |
| #op " %" #src2 ", %" #src ", %" #dst " \n\t"\ |
| __FOP_RET(#op "_" #dst "_" #src "_" #src2) |
| |
| /* 3-operand, word-only, src2=cl */ |
| #define FASTOP3WCL(op) \ |
| FOP_START(op) \ |
| FOPNOP() \ |
| FOP3E(op##w, ax, dx, cl) \ |
| FOP3E(op##l, eax, edx, cl) \ |
| ON64(FOP3E(op##q, rax, rdx, cl)) \ |
| FOP_END |
| |
| /* Special case for SETcc - 1 instruction per cc */ |
| #define FOP_SETCC(op) \ |
| ".align 4 \n\t" \ |
| ".type " #op ", @function \n\t" \ |
| #op ": \n\t" \ |
| #op " %al \n\t" \ |
| __FOP_RET(#op) |
| |
| asm(".pushsection .fixup, \"ax\"\n" |
| ".global kvm_fastop_exception \n" |
| "kvm_fastop_exception: xor %esi, %esi; ret\n" |
| ".popsection"); |
| |
| FOP_START(setcc) |
| FOP_SETCC(seto) |
| FOP_SETCC(setno) |
| FOP_SETCC(setc) |
| FOP_SETCC(setnc) |
| FOP_SETCC(setz) |
| FOP_SETCC(setnz) |
| FOP_SETCC(setbe) |
| FOP_SETCC(setnbe) |
| FOP_SETCC(sets) |
| FOP_SETCC(setns) |
| FOP_SETCC(setp) |
| FOP_SETCC(setnp) |
| FOP_SETCC(setl) |
| FOP_SETCC(setnl) |
| FOP_SETCC(setle) |
| FOP_SETCC(setnle) |
| FOP_END; |
| |
| FOP_START(salc) |
| FOP_FUNC(salc) |
| "pushf; sbb %al, %al; popf \n\t" |
| FOP_RET(salc) |
| FOP_END; |
| |
| /* |
| * XXX: inoutclob user must know where the argument is being expanded. |
| * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault. |
| */ |
| #define asm_safe(insn, inoutclob...) \ |
| ({ \ |
| int _fault = 0; \ |
| \ |
| asm volatile("1:" insn "\n" \ |
| "2:\n" \ |
| ".pushsection .fixup, \"ax\"\n" \ |
| "3: movl $1, %[_fault]\n" \ |
| " jmp 2b\n" \ |
| ".popsection\n" \ |
| _ASM_EXTABLE(1b, 3b) \ |
| : [_fault] "+qm"(_fault) inoutclob ); \ |
| \ |
| _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \ |
| }) |
| |
| static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt, |
| enum x86_intercept intercept, |
| enum x86_intercept_stage stage) |
| { |
| struct x86_instruction_info info = { |
| .intercept = intercept, |
| .rep_prefix = ctxt->rep_prefix, |
| .modrm_mod = ctxt->modrm_mod, |
| .modrm_reg = ctxt->modrm_reg, |
| .modrm_rm = ctxt->modrm_rm, |
| .src_val = ctxt->src.val64, |
| .dst_val = ctxt->dst.val64, |
| .src_bytes = ctxt->src.bytes, |
| .dst_bytes = ctxt->dst.bytes, |
| .ad_bytes = ctxt->ad_bytes, |
| .next_rip = ctxt->eip, |
| }; |
| |
| return ctxt->ops->intercept(ctxt, &info, stage); |
| } |
| |
| static void assign_masked(ulong *dest, ulong src, ulong mask) |
| { |
| *dest = (*dest & ~mask) | (src & mask); |
| } |
| |
| static void assign_register(unsigned long *reg, u64 val, int bytes) |
| { |
| /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ |
| switch (bytes) { |
| case 1: |
| *(u8 *)reg = (u8)val; |
| break; |
| case 2: |
| *(u16 *)reg = (u16)val; |
| break; |
| case 4: |
| *reg = (u32)val; |
| break; /* 64b: zero-extend */ |
| case 8: |
| *reg = val; |
| break; |
| } |
| } |
| |
| static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt) |
| { |
| return (1UL << (ctxt->ad_bytes << 3)) - 1; |
| } |
| |
| static ulong stack_mask(struct x86_emulate_ctxt *ctxt) |
| { |
| u16 sel; |
| struct desc_struct ss; |
| |
| if (ctxt->mode == X86EMUL_MODE_PROT64) |
| return ~0UL; |
| ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS); |
| return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */ |
| } |
| |
| static int stack_size(struct x86_emulate_ctxt *ctxt) |
| { |
| return (__fls(stack_mask(ctxt)) + 1) >> 3; |
| } |
| |
| /* Access/update address held in a register, based on addressing mode. */ |
| static inline unsigned long |
| address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg) |
| { |
| if (ctxt->ad_bytes == sizeof(unsigned long)) |
| return reg; |
| else |
| return reg & ad_mask(ctxt); |
| } |
| |
| static inline unsigned long |
| register_address(struct x86_emulate_ctxt *ctxt, int reg) |
| { |
| return address_mask(ctxt, reg_read(ctxt, reg)); |
| } |
| |
| static void masked_increment(ulong *reg, ulong mask, int inc) |
| { |
| assign_masked(reg, *reg + inc, mask); |
| } |
| |
| static inline void |
| register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc) |
| { |
| ulong *preg = reg_rmw(ctxt, reg); |
| |
| assign_register(preg, *preg + inc, ctxt->ad_bytes); |
| } |
| |
| static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) |
| { |
| masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc); |
| } |
| |
| static u32 desc_limit_scaled(struct desc_struct *desc) |
| { |
| u32 limit = get_desc_limit(desc); |
| |
| return desc->g ? (limit << 12) | 0xfff : limit; |
| } |
| |
| static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg) |
| { |
| if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS) |
| return 0; |
| |
| return ctxt->ops->get_cached_segment_base(ctxt, seg); |
| } |
| |
| static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec, |
| u32 error, bool valid) |
| { |
| WARN_ON(vec > 0x1f); |
| ctxt->exception.vector = vec; |
| ctxt->exception.error_code = error; |
| ctxt->exception.error_code_valid = valid; |
| return X86EMUL_PROPAGATE_FAULT; |
| } |
| |
| static int emulate_db(struct x86_emulate_ctxt *ctxt) |
| { |
| return emulate_exception(ctxt, DB_VECTOR, 0, false); |
| } |
| |
| static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err) |
| { |
| return emulate_exception(ctxt, GP_VECTOR, err, true); |
| } |
| |
| static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err) |
| { |
| return emulate_exception(ctxt, SS_VECTOR, err, true); |
| } |
| |
| static int emulate_ud(struct x86_emulate_ctxt *ctxt) |
| { |
| return emulate_exception(ctxt, UD_VECTOR, 0, false); |
| } |
| |
| static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err) |
| { |
| return emulate_exception(ctxt, TS_VECTOR, err, true); |
| } |
| |
| static int emulate_de(struct x86_emulate_ctxt *ctxt) |
| { |
| return emulate_exception(ctxt, DE_VECTOR, 0, false); |
| } |
| |
| static int emulate_nm(struct x86_emulate_ctxt *ctxt) |
| { |
| return emulate_exception(ctxt, NM_VECTOR, 0, false); |
| } |
| |
| static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg) |
| { |
| u16 selector; |
| struct desc_struct desc; |
| |
| ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg); |
| return selector; |
| } |
| |
| static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector, |
| unsigned seg) |
| { |
| u16 dummy; |
| u32 base3; |
| struct desc_struct desc; |
| |
| ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg); |
| ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg); |
| } |
| |
| static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) |
| { |
| return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; |
| } |
| |
| static inline bool emul_is_noncanonical_address(u64 la, |
| struct x86_emulate_ctxt *ctxt) |
| { |
| return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; |
| } |
| |
| /* |
| * x86 defines three classes of vector instructions: explicitly |
| * aligned, explicitly unaligned, and the rest, which change behaviour |
| * depending on whether they're AVX encoded or not. |
| * |
| * Also included is CMPXCHG16B which is not a vector instruction, yet it is |
| * subject to the same check. FXSAVE and FXRSTOR are checked here too as their |
| * 512 bytes of data must be aligned to a 16 byte boundary. |
| */ |
| static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size) |
| { |
| u64 alignment = ctxt->d & AlignMask; |
| |
| if (likely(size < 16)) |
| return 1; |
| |
| switch (alignment) { |
| case Unaligned: |
| case Avx: |
| return 1; |
| case Aligned16: |
| return 16; |
| case Aligned: |
| default: |
| return size; |
| } |
| } |
| |
| static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| unsigned *max_size, unsigned size, |
| bool write, bool fetch, |
| enum x86emul_mode mode, ulong *linear) |
| { |
| struct desc_struct desc; |
| bool usable; |
| ulong la; |
| u32 lim; |
| u16 sel; |
| u8 va_bits; |
| |
| la = seg_base(ctxt, addr.seg) + addr.ea; |
| *max_size = 0; |
| switch (mode) { |
| case X86EMUL_MODE_PROT64: |
| *linear = la; |
| va_bits = ctxt_virt_addr_bits(ctxt); |
| if (get_canonical(la, va_bits) != la) |
| goto bad; |
| |
| *max_size = min_t(u64, ~0u, (1ull << va_bits) - la); |
| if (size > *max_size) |
| goto bad; |
| break; |
| default: |
| *linear = la = (u32)la; |
| usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL, |
| addr.seg); |
| if (!usable) |
| goto bad; |
| /* code segment in protected mode or read-only data segment */ |
| if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8)) |
| || !(desc.type & 2)) && write) |
| goto bad; |
| /* unreadable code segment */ |
| if (!fetch && (desc.type & 8) && !(desc.type & 2)) |
| goto bad; |
| lim = desc_limit_scaled(&desc); |
| if (!(desc.type & 8) && (desc.type & 4)) { |
| /* expand-down segment */ |
| if (addr.ea <= lim) |
| goto bad; |
| lim = desc.d ? 0xffffffff : 0xffff; |
| } |
| if (addr.ea > lim) |
| goto bad; |
| if (lim == 0xffffffff) |
| *max_size = ~0u; |
| else { |
| *max_size = (u64)lim + 1 - addr.ea; |
| if (size > *max_size) |
| goto bad; |
| } |
| break; |
| } |
| if (la & (insn_alignment(ctxt, size) - 1)) |
| return emulate_gp(ctxt, 0); |
| return X86EMUL_CONTINUE; |
| bad: |
| if (addr.seg == VCPU_SREG_SS) |
| return emulate_ss(ctxt, 0); |
| else |
| return emulate_gp(ctxt, 0); |
| } |
| |
| static int linearize(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| unsigned size, bool write, |
| ulong *linear) |
| { |
| unsigned max_size; |
| return __linearize(ctxt, addr, &max_size, size, write, false, |
| ctxt->mode, linear); |
| } |
| |
| static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst, |
| enum x86emul_mode mode) |
| { |
| ulong linear; |
| int rc; |
| unsigned max_size; |
| struct segmented_address addr = { .seg = VCPU_SREG_CS, |
| .ea = dst }; |
| |
| if (ctxt->op_bytes != sizeof(unsigned long)) |
| addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); |
| rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear); |
| if (rc == X86EMUL_CONTINUE) |
| ctxt->_eip = addr.ea; |
| return rc; |
| } |
| |
| static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst) |
| { |
| return assign_eip(ctxt, dst, ctxt->mode); |
| } |
| |
| static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst, |
| const struct desc_struct *cs_desc) |
| { |
| enum x86emul_mode mode = ctxt->mode; |
| int rc; |
| |
| #ifdef CONFIG_X86_64 |
| if (ctxt->mode >= X86EMUL_MODE_PROT16) { |
| if (cs_desc->l) { |
| u64 efer = 0; |
| |
| ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); |
| if (efer & EFER_LMA) |
| mode = X86EMUL_MODE_PROT64; |
| } else |
| mode = X86EMUL_MODE_PROT32; /* temporary value */ |
| } |
| #endif |
| if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32) |
| mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; |
| rc = assign_eip(ctxt, dst, mode); |
| if (rc == X86EMUL_CONTINUE) |
| ctxt->mode = mode; |
| return rc; |
| } |
| |
| static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel) |
| { |
| return assign_eip_near(ctxt, ctxt->_eip + rel); |
| } |
| |
| static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear, |
| void *data, unsigned size) |
| { |
| return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true); |
| } |
| |
| static int linear_write_system(struct x86_emulate_ctxt *ctxt, |
| ulong linear, void *data, |
| unsigned int size) |
| { |
| return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true); |
| } |
| |
| static int segmented_read_std(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| void *data, |
| unsigned size) |
| { |
| int rc; |
| ulong linear; |
| |
| rc = linearize(ctxt, addr, size, false, &linear); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false); |
| } |
| |
| static int segmented_write_std(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| void *data, |
| unsigned int size) |
| { |
| int rc; |
| ulong linear; |
| |
| rc = linearize(ctxt, addr, size, true, &linear); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false); |
| } |
| |
| /* |
| * Prefetch the remaining bytes of the instruction without crossing page |
| * boundary if they are not in fetch_cache yet. |
| */ |
| static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size) |
| { |
| int rc; |
| unsigned size, max_size; |
| unsigned long linear; |
| int cur_size = ctxt->fetch.end - ctxt->fetch.data; |
| struct segmented_address addr = { .seg = VCPU_SREG_CS, |
| .ea = ctxt->eip + cur_size }; |
| |
| /* |
| * We do not know exactly how many bytes will be needed, and |
| * __linearize is expensive, so fetch as much as possible. We |
| * just have to avoid going beyond the 15 byte limit, the end |
| * of the segment, or the end of the page. |
| * |
| * __linearize is called with size 0 so that it does not do any |
| * boundary check itself. Instead, we use max_size to check |
| * against op_size. |
| */ |
| rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode, |
| &linear); |
| if (unlikely(rc != X86EMUL_CONTINUE)) |
| return rc; |
| |
| size = min_t(unsigned, 15UL ^ cur_size, max_size); |
| size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear)); |
| |
| /* |
| * One instruction can only straddle two pages, |
| * and one has been loaded at the beginning of |
| * x86_decode_insn. So, if not enough bytes |
| * still, we must have hit the 15-byte boundary. |
| */ |
| if (unlikely(size < op_size)) |
| return emulate_gp(ctxt, 0); |
| |
| rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end, |
| size, &ctxt->exception); |
| if (unlikely(rc != X86EMUL_CONTINUE)) |
| return rc; |
| ctxt->fetch.end += size; |
| return X86EMUL_CONTINUE; |
| } |
| |
| static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, |
| unsigned size) |
| { |
| unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr; |
| |
| if (unlikely(done_size < size)) |
| return __do_insn_fetch_bytes(ctxt, size - done_size); |
| else |
| return X86EMUL_CONTINUE; |
| } |
| |
| /* Fetch next part of the instruction being emulated. */ |
| #define insn_fetch(_type, _ctxt) \ |
| ({ _type _x; \ |
| \ |
| rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \ |
| if (rc != X86EMUL_CONTINUE) \ |
| goto done; \ |
| ctxt->_eip += sizeof(_type); \ |
| memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \ |
| ctxt->fetch.ptr += sizeof(_type); \ |
| _x; \ |
| }) |
| |
| #define insn_fetch_arr(_arr, _size, _ctxt) \ |
| ({ \ |
| rc = do_insn_fetch_bytes(_ctxt, _size); \ |
| if (rc != X86EMUL_CONTINUE) \ |
| goto done; \ |
| ctxt->_eip += (_size); \ |
| memcpy(_arr, ctxt->fetch.ptr, _size); \ |
| ctxt->fetch.ptr += (_size); \ |
| }) |
| |
| /* |
| * Given the 'reg' portion of a ModRM byte, and a register block, return a |
| * pointer into the block that addresses the relevant register. |
| * @highbyte_regs specifies whether to decode AH,CH,DH,BH. |
| */ |
| static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg, |
| int byteop) |
| { |
| void *p; |
| int highbyte_regs = (ctxt->rex_prefix == 0) && byteop; |
| |
| if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) |
| p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1; |
| else |
| p = reg_rmw(ctxt, modrm_reg); |
| return p; |
| } |
| |
| static int read_descriptor(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| u16 *size, unsigned long *address, int op_bytes) |
| { |
| int rc; |
| |
| if (op_bytes == 2) |
| op_bytes = 3; |
| *address = 0; |
| rc = segmented_read_std(ctxt, addr, size, 2); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| addr.ea += 2; |
| rc = segmented_read_std(ctxt, addr, address, op_bytes); |
| return rc; |
| } |
| |
| FASTOP2(add); |
| FASTOP2(or); |
| FASTOP2(adc); |
| FASTOP2(sbb); |
| FASTOP2(and); |
| FASTOP2(sub); |
| FASTOP2(xor); |
| FASTOP2(cmp); |
| FASTOP2(test); |
| |
| FASTOP1SRC2(mul, mul_ex); |
| FASTOP1SRC2(imul, imul_ex); |
| FASTOP1SRC2EX(div, div_ex); |
| FASTOP1SRC2EX(idiv, idiv_ex); |
| |
| FASTOP3WCL(shld); |
| FASTOP3WCL(shrd); |
| |
| FASTOP2W(imul); |
| |
| FASTOP1(not); |
| FASTOP1(neg); |
| FASTOP1(inc); |
| FASTOP1(dec); |
| |
| FASTOP2CL(rol); |
| FASTOP2CL(ror); |
| FASTOP2CL(rcl); |
| FASTOP2CL(rcr); |
| FASTOP2CL(shl); |
| FASTOP2CL(shr); |
| FASTOP2CL(sar); |
| |
| FASTOP2W(bsf); |
| FASTOP2W(bsr); |
| FASTOP2W(bt); |
| FASTOP2W(bts); |
| FASTOP2W(btr); |
| FASTOP2W(btc); |
| |
| FASTOP2(xadd); |
| |
| FASTOP2R(cmp, cmp_r); |
| |
| static int em_bsf_c(struct x86_emulate_ctxt *ctxt) |
| { |
| /* If src is zero, do not writeback, but update flags */ |
| if (ctxt->src.val == 0) |
| ctxt->dst.type = OP_NONE; |
| return fastop(ctxt, em_bsf); |
| } |
| |
| static int em_bsr_c(struct x86_emulate_ctxt *ctxt) |
| { |
| /* If src is zero, do not writeback, but update flags */ |
| if (ctxt->src.val == 0) |
| ctxt->dst.type = OP_NONE; |
| return fastop(ctxt, em_bsr); |
| } |
| |
| static __always_inline u8 test_cc(unsigned int condition, unsigned long flags) |
| { |
| u8 rc; |
| void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf); |
| |
| flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF; |
| asm("push %[flags]; popf; " CALL_NOSPEC |
| : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags)); |
| return rc; |
| } |
| |
| static void fetch_register_operand(struct operand *op) |
| { |
| switch (op->bytes) { |
| case 1: |
| op->val = *(u8 *)op->addr.reg; |
| break; |
| case 2: |
| op->val = *(u16 *)op->addr.reg; |
| break; |
| case 4: |
| op->val = *(u32 *)op->addr.reg; |
| break; |
| case 8: |
| op->val = *(u64 *)op->addr.reg; |
| break; |
| } |
| } |
| |
| static void emulator_get_fpu(void) |
| { |
| fpregs_lock(); |
| |
| fpregs_assert_state_consistent(); |
| if (test_thread_flag(TIF_NEED_FPU_LOAD)) |
| switch_fpu_return(); |
| } |
| |
| static void emulator_put_fpu(void) |
| { |
| fpregs_unlock(); |
| } |
| |
| static void read_sse_reg(sse128_t *data, int reg) |
| { |
| emulator_get_fpu(); |
| switch (reg) { |
| case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break; |
| case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break; |
| case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break; |
| case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break; |
| case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break; |
| case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break; |
| case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break; |
| case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break; |
| #ifdef CONFIG_X86_64 |
| case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break; |
| case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break; |
| case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break; |
| case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break; |
| case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break; |
| case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break; |
| case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break; |
| case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break; |
| #endif |
| default: BUG(); |
| } |
| emulator_put_fpu(); |
| } |
| |
| static void write_sse_reg(sse128_t *data, int reg) |
| { |
| emulator_get_fpu(); |
| switch (reg) { |
| case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break; |
| case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break; |
| case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break; |
| case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break; |
| case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break; |
| case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break; |
| case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break; |
| case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break; |
| #ifdef CONFIG_X86_64 |
| case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break; |
| case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break; |
| case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break; |
| case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break; |
| case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break; |
| case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break; |
| case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break; |
| case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break; |
| #endif |
| default: BUG(); |
| } |
| emulator_put_fpu(); |
| } |
| |
| static void read_mmx_reg(u64 *data, int reg) |
| { |
| emulator_get_fpu(); |
| switch (reg) { |
| case 0: asm("movq %%mm0, %0" : "=m"(*data)); break; |
| case 1: asm("movq %%mm1, %0" : "=m"(*data)); break; |
| case 2: asm("movq %%mm2, %0" : "=m"(*data)); break; |
| case 3: asm("movq %%mm3, %0" : "=m"(*data)); break; |
| case 4: asm("movq %%mm4, %0" : "=m"(*data)); break; |
| case 5: asm("movq %%mm5, %0" : "=m"(*data)); break; |
| case 6: asm("movq %%mm6, %0" : "=m"(*data)); break; |
| case 7: asm("movq %%mm7, %0" : "=m"(*data)); break; |
| default: BUG(); |
| } |
| emulator_put_fpu(); |
| } |
| |
| static void write_mmx_reg(u64 *data, int reg) |
| { |
| emulator_get_fpu(); |
| switch (reg) { |
| case 0: asm("movq %0, %%mm0" : : "m"(*data)); break; |
| case 1: asm("movq %0, %%mm1" : : "m"(*data)); break; |
| case 2: asm("movq %0, %%mm2" : : "m"(*data)); break; |
| case 3: asm("movq %0, %%mm3" : : "m"(*data)); break; |
| case 4: asm("movq %0, %%mm4" : : "m"(*data)); break; |
| case 5: asm("movq %0, %%mm5" : : "m"(*data)); break; |
| case 6: asm("movq %0, %%mm6" : : "m"(*data)); break; |
| case 7: asm("movq %0, %%mm7" : : "m"(*data)); break; |
| default: BUG(); |
| } |
| emulator_put_fpu(); |
| } |
| |
| static int em_fninit(struct x86_emulate_ctxt *ctxt) |
| { |
| if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) |
| return emulate_nm(ctxt); |
| |
| emulator_get_fpu(); |
| asm volatile("fninit"); |
| emulator_put_fpu(); |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_fnstcw(struct x86_emulate_ctxt *ctxt) |
| { |
| u16 fcw; |
| |
| if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) |
| return emulate_nm(ctxt); |
| |
| emulator_get_fpu(); |
| asm volatile("fnstcw %0": "+m"(fcw)); |
| emulator_put_fpu(); |
| |
| ctxt->dst.val = fcw; |
| |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_fnstsw(struct x86_emulate_ctxt *ctxt) |
| { |
| u16 fsw; |
| |
| if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) |
| return emulate_nm(ctxt); |
| |
| emulator_get_fpu(); |
| asm volatile("fnstsw %0": "+m"(fsw)); |
| emulator_put_fpu(); |
| |
| ctxt->dst.val = fsw; |
| |
| return X86EMUL_CONTINUE; |
| } |
| |
| static void decode_register_operand(struct x86_emulate_ctxt *ctxt, |
| struct operand *op) |
| { |
| unsigned reg = ctxt->modrm_reg; |
| |
| if (!(ctxt->d & ModRM)) |
| reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3); |
| |
| if (ctxt->d & Sse) { |
| op->type = OP_XMM; |
| op->bytes = 16; |
| op->addr.xmm = reg; |
| read_sse_reg(&op->vec_val, reg); |
| return; |
| } |
| if (ctxt->d & Mmx) { |
| reg &= 7; |
| op->type = OP_MM; |
| op->bytes = 8; |
| op->addr.mm = reg; |
| return; |
| } |
| |
| op->type = OP_REG; |
| op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp); |
| |
| fetch_register_operand(op); |
| op->orig_val = op->val; |
| } |
| |
| static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg) |
| { |
| if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP) |
| ctxt->modrm_seg = VCPU_SREG_SS; |
| } |
| |
| static int decode_modrm(struct x86_emulate_ctxt *ctxt, |
| struct operand *op) |
| { |
| u8 sib; |
| int index_reg, base_reg, scale; |
| int rc = X86EMUL_CONTINUE; |
| ulong modrm_ea = 0; |
| |
| ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */ |
| index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */ |
| base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */ |
| |
| ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6; |
| ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3; |
| ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07); |
| ctxt->modrm_seg = VCPU_SREG_DS; |
| |
| if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) { |
| op->type = OP_REG; |
| op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; |
| op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, |
| ctxt->d & ByteOp); |
| if (ctxt->d & Sse) { |
| op->type = OP_XMM; |
| op->bytes = 16; |
| op->addr.xmm = ctxt->modrm_rm; |
| read_sse_reg(&op->vec_val, ctxt->modrm_rm); |
| return rc; |
| } |
| if (ctxt->d & Mmx) { |
| op->type = OP_MM; |
| op->bytes = 8; |
| op->addr.mm = ctxt->modrm_rm & 7; |
| return rc; |
| } |
| fetch_register_operand(op); |
| return rc; |
| } |
| |
| op->type = OP_MEM; |
| |
| if (ctxt->ad_bytes == 2) { |
| unsigned bx = reg_read(ctxt, VCPU_REGS_RBX); |
| unsigned bp = reg_read(ctxt, VCPU_REGS_RBP); |
| unsigned si = reg_read(ctxt, VCPU_REGS_RSI); |
| unsigned di = reg_read(ctxt, VCPU_REGS_RDI); |
| |
| /* 16-bit ModR/M decode. */ |
| switch (ctxt->modrm_mod) { |
| case 0: |
| if (ctxt->modrm_rm == 6) |
| modrm_ea += insn_fetch(u16, ctxt); |
| break; |
| case 1: |
| modrm_ea += insn_fetch(s8, ctxt); |
| break; |
| case 2: |
| modrm_ea += insn_fetch(u16, ctxt); |
| break; |
| } |
| switch (ctxt->modrm_rm) { |
| case 0: |
| modrm_ea += bx + si; |
| break; |
| case 1: |
| modrm_ea += bx + di; |
| break; |
| case 2: |
| modrm_ea += bp + si; |
| break; |
| case 3: |
| modrm_ea += bp + di; |
| break; |
| case 4: |
| modrm_ea += si; |
| break; |
| case 5: |
| modrm_ea += di; |
| break; |
| case 6: |
| if (ctxt->modrm_mod != 0) |
| modrm_ea += bp; |
| break; |
| case 7: |
| modrm_ea += bx; |
| break; |
| } |
| if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 || |
| (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0)) |
| ctxt->modrm_seg = VCPU_SREG_SS; |
| modrm_ea = (u16)modrm_ea; |
| } else { |
| /* 32/64-bit ModR/M decode. */ |
| if ((ctxt->modrm_rm & 7) == 4) { |
| sib = insn_fetch(u8, ctxt); |
| index_reg |= (sib >> 3) & 7; |
| base_reg |= sib & 7; |
| scale = sib >> 6; |
| |
| if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) |
| modrm_ea += insn_fetch(s32, ctxt); |
| else { |
| modrm_ea += reg_read(ctxt, base_reg); |
| adjust_modrm_seg(ctxt, base_reg); |
| /* Increment ESP on POP [ESP] */ |
| if ((ctxt->d & IncSP) && |
| base_reg == VCPU_REGS_RSP) |
| modrm_ea += ctxt->op_bytes; |
| } |
| if (index_reg != 4) |
| modrm_ea += reg_read(ctxt, index_reg) << scale; |
| } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { |
| modrm_ea += insn_fetch(s32, ctxt); |
| if (ctxt->mode == X86EMUL_MODE_PROT64) |
| ctxt->rip_relative = 1; |
| } else { |
| base_reg = ctxt->modrm_rm; |
| modrm_ea += reg_read(ctxt, base_reg); |
| adjust_modrm_seg(ctxt, base_reg); |
| } |
| switch (ctxt->modrm_mod) { |
| case 1: |
| modrm_ea += insn_fetch(s8, ctxt); |
| break; |
| case 2: |
| modrm_ea += insn_fetch(s32, ctxt); |
| break; |
| } |
| } |
| op->addr.mem.ea = modrm_ea; |
| if (ctxt->ad_bytes != 8) |
| ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea; |
| |
| done: |
| return rc; |
| } |
| |
| static int decode_abs(struct x86_emulate_ctxt *ctxt, |
| struct operand *op) |
| { |
| int rc = X86EMUL_CONTINUE; |
| |
| op->type = OP_MEM; |
| switch (ctxt->ad_bytes) { |
| case 2: |
| op->addr.mem.ea = insn_fetch(u16, ctxt); |
| break; |
| case 4: |
| op->addr.mem.ea = insn_fetch(u32, ctxt); |
| break; |
| case 8: |
| op->addr.mem.ea = insn_fetch(u64, ctxt); |
| break; |
| } |
| done: |
| return rc; |
| } |
| |
| static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt) |
| { |
| long sv = 0, mask; |
| |
| if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) { |
| mask = ~((long)ctxt->dst.bytes * 8 - 1); |
| |
| if (ctxt->src.bytes == 2) |
| sv = (s16)ctxt->src.val & (s16)mask; |
| else if (ctxt->src.bytes == 4) |
| sv = (s32)ctxt->src.val & (s32)mask; |
| else |
| sv = (s64)ctxt->src.val & (s64)mask; |
| |
| ctxt->dst.addr.mem.ea = address_mask(ctxt, |
| ctxt->dst.addr.mem.ea + (sv >> 3)); |
| } |
| |
| /* only subword offset */ |
| ctxt->src.val &= (ctxt->dst.bytes << 3) - 1; |
| } |
| |
| static int read_emulated(struct x86_emulate_ctxt *ctxt, |
| unsigned long addr, void *dest, unsigned size) |
| { |
| int rc; |
| struct read_cache *mc = &ctxt->mem_read; |
| |
| if (mc->pos < mc->end) |
| goto read_cached; |
| |
| WARN_ON((mc->end + size) >= sizeof(mc->data)); |
| |
| rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size, |
| &ctxt->exception); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| mc->end += size; |
| |
| read_cached: |
| memcpy(dest, mc->data + mc->pos, size); |
| mc->pos += size; |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int segmented_read(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| void *data, |
| unsigned size) |
| { |
| int rc; |
| ulong linear; |
| |
| rc = linearize(ctxt, addr, size, false, &linear); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| return read_emulated(ctxt, linear, data, size); |
| } |
| |
| static int segmented_write(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| const void *data, |
| unsigned size) |
| { |
| int rc; |
| ulong linear; |
| |
| rc = linearize(ctxt, addr, size, true, &linear); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| return ctxt->ops->write_emulated(ctxt, linear, data, size, |
| &ctxt->exception); |
| } |
| |
| static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt, |
| struct segmented_address addr, |
| const void *orig_data, const void *data, |
| unsigned size) |
| { |
| int rc; |
| ulong linear; |
| |
| rc = linearize(ctxt, addr, size, true, &linear); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data, |
| size, &ctxt->exception); |
| } |
| |
| static int pio_in_emulated(struct x86_emulate_ctxt *ctxt, |
| unsigned int size, unsigned short port, |
| void *dest) |
| { |
| struct read_cache *rc = &ctxt->io_read; |
| |
| if (rc->pos == rc->end) { /* refill pio read ahead */ |
| unsigned int in_page, n; |
| unsigned int count = ctxt->rep_prefix ? |
| address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1; |
| in_page = (ctxt->eflags & X86_EFLAGS_DF) ? |
| offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) : |
| PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)); |
| n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count); |
| if (n == 0) |
| n = 1; |
| rc->pos = rc->end = 0; |
| if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n)) |
| return 0; |
| rc->end = n * size; |
| } |
| |
| if (ctxt->rep_prefix && (ctxt->d & String) && |
| !(ctxt->eflags & X86_EFLAGS_DF)) { |
| ctxt->dst.data = rc->data + rc->pos; |
| ctxt->dst.type = OP_MEM_STR; |
| ctxt->dst.count = (rc->end - rc->pos) / size; |
| rc->pos = rc->end; |
| } else { |
| memcpy(dest, rc->data + rc->pos, size); |
| rc->pos += size; |
| } |
| return 1; |
| } |
| |
| static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt, |
| u16 index, struct desc_struct *desc) |
| { |
| struct desc_ptr dt; |
| ulong addr; |
| |
| ctxt->ops->get_idt(ctxt, &dt); |
| |
| if (dt.size < index * 8 + 7) |
| return emulate_gp(ctxt, index << 3 | 0x2); |
| |
| addr = dt.address + index * 8; |
| return linear_read_system(ctxt, addr, desc, sizeof(*desc)); |
| } |
| |
| static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt, |
| u16 selector, struct desc_ptr *dt) |
| { |
| const struct x86_emulate_ops *ops = ctxt->ops; |
| u32 base3 = 0; |
| |
| if (selector & 1 << 2) { |
| struct desc_struct desc; |
| u16 sel; |
| |
| memset(dt, 0, sizeof(*dt)); |
| if (!ops->get_segment(ctxt, &sel, &desc, &base3, |
| VCPU_SREG_LDTR)) |
| return; |
| |
| dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */ |
| dt->address = get_desc_base(&desc) | ((u64)base3 << 32); |
| } else |
| ops->get_gdt(ctxt, dt); |
| } |
| |
| static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt, |
| u16 selector, ulong *desc_addr_p) |
| { |
| struct desc_ptr dt; |
| u16 index = selector >> 3; |
| ulong addr; |
| |
| get_descriptor_table_ptr(ctxt, selector, &dt); |
| |
| if (dt.size < index * 8 + 7) |
| return emulate_gp(ctxt, selector & 0xfffc); |
| |
| addr = dt.address + index * 8; |
| |
| #ifdef CONFIG_X86_64 |
| if (addr >> 32 != 0) { |
| u64 efer = 0; |
| |
| ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); |
| if (!(efer & EFER_LMA)) |
| addr &= (u32)-1; |
| } |
| #endif |
| |
| *desc_addr_p = addr; |
| return X86EMUL_CONTINUE; |
| } |
| |
| /* allowed just for 8 bytes segments */ |
| static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt, |
| u16 selector, struct desc_struct *desc, |
| ulong *desc_addr_p) |
| { |
| int rc; |
| |
| rc = get_descriptor_ptr(ctxt, selector, desc_addr_p); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc)); |
| } |
| |
| /* allowed just for 8 bytes segments */ |
| static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt, |
| u16 selector, struct desc_struct *desc) |
| { |
| int rc; |
| ulong addr; |
| |
| rc = get_descriptor_ptr(ctxt, selector, &addr); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| return linear_write_system(ctxt, addr, desc, sizeof(*desc)); |
| } |
| |
| static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt, |
| u16 selector, int seg, u8 cpl, |
| enum x86_transfer_type transfer, |
| struct desc_struct *desc) |
| { |
| struct desc_struct seg_desc, old_desc; |
| u8 dpl, rpl; |
| unsigned err_vec = GP_VECTOR; |
| u32 err_code = 0; |
| bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ |
| ulong desc_addr; |
| int ret; |
| u16 dummy; |
| u32 base3 = 0; |
| |
| memset(&seg_desc, 0, sizeof(seg_desc)); |
| |
| if (ctxt->mode == X86EMUL_MODE_REAL) { |
| /* set real mode segment descriptor (keep limit etc. for |
| * unreal mode) */ |
| ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg); |
| set_desc_base(&seg_desc, selector << 4); |
| goto load; |
| } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) { |
| /* VM86 needs a clean new segment descriptor */ |
| set_desc_base(&seg_desc, selector << 4); |
| set_desc_limit(&seg_desc, 0xffff); |
| seg_desc.type = 3; |
| seg_desc.p = 1; |
| seg_desc.s = 1; |
| seg_desc.dpl = 3; |
| goto load; |
| } |
| |
| rpl = selector & 3; |
| |
| /* TR should be in GDT only */ |
| if (seg == VCPU_SREG_TR && (selector & (1 << 2))) |
| goto exception; |
| |
| /* NULL selector is not valid for TR, CS and (except for long mode) SS */ |
| if (null_selector) { |
| if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR) |
| goto exception; |
| |
| if (seg == VCPU_SREG_SS) { |
| if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl) |
| goto exception; |
| |
| /* |
| * ctxt->ops->set_segment expects the CPL to be in |
| * SS.DPL, so fake an expand-up 32-bit data segment. |
| */ |
| seg_desc.type = 3; |
| seg_desc.p = 1; |
| seg_desc.s = 1; |
| seg_desc.dpl = cpl; |
| seg_desc.d = 1; |
| seg_desc.g = 1; |
| } |
| |
| /* Skip all following checks */ |
| goto load; |
| } |
| |
| ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr); |
| if (ret != X86EMUL_CONTINUE) |
| return ret; |
| |
| err_code = selector & 0xfffc; |
| err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR : |
| GP_VECTOR; |
| |
| /* can't load system descriptor into segment selector */ |
| if (seg <= VCPU_SREG_GS && !seg_desc.s) { |
| if (transfer == X86_TRANSFER_CALL_JMP) |
| return X86EMUL_UNHANDLEABLE; |
| goto exception; |
| } |
| |
| if (!seg_desc.p) { |
| err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR; |
| goto exception; |
| } |
| |
| dpl = seg_desc.dpl; |
| |
| switch (seg) { |
| case VCPU_SREG_SS: |
| /* |
| * segment is not a writable data segment or segment |
| * selector's RPL != CPL or segment selector's RPL != CPL |
| */ |
| if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl) |
| goto exception; |
| break; |
| case VCPU_SREG_CS: |
| if (!(seg_desc.type & 8)) |
| goto exception; |
| |
| if (seg_desc.type & 4) { |
| /* conforming */ |
| if (dpl > cpl) |
| goto exception; |
| } else { |
| /* nonconforming */ |
| if (rpl > cpl || dpl != cpl) |
| goto exception; |
| } |
| /* in long-mode d/b must be clear if l is set */ |
| if (seg_desc.d && seg_desc.l) { |
| u64 efer = 0; |
| |
| ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); |
| if (efer & EFER_LMA) |
| goto exception; |
| } |
| |
| /* CS(RPL) <- CPL */ |
| selector = (selector & 0xfffc) | cpl; |
| break; |
| case VCPU_SREG_TR: |
| if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9)) |
| goto exception; |
| old_desc = seg_desc; |
| seg_desc.type |= 2; /* busy */ |
| ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc, |
| sizeof(seg_desc), &ctxt->exception); |
| if (ret != X86EMUL_CONTINUE) |
| return ret; |
| break; |
| case VCPU_SREG_LDTR: |
| if (seg_desc.s || seg_desc.type != 2) |
| goto exception; |
| break; |
| default: /* DS, ES, FS, or GS */ |
| /* |
| * segment is not a data or readable code segment or |
| * ((segment is a data or nonconforming code segment) |
| * and (both RPL and CPL > DPL)) |
| */ |
| if ((seg_desc.type & 0xa) == 0x8 || |
| (((seg_desc.type & 0xc) != 0xc) && |
| (rpl > dpl && cpl > dpl))) |
| goto exception; |
| break; |
| } |
| |
| if (seg_desc.s) { |
| /* mark segment as accessed */ |
| if (!(seg_desc.type & 1)) { |
| seg_desc.type |= 1; |
| ret = write_segment_descriptor(ctxt, selector, |
| &seg_desc); |
| if (ret != X86EMUL_CONTINUE) |
| return ret; |
| } |
| } else if (ctxt->mode == X86EMUL_MODE_PROT64) { |
| ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3)); |
| if (ret != X86EMUL_CONTINUE) |
| return ret; |
| if (emul_is_noncanonical_address(get_desc_base(&seg_desc) | |
| ((u64)base3 << 32), ctxt)) |
| return emulate_gp(ctxt, 0); |
| } |
| load: |
| ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg); |
| if (desc) |
| *desc = seg_desc; |
| return X86EMUL_CONTINUE; |
| exception: |
| return emulate_exception(ctxt, err_vec, err_code, true); |
| } |
| |
| static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, |
| u16 selector, int seg) |
| { |
| u8 cpl = ctxt->ops->cpl(ctxt); |
| |
| /* |
| * None of MOV, POP and LSS can load a NULL selector in CPL=3, but |
| * they can load it at CPL<3 (Intel's manual says only LSS can, |
| * but it's wrong). |
| * |
| * However, the Intel manual says that putting IST=1/DPL=3 in |
| * an interrupt gate will result in SS=3 (the AMD manual instead |
| * says it doesn't), so allow SS=3 in __load_segment_descriptor |
| * and only forbid it here. |
| */ |
| if (seg == VCPU_SREG_SS && selector == 3 && |
| ctxt->mode == X86EMUL_MODE_PROT64) |
| return emulate_exception(ctxt, GP_VECTOR, 0, true); |
| |
| return __load_segment_descriptor(ctxt, selector, seg, cpl, |
| X86_TRANSFER_NONE, NULL); |
| } |
| |
| static void write_register_operand(struct operand *op) |
| { |
| return assign_register(op->addr.reg, op->val, op->bytes); |
| } |
| |
| static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) |
| { |
| switch (op->type) { |
| case OP_REG: |
| write_register_operand(op); |
| break; |
| case OP_MEM: |
| if (ctxt->lock_prefix) |
| return segmented_cmpxchg(ctxt, |
| op->addr.mem, |
| &op->orig_val, |
| &op->val, |
| op->bytes); |
| else |
| return segmented_write(ctxt, |
| op->addr.mem, |
| &op->val, |
| op->bytes); |
| break; |
| case OP_MEM_STR: |
| return segmented_write(ctxt, |
| op->addr.mem, |
| op->data, |
| op->bytes * op->count); |
| break; |
| case OP_XMM: |
| write_sse_reg(&op->vec_val, op->addr.xmm); |
| break; |
| case OP_MM: |
| write_mmx_reg(&op->mm_val, op->addr.mm); |
| break; |
| case OP_NONE: |
| /* no writeback */ |
| break; |
| default: |
| break; |
| } |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes) |
| { |
| struct segmented_address addr; |
| |
| rsp_increment(ctxt, -bytes); |
| addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt); |
| addr.seg = VCPU_SREG_SS; |
| |
| return segmented_write(ctxt, addr, data, bytes); |
| } |
| |
| static int em_push(struct x86_emulate_ctxt *ctxt) |
| { |
| /* Disable writeback. */ |
| ctxt->dst.type = OP_NONE; |
| return push(ctxt, &ctxt->src.val, ctxt->op_bytes); |
| } |
| |
| static int emulate_pop(struct x86_emulate_ctxt *ctxt, |
| void *dest, int len) |
| { |
| int rc; |
| struct segmented_address addr; |
| |
| addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt); |
| addr.seg = VCPU_SREG_SS; |
| rc = segmented_read(ctxt, addr, dest, len); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rsp_increment(ctxt, len); |
| return rc; |
| } |
| |
| static int em_pop(struct x86_emulate_ctxt *ctxt) |
| { |
| return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes); |
| } |
| |
| static int emulate_popf(struct x86_emulate_ctxt *ctxt, |
| void *dest, int len) |
| { |
| int rc; |
| unsigned long val, change_mask; |
| int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT; |
| int cpl = ctxt->ops->cpl(ctxt); |
| |
| rc = emulate_pop(ctxt, &val, len); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | |
| X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF | |
| X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT | |
| X86_EFLAGS_AC | X86_EFLAGS_ID; |
| |
| switch(ctxt->mode) { |
| case X86EMUL_MODE_PROT64: |
| case X86EMUL_MODE_PROT32: |
| case X86EMUL_MODE_PROT16: |
| if (cpl == 0) |
| change_mask |= X86_EFLAGS_IOPL; |
| if (cpl <= iopl) |
| change_mask |= X86_EFLAGS_IF; |
| break; |
| case X86EMUL_MODE_VM86: |
| if (iopl < 3) |
| return emulate_gp(ctxt, 0); |
| change_mask |= X86_EFLAGS_IF; |
| break; |
| default: /* real mode */ |
| change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF); |
| break; |
| } |
| |
| *(unsigned long *)dest = |
| (ctxt->eflags & ~change_mask) | (val & change_mask); |
| |
| return rc; |
| } |
| |
| static int em_popf(struct x86_emulate_ctxt *ctxt) |
| { |
| ctxt->dst.type = OP_REG; |
| ctxt->dst.addr.reg = &ctxt->eflags; |
| ctxt->dst.bytes = ctxt->op_bytes; |
| return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes); |
| } |
| |
| static int em_enter(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| unsigned frame_size = ctxt->src.val; |
| unsigned nesting_level = ctxt->src2.val & 31; |
| ulong rbp; |
| |
| if (nesting_level) |
| return X86EMUL_UNHANDLEABLE; |
| |
| rbp = reg_read(ctxt, VCPU_REGS_RBP); |
| rc = push(ctxt, &rbp, stack_size(ctxt)); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP), |
| stack_mask(ctxt)); |
| assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), |
| reg_read(ctxt, VCPU_REGS_RSP) - frame_size, |
| stack_mask(ctxt)); |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_leave(struct x86_emulate_ctxt *ctxt) |
| { |
| assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP), |
| stack_mask(ctxt)); |
| return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes); |
| } |
| |
| static int em_push_sreg(struct x86_emulate_ctxt *ctxt) |
| { |
| int seg = ctxt->src2.val; |
| |
| ctxt->src.val = get_segment_selector(ctxt, seg); |
| if (ctxt->op_bytes == 4) { |
| rsp_increment(ctxt, -2); |
| ctxt->op_bytes = 2; |
| } |
| |
| return em_push(ctxt); |
| } |
| |
| static int em_pop_sreg(struct x86_emulate_ctxt *ctxt) |
| { |
| int seg = ctxt->src2.val; |
| unsigned long selector; |
| int rc; |
| |
| rc = emulate_pop(ctxt, &selector, 2); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| if (ctxt->modrm_reg == VCPU_SREG_SS) |
| ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; |
| if (ctxt->op_bytes > 2) |
| rsp_increment(ctxt, ctxt->op_bytes - 2); |
| |
| rc = load_segment_descriptor(ctxt, (u16)selector, seg); |
| return rc; |
| } |
| |
| static int em_pusha(struct x86_emulate_ctxt *ctxt) |
| { |
| unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP); |
| int rc = X86EMUL_CONTINUE; |
| int reg = VCPU_REGS_RAX; |
| |
| while (reg <= VCPU_REGS_RDI) { |
| (reg == VCPU_REGS_RSP) ? |
| (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg)); |
| |
| rc = em_push(ctxt); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ++reg; |
| } |
| |
| return rc; |
| } |
| |
| static int em_pushf(struct x86_emulate_ctxt *ctxt) |
| { |
| ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM; |
| return em_push(ctxt); |
| } |
| |
| static int em_popa(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc = X86EMUL_CONTINUE; |
| int reg = VCPU_REGS_RDI; |
| u32 val; |
| |
| while (reg >= VCPU_REGS_RAX) { |
| if (reg == VCPU_REGS_RSP) { |
| rsp_increment(ctxt, ctxt->op_bytes); |
| --reg; |
| } |
| |
| rc = emulate_pop(ctxt, &val, ctxt->op_bytes); |
| if (rc != X86EMUL_CONTINUE) |
| break; |
| assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); |
| --reg; |
| } |
| return rc; |
| } |
| |
| static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) |
| { |
| const struct x86_emulate_ops *ops = ctxt->ops; |
| int rc; |
| struct desc_ptr dt; |
| gva_t cs_addr; |
| gva_t eip_addr; |
| u16 cs, eip; |
| |
| /* TODO: Add limit checks */ |
| ctxt->src.val = ctxt->eflags; |
| rc = em_push(ctxt); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC); |
| |
| ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS); |
| rc = em_push(ctxt); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ctxt->src.val = ctxt->_eip; |
| rc = em_push(ctxt); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ops->get_idt(ctxt, &dt); |
| |
| eip_addr = dt.address + (irq << 2); |
| cs_addr = dt.address + (irq << 2) + 2; |
| |
| rc = linear_read_system(ctxt, cs_addr, &cs, 2); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rc = linear_read_system(ctxt, eip_addr, &eip, 2); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ctxt->_eip = eip; |
| |
| return rc; |
| } |
| |
| int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) |
| { |
| int rc; |
| |
| invalidate_registers(ctxt); |
| rc = __emulate_int_real(ctxt, irq); |
| if (rc == X86EMUL_CONTINUE) |
| writeback_registers(ctxt); |
| return rc; |
| } |
| |
| static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq) |
| { |
| switch(ctxt->mode) { |
| case X86EMUL_MODE_REAL: |
| return __emulate_int_real(ctxt, irq); |
| case X86EMUL_MODE_VM86: |
| case X86EMUL_MODE_PROT16: |
| case X86EMUL_MODE_PROT32: |
| case X86EMUL_MODE_PROT64: |
| default: |
| /* Protected mode interrupts unimplemented yet */ |
| return X86EMUL_UNHANDLEABLE; |
| } |
| } |
| |
| static int emulate_iret_real(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc = X86EMUL_CONTINUE; |
| unsigned long temp_eip = 0; |
| unsigned long temp_eflags = 0; |
| unsigned long cs = 0; |
| unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | |
| X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF | |
| X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF | |
| X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF | |
| X86_EFLAGS_AC | X86_EFLAGS_ID | |
| X86_EFLAGS_FIXED; |
| unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF | |
| X86_EFLAGS_VIP; |
| |
| /* TODO: Add stack limit check */ |
| |
| rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes); |
| |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| if (temp_eip & ~0xffff) |
| return emulate_gp(ctxt, 0); |
| |
| rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); |
| |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes); |
| |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS); |
| |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ctxt->_eip = temp_eip; |
| |
| if (ctxt->op_bytes == 4) |
| ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask)); |
| else if (ctxt->op_bytes == 2) { |
| ctxt->eflags &= ~0xffff; |
| ctxt->eflags |= temp_eflags; |
| } |
| |
| ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */ |
| ctxt->eflags |= X86_EFLAGS_FIXED; |
| ctxt->ops->set_nmi_mask(ctxt, false); |
| |
| return rc; |
| } |
| |
| static int em_iret(struct x86_emulate_ctxt *ctxt) |
| { |
| switch(ctxt->mode) { |
| case X86EMUL_MODE_REAL: |
| return emulate_iret_real(ctxt); |
| case X86EMUL_MODE_VM86: |
| case X86EMUL_MODE_PROT16: |
| case X86EMUL_MODE_PROT32: |
| case X86EMUL_MODE_PROT64: |
| default: |
| /* iret from protected mode unimplemented yet */ |
| return X86EMUL_UNHANDLEABLE; |
| } |
| } |
| |
| static int em_jmp_far(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| unsigned short sel; |
| struct desc_struct new_desc; |
| u8 cpl = ctxt->ops->cpl(ctxt); |
| |
| memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); |
| |
| rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl, |
| X86_TRANSFER_CALL_JMP, |
| &new_desc); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc); |
| /* Error handling is not implemented. */ |
| if (rc != X86EMUL_CONTINUE) |
| return X86EMUL_UNHANDLEABLE; |
| |
| return rc; |
| } |
| |
| static int em_jmp_abs(struct x86_emulate_ctxt *ctxt) |
| { |
| return assign_eip_near(ctxt, ctxt->src.val); |
| } |
| |
| static int em_call_near_abs(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| long int old_eip; |
| |
| old_eip = ctxt->_eip; |
| rc = assign_eip_near(ctxt, ctxt->src.val); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| ctxt->src.val = old_eip; |
| rc = em_push(ctxt); |
| return rc; |
| } |
| |
| static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt) |
| { |
| u64 old = ctxt->dst.orig_val64; |
| |
| if (ctxt->dst.bytes == 16) |
| return X86EMUL_UNHANDLEABLE; |
| |
| if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) || |
| ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) { |
| *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0); |
| *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32); |
| ctxt->eflags &= ~X86_EFLAGS_ZF; |
| } else { |
| ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) | |
| (u32) reg_read(ctxt, VCPU_REGS_RBX); |
| |
| ctxt->eflags |= X86_EFLAGS_ZF; |
| } |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_ret(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| unsigned long eip; |
| |
| rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| return assign_eip_near(ctxt, eip); |
| } |
| |
| static int em_ret_far(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| unsigned long eip, cs; |
| int cpl = ctxt->ops->cpl(ctxt); |
| struct desc_struct new_desc; |
| |
| rc = emulate_pop(ctxt, &eip, ctxt->op_bytes); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| rc = emulate_pop(ctxt, &cs, ctxt->op_bytes); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| /* Outer-privilege level return is not implemented */ |
| if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl) |
| return X86EMUL_UNHANDLEABLE; |
| rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl, |
| X86_TRANSFER_RET, |
| &new_desc); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| rc = assign_eip_far(ctxt, eip, &new_desc); |
| /* Error handling is not implemented. */ |
| if (rc != X86EMUL_CONTINUE) |
| return X86EMUL_UNHANDLEABLE; |
| |
| return rc; |
| } |
| |
| static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt) |
| { |
| int rc; |
| |
| rc = em_ret_far(ctxt); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| rsp_increment(ctxt, ctxt->src.val); |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) |
| { |
| /* Save real source value, then compare EAX against destination. */ |
| ctxt->dst.orig_val = ctxt->dst.val; |
| ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX); |
| ctxt->src.orig_val = ctxt->src.val; |
| ctxt->src.val = ctxt->dst.orig_val; |
| fastop(ctxt, em_cmp); |
| |
| if (ctxt->eflags & X86_EFLAGS_ZF) { |
| /* Success: write back to memory; no update of EAX */ |
| ctxt->src.type = OP_NONE; |
| ctxt->dst.val = ctxt->src.orig_val; |
| } else { |
| /* Failure: write the value we saw to EAX. */ |
| ctxt->src.type = OP_REG; |
| ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX); |
| ctxt->src.val = ctxt->dst.orig_val; |
| /* Create write-cycle to dest by writing the same value */ |
| ctxt->dst.val = ctxt->dst.orig_val; |
| } |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int em_lseg(struct x86_emulate_ctxt *ctxt) |
| { |
| int seg = ctxt->src2.val; |
| unsigned short sel; |
| int rc; |
| |
| memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2); |
| |
| rc = load_segment_descriptor(ctxt, sel, seg); |
| if (rc != X86EMUL_CONTINUE) |
| return rc; |
| |
| ctxt->dst.val = ctxt->src.val; |
| return rc; |
| } |
| |
| static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt) |
| { |
| #ifdef CONFIG_X86_64 |
| return ctxt->ops->guest_has_long_mode(ctxt); |
| #else |
| return false; |
| #endif |
| } |
| |
| static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags) |
| { |
| desc->g = (flags >> 23) & 1; |
| desc->d = (flags >> 22) & 1; |
| desc->l = (flags >> 21) & 1; |
| desc->avl = (flags >> 20) & 1; |
| desc->p = (flags >> 15) & 1; |
| desc->dpl = (flags >> 13) & 3; |
| desc->s = (flags >> 12) & 1; |
| desc->type = (flags >> 8) & 15; |
| } |
| |
| static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, const char *smstate, |
| int n) |
| { |
| struct desc_struct desc; |
| int offset; |
| u16 selector; |
| |
| selector = GET_SMSTATE(u32, smstate, 0x7fa8 + n * 4); |
| |
| if (n < 3) |
| offset = 0x7f84 + n * 12; |
| else |
| offset = 0x7f2c + (n - 3) * 12; |
| |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8)); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4)); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, offset)); |
| ctxt->ops->set_segment(ctxt, selector, &desc, 0, n); |
| return X86EMUL_CONTINUE; |
| } |
| |
| #ifdef CONFIG_X86_64 |
| static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, const char *smstate, |
| int n) |
| { |
| struct desc_struct desc; |
| int offset; |
| u16 selector; |
| u32 base3; |
| |
| offset = 0x7e00 + n * 16; |
| |
| selector = GET_SMSTATE(u16, smstate, offset); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smstate, offset + 2) << 8); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, offset + 4)); |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, offset + 8)); |
| base3 = GET_SMSTATE(u32, smstate, offset + 12); |
| |
| ctxt->ops->set_segment(ctxt, selector, &desc, base3, n); |
| return X86EMUL_CONTINUE; |
| } |
| #endif |
| |
| static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt, |
| u64 cr0, u64 cr3, u64 cr4) |
| { |
| int bad; |
| u64 pcid; |
| |
| /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */ |
| pcid = 0; |
| if (cr4 & X86_CR4_PCIDE) { |
| pcid = cr3 & 0xfff; |
| cr3 &= ~0xfff; |
| } |
| |
| bad = ctxt->ops->set_cr(ctxt, 3, cr3); |
| if (bad) |
| return X86EMUL_UNHANDLEABLE; |
| |
| /* |
| * First enable PAE, long mode needs it before CR0.PG = 1 is set. |
| * Then enable protected mode. However, PCID cannot be enabled |
| * if EFER.LMA=0, so set it separately. |
| */ |
| bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE); |
| if (bad) |
| return X86EMUL_UNHANDLEABLE; |
| |
| bad = ctxt->ops->set_cr(ctxt, 0, cr0); |
| if (bad) |
| return X86EMUL_UNHANDLEABLE; |
| |
| if (cr4 & X86_CR4_PCIDE) { |
| bad = ctxt->ops->set_cr(ctxt, 4, cr4); |
| if (bad) |
| return X86EMUL_UNHANDLEABLE; |
| if (pcid) { |
| bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid); |
| if (bad) |
| return X86EMUL_UNHANDLEABLE; |
| } |
| |
| } |
| |
| return X86EMUL_CONTINUE; |
| } |
| |
| static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, |
| const char *smstate) |
| { |
| struct desc_struct desc; |
| struct desc_ptr dt; |
| u16 selector; |
| u32 val, cr0, cr3, cr4; |
| int i; |
| |
| cr0 = GET_SMSTATE(u32, smstate, 0x7ffc); |
| cr3 = GET_SMSTATE(u32, smstate, 0x7ff8); |
| ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED; |
| ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0); |
| |
| for (i = 0; i < 8; i++) |
| *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4); |
| |
| val = GET_SMSTATE(u32, smstate, 0x7fcc); |
| |
| if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| val = GET_SMSTATE(u32, smstate, 0x7fc8); |
| |
| if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| selector = GET_SMSTATE(u32, smstate, 0x7fc4); |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f64)); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f60)); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f5c)); |
| ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR); |
| |
| selector = GET_SMSTATE(u32, smstate, 0x7fc0); |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7f80)); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7f7c)); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7f78)); |
| ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR); |
| |
| dt.address = GET_SMSTATE(u32, smstate, 0x7f74); |
| dt.size = GET_SMSTATE(u32, smstate, 0x7f70); |
| ctxt->ops->set_gdt(ctxt, &dt); |
| |
| dt.address = GET_SMSTATE(u32, smstate, 0x7f58); |
| dt.size = GET_SMSTATE(u32, smstate, 0x7f54); |
| ctxt->ops->set_idt(ctxt, &dt); |
| |
| for (i = 0; i < 6; i++) { |
| int r = rsm_load_seg_32(ctxt, smstate, i); |
| if (r != X86EMUL_CONTINUE) |
| return r; |
| } |
| |
| cr4 = GET_SMSTATE(u32, smstate, 0x7f14); |
| |
| ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7ef8)); |
| |
| return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4); |
| } |
| |
| #ifdef CONFIG_X86_64 |
| static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, |
| const char *smstate) |
| { |
| struct desc_struct desc; |
| struct desc_ptr dt; |
| u64 val, cr0, cr3, cr4; |
| u32 base3; |
| u16 selector; |
| int i, r; |
| |
| for (i = 0; i < 16; i++) |
| *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8); |
| |
| ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78); |
| ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7f70) | X86_EFLAGS_FIXED; |
| |
| val = GET_SMSTATE(u32, smstate, 0x7f68); |
| |
| if (ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| val = GET_SMSTATE(u32, smstate, 0x7f60); |
| |
| if (ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| cr0 = GET_SMSTATE(u64, smstate, 0x7f58); |
| cr3 = GET_SMSTATE(u64, smstate, 0x7f50); |
| cr4 = GET_SMSTATE(u64, smstate, 0x7f48); |
| ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smstate, 0x7f00)); |
| val = GET_SMSTATE(u64, smstate, 0x7ed0); |
| |
| if (ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| selector = GET_SMSTATE(u32, smstate, 0x7e90); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e92) << 8); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e94)); |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e98)); |
| base3 = GET_SMSTATE(u32, smstate, 0x7e9c); |
| ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR); |
| |
| dt.size = GET_SMSTATE(u32, smstate, 0x7e84); |
| dt.address = GET_SMSTATE(u64, smstate, 0x7e88); |
| ctxt->ops->set_idt(ctxt, &dt); |
| |
| selector = GET_SMSTATE(u32, smstate, 0x7e70); |
| rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smstate, 0x7e72) << 8); |
| set_desc_limit(&desc, GET_SMSTATE(u32, smstate, 0x7e74)); |
| set_desc_base(&desc, GET_SMSTATE(u32, smstate, 0x7e78)); |
| base3 = GET_SMSTATE(u32, smstate, 0x7e7c); |
| ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR); |
| |
| dt.size = GET_SMSTATE(u32, smstate, 0x7e64); |
| dt.address = GET_SMSTATE(u64, smstate, 0x7e68); |
| ctxt->ops->set_gdt(ctxt, &dt); |
| |
| r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4); |
| if (r != X86EMUL_CONTINUE) |
| return r; |
| |
| for (i = 0; i < 6; i++) { |
| r = rsm_load_seg_64(ctxt, smstate, i); |
| if (r != X86EMUL_CONTINUE) |
| return r; |
| } |
| |
| return X86EMUL_CONTINUE; |
| } |
| #endif |
| |
| static int em_rsm(struct x86_emulate_ctxt *ctxt) |
| { |
| unsigned long cr0, cr4, efer; |
| char buf[512]; |
| u64 smbase; |
| int ret; |
| |
| if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0) |
| return emulate_ud(ctxt); |
| |
| smbase = ctxt->ops->get_smbase(ctxt); |
| |
| ret = ctxt->ops->read_phys(ctxt, smbase + 0xfe00, buf, sizeof(buf)); |
| if (ret != X86EMUL_CONTINUE) |
| return X86EMUL_UNHANDLEABLE; |
| |
| if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0) |
| ctxt->ops->set_nmi_mask(ctxt, false); |
| |
| ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) & |
| ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK)); |
| |
| /* |
| * Get back to real mode, to prepare a safe state in which to load |
| * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU |
| * supports long mode. |
| */ |
| if (emulator_has_longmode(ctxt)) { |
| struct desc_struct cs_desc; |
| |
| /* Zero CR4.PCIDE before CR0.PG. */ |
| cr4 = ctxt->ops->get_cr(ctxt, 4); |
| if (cr4 & X86_CR4_PCIDE) |
| ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE); |
| |
| /* A 32-bit code segment is required to clear EFER.LMA. */ |
| memset(&cs_desc, 0, sizeof(cs_desc)); |
| cs_desc.type = 0xb; |
| cs_desc.s = cs_desc.g = cs_desc.p = 1; |
| ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS); |
| } |
| |
| /* For the 64-bit case, this will clear EFER.LMA. */ |
| cr0 = ctxt->ops->get_cr(ctxt, 0); |
| if (cr0 & X86_CR0_PE) |
| ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE)); |
| |
| if (emulator_has_longmode(ctxt)) { |
| /* Clear CR4.PAE before clearing EFER.LME. */ |
| cr4 = ctxt->ops->get_cr(ctxt, 4); |
| if (cr4 & X86_CR4_PAE) |
| ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE); |
| |
| /* And finally go back to 32-bit mode. */ |
| efer = 0; |
| ctxt->ops->set_msr(ctxt, MSR_EFER, efer); |
| } |
| |
| /* |
| * Give pre_leave_smm() a chance to make ISA-specific changes to the |
| * vCPU state (e.g. enter guest mode) before loading state from the SMM |
| * state-save area. |
| */ |
| if (ctxt->ops->pre_leave_smm(ctxt, buf)) |
| return X86EMUL_UNHANDLEABLE; |
| |
| #ifdef CONFIG_X86_64 |
| if (emulator_has_longmode(ctxt)) |
| ret = rsm_load_state_64(ctxt, buf); |
| else |
| #endif |
| ret = rsm_load_state_32(ctxt, buf); |
| |
| if (ret != X86EMUL_CONTINUE) { |
| /* FIXME: should triple fault */ |
| return X86EMUL_UNHANDLEABLE; |
| } |
| |
| ctxt->ops->post_leave_smm(ctxt); |
| |
| return X86EMUL_CONTINUE; |
| } |
| |
| static void |
| setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, |
| struct desc_struct *cs, struct desc_struct *ss) |
| { |
| cs->l = 0; /* will be adjusted later */ |
| set_desc_base(cs, 0); /* flat segment */ |
| cs->g = 1; /* 4kb granularity */ |
| set_desc_limit(cs, 0xfffff); /* 4GB limit */ |
| cs->type = 0x0b; /* Read, Execute, Accessed */ |
| cs->s = 1; |
| cs->dpl = 0; /* will be adjusted later */ |
| cs->p = 1; |
| cs->d = 1; |
| cs->avl = 0; |
| |
| set_desc_base(ss, 0); /* flat segment */ |
| set_desc_limit(ss, 0xfffff); /* 4GB limit */ |
| ss->g = 1; /* 4kb granularity */ |
| ss->s = 1; |
| ss->type = 0x03; /* Read/Write, Accessed */ |
| ss->d = 1; /* 32bit stack segment */ |
| ss->dpl = 0; |
| ss->p = 1; |
| ss->l = 0; |
| ss->avl = 0; |
| } |
| |
| static bool vendor_intel(struct x86_emulate_ctxt *ctxt) |
| { |
| u32 eax, ebx, ecx, edx; |
| |
| eax = ecx = 0; |
| ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); |
| return is_guest_vendor_intel(ebx, ecx, edx); |
| } |
| |
| static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) |
| { |
| const struct x86_emulate_ops *ops = ctxt->ops; |
| u32 eax, ebx, ecx, edx; |
| |
| /* |
| * syscall should always be enabled in longmode - so only become |
| * vendor specific (cpuid) if other modes are active... |
| */ |
| if (ctxt->mode == X86EMUL_MODE_PROT64) |
| return true; |
| |
| eax = 0x00000000; |
| ecx = 0x00000000; |
| ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true); |
| /* |
| * remark: Intel CPUs only support "syscall" in 64bit longmode. Also a |
| * 64bit guest with a 32bit compat-app running will #UD !! While this |
| * behaviour can be fixed (by emulating) into AMD response - CPUs of |
| * AMD can't behave like Intel. |
| */ |
| if (is_guest_vendor_intel(ebx, ecx, edx)) |
| return false; |
| |
| if (is_guest_vendor_amd(ebx, ecx, edx) || |
| is_guest_vendor_hygon(ebx, ecx, edx)) |
| return true; |
| |
| /* |
| * default: (not Intel, not AMD, not Hygon), apply Intel's |
| * stricter rules... |
| */ |
| return false; |
| } |
| |
| static int em_syscall(struct x86_emulate_ctxt *ctxt) |
| { |
| const struct x86_emulate_ops *ops = ctxt->ops; |
| struct desc_struct cs, ss; |
| u64 msr_data; |
| u16 cs_sel, ss_sel; |
| u64 efer = 0; |
| |
| /* syscall is not available in real mode */ |
| if (ctxt->mode == X86EMUL_MODE_REAL || |
| ctxt->mode == X86EMUL_MODE_VM86) |
| return emulate_ud(ctxt); |
| |
| if (!(em_syscall_is_enabled(ctxt))) |
| return emulate_ud(ctxt); |
| |
| ops->get_msr(ctxt, MSR_EFER, &efer); |
| if (!(efer & EFER_SCE)) |
| return emulate_ud(ctxt); |
| |
| setup_syscalls_segments(ctxt, &cs, &ss); |
| ops->get_msr(ctxt, MSR_STAR, &msr_data); |
| msr_data >>= 32; |
| cs_sel = (u16)(msr_data & 0xfffc); |
| ss_sel = (u16)(msr_data + 8); |
| |
| if (efer & EFER_LMA) { |
| cs.d = 0; |
| cs.l = 1; |
| } |
| ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); |
| ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); |
| |
| *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip; |
| if (efer & EFER_LMA) { |
| #ifdef CONFIG_X86_64 |
| *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags; |
| |
| ops->get_msr(ctxt, |
| ctxt->mode == X86EMUL_MODE_PROT64 ? |
| MSR_LSTAR : MSR_CSTAR, &msr_data); |
| ctxt->_eip = msr_data; |
| |
| ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data); |
| ctxt->eflags &= ~msr_data; |
| ctxt->eflags |= X86_EFLAGS_FIXED; |
| #endif |
| } else { |
| /* legacy mode */ |
| ops->get_msr(ctxt, MSR_STAR, &msr_data); |
| ctxt->_eip = (u32)msr_data; |
| |
| ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF); |
| } |
| |
| ctxt->tf = (ctxt-> |