VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 71204

最後變更 在這個檔案從71204是 71108,由 vboxsync 提交於 7 年 前

Added speculation control settings to API, refined implementation.
  • 屬性 svn:eol-style 設為 native
  • 屬性 svn:keywords 設為 Author Date Id Revision
檔案大小: 599.9 KB
行 
1/* $Id: HMVMXR0.cpp 71108 2018-02-22 15:38:35Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.alldomusa.eu.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#define HMVMX_USE_IEM_EVENT_REFLECTION
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_SWAP_FPU_STATE
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name Updated-guest-state flags.
70 * @{ */
71#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92 | HMVMX_UPDATED_GUEST_RSP \
93 | HMVMX_UPDATED_GUEST_RFLAGS \
94 | HMVMX_UPDATED_GUEST_CR0 \
95 | HMVMX_UPDATED_GUEST_CR3 \
96 | HMVMX_UPDATED_GUEST_CR4 \
97 | HMVMX_UPDATED_GUEST_GDTR \
98 | HMVMX_UPDATED_GUEST_IDTR \
99 | HMVMX_UPDATED_GUEST_LDTR \
100 | HMVMX_UPDATED_GUEST_TR \
101 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102 | HMVMX_UPDATED_GUEST_DR7 \
103 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
108 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109 | HMVMX_UPDATED_GUEST_INTR_STATE \
110 | HMVMX_UPDATED_GUEST_APIC_STATE)
111/** @} */
112
113/** @name
114 * Flags to skip redundant reads of some common VMCS fields that are not part of
115 * the guest-CPU state but are in the transient structure.
116 */
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124/** @} */
125
126/** @name
127 * States of the VMCS.
128 *
129 * This does not reflect all possible VMCS states but currently only those
130 * needed for maintaining the VMCS consistently even when thread-context hooks
131 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
132 */
133#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136/** @} */
137
138/**
139 * Exception bitmap mask for real-mode guests (real-on-v86).
140 *
141 * We need to intercept all exceptions manually except:
142 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144 * due to bugs in Intel CPUs.
145 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
146 * support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*********************************************************************************************************************************
197* Structures and Typedefs *
198*********************************************************************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG fEFlags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit code qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u7Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 /** INVEPT, INVVPID, INVPCID information. */
247 struct
248 {
249 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250 uint32_t u2Scaling : 2;
251 uint32_t u5Reserved0 : 5;
252 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253 uint32_t u3AddrSize : 3;
254 uint32_t u1Reserved0 : 1;
255 uint32_t u4Reserved0 : 4;
256 /** The segment register (X86_SREG_XXX). */
257 uint32_t iSegReg : 3;
258 /** The index register (X86_GREG_XXX). */
259 uint32_t iIdxReg : 4;
260 /** Set if index register is invalid. */
261 uint32_t fIdxRegValid : 1;
262 /** The base register (X86_GREG_XXX). */
263 uint32_t iBaseReg : 4;
264 /** Set if base register is invalid. */
265 uint32_t fBaseRegValid : 1;
266 /** Register 2 (X86_GREG_XXX). */
267 uint32_t iReg2 : 4;
268 } Inv;
269 } ExitInstrInfo;
270 /** Whether the VM-entry failed or not. */
271 bool fVMEntryFailed;
272 /** Alignment. */
273 uint8_t abAlignment1[3];
274
275 /** The VM-entry interruption-information field. */
276 uint32_t uEntryIntInfo;
277 /** The VM-entry exception error code field. */
278 uint32_t uEntryXcptErrorCode;
279 /** The VM-entry instruction length field. */
280 uint32_t cbEntryInstr;
281
282 /** IDT-vectoring information field. */
283 uint32_t uIdtVectoringInfo;
284 /** IDT-vectoring error code. */
285 uint32_t uIdtVectoringErrorCode;
286
287 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
288 uint32_t fVmcsFieldsRead;
289
290 /** Whether the guest FPU was active at the time of VM-exit. */
291 bool fWasGuestFPUStateActive;
292 /** Whether the guest debug state was active at the time of VM-exit. */
293 bool fWasGuestDebugStateActive;
294 /** Whether the hyper debug state was active at the time of VM-exit. */
295 bool fWasHyperDebugStateActive;
296 /** Whether TSC-offsetting should be setup before VM-entry. */
297 bool fUpdateTscOffsettingAndPreemptTimer;
298 /** Whether the VM-exit was caused by a page-fault during delivery of a
299 * contributory exception or a page-fault. */
300 bool fVectoringDoublePF;
301 /** Whether the VM-exit was caused by a page-fault during delivery of an
302 * external interrupt or NMI. */
303 bool fVectoringPF;
304} VMXTRANSIENT;
305AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310/** Pointer to VMX transient state. */
311typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314/**
315 * MSR-bitmap read permissions.
316 */
317typedef enum VMXMSREXITREAD
318{
319 /** Reading this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_READ = 0xb,
321 /** Reading this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_READ
323} VMXMSREXITREAD;
324/** Pointer to MSR-bitmap read permissions. */
325typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327/**
328 * MSR-bitmap write permissions.
329 */
330typedef enum VMXMSREXITWRITE
331{
332 /** Writing to this MSR causes a VM-exit. */
333 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334 /** Writing to this MSR does not cause a VM-exit. */
335 VMXMSREXIT_PASSTHRU_WRITE
336} VMXMSREXITWRITE;
337/** Pointer to MSR-bitmap write permissions. */
338typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341/**
342 * VMX VM-exit handler.
343 *
344 * @returns Strict VBox status code (i.e. informational status codes too).
345 * @param pVCpu The cross context virtual CPU structure.
346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
347 * out-of-sync. Make sure to update the required
348 * fields before using them.
349 * @param pVmxTransient Pointer to the VMX-transient structure.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355/** Pointer to VM-exit handler. */
356typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357#endif
358
359/**
360 * VMX VM-exit handler, non-strict status code.
361 *
362 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363 *
364 * @returns VBox status code, no informational status code returned.
365 * @param pVCpu The cross context virtual CPU structure.
366 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
367 * out-of-sync. Make sure to update the required
368 * fields before using them.
369 * @param pVmxTransient Pointer to the VMX-transient structure.
370 *
371 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372 * use of that status code will be replaced with VINF_EM_SOMETHING
373 * later when switching over to IEM.
374 */
375#ifndef HMVMX_USE_FUNCTION_TABLE
376typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377#else
378typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379#endif
380
381
382/*********************************************************************************************************************************
383* Internal Functions *
384*********************************************************************************************************************************/
385static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390 bool fStepping, uint32_t *puIntState);
391#if HC_ARCH_BITS == 32
392static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393#endif
394#ifndef HMVMX_USE_FUNCTION_TABLE
395DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398#else
399# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401#endif
402
403
404/** @name VM-exit handlers.
405 * @{
406 */
407static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419static FNVMXEXITHANDLER hmR0VmxExitHlt;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435static FNVMXEXITHANDLER hmR0VmxExitMwait;
436static FNVMXEXITHANDLER hmR0VmxExitMtf;
437static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438static FNVMXEXITHANDLER hmR0VmxExitPause;
439static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452/** @} */
453
454static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465/*********************************************************************************************************************************
466* Global Variables *
467*********************************************************************************************************************************/
468#ifdef HMVMX_USE_FUNCTION_TABLE
469
470/**
471 * VMX_EXIT dispatch table.
472 */
473static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474{
475 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515 /* 40 UNDEFINED */ hmR0VmxExitPause,
516 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
537 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540};
541#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543#ifdef VBOX_STRICT
544static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545{
546 /* 0 */ "(Not Used)",
547 /* 1 */ "VMCALL executed in VMX root operation.",
548 /* 2 */ "VMCLEAR with invalid physical address.",
549 /* 3 */ "VMCLEAR with VMXON pointer.",
550 /* 4 */ "VMLAUNCH with non-clear VMCS.",
551 /* 5 */ "VMRESUME with non-launched VMCS.",
552 /* 6 */ "VMRESUME after VMXOFF",
553 /* 7 */ "VM-entry with invalid control fields.",
554 /* 8 */ "VM-entry with invalid host state fields.",
555 /* 9 */ "VMPTRLD with invalid physical address.",
556 /* 10 */ "VMPTRLD with VMXON pointer.",
557 /* 11 */ "VMPTRLD with incorrect revision identifier.",
558 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559 /* 13 */ "VMWRITE to read-only VMCS component.",
560 /* 14 */ "(Not Used)",
561 /* 15 */ "VMXON executed in VMX root operation.",
562 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563 /* 17 */ "VM-entry with non-launched executing VMCS.",
564 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565 /* 19 */ "VMCALL with non-clear VMCS.",
566 /* 20 */ "VMCALL with invalid VM-exit control fields.",
567 /* 21 */ "(Not Used)",
568 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570 /* 24 */ "VMCALL with invalid SMM-monitor features.",
571 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572 /* 26 */ "VM-entry with events blocked by MOV SS.",
573 /* 27 */ "(Not Used)",
574 /* 28 */ "Invalid operand to INVEPT/INVVPID."
575};
576#endif /* VBOX_STRICT */
577
578
579
580/**
581 * Updates the VM's last error record.
582 *
583 * If there was a VMX instruction error, reads the error data from the VMCS and
584 * updates VCPU's last error record as well.
585 *
586 * @param pVM The cross context VM structure.
587 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
588 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589 * VERR_VMX_INVALID_VMCS_FIELD.
590 * @param rc The error code.
591 */
592static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593{
594 AssertPtr(pVM);
595 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596 || rc == VERR_VMX_UNABLE_TO_START_VM)
597 {
598 AssertPtrReturnVoid(pVCpu);
599 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600 }
601 pVM->hm.s.lLastError = rc;
602}
603
604
605/**
606 * Reads the VM-entry interruption-information field from the VMCS into the VMX
607 * transient structure.
608 *
609 * @returns VBox status code.
610 * @param pVmxTransient Pointer to the VMX transient structure.
611 *
612 * @remarks No-long-jump zone!!!
613 */
614DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615{
616 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617 AssertRCReturn(rc, rc);
618 return VINF_SUCCESS;
619}
620
621
622#ifdef VBOX_STRICT
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641#ifdef VBOX_STRICT
642/**
643 * Reads the VM-entry exception error code field from the VMCS into
644 * the VMX transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVmxTransient Pointer to the VMX transient structure.
648 *
649 * @remarks No-long-jump zone!!!
650 */
651DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652{
653 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654 AssertRCReturn(rc, rc);
655 return VINF_SUCCESS;
656}
657#endif /* VBOX_STRICT */
658
659
660/**
661 * Reads the VM-exit interruption-information field from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit interruption error code from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction length field from the VMCS into the VMX
700 * transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the VM-exit instruction-information field from the VMCS into
719 * the VMX transient structure.
720 *
721 * @returns VBox status code.
722 * @param pVmxTransient Pointer to the VMX transient structure.
723 */
724DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725{
726 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727 {
728 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729 AssertRCReturn(rc, rc);
730 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731 }
732 return VINF_SUCCESS;
733}
734
735
736/**
737 * Reads the exit code qualification from the VMCS into the VMX transient
738 * structure.
739 *
740 * @returns VBox status code.
741 * @param pVCpu The cross context virtual CPU structure of the
742 * calling EMT. (Required for the VMCS cache case.)
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 */
745DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746{
747 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748 {
749 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750 AssertRCReturn(rc, rc);
751 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752 }
753 return VINF_SUCCESS;
754}
755
756
757/**
758 * Reads the IDT-vectoring information field from the VMCS into the VMX
759 * transient structure.
760 *
761 * @returns VBox status code.
762 * @param pVmxTransient Pointer to the VMX transient structure.
763 *
764 * @remarks No-long-jump zone!!!
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Reads the IDT-vectoring error code from the VMCS into the VMX
780 * transient structure.
781 *
782 * @returns VBox status code.
783 * @param pVmxTransient Pointer to the VMX transient structure.
784 */
785DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786{
787 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788 {
789 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790 AssertRCReturn(rc, rc);
791 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792 }
793 return VINF_SUCCESS;
794}
795
796
797/**
798 * Enters VMX root mode operation on the current CPU.
799 *
800 * @returns VBox status code.
801 * @param pVM The cross context VM structure. Can be
802 * NULL, after a resume.
803 * @param HCPhysCpuPage Physical address of the VMXON region.
804 * @param pvCpuPage Pointer to the VMXON region.
805 */
806static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807{
808 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810 Assert(pvCpuPage);
811 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813 if (pVM)
814 {
815 /* Write the VMCS revision dword to the VMXON region. */
816 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817 }
818
819 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820 RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822 /* Enable the VMX bit in CR4 if necessary. */
823 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825 /* Enter VMX root mode. */
826 int rc = VMXEnable(HCPhysCpuPage);
827 if (RT_FAILURE(rc))
828 {
829 if (!(uOldCr4 & X86_CR4_VMXE))
830 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832 if (pVM)
833 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834 }
835
836 /* Restore interrupts. */
837 ASMSetFlags(fEFlags);
838 return rc;
839}
840
841
842/**
843 * Exits VMX root mode operation on the current CPU.
844 *
845 * @returns VBox status code.
846 */
847static int hmR0VmxLeaveRootMode(void)
848{
849 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852 RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854 /* If we're for some reason not in VMX root mode, then don't leave it. */
855 RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857 int rc;
858 if (uHostCR4 & X86_CR4_VMXE)
859 {
860 /* Exit VMX root mode and clear the VMX bit in CR4. */
861 VMXDisable();
862 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863 rc = VINF_SUCCESS;
864 }
865 else
866 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868 /* Restore interrupts. */
869 ASMSetFlags(fEFlags);
870 return rc;
871}
872
873
874/**
875 * Allocates and maps one physically contiguous page. The allocated page is
876 * zero'd out. (Used by various VT-x structures).
877 *
878 * @returns IPRT status code.
879 * @param pMemObj Pointer to the ring-0 memory object.
880 * @param ppVirt Where to store the virtual address of the
881 * allocation.
882 * @param pHCPhys Where to store the physical address of the
883 * allocation.
884 */
885DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886{
887 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892 if (RT_FAILURE(rc))
893 return rc;
894 *ppVirt = RTR0MemObjAddress(*pMemObj);
895 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
896 ASMMemZero32(*ppVirt, PAGE_SIZE);
897 return VINF_SUCCESS;
898}
899
900
901/**
902 * Frees and unmaps an allocated physical page.
903 *
904 * @param pMemObj Pointer to the ring-0 memory object.
905 * @param ppVirt Where to re-initialize the virtual address of
906 * allocation as 0.
907 * @param pHCPhys Where to re-initialize the physical address of the
908 * allocation as 0.
909 */
910DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911{
912 AssertPtr(pMemObj);
913 AssertPtr(ppVirt);
914 AssertPtr(pHCPhys);
915 if (*pMemObj != NIL_RTR0MEMOBJ)
916 {
917 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
918 AssertRC(rc);
919 *pMemObj = NIL_RTR0MEMOBJ;
920 *ppVirt = 0;
921 *pHCPhys = 0;
922 }
923}
924
925
926/**
927 * Worker function to free VT-x related structures.
928 *
929 * @returns IPRT status code.
930 * @param pVM The cross context VM structure.
931 */
932static void hmR0VmxStructsFree(PVM pVM)
933{
934 for (VMCPUID i = 0; i < pVM->cCpus; i++)
935 {
936 PVMCPU pVCpu = &pVM->aCpus[i];
937 AssertPtr(pVCpu);
938
939 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946 }
947
948 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951#endif
952}
953
954
955/**
956 * Worker function to allocate VT-x related VM structures.
957 *
958 * @returns IPRT status code.
959 * @param pVM The cross context VM structure.
960 */
961static int hmR0VmxStructsAlloc(PVM pVM)
962{
963 /*
964 * Initialize members up-front so we can cleanup properly on allocation failure.
965 */
966#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969 pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978#endif
979 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982 for (VMCPUID i = 0; i < pVM->cCpus; i++)
983 {
984 PVMCPU pVCpu = &pVM->aCpus[i];
985 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989 }
990#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998 /*
999 * Allocate all the VT-x structures.
1000 */
1001 int rc = VINF_SUCCESS;
1002#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004 if (RT_FAILURE(rc))
1005 goto cleanup;
1006 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008#endif
1009
1010 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014 &pVM->hm.s.vmx.HCPhysApicAccess);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 }
1018
1019 /*
1020 * Initialize per-VCPU VT-x structures.
1021 */
1022 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023 {
1024 PVMCPU pVCpu = &pVM->aCpus[i];
1025 AssertPtr(pVCpu);
1026
1027 /* Allocate the VM control structure (VMCS). */
1028 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031
1032 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033 if ( PDMHasApic(pVM)
1034 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035 {
1036 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1038 if (RT_FAILURE(rc))
1039 goto cleanup;
1040 }
1041
1042 /*
1043 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044 * transparent accesses of specific MSRs.
1045 *
1046 * If the condition for enabling MSR bitmaps changes here, don't forget to
1047 * update HMAreMsrBitmapsAvailable().
1048 */
1049 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050 {
1051 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053 if (RT_FAILURE(rc))
1054 goto cleanup;
1055 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056 }
1057
1058 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060 if (RT_FAILURE(rc))
1061 goto cleanup;
1062
1063 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1064 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065 if (RT_FAILURE(rc))
1066 goto cleanup;
1067 }
1068
1069 return VINF_SUCCESS;
1070
1071cleanup:
1072 hmR0VmxStructsFree(pVM);
1073 return rc;
1074}
1075
1076
1077/**
1078 * Does global VT-x initialization (called during module initialization).
1079 *
1080 * @returns VBox status code.
1081 */
1082VMMR0DECL(int) VMXR0GlobalInit(void)
1083{
1084#ifdef HMVMX_USE_FUNCTION_TABLE
1085 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086# ifdef VBOX_STRICT
1087 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088 Assert(g_apfnVMExitHandlers[i]);
1089# endif
1090#endif
1091 return VINF_SUCCESS;
1092}
1093
1094
1095/**
1096 * Does global VT-x termination (called during module termination).
1097 */
1098VMMR0DECL(void) VMXR0GlobalTerm()
1099{
1100 /* Nothing to do currently. */
1101}
1102
1103
1104/**
1105 * Sets up and activates VT-x on the current CPU.
1106 *
1107 * @returns VBox status code.
1108 * @param pCpu Pointer to the global CPU info struct.
1109 * @param pVM The cross context VM structure. Can be
1110 * NULL after a host resume operation.
1111 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112 * fEnabledByHost is @c true).
1113 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114 * @a fEnabledByHost is @c true).
1115 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116 * enable VT-x on the host.
1117 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1118 */
1119VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120 void *pvMsrs)
1121{
1122 Assert(pCpu);
1123 Assert(pvMsrs);
1124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126 /* Enable VT-x if it's not already enabled by the host. */
1127 if (!fEnabledByHost)
1128 {
1129 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130 if (RT_FAILURE(rc))
1131 return rc;
1132 }
1133
1134 /*
1135 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137 */
1138 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140 {
1141 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142 pCpu->fFlushAsidBeforeUse = false;
1143 }
1144 else
1145 pCpu->fFlushAsidBeforeUse = true;
1146
1147 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148 ++pCpu->cTlbFlushes;
1149
1150 return VINF_SUCCESS;
1151}
1152
1153
1154/**
1155 * Deactivates VT-x on the current CPU.
1156 *
1157 * @returns VBox status code.
1158 * @param pCpu Pointer to the global CPU info struct.
1159 * @param pvCpuPage Pointer to the VMXON region.
1160 * @param HCPhysCpuPage Physical address of the VMXON region.
1161 *
1162 * @remarks This function should never be called when SUPR0EnableVTx() or
1163 * similar was used to enable VT-x on the host.
1164 */
1165VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166{
1167 NOREF(pCpu);
1168 NOREF(pvCpuPage);
1169 NOREF(HCPhysCpuPage);
1170
1171 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172 return hmR0VmxLeaveRootMode();
1173}
1174
1175
1176/**
1177 * Sets the permission bits for the specified MSR in the MSR bitmap.
1178 *
1179 * @param pVCpu The cross context virtual CPU structure.
1180 * @param uMsr The MSR value.
1181 * @param enmRead Whether reading this MSR causes a VM-exit.
1182 * @param enmWrite Whether writing this MSR causes a VM-exit.
1183 */
1184static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185{
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /*
1190 * Layout:
1191 * 0x000 - 0x3ff - Low MSR read bits
1192 * 0x400 - 0x7ff - High MSR read bits
1193 * 0x800 - 0xbff - Low MSR write bits
1194 * 0xc00 - 0xfff - High MSR write bits
1195 */
1196 if (uMsr <= 0x00001FFF)
1197 iBit = uMsr;
1198 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199 {
1200 iBit = uMsr - UINT32_C(0xC0000000);
1201 pbMsrBitmap += 0x400;
1202 }
1203 else
1204 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206 Assert(iBit <= 0x1fff);
1207 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208 ASMBitSet(pbMsrBitmap, iBit);
1209 else
1210 ASMBitClear(pbMsrBitmap, iBit);
1211
1212 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214 else
1215 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216}
1217
1218
1219#ifdef VBOX_STRICT
1220/**
1221 * Gets the permission bits for the specified MSR in the MSR bitmap.
1222 *
1223 * @returns VBox status code.
1224 * @retval VINF_SUCCESS if the specified MSR is found.
1225 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1226 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227 *
1228 * @param pVCpu The cross context virtual CPU structure.
1229 * @param uMsr The MSR.
1230 * @param penmRead Where to store the read permissions.
1231 * @param penmWrite Where to store the write permissions.
1232 */
1233static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234{
1235 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237 int32_t iBit;
1238 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240 /* See hmR0VmxSetMsrPermission() for the layout. */
1241 if (uMsr <= 0x00001FFF)
1242 iBit = uMsr;
1243 else if ( uMsr >= 0xC0000000
1244 && uMsr <= 0xC0001FFF)
1245 {
1246 iBit = (uMsr - 0xC0000000);
1247 pbMsrBitmap += 0x400;
1248 }
1249 else
1250 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252 Assert(iBit <= 0x1fff);
1253 if (ASMBitTest(pbMsrBitmap, iBit))
1254 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1255 else
1256 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260 else
1261 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262 return VINF_SUCCESS;
1263}
1264#endif /* VBOX_STRICT */
1265
1266
1267/**
1268 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269 * area.
1270 *
1271 * @returns VBox status code.
1272 * @param pVCpu The cross context virtual CPU structure.
1273 * @param cMsrs The number of MSRs.
1274 */
1275DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276{
1277 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280 {
1281 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284 }
1285
1286 /* Update number of guest MSRs to load/store across the world-switch. */
1287 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292 AssertRCReturn(rc, rc);
1293
1294 /* Update the VCPU's copy of the MSR count. */
1295 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297 return VINF_SUCCESS;
1298}
1299
1300
1301/**
1302 * Adds a new (or updates the value of an existing) guest/host MSR
1303 * pair to be swapped during the world-switch as part of the
1304 * auto-load/store MSR area in the VMCS.
1305 *
1306 * @returns VBox status code.
1307 * @param pVCpu The cross context virtual CPU structure.
1308 * @param uMsr The MSR.
1309 * @param uGuestMsrValue Value of the guest MSR.
1310 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1311 * necessary.
1312 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313 * its value was updated. Optional, can be NULL.
1314 */
1315static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316 bool *pfAddedAndUpdated)
1317{
1318 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320 uint32_t i;
1321 for (i = 0; i < cMsrs; i++)
1322 {
1323 if (pGuestMsr->u32Msr == uMsr)
1324 break;
1325 pGuestMsr++;
1326 }
1327
1328 bool fAdded = false;
1329 if (i == cMsrs)
1330 {
1331 ++cMsrs;
1332 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339 fAdded = true;
1340 }
1341
1342 /* Update the MSR values in the auto-load/store MSR area. */
1343 pGuestMsr->u32Msr = uMsr;
1344 pGuestMsr->u64Value = uGuestMsrValue;
1345
1346 /* Create/update the MSR slot in the host MSR area. */
1347 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 pHostMsr += i;
1349 pHostMsr->u32Msr = uMsr;
1350
1351 /*
1352 * Update the host MSR only when requested by the caller AND when we're
1353 * adding it to the auto-load/store area. Otherwise, it would have been
1354 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355 */
1356 bool fUpdatedMsrValue = false;
1357 if ( fAdded
1358 && fUpdateHostMsr)
1359 {
1360 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363 fUpdatedMsrValue = true;
1364 }
1365
1366 if (pfAddedAndUpdated)
1367 *pfAddedAndUpdated = fUpdatedMsrValue;
1368 return VINF_SUCCESS;
1369}
1370
1371
1372/**
1373 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1374 * auto-load/store MSR area in the VMCS.
1375 *
1376 * @returns VBox status code.
1377 * @param pVCpu The cross context virtual CPU structure.
1378 * @param uMsr The MSR.
1379 */
1380static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381{
1382 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384 for (uint32_t i = 0; i < cMsrs; i++)
1385 {
1386 /* Find the MSR. */
1387 if (pGuestMsr->u32Msr == uMsr)
1388 {
1389 /* If it's the last MSR, simply reduce the count. */
1390 if (i == cMsrs - 1)
1391 {
1392 --cMsrs;
1393 break;
1394 }
1395
1396 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398 pLastGuestMsr += cMsrs - 1;
1399 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404 pLastHostMsr += cMsrs - 1;
1405 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406 pHostMsr->u64Value = pLastHostMsr->u64Value;
1407 --cMsrs;
1408 break;
1409 }
1410 pGuestMsr++;
1411 }
1412
1413 /* Update the VMCS if the count changed (meaning the MSR was found). */
1414 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415 {
1416 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417 AssertRCReturn(rc, rc);
1418
1419 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424 return VINF_SUCCESS;
1425 }
1426
1427 return VERR_NOT_FOUND;
1428}
1429
1430
1431/**
1432 * Checks if the specified guest MSR is part of the auto-load/store area in
1433 * the VMCS.
1434 *
1435 * @returns true if found, false otherwise.
1436 * @param pVCpu The cross context virtual CPU structure.
1437 * @param uMsr The MSR to find.
1438 */
1439static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440{
1441 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445 {
1446 if (pGuestMsr->u32Msr == uMsr)
1447 return true;
1448 }
1449 return false;
1450}
1451
1452
1453/**
1454 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455 *
1456 * @param pVCpu The cross context virtual CPU structure.
1457 *
1458 * @remarks No-long-jump zone!!!
1459 */
1460static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461{
1462 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468 {
1469 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471 /*
1472 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474 */
1475 if (pHostMsr->u32Msr == MSR_K6_EFER)
1476 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477 else
1478 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479 }
1480
1481 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482}
1483
1484
1485/**
1486 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1487 * perform lazy restoration of the host MSRs while leaving VT-x.
1488 *
1489 * @param pVCpu The cross context virtual CPU structure.
1490 *
1491 * @remarks No-long-jump zone!!!
1492 */
1493static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494{
1495 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497 /*
1498 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499 */
1500 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501 {
1502 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510 }
1511#endif
1512 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1513 }
1514}
1515
1516
1517/**
1518 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1519 * lazily while leaving VT-x.
1520 *
1521 * @returns true if it does, false otherwise.
1522 * @param pVCpu The cross context virtual CPU structure.
1523 * @param uMsr The MSR to check.
1524 */
1525static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526{
1527 NOREF(pVCpu);
1528#if HC_ARCH_BITS == 64
1529 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530 {
1531 switch (uMsr)
1532 {
1533 case MSR_K8_LSTAR:
1534 case MSR_K6_STAR:
1535 case MSR_K8_SF_MASK:
1536 case MSR_K8_KERNEL_GS_BASE:
1537 return true;
1538 }
1539 }
1540#else
1541 RT_NOREF(pVCpu, uMsr);
1542#endif
1543 return false;
1544}
1545
1546
1547/**
1548 * Saves a set of guest MSRs back into the guest-CPU context.
1549 *
1550 * @param pVCpu The cross context virtual CPU structure.
1551 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552 * out-of-sync. Make sure to update the required fields
1553 * before using them.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 */
1557static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558{
1559 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563 {
1564 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565#if HC_ARCH_BITS == 64
1566 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567 {
1568 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572 }
1573#else
1574 NOREF(pMixedCtx);
1575#endif
1576 }
1577}
1578
1579
1580/**
1581 * Loads a set of guests MSRs to allow read/passthru to the guest.
1582 *
1583 * The name of this function is slightly confusing. This function does NOT
1584 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585 * common prefix for functions dealing with "lazy restoration" of the shared
1586 * MSRs.
1587 *
1588 * @param pVCpu The cross context virtual CPU structure.
1589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590 * out-of-sync. Make sure to update the required fields
1591 * before using them.
1592 *
1593 * @remarks No-long-jump zone!!!
1594 */
1595static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596{
1597 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601#if HC_ARCH_BITS == 64
1602 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603 {
1604 /*
1605 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607 * we can skip a few MSR writes.
1608 *
1609 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610 * guest MSR values in the guest-CPU context might be different to what's currently
1611 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612 * CPU, see @bugref{8728}.
1613 */
1614 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619 {
1620#ifdef VBOX_STRICT
1621 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625#endif
1626 }
1627 else
1628 {
1629 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633 }
1634 }
1635#else
1636 RT_NOREF(pMixedCtx);
1637#endif
1638 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1639}
1640
1641
1642/**
1643 * Performs lazy restoration of the set of host MSRs if they were previously
1644 * loaded with guest MSR values.
1645 *
1646 * @param pVCpu The cross context virtual CPU structure.
1647 *
1648 * @remarks No-long-jump zone!!!
1649 * @remarks The guest MSRs should have been saved back into the guest-CPU
1650 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1651 */
1652static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653{
1654 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658 {
1659 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660#if HC_ARCH_BITS == 64
1661 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662 {
1663 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667 }
1668#endif
1669 }
1670 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1671}
1672
1673
1674/**
1675 * Verifies that our cached values of the VMCS controls are all
1676 * consistent with what's actually present in the VMCS.
1677 *
1678 * @returns VBox status code.
1679 * @param pVCpu The cross context virtual CPU structure.
1680 */
1681static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682{
1683 uint32_t u32Val;
1684 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685 AssertRCReturn(rc, rc);
1686 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690 AssertRCReturn(rc, rc);
1691 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695 AssertRCReturn(rc, rc);
1696 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700 AssertRCReturn(rc, rc);
1701 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705 {
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707 AssertRCReturn(rc, rc);
1708 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711 }
1712
1713 return VINF_SUCCESS;
1714}
1715
1716
1717#ifdef VBOX_STRICT
1718/**
1719 * Verifies that our cached host EFER value has not changed
1720 * since we cached it.
1721 *
1722 * @param pVCpu The cross context virtual CPU structure.
1723 */
1724static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725{
1726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729 {
1730 uint64_t u64Val;
1731 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732 AssertRC(rc);
1733
1734 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736 }
1737}
1738
1739
1740/**
1741 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742 * VMCS are correct.
1743 *
1744 * @param pVCpu The cross context virtual CPU structure.
1745 */
1746static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747{
1748 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750 /* Verify MSR counts in the VMCS are what we think it should be. */
1751 uint32_t cMsrs;
1752 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764 {
1765 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767 pGuestMsr->u32Msr, cMsrs));
1768
1769 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773 /* Verify that the permissions are as expected in the MSR bitmap. */
1774 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775 {
1776 VMXMSREXITREAD enmRead;
1777 VMXMSREXITWRITE enmWrite;
1778 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781 {
1782 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784 }
1785 else
1786 {
1787 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788 pGuestMsr->u32Msr, cMsrs));
1789 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790 pGuestMsr->u32Msr, cMsrs));
1791 }
1792 }
1793 }
1794}
1795#endif /* VBOX_STRICT */
1796
1797
1798/**
1799 * Flushes the TLB using EPT.
1800 *
1801 * @returns VBox status code.
1802 * @param pVCpu The cross context virtual CPU structure of the calling
1803 * EMT. Can be NULL depending on @a enmFlush.
1804 * @param enmFlush Type of flush.
1805 *
1806 * @remarks Caller is responsible for making sure this function is called only
1807 * when NestedPaging is supported and providing @a enmFlush that is
1808 * supported by the CPU.
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812{
1813 uint64_t au64Descriptor[2];
1814 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815 au64Descriptor[0] = 0;
1816 else
1817 {
1818 Assert(pVCpu);
1819 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820 }
1821 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825 rc));
1826 if ( RT_SUCCESS(rc)
1827 && pVCpu)
1828 {
1829 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830 }
1831}
1832
1833
1834/**
1835 * Flushes the TLB using VPID.
1836 *
1837 * @returns VBox status code.
1838 * @param pVM The cross context VM structure.
1839 * @param pVCpu The cross context virtual CPU structure of the calling
1840 * EMT. Can be NULL depending on @a enmFlush.
1841 * @param enmFlush Type of flush.
1842 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1843 * on @a enmFlush).
1844 *
1845 * @remarks Can be called with interrupts disabled.
1846 */
1847static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848{
1849 NOREF(pVM);
1850 AssertPtr(pVM);
1851 Assert(pVM->hm.s.vmx.fVpid);
1852
1853 uint64_t au64Descriptor[2];
1854 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855 {
1856 au64Descriptor[0] = 0;
1857 au64Descriptor[1] = 0;
1858 }
1859 else
1860 {
1861 AssertPtr(pVCpu);
1862 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865 au64Descriptor[1] = GCPtr;
1866 }
1867
1868 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869 AssertMsg(rc == VINF_SUCCESS,
1870 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871 if ( RT_SUCCESS(rc)
1872 && pVCpu)
1873 {
1874 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875 }
1876}
1877
1878
1879/**
1880 * Invalidates a guest page by guest virtual address. Only relevant for
1881 * EPT/VPID, otherwise there is nothing really to invalidate.
1882 *
1883 * @returns VBox status code.
1884 * @param pVM The cross context VM structure.
1885 * @param pVCpu The cross context virtual CPU structure.
1886 * @param GCVirt Guest virtual address of the page to invalidate.
1887 */
1888VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889{
1890 AssertPtr(pVM);
1891 AssertPtr(pVCpu);
1892 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895 if (!fFlushPending)
1896 {
1897 /*
1898 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899 * See @bugref{6043} and @bugref{6177}.
1900 *
1901 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902 * function maybe called in a loop with individual addresses.
1903 */
1904 if (pVM->hm.s.vmx.fVpid)
1905 {
1906 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1907 {
1908 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1909 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1910 }
1911 else
1912 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1913 }
1914 else if (pVM->hm.s.fNestedPaging)
1915 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1916 }
1917
1918 return VINF_SUCCESS;
1919}
1920
1921
1922/**
1923 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1924 * otherwise there is nothing really to invalidate.
1925 *
1926 * @returns VBox status code.
1927 * @param pVM The cross context VM structure.
1928 * @param pVCpu The cross context virtual CPU structure.
1929 * @param GCPhys Guest physical address of the page to invalidate.
1930 */
1931VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1932{
1933 NOREF(pVM); NOREF(GCPhys);
1934 LogFlowFunc(("%RGp\n", GCPhys));
1935
1936 /*
1937 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1938 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1939 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1940 */
1941 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1942 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1943 return VINF_SUCCESS;
1944}
1945
1946
1947/**
1948 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1949 * case where neither EPT nor VPID is supported by the CPU.
1950 *
1951 * @param pVM The cross context VM structure.
1952 * @param pVCpu The cross context virtual CPU structure.
1953 * @param pCpu Pointer to the global HM struct.
1954 *
1955 * @remarks Called with interrupts disabled.
1956 */
1957static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1958{
1959 AssertPtr(pVCpu);
1960 AssertPtr(pCpu);
1961 NOREF(pVM);
1962
1963 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1964
1965 Assert(pCpu->idCpu != NIL_RTCPUID);
1966 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968 pVCpu->hm.s.fForceTLBFlush = false;
1969 return;
1970}
1971
1972
1973/**
1974 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1975 *
1976 * @param pVM The cross context VM structure.
1977 * @param pVCpu The cross context virtual CPU structure.
1978 * @param pCpu Pointer to the global HM CPU struct.
1979 * @remarks All references to "ASID" in this function pertains to "VPID" in
1980 * Intel's nomenclature. The reason is, to avoid confusion in compare
1981 * statements since the host-CPU copies are named "ASID".
1982 *
1983 * @remarks Called with interrupts disabled.
1984 */
1985static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1986{
1987#ifdef VBOX_WITH_STATISTICS
1988 bool fTlbFlushed = false;
1989# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1990# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1991 if (!fTlbFlushed) \
1992 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1993 } while (0)
1994#else
1995# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1996# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1997#endif
1998
1999 AssertPtr(pVM);
2000 AssertPtr(pCpu);
2001 AssertPtr(pVCpu);
2002 Assert(pCpu->idCpu != NIL_RTCPUID);
2003
2004 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2005 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2006 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2007
2008 /*
2009 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2010 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2011 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2012 */
2013 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2014 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2015 {
2016 ++pCpu->uCurrentAsid;
2017 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2018 {
2019 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2020 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2021 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2022 }
2023
2024 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2025 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2026 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2027
2028 /*
2029 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2030 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2031 */
2032 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 HMVMX_SET_TAGGED_TLB_FLUSHED();
2035 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2036 }
2037
2038 /* Check for explicit TLB flushes. */
2039 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040 {
2041 /*
2042 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2043 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2044 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2045 * but not guest-physical mappings.
2046 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2047 */
2048 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 HMVMX_SET_TAGGED_TLB_FLUSHED();
2051 }
2052
2053 pVCpu->hm.s.fForceTLBFlush = false;
2054 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2055
2056 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2057 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2058 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2059 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2060 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2061 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2062 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2063 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2064 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2065
2066 /* Update VMCS with the VPID. */
2067 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2068 AssertRC(rc);
2069
2070#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2071}
2072
2073
2074/**
2075 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2076 *
2077 * @returns VBox status code.
2078 * @param pVM The cross context VM structure.
2079 * @param pVCpu The cross context virtual CPU structure.
2080 * @param pCpu Pointer to the global HM CPU struct.
2081 *
2082 * @remarks Called with interrupts disabled.
2083 */
2084static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085{
2086 AssertPtr(pVM);
2087 AssertPtr(pVCpu);
2088 AssertPtr(pCpu);
2089 Assert(pCpu->idCpu != NIL_RTCPUID);
2090 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2091 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2092
2093 /*
2094 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2095 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2096 */
2097 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099 {
2100 pVCpu->hm.s.fForceTLBFlush = true;
2101 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102 }
2103
2104 /* Check for explicit TLB flushes. */
2105 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109 }
2110
2111 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2117 pVCpu->hm.s.fForceTLBFlush = false;
2118 }
2119}
2120
2121
2122/**
2123 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2124 *
2125 * @returns VBox status code.
2126 * @param pVM The cross context VM structure.
2127 * @param pVCpu The cross context virtual CPU structure.
2128 * @param pCpu Pointer to the global HM CPU struct.
2129 *
2130 * @remarks Called with interrupts disabled.
2131 */
2132static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2133{
2134 AssertPtr(pVM);
2135 AssertPtr(pVCpu);
2136 AssertPtr(pCpu);
2137 Assert(pCpu->idCpu != NIL_RTCPUID);
2138 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2139 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2140
2141 /*
2142 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2143 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2144 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2145 */
2146 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2147 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2148 {
2149 pVCpu->hm.s.fForceTLBFlush = true;
2150 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2151 }
2152
2153 /* Check for explicit TLB flushes. */
2154 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2155 {
2156 /*
2157 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2158 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2159 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2160 */
2161 pVCpu->hm.s.fForceTLBFlush = true;
2162 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2163 }
2164
2165 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2166 if (pVCpu->hm.s.fForceTLBFlush)
2167 {
2168 ++pCpu->uCurrentAsid;
2169 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2170 {
2171 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2172 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2173 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2174 }
2175
2176 pVCpu->hm.s.fForceTLBFlush = false;
2177 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2178 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2179 if (pCpu->fFlushAsidBeforeUse)
2180 {
2181 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2182 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2183 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2184 {
2185 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2186 pCpu->fFlushAsidBeforeUse = false;
2187 }
2188 else
2189 {
2190 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2191 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2192 }
2193 }
2194 }
2195
2196 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2197 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2198 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2199 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2200 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2201 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2202 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2203
2204 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2205 AssertRC(rc);
2206}
2207
2208
2209/**
2210 * Flushes the guest TLB entry based on CPU capabilities.
2211 *
2212 * @param pVCpu The cross context virtual CPU structure.
2213 * @param pCpu Pointer to the global HM CPU struct.
2214 */
2215DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2216{
2217#ifdef HMVMX_ALWAYS_FLUSH_TLB
2218 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2219#endif
2220 PVM pVM = pVCpu->CTX_SUFF(pVM);
2221 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2222 {
2223 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2224 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2225 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2226 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2227 default:
2228 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2229 break;
2230 }
2231
2232 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2233}
2234
2235
2236/**
2237 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2238 * TLB entries from the host TLB before VM-entry.
2239 *
2240 * @returns VBox status code.
2241 * @param pVM The cross context VM structure.
2242 */
2243static int hmR0VmxSetupTaggedTlb(PVM pVM)
2244{
2245 /*
2246 * Determine optimal flush type for Nested Paging.
2247 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2248 * guest execution (see hmR3InitFinalizeR0()).
2249 */
2250 if (pVM->hm.s.fNestedPaging)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2261 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264 }
2265
2266 /* Make sure the write-back cacheable memory type for EPT is supported. */
2267 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2268 {
2269 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272 }
2273
2274 /* EPT requires a page-walk length of 4. */
2275 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2276 {
2277 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2279 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280 }
2281 }
2282 else
2283 {
2284 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2285 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2287 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288 }
2289 }
2290
2291 /*
2292 * Determine optimal flush type for VPID.
2293 */
2294 if (pVM->hm.s.vmx.fVpid)
2295 {
2296 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2297 {
2298 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2299 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2300 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2301 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2302 else
2303 {
2304 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2305 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2306 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2307 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2308 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2309 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2310 pVM->hm.s.vmx.fVpid = false;
2311 }
2312 }
2313 else
2314 {
2315 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2316 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2317 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318 pVM->hm.s.vmx.fVpid = false;
2319 }
2320 }
2321
2322 /*
2323 * Setup the handler for flushing tagged-TLBs.
2324 */
2325 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2326 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2327 else if (pVM->hm.s.fNestedPaging)
2328 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2329 else if (pVM->hm.s.vmx.fVpid)
2330 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2331 else
2332 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2333 return VINF_SUCCESS;
2334}
2335
2336
2337/**
2338 * Sets up pin-based VM-execution controls in the VMCS.
2339 *
2340 * @returns VBox status code.
2341 * @param pVM The cross context VM structure.
2342 * @param pVCpu The cross context virtual CPU structure.
2343 */
2344static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2345{
2346 AssertPtr(pVM);
2347 AssertPtr(pVCpu);
2348
2349 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2350 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2351
2352 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2353 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2354
2355 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2356 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2357
2358 /* Enable the VMX preemption timer. */
2359 if (pVM->hm.s.vmx.fUsePreemptTimer)
2360 {
2361 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2362 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2363 }
2364
2365#if 0
2366 /* Enable posted-interrupt processing. */
2367 if (pVM->hm.s.fPostedIntrs)
2368 {
2369 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2370 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2371 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2372 }
2373#endif
2374
2375 if ((val & zap) != val)
2376 {
2377 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2378 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2379 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2380 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381 }
2382
2383 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2384 AssertRCReturn(rc, rc);
2385
2386 pVCpu->hm.s.vmx.u32PinCtls = val;
2387 return rc;
2388}
2389
2390
2391/**
2392 * Sets up processor-based VM-execution controls in the VMCS.
2393 *
2394 * @returns VBox status code.
2395 * @param pVM The cross context VM structure.
2396 * @param pVCpu The cross context virtual CPU structure.
2397 */
2398static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2399{
2400 AssertPtr(pVM);
2401 AssertPtr(pVCpu);
2402
2403 int rc = VERR_INTERNAL_ERROR_5;
2404 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2405 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2406
2407 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2408 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2409 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2410 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2411 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2412 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2413 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2414
2415 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2416 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2417 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2418 {
2419 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2420 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2421 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422 }
2423
2424 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2425 if (!pVM->hm.s.fNestedPaging)
2426 {
2427 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2428 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2429 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2430 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2431 }
2432
2433 /* Use TPR shadowing if supported by the CPU. */
2434 if ( PDMHasApic(pVM)
2435 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2436 {
2437 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2438 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2439 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2440 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2441 AssertRCReturn(rc, rc);
2442
2443 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2444 /* CR8 writes cause a VM-exit based on TPR threshold. */
2445 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2446 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2447 }
2448 else
2449 {
2450 /*
2451 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2452 * Set this control only for 64-bit guests.
2453 */
2454 if (pVM->hm.s.fAllow64BitGuests)
2455 {
2456 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2458 }
2459 }
2460
2461 /* Use MSR-bitmaps if supported by the CPU. */
2462 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2463 {
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2465
2466 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2467 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2468 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2469 AssertRCReturn(rc, rc);
2470
2471 /*
2472 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2473 * automatically using dedicated fields in the VMCS.
2474 */
2475 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2476 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2478 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2479 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2480
2481#if HC_ARCH_BITS == 64
2482 /*
2483 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2484 */
2485 if (pVM->hm.s.fAllow64BitGuests)
2486 {
2487 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491 }
2492#endif
2493 /*
2494 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2495 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2496 */
2497 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2498 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2499
2500 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2501 }
2502
2503 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2504 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2505 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2506
2507 if ((val & zap) != val)
2508 {
2509 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2510 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2511 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2512 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2513 }
2514
2515 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2516 AssertRCReturn(rc, rc);
2517
2518 pVCpu->hm.s.vmx.u32ProcCtls = val;
2519
2520 /*
2521 * Secondary processor-based VM-execution controls.
2522 */
2523 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2524 {
2525 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2526 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2527
2528 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2529 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2530
2531 if (pVM->hm.s.fNestedPaging)
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2533
2534 /*
2535 * Enable the INVPCID instruction if supported by the hardware and we expose
2536 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2537 */
2538 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2539 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2540 {
2541 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2542 }
2543
2544 if (pVM->hm.s.vmx.fVpid)
2545 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2546
2547 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2548 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2549
2550#if 0
2551 if (pVM->hm.s.fVirtApicRegs)
2552 {
2553 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2554 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2555
2556 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2557 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2558 }
2559#endif
2560
2561 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2562 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2563 * done dynamically. */
2564 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2565 {
2566 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2567 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2568 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2570 AssertRCReturn(rc, rc);
2571 }
2572
2573 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2574 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2575
2576 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2577 && pVM->hm.s.vmx.cPleGapTicks
2578 && pVM->hm.s.vmx.cPleWindowTicks)
2579 {
2580 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2581
2582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2583 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2584 AssertRCReturn(rc, rc);
2585 }
2586
2587 if ((val & zap) != val)
2588 {
2589 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2590 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2591 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2592 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2593 }
2594
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2596 AssertRCReturn(rc, rc);
2597
2598 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2599 }
2600 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2601 {
2602 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2603 "available\n"));
2604 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2605 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2606 }
2607
2608 return VINF_SUCCESS;
2609}
2610
2611
2612/**
2613 * Sets up miscellaneous (everything other than Pin & Processor-based
2614 * VM-execution) control fields in the VMCS.
2615 *
2616 * @returns VBox status code.
2617 * @param pVM The cross context VM structure.
2618 * @param pVCpu The cross context virtual CPU structure.
2619 */
2620static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2621{
2622 NOREF(pVM);
2623 AssertPtr(pVM);
2624 AssertPtr(pVCpu);
2625
2626 int rc = VERR_GENERAL_FAILURE;
2627
2628 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2629#if 0
2630 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2631 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2632 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2633
2634 /*
2635 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2636 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2637 * We thus use the exception bitmap to control it rather than use both.
2638 */
2639 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2640 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2641
2642 /** @todo Explore possibility of using IO-bitmaps. */
2643 /* All IO & IOIO instructions cause VM-exits. */
2644 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2645 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2646
2647 /* Initialize the MSR-bitmap area. */
2648 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2649 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2650 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2651 AssertRCReturn(rc, rc);
2652#endif
2653
2654 /* Setup MSR auto-load/store area. */
2655 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2656 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2657 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2658 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2659 AssertRCReturn(rc, rc);
2660
2661 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2662 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2663 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2664 AssertRCReturn(rc, rc);
2665
2666 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2667 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2668 AssertRCReturn(rc, rc);
2669
2670 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2671#if 0
2672 /* Setup debug controls */
2673 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2674 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2675 AssertRCReturn(rc, rc);
2676#endif
2677
2678 return rc;
2679}
2680
2681
2682/**
2683 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2684 *
2685 * We shall setup those exception intercepts that don't change during the
2686 * lifetime of the VM here. The rest are done dynamically while loading the
2687 * guest state.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM The cross context VM structure.
2691 * @param pVCpu The cross context virtual CPU structure.
2692 */
2693static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2694{
2695 AssertPtr(pVM);
2696 AssertPtr(pVCpu);
2697
2698 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2699
2700 uint32_t u32XcptBitmap = 0;
2701
2702 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2703 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2704
2705 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2706 and writes, and because recursive #DBs can cause the CPU hang, we must always
2707 intercept #DB. */
2708 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2709
2710 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2711 if (!pVM->hm.s.fNestedPaging)
2712 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2713
2714 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2715 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2716 AssertRCReturn(rc, rc);
2717 return rc;
2718}
2719
2720
2721/**
2722 * Sets up the initial guest-state mask. The guest-state mask is consulted
2723 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2724 * for the nested virtualization case (as it would cause a VM-exit).
2725 *
2726 * @param pVCpu The cross context virtual CPU structure.
2727 */
2728static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2729{
2730 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2731 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2732 return VINF_SUCCESS;
2733}
2734
2735
2736/**
2737 * Does per-VM VT-x initialization.
2738 *
2739 * @returns VBox status code.
2740 * @param pVM The cross context VM structure.
2741 */
2742VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2743{
2744 LogFlowFunc(("pVM=%p\n", pVM));
2745
2746 int rc = hmR0VmxStructsAlloc(pVM);
2747 if (RT_FAILURE(rc))
2748 {
2749 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2750 return rc;
2751 }
2752
2753 return VINF_SUCCESS;
2754}
2755
2756
2757/**
2758 * Does per-VM VT-x termination.
2759 *
2760 * @returns VBox status code.
2761 * @param pVM The cross context VM structure.
2762 */
2763VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2764{
2765 LogFlowFunc(("pVM=%p\n", pVM));
2766
2767#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2768 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2769 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2770#endif
2771 hmR0VmxStructsFree(pVM);
2772 return VINF_SUCCESS;
2773}
2774
2775
2776/**
2777 * Sets up the VM for execution under VT-x.
2778 * This function is only called once per-VM during initialization.
2779 *
2780 * @returns VBox status code.
2781 * @param pVM The cross context VM structure.
2782 */
2783VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2784{
2785 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2786 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2787
2788 LogFlowFunc(("pVM=%p\n", pVM));
2789
2790 /*
2791 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2792 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2793 */
2794 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2795 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2796 || !pVM->hm.s.vmx.pRealModeTSS))
2797 {
2798 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2799 return VERR_INTERNAL_ERROR;
2800 }
2801
2802 /* Initialize these always, see hmR3InitFinalizeR0().*/
2803 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2804 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2805
2806 /* Setup the tagged-TLB flush handlers. */
2807 int rc = hmR0VmxSetupTaggedTlb(pVM);
2808 if (RT_FAILURE(rc))
2809 {
2810 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2811 return rc;
2812 }
2813
2814 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2815 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2816#if HC_ARCH_BITS == 64
2817 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2818 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2819 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2820 {
2821 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2822 }
2823#endif
2824
2825 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2826 RTCCUINTREG uHostCR4 = ASMGetCR4();
2827 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2828 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2829
2830 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2831 {
2832 PVMCPU pVCpu = &pVM->aCpus[i];
2833 AssertPtr(pVCpu);
2834 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2835
2836 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2837 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2838
2839 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2840 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2841 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2842
2843 /* Set revision dword at the beginning of the VMCS structure. */
2844 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2845
2846 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2847 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2848 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2849 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2850
2851 /* Load this VMCS as the current VMCS. */
2852 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2853 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2854 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2855
2856 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2857 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2858 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2859
2860 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2861 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2862 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2863
2864 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2865 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2866 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2867
2868 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2869 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2870 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2871
2872 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2873 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2874 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2875
2876#if HC_ARCH_BITS == 32
2877 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2878 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2879 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2880#endif
2881
2882 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2883 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2884 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2885 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2886
2887 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2888
2889 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2890 }
2891
2892 return VINF_SUCCESS;
2893}
2894
2895
2896/**
2897 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2898 * the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM The cross context VM structure.
2902 * @param pVCpu The cross context virtual CPU structure.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 NOREF(pVM); NOREF(pVCpu);
2907
2908 RTCCUINTREG uReg = ASMGetCR0();
2909 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2910 AssertRCReturn(rc, rc);
2911
2912 uReg = ASMGetCR3();
2913 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2914 AssertRCReturn(rc, rc);
2915
2916 uReg = ASMGetCR4();
2917 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2918 AssertRCReturn(rc, rc);
2919 return rc;
2920}
2921
2922
2923#if HC_ARCH_BITS == 64
2924/**
2925 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2926 * requirements. See hmR0VmxSaveHostSegmentRegs().
2927 */
2928# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2929 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2930 { \
2931 bool fValidSelector = true; \
2932 if ((selValue) & X86_SEL_LDT) \
2933 { \
2934 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2935 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2936 } \
2937 if (fValidSelector) \
2938 { \
2939 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2940 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2941 } \
2942 (selValue) = 0; \
2943 }
2944#endif
2945
2946
2947/**
2948 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2949 * the host-state area in the VMCS.
2950 *
2951 * @returns VBox status code.
2952 * @param pVM The cross context VM structure.
2953 * @param pVCpu The cross context virtual CPU structure.
2954 */
2955DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2956{
2957 int rc = VERR_INTERNAL_ERROR_5;
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2962 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2963 *
2964 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2965 * Was observed booting Solaris10u10 32-bit guest.
2966 */
2967 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2968 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2969 {
2970 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2971 pVCpu->idCpu));
2972 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2973 }
2974 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2975#else
2976 RT_NOREF(pVCpu);
2977#endif
2978
2979 /*
2980 * Host DS, ES, FS and GS segment registers.
2981 */
2982#if HC_ARCH_BITS == 64
2983 RTSEL uSelDS = ASMGetDS();
2984 RTSEL uSelES = ASMGetES();
2985 RTSEL uSelFS = ASMGetFS();
2986 RTSEL uSelGS = ASMGetGS();
2987#else
2988 RTSEL uSelDS = 0;
2989 RTSEL uSelES = 0;
2990 RTSEL uSelFS = 0;
2991 RTSEL uSelGS = 0;
2992#endif
2993
2994 /*
2995 * Host CS and SS segment registers.
2996 */
2997 RTSEL uSelCS = ASMGetCS();
2998 RTSEL uSelSS = ASMGetSS();
2999
3000 /*
3001 * Host TR segment register.
3002 */
3003 RTSEL uSelTR = ASMGetTR();
3004
3005#if HC_ARCH_BITS == 64
3006 /*
3007 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3008 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3009 */
3010 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3011 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3012 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3013 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3014# undef VMXLOCAL_ADJUST_HOST_SEG
3015#endif
3016
3017 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3018 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3019 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3020 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3021 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3022 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3023 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3024 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3025 Assert(uSelCS);
3026 Assert(uSelTR);
3027
3028 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3029#if 0
3030 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3031 Assert(uSelSS != 0);
3032#endif
3033
3034 /* Write these host selector fields into the host-state area in the VMCS. */
3035 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3036 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3037#if HC_ARCH_BITS == 64
3038 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3039 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3040 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3041 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3042#else
3043 NOREF(uSelDS);
3044 NOREF(uSelES);
3045 NOREF(uSelFS);
3046 NOREF(uSelGS);
3047#endif
3048 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3049 AssertRCReturn(rc, rc);
3050
3051 /*
3052 * Host GDTR and IDTR.
3053 */
3054 RTGDTR Gdtr;
3055 RTIDTR Idtr;
3056 RT_ZERO(Gdtr);
3057 RT_ZERO(Idtr);
3058 ASMGetGDTR(&Gdtr);
3059 ASMGetIDTR(&Idtr);
3060 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3061 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3062 AssertRCReturn(rc, rc);
3063
3064#if HC_ARCH_BITS == 64
3065 /*
3066 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3067 * maximum limit (0xffff) on every VM-exit.
3068 */
3069 if (Gdtr.cbGdt != 0xffff)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3071
3072 /*
3073 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3074 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3075 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3076 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3077 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3078 * hosts where we are pretty sure it won't cause trouble.
3079 */
3080# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3081 if (Idtr.cbIdt < 0x0fff)
3082# else
3083 if (Idtr.cbIdt != 0xffff)
3084# endif
3085 {
3086 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3087 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3088 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3089 }
3090#endif
3091
3092 /*
3093 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3094 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3095 */
3096 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3097 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3098 VERR_VMX_INVALID_HOST_STATE);
3099
3100 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3101#if HC_ARCH_BITS == 64
3102 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3103
3104 /*
3105 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3106 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3107 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3108 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3109 *
3110 * [1] See Intel spec. 3.5 "System Descriptor Types".
3111 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3112 */
3113 Assert(pDesc->System.u4Type == 11);
3114 if ( pDesc->System.u16LimitLow != 0x67
3115 || pDesc->System.u4LimitHigh)
3116 {
3117 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3118 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3119 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3120 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3121 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3122 }
3123
3124 /*
3125 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3126 */
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3128 {
3129 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3130 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3131 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3132 {
3133 /* The GDT is read-only but the writable GDT is available. */
3134 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3135 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3136 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3137 AssertRCReturn(rc, rc);
3138 }
3139 }
3140#else
3141 NOREF(pVM);
3142 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3143#endif
3144 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3145 AssertRCReturn(rc, rc);
3146
3147 /*
3148 * Host FS base and GS base.
3149 */
3150#if HC_ARCH_BITS == 64
3151 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3152 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3153 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3154 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3155 AssertRCReturn(rc, rc);
3156
3157 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3158 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3159 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3160 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3161 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3162#endif
3163 return rc;
3164}
3165
3166
3167/**
3168 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3169 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3170 * the host after every successful VM-exit.
3171 *
3172 * @returns VBox status code.
3173 * @param pVM The cross context VM structure.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 *
3176 * @remarks No-long-jump zone!!!
3177 */
3178DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3179{
3180 NOREF(pVM);
3181
3182 AssertPtr(pVCpu);
3183 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3184
3185 /*
3186 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3187 * rather than swapping them on every VM-entry.
3188 */
3189 hmR0VmxLazySaveHostMsrs(pVCpu);
3190
3191 /*
3192 * Host Sysenter MSRs.
3193 */
3194 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3195#if HC_ARCH_BITS == 32
3196 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3197 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3198#else
3199 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3200 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3201#endif
3202 AssertRCReturn(rc, rc);
3203
3204 /*
3205 * Host EFER MSR.
3206 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3207 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3208 */
3209 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3210 {
3211 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3212 AssertRCReturn(rc, rc);
3213 }
3214
3215 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3216 * hmR0VmxLoadGuestExitCtls() !! */
3217
3218 return rc;
3219}
3220
3221
3222/**
3223 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3224 *
3225 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3226 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3227 * hmR0VMxLoadGuestEntryCtls().
3228 *
3229 * @returns true if we need to load guest EFER, false otherwise.
3230 * @param pVCpu The cross context virtual CPU structure.
3231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3232 * out-of-sync. Make sure to update the required fields
3233 * before using them.
3234 *
3235 * @remarks Requires EFER, CR4.
3236 * @remarks No-long-jump zone!!!
3237 */
3238static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3239{
3240#ifdef HMVMX_ALWAYS_SWAP_EFER
3241 return true;
3242#endif
3243
3244#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3245 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3246 if (CPUMIsGuestInLongMode(pVCpu))
3247 return false;
3248#endif
3249
3250 PVM pVM = pVCpu->CTX_SUFF(pVM);
3251 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3252 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3253
3254 /*
3255 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3256 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3257 */
3258 if ( CPUMIsGuestInLongMode(pVCpu)
3259 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3260 {
3261 return true;
3262 }
3263
3264 /*
3265 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3266 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3267 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3268 */
3269 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3270 && (pMixedCtx->cr0 & X86_CR0_PG)
3271 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3272 {
3273 /* Assert that host is PAE capable. */
3274 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3275 return true;
3276 }
3277
3278 /** @todo Check the latest Intel spec. for any other bits,
3279 * like SMEP/SMAP? */
3280 return false;
3281}
3282
3283
3284/**
3285 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3286 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3287 * controls".
3288 *
3289 * @returns VBox status code.
3290 * @param pVCpu The cross context virtual CPU structure.
3291 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3292 * out-of-sync. Make sure to update the required fields
3293 * before using them.
3294 *
3295 * @remarks Requires EFER.
3296 * @remarks No-long-jump zone!!!
3297 */
3298DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3299{
3300 int rc = VINF_SUCCESS;
3301 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3302 {
3303 PVM pVM = pVCpu->CTX_SUFF(pVM);
3304 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3305 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3306
3307 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3308 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3309
3310 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3311 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3312 {
3313 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3314 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3315 }
3316 else
3317 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3318
3319 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3320 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322 {
3323 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3324 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3325 }
3326
3327 /*
3328 * The following should -not- be set (since we're not in SMM mode):
3329 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3330 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3331 */
3332
3333 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3334 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3335
3336 if ((val & zap) != val)
3337 {
3338 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3339 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3340 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3341 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3342 }
3343
3344 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3345 AssertRCReturn(rc, rc);
3346
3347 pVCpu->hm.s.vmx.u32EntryCtls = val;
3348 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3349 }
3350 return rc;
3351}
3352
3353
3354/**
3355 * Sets up the VM-exit controls in the VMCS.
3356 *
3357 * @returns VBox status code.
3358 * @param pVCpu The cross context virtual CPU structure.
3359 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360 * out-of-sync. Make sure to update the required fields
3361 * before using them.
3362 *
3363 * @remarks Requires EFER.
3364 */
3365DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366{
3367 NOREF(pMixedCtx);
3368
3369 int rc = VINF_SUCCESS;
3370 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3371 {
3372 PVM pVM = pVCpu->CTX_SUFF(pVM);
3373 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3374 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3375
3376 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3377 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3378
3379 /*
3380 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3381 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3382 */
3383#if HC_ARCH_BITS == 64
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386#else
3387 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3388 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3389 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3390 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3391 {
3392 /* The switcher returns to long mode, EFER is managed by the switcher. */
3393 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3394 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3395 }
3396 else
3397 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3398#endif
3399
3400 /* If the newer VMCS fields for managing EFER exists, use it. */
3401 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3402 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3403 {
3404 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3405 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3406 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3407 }
3408
3409 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3410 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3411
3412 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3413 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3414 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3415
3416 if ( pVM->hm.s.vmx.fUsePreemptTimer
3417 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3418 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3419
3420 if ((val & zap) != val)
3421 {
3422 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3423 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3424 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3425 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3426 }
3427
3428 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3429 AssertRCReturn(rc, rc);
3430
3431 pVCpu->hm.s.vmx.u32ExitCtls = val;
3432 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3433 }
3434 return rc;
3435}
3436
3437
3438/**
3439 * Sets the TPR threshold in the VMCS.
3440 *
3441 * @returns VBox status code.
3442 * @param pVCpu The cross context virtual CPU structure.
3443 * @param u32TprThreshold The TPR threshold (task-priority class only).
3444 */
3445DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3446{
3447 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3448 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3449 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3450}
3451
3452
3453/**
3454 * Loads the guest APIC and related state.
3455 *
3456 * @returns VBox status code.
3457 * @param pVCpu The cross context virtual CPU structure.
3458 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3459 * out-of-sync. Make sure to update the required fields
3460 * before using them.
3461 *
3462 * @remarks No-long-jump zone!!!
3463 */
3464DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3465{
3466 NOREF(pMixedCtx);
3467
3468 int rc = VINF_SUCCESS;
3469 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3470 {
3471 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3472 && APICIsEnabled(pVCpu))
3473 {
3474 /*
3475 * Setup TPR shadowing.
3476 */
3477 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3478 {
3479 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3480
3481 bool fPendingIntr = false;
3482 uint8_t u8Tpr = 0;
3483 uint8_t u8PendingIntr = 0;
3484 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3485 AssertRCReturn(rc, rc);
3486
3487 /*
3488 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3489 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3490 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3491 */
3492 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3493 uint32_t u32TprThreshold = 0;
3494 if (fPendingIntr)
3495 {
3496 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3497 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3498 const uint8_t u8TprPriority = u8Tpr >> 4;
3499 if (u8PendingPriority <= u8TprPriority)
3500 u32TprThreshold = u8PendingPriority;
3501 }
3502
3503 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3504 AssertRCReturn(rc, rc);
3505 }
3506 }
3507 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3508 }
3509
3510 return rc;
3511}
3512
3513
3514/**
3515 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3516 *
3517 * @returns Guest's interruptibility-state.
3518 * @param pVCpu The cross context virtual CPU structure.
3519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3520 * out-of-sync. Make sure to update the required fields
3521 * before using them.
3522 *
3523 * @remarks No-long-jump zone!!!
3524 */
3525DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3526{
3527 /*
3528 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3529 */
3530 uint32_t uIntrState = 0;
3531 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3532 {
3533 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3534 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3535 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3536 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3537 {
3538 if (pMixedCtx->eflags.Bits.u1IF)
3539 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3540 else
3541 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3542 }
3543 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3544 {
3545 /*
3546 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3547 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3548 */
3549 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3550 }
3551 }
3552
3553 /*
3554 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3555 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3556 * setting this would block host-NMIs and IRET will not clear the blocking.
3557 *
3558 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3559 */
3560 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3561 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3562 {
3563 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3564 }
3565
3566 return uIntrState;
3567}
3568
3569
3570/**
3571 * Loads the guest's interruptibility-state into the guest-state area in the
3572 * VMCS.
3573 *
3574 * @returns VBox status code.
3575 * @param pVCpu The cross context virtual CPU structure.
3576 * @param uIntrState The interruptibility-state to set.
3577 */
3578static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3579{
3580 NOREF(pVCpu);
3581 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3582 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3583 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3584 AssertRC(rc);
3585 return rc;
3586}
3587
3588
3589/**
3590 * Loads the exception intercepts required for guest execution in the VMCS.
3591 *
3592 * @returns VBox status code.
3593 * @param pVCpu The cross context virtual CPU structure.
3594 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3595 * out-of-sync. Make sure to update the required fields
3596 * before using them.
3597 */
3598static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3599{
3600 NOREF(pMixedCtx);
3601 int rc = VINF_SUCCESS;
3602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3603 {
3604 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3605 if (pVCpu->hm.s.fGIMTrapXcptUD)
3606 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3607#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3608 else
3609 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3610#endif
3611
3612 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3613 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3614
3615 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3616 AssertRCReturn(rc, rc);
3617
3618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3619 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3620 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3621 }
3622 return rc;
3623}
3624
3625
3626/**
3627 * Loads the guest's RIP into the guest-state area in the VMCS.
3628 *
3629 * @returns VBox status code.
3630 * @param pVCpu The cross context virtual CPU structure.
3631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3632 * out-of-sync. Make sure to update the required fields
3633 * before using them.
3634 *
3635 * @remarks No-long-jump zone!!!
3636 */
3637static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3638{
3639 int rc = VINF_SUCCESS;
3640 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3641 {
3642 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3643 AssertRCReturn(rc, rc);
3644
3645 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3646 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3647 HMCPU_CF_VALUE(pVCpu)));
3648 }
3649 return rc;
3650}
3651
3652
3653/**
3654 * Loads the guest's RSP into the guest-state area in the VMCS.
3655 *
3656 * @returns VBox status code.
3657 * @param pVCpu The cross context virtual CPU structure.
3658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3659 * out-of-sync. Make sure to update the required fields
3660 * before using them.
3661 *
3662 * @remarks No-long-jump zone!!!
3663 */
3664static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665{
3666 int rc = VINF_SUCCESS;
3667 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3668 {
3669 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3670 AssertRCReturn(rc, rc);
3671
3672 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3673 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3674 }
3675 return rc;
3676}
3677
3678
3679/**
3680 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3681 *
3682 * @returns VBox status code.
3683 * @param pVCpu The cross context virtual CPU structure.
3684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685 * out-of-sync. Make sure to update the required fields
3686 * before using them.
3687 *
3688 * @remarks No-long-jump zone!!!
3689 */
3690static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3691{
3692 int rc = VINF_SUCCESS;
3693 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3694 {
3695 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3696 Let us assert it as such and use 32-bit VMWRITE. */
3697 Assert(!(pMixedCtx->rflags.u64 >> 32));
3698 X86EFLAGS Eflags = pMixedCtx->eflags;
3699 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3700 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3701 * These will never be cleared/set, unless some other part of the VMM
3702 * code is buggy - in which case we're better of finding and fixing
3703 * those bugs than hiding them. */
3704 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3705 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3706 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3707 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3708
3709 /*
3710 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3711 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3712 */
3713 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3714 {
3715 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3716 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3717 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3718 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3719 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3720 }
3721
3722 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3723 AssertRCReturn(rc, rc);
3724
3725 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3726 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3727 }
3728 return rc;
3729}
3730
3731
3732/**
3733 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3734 *
3735 * @returns VBox status code.
3736 * @param pVCpu The cross context virtual CPU structure.
3737 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3738 * out-of-sync. Make sure to update the required fields
3739 * before using them.
3740 *
3741 * @remarks No-long-jump zone!!!
3742 */
3743DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3744{
3745 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3746 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3747 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3748 AssertRCReturn(rc, rc);
3749 return rc;
3750}
3751
3752
3753/**
3754 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3755 * CR0 is partially shared with the host and we have to consider the FPU bits.
3756 *
3757 * @returns VBox status code.
3758 * @param pVCpu The cross context virtual CPU structure.
3759 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3760 * out-of-sync. Make sure to update the required fields
3761 * before using them.
3762 *
3763 * @remarks No-long-jump zone!!!
3764 */
3765static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3766{
3767 /*
3768 * Guest CR0.
3769 * Guest FPU.
3770 */
3771 int rc = VINF_SUCCESS;
3772 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3773 {
3774 Assert(!(pMixedCtx->cr0 >> 32));
3775 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3776 PVM pVM = pVCpu->CTX_SUFF(pVM);
3777
3778 /* The guest's view (read access) of its CR0 is unblemished. */
3779 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3780 AssertRCReturn(rc, rc);
3781 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3782
3783 /* Setup VT-x's view of the guest CR0. */
3784 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3785 if (pVM->hm.s.fNestedPaging)
3786 {
3787 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3788 {
3789 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3790 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3791 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3792 }
3793 else
3794 {
3795 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3796 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3797 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3798 }
3799
3800 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3801 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3802 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3803
3804 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3805 AssertRCReturn(rc, rc);
3806 }
3807 else
3808 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3809
3810 /*
3811 * Guest FPU bits.
3812 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3813 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3814 */
3815 u32GuestCR0 |= X86_CR0_NE;
3816 bool fInterceptNM = false;
3817 if (CPUMIsGuestFPUStateActive(pVCpu))
3818 {
3819 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3820 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3821 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3822 }
3823 else
3824 {
3825 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3826 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3827 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3828 }
3829
3830 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3831 bool fInterceptMF = false;
3832 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3833 fInterceptMF = true;
3834
3835 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3836 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3837 {
3838 Assert(PDMVmmDevHeapIsEnabled(pVM));
3839 Assert(pVM->hm.s.vmx.pRealModeTSS);
3840 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3841 fInterceptNM = true;
3842 fInterceptMF = true;
3843 }
3844 else
3845 {
3846 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3847 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3848 }
3849 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3850
3851 if (fInterceptNM)
3852 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3853 else
3854 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3855
3856 if (fInterceptMF)
3857 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3858 else
3859 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3860
3861 /* Additional intercepts for debugging, define these yourself explicitly. */
3862#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3863 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3864 | RT_BIT(X86_XCPT_BP)
3865 | RT_BIT(X86_XCPT_DE)
3866 | RT_BIT(X86_XCPT_NM)
3867 | RT_BIT(X86_XCPT_TS)
3868 | RT_BIT(X86_XCPT_UD)
3869 | RT_BIT(X86_XCPT_NP)
3870 | RT_BIT(X86_XCPT_SS)
3871 | RT_BIT(X86_XCPT_GP)
3872 | RT_BIT(X86_XCPT_PF)
3873 | RT_BIT(X86_XCPT_MF)
3874 ;
3875#elif defined(HMVMX_ALWAYS_TRAP_PF)
3876 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3877#endif
3878
3879 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3880
3881 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3882 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3883 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3884 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3885 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3886 else
3887 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3888
3889 u32GuestCR0 |= uSetCR0;
3890 u32GuestCR0 &= uZapCR0;
3891 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3892
3893 /* Write VT-x's view of the guest CR0 into the VMCS. */
3894 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3895 AssertRCReturn(rc, rc);
3896 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3897 uZapCR0));
3898
3899 /*
3900 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3901 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3902 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3903 */
3904 uint32_t u32CR0Mask = 0;
3905 u32CR0Mask = X86_CR0_PE
3906 | X86_CR0_NE
3907 | X86_CR0_WP
3908 | X86_CR0_PG
3909 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3910 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3911 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3912
3913 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3914 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3915 * and @bugref{6944}. */
3916#if 0
3917 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3918 u32CR0Mask &= ~X86_CR0_PE;
3919#endif
3920 if (pVM->hm.s.fNestedPaging)
3921 u32CR0Mask &= ~X86_CR0_WP;
3922
3923 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3924 if (fInterceptNM)
3925 {
3926 u32CR0Mask |= X86_CR0_TS
3927 | X86_CR0_MP;
3928 }
3929
3930 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3931 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3932 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3933 AssertRCReturn(rc, rc);
3934 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3935
3936 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3937 }
3938 return rc;
3939}
3940
3941
3942/**
3943 * Loads the guest control registers (CR3, CR4) into the guest-state area
3944 * in the VMCS.
3945 *
3946 * @returns VBox strict status code.
3947 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3948 * without unrestricted guest access and the VMMDev is not presently
3949 * mapped (e.g. EFI32).
3950 *
3951 * @param pVCpu The cross context virtual CPU structure.
3952 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3953 * out-of-sync. Make sure to update the required fields
3954 * before using them.
3955 *
3956 * @remarks No-long-jump zone!!!
3957 */
3958static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3959{
3960 int rc = VINF_SUCCESS;
3961 PVM pVM = pVCpu->CTX_SUFF(pVM);
3962
3963 /*
3964 * Guest CR2.
3965 * It's always loaded in the assembler code. Nothing to do here.
3966 */
3967
3968 /*
3969 * Guest CR3.
3970 */
3971 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3972 {
3973 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3974 if (pVM->hm.s.fNestedPaging)
3975 {
3976 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3977
3978 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3979 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3980 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3981 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3982
3983 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3984 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3985 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3986
3987 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3988 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3989 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3990 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3991 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3992 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3993 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3994
3995 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3996 AssertRCReturn(rc, rc);
3997 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3998
3999 if ( pVM->hm.s.vmx.fUnrestrictedGuest
4000 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
4001 {
4002 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4003 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
4004 {
4005 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4006 AssertRCReturn(rc, rc);
4007 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4008 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4009 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4010 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4011 AssertRCReturn(rc, rc);
4012 }
4013
4014 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4015 have Unrestricted Execution to handle the guest when it's not using paging. */
4016 GCPhysGuestCR3 = pMixedCtx->cr3;
4017 }
4018 else
4019 {
4020 /*
4021 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4022 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4023 * EPT takes care of translating it to host-physical addresses.
4024 */
4025 RTGCPHYS GCPhys;
4026 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4027
4028 /* We obtain it here every time as the guest could have relocated this PCI region. */
4029 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4030 if (RT_SUCCESS(rc))
4031 { /* likely */ }
4032 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4033 {
4034 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4035 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4036 }
4037 else
4038 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4039
4040 GCPhysGuestCR3 = GCPhys;
4041 }
4042
4043 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4044 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4045 }
4046 else
4047 {
4048 /* Non-nested paging case, just use the hypervisor's CR3. */
4049 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4050
4051 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4052 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4053 }
4054 AssertRCReturn(rc, rc);
4055
4056 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4057 }
4058
4059 /*
4060 * Guest CR4.
4061 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4062 */
4063 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4064 {
4065 Assert(!(pMixedCtx->cr4 >> 32));
4066 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4067
4068 /* The guest's view of its CR4 is unblemished. */
4069 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4070 AssertRCReturn(rc, rc);
4071 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4072
4073 /* Setup VT-x's view of the guest CR4. */
4074 /*
4075 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4076 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4077 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4078 */
4079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4080 {
4081 Assert(pVM->hm.s.vmx.pRealModeTSS);
4082 Assert(PDMVmmDevHeapIsEnabled(pVM));
4083 u32GuestCR4 &= ~X86_CR4_VME;
4084 }
4085
4086 if (pVM->hm.s.fNestedPaging)
4087 {
4088 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4089 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4090 {
4091 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4092 u32GuestCR4 |= X86_CR4_PSE;
4093 /* Our identity mapping is a 32-bit page directory. */
4094 u32GuestCR4 &= ~X86_CR4_PAE;
4095 }
4096 /* else use guest CR4.*/
4097 }
4098 else
4099 {
4100 /*
4101 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4102 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4103 */
4104 switch (pVCpu->hm.s.enmShadowMode)
4105 {
4106 case PGMMODE_REAL: /* Real-mode. */
4107 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4108 case PGMMODE_32_BIT: /* 32-bit paging. */
4109 {
4110 u32GuestCR4 &= ~X86_CR4_PAE;
4111 break;
4112 }
4113
4114 case PGMMODE_PAE: /* PAE paging. */
4115 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4116 {
4117 u32GuestCR4 |= X86_CR4_PAE;
4118 break;
4119 }
4120
4121 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4122 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4123#ifdef VBOX_ENABLE_64_BITS_GUESTS
4124 break;
4125#endif
4126 default:
4127 AssertFailed();
4128 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4129 }
4130 }
4131
4132 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4133 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4134 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4135 u32GuestCR4 |= uSetCR4;
4136 u32GuestCR4 &= uZapCR4;
4137
4138 /* Write VT-x's view of the guest CR4 into the VMCS. */
4139 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4140 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4141 AssertRCReturn(rc, rc);
4142
4143 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4144 uint32_t u32CR4Mask = X86_CR4_VME
4145 | X86_CR4_PAE
4146 | X86_CR4_PGE
4147 | X86_CR4_PSE
4148 | X86_CR4_VMXE;
4149 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4150 u32CR4Mask |= X86_CR4_OSXSAVE;
4151 if (pVM->cpum.ro.GuestFeatures.fPcid)
4152 u32CR4Mask |= X86_CR4_PCIDE;
4153 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4154 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4155 AssertRCReturn(rc, rc);
4156
4157 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4158 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4159
4160 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4161 }
4162 return rc;
4163}
4164
4165
4166/**
4167 * Loads the guest debug registers into the guest-state area in the VMCS.
4168 *
4169 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4170 *
4171 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4172 *
4173 * @returns VBox status code.
4174 * @param pVCpu The cross context virtual CPU structure.
4175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4176 * out-of-sync. Make sure to update the required fields
4177 * before using them.
4178 *
4179 * @remarks No-long-jump zone!!!
4180 */
4181static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4182{
4183 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4184 return VINF_SUCCESS;
4185
4186#ifdef VBOX_STRICT
4187 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4188 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4189 {
4190 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4191 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4192 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4193 }
4194#endif
4195
4196 int rc;
4197 PVM pVM = pVCpu->CTX_SUFF(pVM);
4198 bool fSteppingDB = false;
4199 bool fInterceptMovDRx = false;
4200 if (pVCpu->hm.s.fSingleInstruction)
4201 {
4202 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4203 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4204 {
4205 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4206 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4207 AssertRCReturn(rc, rc);
4208 Assert(fSteppingDB == false);
4209 }
4210 else
4211 {
4212 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4213 pVCpu->hm.s.fClearTrapFlag = true;
4214 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4215 fSteppingDB = true;
4216 }
4217 }
4218
4219 if ( fSteppingDB
4220 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4221 {
4222 /*
4223 * Use the combined guest and host DRx values found in the hypervisor
4224 * register set because the debugger has breakpoints active or someone
4225 * is single stepping on the host side without a monitor trap flag.
4226 *
4227 * Note! DBGF expects a clean DR6 state before executing guest code.
4228 */
4229#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4230 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4231 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4232 {
4233 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4234 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4235 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4236 }
4237 else
4238#endif
4239 if (!CPUMIsHyperDebugStateActive(pVCpu))
4240 {
4241 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4242 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4243 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4244 }
4245
4246 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4247 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4248 AssertRCReturn(rc, rc);
4249
4250 pVCpu->hm.s.fUsingHyperDR7 = true;
4251 fInterceptMovDRx = true;
4252 }
4253 else
4254 {
4255 /*
4256 * If the guest has enabled debug registers, we need to load them prior to
4257 * executing guest code so they'll trigger at the right time.
4258 */
4259 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4260 {
4261#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4262 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4263 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4264 {
4265 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4266 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4267 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4268 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4269 }
4270 else
4271#endif
4272 if (!CPUMIsGuestDebugStateActive(pVCpu))
4273 {
4274 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4275 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4276 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4277 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4278 }
4279 Assert(!fInterceptMovDRx);
4280 }
4281 /*
4282 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4283 * must intercept #DB in order to maintain a correct DR6 guest value, and
4284 * because we need to intercept it to prevent nested #DBs from hanging the
4285 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4286 */
4287#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4288 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4289 && !CPUMIsGuestDebugStateActive(pVCpu))
4290#else
4291 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4292#endif
4293 {
4294 fInterceptMovDRx = true;
4295 }
4296
4297 /* Update guest DR7. */
4298 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4299 AssertRCReturn(rc, rc);
4300
4301 pVCpu->hm.s.fUsingHyperDR7 = false;
4302 }
4303
4304 /*
4305 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4306 */
4307 if (fInterceptMovDRx)
4308 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4309 else
4310 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4311 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4312 AssertRCReturn(rc, rc);
4313
4314 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4315 return VINF_SUCCESS;
4316}
4317
4318
4319#ifdef VBOX_STRICT
4320/**
4321 * Strict function to validate segment registers.
4322 *
4323 * @remarks ASSUMES CR0 is up to date.
4324 */
4325static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4326{
4327 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4328 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4329 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4330 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4331 && ( !CPUMIsGuestInRealModeEx(pCtx)
4332 && !CPUMIsGuestInV86ModeEx(pCtx)))
4333 {
4334 /* Protected mode checks */
4335 /* CS */
4336 Assert(pCtx->cs.Attr.n.u1Present);
4337 Assert(!(pCtx->cs.Attr.u & 0xf00));
4338 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->cs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4342 || (pCtx->cs.Attr.n.u1Granularity));
4343 /* CS cannot be loaded with NULL in protected mode. */
4344 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4345 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4346 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4347 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4348 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4349 else
4350 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4351 /* SS */
4352 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4353 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4354 if ( !(pCtx->cr0 & X86_CR0_PE)
4355 || pCtx->cs.Attr.n.u4Type == 3)
4356 {
4357 Assert(!pCtx->ss.Attr.n.u2Dpl);
4358 }
4359 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4360 {
4361 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4362 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4363 Assert(pCtx->ss.Attr.n.u1Present);
4364 Assert(!(pCtx->ss.Attr.u & 0xf00));
4365 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4366 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4367 || !(pCtx->ss.Attr.n.u1Granularity));
4368 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4369 || (pCtx->ss.Attr.n.u1Granularity));
4370 }
4371 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4372 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4373 {
4374 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4375 Assert(pCtx->ds.Attr.n.u1Present);
4376 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4377 Assert(!(pCtx->ds.Attr.u & 0xf00));
4378 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4379 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4380 || !(pCtx->ds.Attr.n.u1Granularity));
4381 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4382 || (pCtx->ds.Attr.n.u1Granularity));
4383 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4384 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4385 }
4386 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4387 {
4388 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4389 Assert(pCtx->es.Attr.n.u1Present);
4390 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4391 Assert(!(pCtx->es.Attr.u & 0xf00));
4392 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4393 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4394 || !(pCtx->es.Attr.n.u1Granularity));
4395 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4396 || (pCtx->es.Attr.n.u1Granularity));
4397 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4398 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4399 }
4400 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4401 {
4402 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4403 Assert(pCtx->fs.Attr.n.u1Present);
4404 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4405 Assert(!(pCtx->fs.Attr.u & 0xf00));
4406 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4407 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4408 || !(pCtx->fs.Attr.n.u1Granularity));
4409 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4410 || (pCtx->fs.Attr.n.u1Granularity));
4411 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4412 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4413 }
4414 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4415 {
4416 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4417 Assert(pCtx->gs.Attr.n.u1Present);
4418 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4419 Assert(!(pCtx->gs.Attr.u & 0xf00));
4420 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4421 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4422 || !(pCtx->gs.Attr.n.u1Granularity));
4423 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4424 || (pCtx->gs.Attr.n.u1Granularity));
4425 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4426 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4427 }
4428 /* 64-bit capable CPUs. */
4429# if HC_ARCH_BITS == 64
4430 Assert(!(pCtx->cs.u64Base >> 32));
4431 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4432 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4433 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4434# endif
4435 }
4436 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4437 || ( CPUMIsGuestInRealModeEx(pCtx)
4438 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4439 {
4440 /* Real and v86 mode checks. */
4441 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4442 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4443 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4444 {
4445 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4446 }
4447 else
4448 {
4449 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4450 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4451 }
4452
4453 /* CS */
4454 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4455 Assert(pCtx->cs.u32Limit == 0xffff);
4456 Assert(u32CSAttr == 0xf3);
4457 /* SS */
4458 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4459 Assert(pCtx->ss.u32Limit == 0xffff);
4460 Assert(u32SSAttr == 0xf3);
4461 /* DS */
4462 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4463 Assert(pCtx->ds.u32Limit == 0xffff);
4464 Assert(u32DSAttr == 0xf3);
4465 /* ES */
4466 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4467 Assert(pCtx->es.u32Limit == 0xffff);
4468 Assert(u32ESAttr == 0xf3);
4469 /* FS */
4470 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4471 Assert(pCtx->fs.u32Limit == 0xffff);
4472 Assert(u32FSAttr == 0xf3);
4473 /* GS */
4474 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4475 Assert(pCtx->gs.u32Limit == 0xffff);
4476 Assert(u32GSAttr == 0xf3);
4477 /* 64-bit capable CPUs. */
4478# if HC_ARCH_BITS == 64
4479 Assert(!(pCtx->cs.u64Base >> 32));
4480 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4481 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4482 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4483# endif
4484 }
4485}
4486#endif /* VBOX_STRICT */
4487
4488
4489/**
4490 * Writes a guest segment register into the guest-state area in the VMCS.
4491 *
4492 * @returns VBox status code.
4493 * @param pVCpu The cross context virtual CPU structure.
4494 * @param idxSel Index of the selector in the VMCS.
4495 * @param idxLimit Index of the segment limit in the VMCS.
4496 * @param idxBase Index of the segment base in the VMCS.
4497 * @param idxAccess Index of the access rights of the segment in the VMCS.
4498 * @param pSelReg Pointer to the segment selector.
4499 *
4500 * @remarks No-long-jump zone!!!
4501 */
4502static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4503 uint32_t idxAccess, PCPUMSELREG pSelReg)
4504{
4505 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4506 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4507 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4508 AssertRCReturn(rc, rc);
4509
4510 uint32_t u32Access = pSelReg->Attr.u;
4511 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4512 {
4513 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4514 u32Access = 0xf3;
4515 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4516 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4517 }
4518 else
4519 {
4520 /*
4521 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4522 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4523 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4524 * loaded in protected-mode have their attribute as 0.
4525 */
4526 if (!u32Access)
4527 u32Access = X86DESCATTR_UNUSABLE;
4528 }
4529
4530 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4531 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4532 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4533
4534 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4535 AssertRCReturn(rc, rc);
4536 return rc;
4537}
4538
4539
4540/**
4541 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4542 * into the guest-state area in the VMCS.
4543 *
4544 * @returns VBox status code.
4545 * @param pVCpu The cross context virtual CPU structure.
4546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4547 * out-of-sync. Make sure to update the required fields
4548 * before using them.
4549 *
4550 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4551 * @remarks No-long-jump zone!!!
4552 */
4553static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4554{
4555 int rc = VERR_INTERNAL_ERROR_5;
4556 PVM pVM = pVCpu->CTX_SUFF(pVM);
4557
4558 /*
4559 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4560 */
4561 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4562 {
4563 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4564 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4567 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4568 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4569 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4570 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4571 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4572 }
4573
4574#ifdef VBOX_WITH_REM
4575 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4576 {
4577 Assert(pVM->hm.s.vmx.pRealModeTSS);
4578 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4579 if ( pVCpu->hm.s.vmx.fWasInRealMode
4580 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4581 {
4582 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4583 in real-mode (e.g. OpenBSD 4.0) */
4584 REMFlushTBs(pVM);
4585 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4586 pVCpu->hm.s.vmx.fWasInRealMode = false;
4587 }
4588 }
4589#endif
4590 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4591 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4592 AssertRCReturn(rc, rc);
4593 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4594 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4595 AssertRCReturn(rc, rc);
4596 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4597 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4598 AssertRCReturn(rc, rc);
4599 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4600 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4601 AssertRCReturn(rc, rc);
4602 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4603 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4604 AssertRCReturn(rc, rc);
4605 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4606 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4607 AssertRCReturn(rc, rc);
4608
4609#ifdef VBOX_STRICT
4610 /* Validate. */
4611 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4612#endif
4613
4614 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4615 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4616 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4617 }
4618
4619 /*
4620 * Guest TR.
4621 */
4622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4623 {
4624 /*
4625 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4626 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4627 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4628 */
4629 uint16_t u16Sel = 0;
4630 uint32_t u32Limit = 0;
4631 uint64_t u64Base = 0;
4632 uint32_t u32AccessRights = 0;
4633
4634 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4635 {
4636 u16Sel = pMixedCtx->tr.Sel;
4637 u32Limit = pMixedCtx->tr.u32Limit;
4638 u64Base = pMixedCtx->tr.u64Base;
4639 u32AccessRights = pMixedCtx->tr.Attr.u;
4640 }
4641 else
4642 {
4643 Assert(pVM->hm.s.vmx.pRealModeTSS);
4644 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4645
4646 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4647 RTGCPHYS GCPhys;
4648 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4649 AssertRCReturn(rc, rc);
4650
4651 X86DESCATTR DescAttr;
4652 DescAttr.u = 0;
4653 DescAttr.n.u1Present = 1;
4654 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4655
4656 u16Sel = 0;
4657 u32Limit = HM_VTX_TSS_SIZE;
4658 u64Base = GCPhys; /* in real-mode phys = virt. */
4659 u32AccessRights = DescAttr.u;
4660 }
4661
4662 /* Validate. */
4663 Assert(!(u16Sel & RT_BIT(2)));
4664 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4665 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4666 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4667 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4668 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4669 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4670 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4671 Assert( (u32Limit & 0xfff) == 0xfff
4672 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4673 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4674 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4675
4676 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4677 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4678 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4679 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4680 AssertRCReturn(rc, rc);
4681
4682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4683 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4684 }
4685
4686 /*
4687 * Guest GDTR.
4688 */
4689 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4690 {
4691 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4692 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4693 AssertRCReturn(rc, rc);
4694
4695 /* Validate. */
4696 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4697
4698 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4699 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4700 }
4701
4702 /*
4703 * Guest LDTR.
4704 */
4705 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4706 {
4707 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4708 uint32_t u32Access = 0;
4709 if (!pMixedCtx->ldtr.Attr.u)
4710 u32Access = X86DESCATTR_UNUSABLE;
4711 else
4712 u32Access = pMixedCtx->ldtr.Attr.u;
4713
4714 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4715 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4716 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4717 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4718 AssertRCReturn(rc, rc);
4719
4720 /* Validate. */
4721 if (!(u32Access & X86DESCATTR_UNUSABLE))
4722 {
4723 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4724 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4725 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4726 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4727 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4728 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4729 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4730 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4731 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4732 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4733 }
4734
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4736 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4737 }
4738
4739 /*
4740 * Guest IDTR.
4741 */
4742 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4743 {
4744 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4745 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4746 AssertRCReturn(rc, rc);
4747
4748 /* Validate. */
4749 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4750
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4752 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4753 }
4754
4755 return VINF_SUCCESS;
4756}
4757
4758
4759/**
4760 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4761 * areas.
4762 *
4763 * These MSRs will automatically be loaded to the host CPU on every successful
4764 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4765 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4766 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4767 *
4768 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4769 *
4770 * @returns VBox status code.
4771 * @param pVCpu The cross context virtual CPU structure.
4772 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4773 * out-of-sync. Make sure to update the required fields
4774 * before using them.
4775 *
4776 * @remarks No-long-jump zone!!!
4777 */
4778static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4779{
4780 AssertPtr(pVCpu);
4781 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4782
4783 /*
4784 * MSRs that we use the auto-load/store MSR area in the VMCS.
4785 */
4786 PVM pVM = pVCpu->CTX_SUFF(pVM);
4787 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4788 {
4789 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4790#if HC_ARCH_BITS == 32
4791 if (pVM->hm.s.fAllow64BitGuests)
4792 {
4793 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4794 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4795 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4796 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4797 AssertRCReturn(rc, rc);
4798# ifdef LOG_ENABLED
4799 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4800 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4801 {
4802 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4803 pMsr->u64Value));
4804 }
4805# endif
4806 }
4807#endif
4808 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4809 }
4810
4811 /*
4812 * Guest Sysenter MSRs.
4813 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4814 * VM-exits on WRMSRs for these MSRs.
4815 */
4816 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4817 {
4818 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4819 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4820 }
4821
4822 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4823 {
4824 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4825 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4826 }
4827
4828 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4829 {
4830 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4831 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4832 }
4833
4834 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4835 {
4836 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4837 {
4838 /*
4839 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4840 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4841 */
4842 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4843 {
4844 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4845 AssertRCReturn(rc,rc);
4846 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4847 }
4848 else
4849 {
4850 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4851 NULL /* pfAddedAndUpdated */);
4852 AssertRCReturn(rc, rc);
4853
4854 /* We need to intercept reads too, see @bugref{7386#c16}. */
4855 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4856 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4857 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4858 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4859 }
4860 }
4861 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4862 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4863 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4864 }
4865
4866 return VINF_SUCCESS;
4867}
4868
4869
4870/**
4871 * Loads the guest activity state into the guest-state area in the VMCS.
4872 *
4873 * @returns VBox status code.
4874 * @param pVCpu The cross context virtual CPU structure.
4875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4876 * out-of-sync. Make sure to update the required fields
4877 * before using them.
4878 *
4879 * @remarks No-long-jump zone!!!
4880 */
4881static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882{
4883 NOREF(pMixedCtx);
4884 /** @todo See if we can make use of other states, e.g.
4885 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4886 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4887 {
4888 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4889 AssertRCReturn(rc, rc);
4890
4891 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4892 }
4893 return VINF_SUCCESS;
4894}
4895
4896
4897#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4898/**
4899 * Check if guest state allows safe use of 32-bit switcher again.
4900 *
4901 * Segment bases and protected mode structures must be 32-bit addressable
4902 * because the 32-bit switcher will ignore high dword when writing these VMCS
4903 * fields. See @bugref{8432} for details.
4904 *
4905 * @returns true if safe, false if must continue to use the 64-bit switcher.
4906 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4907 * out-of-sync. Make sure to update the required fields
4908 * before using them.
4909 *
4910 * @remarks No-long-jump zone!!!
4911 */
4912static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4913{
4914 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4915 return false;
4916 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4917 return false;
4918 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4919 return false;
4920 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4921 return false;
4922 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4923 return false;
4924 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4925 return false;
4926 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4927 return false;
4928 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4929 return false;
4930 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4931 return false;
4932 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4933 return false;
4934 /* All good, bases are 32-bit. */
4935 return true;
4936}
4937#endif
4938
4939
4940/**
4941 * Sets up the appropriate function to run guest code.
4942 *
4943 * @returns VBox status code.
4944 * @param pVCpu The cross context virtual CPU structure.
4945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4946 * out-of-sync. Make sure to update the required fields
4947 * before using them.
4948 *
4949 * @remarks No-long-jump zone!!!
4950 */
4951static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4952{
4953 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4954 {
4955#ifndef VBOX_ENABLE_64_BITS_GUESTS
4956 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4957#endif
4958 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4959#if HC_ARCH_BITS == 32
4960 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4961 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4962 {
4963 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4964 {
4965 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4966 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4967 | HM_CHANGED_VMX_ENTRY_CTLS
4968 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4969 }
4970 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4971
4972 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4973 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4974 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4975 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4976 }
4977#else
4978 /* 64-bit host. */
4979 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4980#endif
4981 }
4982 else
4983 {
4984 /* Guest is not in long mode, use the 32-bit handler. */
4985#if HC_ARCH_BITS == 32
4986 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4987 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4988 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4989 {
4990 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4991 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4992 | HM_CHANGED_VMX_ENTRY_CTLS
4993 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4994 }
4995# ifdef VBOX_ENABLE_64_BITS_GUESTS
4996 /*
4997 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4998 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4999 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
5000 * the much faster 32-bit switcher again.
5001 */
5002 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
5003 {
5004 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5005 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5006 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5007 }
5008 else
5009 {
5010 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5011 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5012 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5013 {
5014 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5015 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5016 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5017 | HM_CHANGED_VMX_ENTRY_CTLS
5018 | HM_CHANGED_VMX_EXIT_CTLS
5019 | HM_CHANGED_HOST_CONTEXT);
5020 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5021 }
5022 }
5023# else
5024 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5025# endif
5026#else
5027 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5028#endif
5029 }
5030 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5031 return VINF_SUCCESS;
5032}
5033
5034
5035/**
5036 * Wrapper for running the guest code in VT-x.
5037 *
5038 * @returns VBox status code, no informational status codes.
5039 * @param pVM The cross context VM structure.
5040 * @param pVCpu The cross context virtual CPU structure.
5041 * @param pCtx Pointer to the guest-CPU context.
5042 *
5043 * @remarks No-long-jump zone!!!
5044 */
5045DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5046{
5047 /*
5048 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5049 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5050 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5051 */
5052 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5053 /** @todo Add stats for resume vs launch. */
5054#ifdef VBOX_WITH_KERNEL_USING_XMM
5055 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5056#else
5057 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5058#endif
5059 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5060 return rc;
5061}
5062
5063
5064/**
5065 * Reports world-switch error and dumps some useful debug info.
5066 *
5067 * @param pVM The cross context VM structure.
5068 * @param pVCpu The cross context virtual CPU structure.
5069 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5070 * @param pCtx Pointer to the guest-CPU context.
5071 * @param pVmxTransient Pointer to the VMX transient structure (only
5072 * exitReason updated).
5073 */
5074static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5075{
5076 Assert(pVM);
5077 Assert(pVCpu);
5078 Assert(pCtx);
5079 Assert(pVmxTransient);
5080 HMVMX_ASSERT_PREEMPT_SAFE();
5081
5082 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5083 switch (rcVMRun)
5084 {
5085 case VERR_VMX_INVALID_VMXON_PTR:
5086 AssertFailed();
5087 break;
5088 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5089 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5090 {
5091 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5092 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5093 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5094 AssertRC(rc);
5095
5096 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5097 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5098 Cannot do it here as we may have been long preempted. */
5099
5100#ifdef VBOX_STRICT
5101 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5102 pVmxTransient->uExitReason));
5103 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5104 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5105 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5106 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5107 else
5108 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5109 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5110 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5111
5112 /* VMX control bits. */
5113 uint32_t u32Val;
5114 uint64_t u64Val;
5115 RTHCUINTREG uHCReg;
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5120 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5121 {
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5124 }
5125 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5126 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5127 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5128 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5129 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5130 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5131 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5132 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5133 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5134 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5135 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5136 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5137 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5138 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5139 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5140 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5141 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5142 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5143 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5144 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5145 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5146 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5147 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5148 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5149 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5150 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5151 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5152 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5153 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5154 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5155 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5156 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5157 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5158 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5159 if (pVM->hm.s.fNestedPaging)
5160 {
5161 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5162 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5163 }
5164
5165 /* Guest bits. */
5166 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5167 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5168 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5169 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5170 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5171 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5172 if (pVM->hm.s.vmx.fVpid)
5173 {
5174 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5175 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5176 }
5177
5178 /* Host bits. */
5179 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5180 Log4(("Host CR0 %#RHr\n", uHCReg));
5181 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5182 Log4(("Host CR3 %#RHr\n", uHCReg));
5183 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5184 Log4(("Host CR4 %#RHr\n", uHCReg));
5185
5186 RTGDTR HostGdtr;
5187 PCX86DESCHC pDesc;
5188 ASMGetGDTR(&HostGdtr);
5189 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5190 Log4(("Host CS %#08x\n", u32Val));
5191 if (u32Val < HostGdtr.cbGdt)
5192 {
5193 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5194 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5195 }
5196
5197 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5198 Log4(("Host DS %#08x\n", u32Val));
5199 if (u32Val < HostGdtr.cbGdt)
5200 {
5201 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5202 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5203 }
5204
5205 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5206 Log4(("Host ES %#08x\n", u32Val));
5207 if (u32Val < HostGdtr.cbGdt)
5208 {
5209 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5210 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5211 }
5212
5213 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5214 Log4(("Host FS %#08x\n", u32Val));
5215 if (u32Val < HostGdtr.cbGdt)
5216 {
5217 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5218 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5219 }
5220
5221 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5222 Log4(("Host GS %#08x\n", u32Val));
5223 if (u32Val < HostGdtr.cbGdt)
5224 {
5225 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5226 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5227 }
5228
5229 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5230 Log4(("Host SS %#08x\n", u32Val));
5231 if (u32Val < HostGdtr.cbGdt)
5232 {
5233 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5234 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5235 }
5236
5237 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5238 Log4(("Host TR %#08x\n", u32Val));
5239 if (u32Val < HostGdtr.cbGdt)
5240 {
5241 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5242 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5243 }
5244
5245 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5246 Log4(("Host TR Base %#RHv\n", uHCReg));
5247 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5248 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5249 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5250 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5251 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5252 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5253 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5254 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5255 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5256 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5257 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5258 Log4(("Host RSP %#RHv\n", uHCReg));
5259 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5260 Log4(("Host RIP %#RHv\n", uHCReg));
5261# if HC_ARCH_BITS == 64
5262 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5263 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5264 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5265 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5266 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5267 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5268# endif
5269#endif /* VBOX_STRICT */
5270 break;
5271 }
5272
5273 default:
5274 /* Impossible */
5275 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5276 break;
5277 }
5278 NOREF(pVM); NOREF(pCtx);
5279}
5280
5281
5282#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5283#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5284# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5285#endif
5286#ifdef VBOX_STRICT
5287static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5288{
5289 switch (idxField)
5290 {
5291 case VMX_VMCS_GUEST_RIP:
5292 case VMX_VMCS_GUEST_RSP:
5293 case VMX_VMCS_GUEST_SYSENTER_EIP:
5294 case VMX_VMCS_GUEST_SYSENTER_ESP:
5295 case VMX_VMCS_GUEST_GDTR_BASE:
5296 case VMX_VMCS_GUEST_IDTR_BASE:
5297 case VMX_VMCS_GUEST_CS_BASE:
5298 case VMX_VMCS_GUEST_DS_BASE:
5299 case VMX_VMCS_GUEST_ES_BASE:
5300 case VMX_VMCS_GUEST_FS_BASE:
5301 case VMX_VMCS_GUEST_GS_BASE:
5302 case VMX_VMCS_GUEST_SS_BASE:
5303 case VMX_VMCS_GUEST_LDTR_BASE:
5304 case VMX_VMCS_GUEST_TR_BASE:
5305 case VMX_VMCS_GUEST_CR3:
5306 return true;
5307 }
5308 return false;
5309}
5310
5311static bool hmR0VmxIsValidReadField(uint32_t idxField)
5312{
5313 switch (idxField)
5314 {
5315 /* Read-only fields. */
5316 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5317 return true;
5318 }
5319 /* Remaining readable fields should also be writable. */
5320 return hmR0VmxIsValidWriteField(idxField);
5321}
5322#endif /* VBOX_STRICT */
5323
5324
5325/**
5326 * Executes the specified handler in 64-bit mode.
5327 *
5328 * @returns VBox status code (no informational status codes).
5329 * @param pVM The cross context VM structure.
5330 * @param pVCpu The cross context virtual CPU structure.
5331 * @param pCtx Pointer to the guest CPU context.
5332 * @param enmOp The operation to perform.
5333 * @param cParams Number of parameters.
5334 * @param paParam Array of 32-bit parameters.
5335 */
5336VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5337 uint32_t cParams, uint32_t *paParam)
5338{
5339 NOREF(pCtx);
5340
5341 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5342 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5343 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5344 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5345
5346#ifdef VBOX_STRICT
5347 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5348 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5349
5350 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5351 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5352#endif
5353
5354 /* Disable interrupts. */
5355 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5356
5357#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5358 RTCPUID idHostCpu = RTMpCpuId();
5359 CPUMR0SetLApic(pVCpu, idHostCpu);
5360#endif
5361
5362 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5363 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5364
5365 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5366 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5367 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5368
5369 /* Leave VMX Root Mode. */
5370 VMXDisable();
5371
5372 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5373
5374 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5375 CPUMSetHyperEIP(pVCpu, enmOp);
5376 for (int i = (int)cParams - 1; i >= 0; i--)
5377 CPUMPushHyper(pVCpu, paParam[i]);
5378
5379 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5380
5381 /* Call the switcher. */
5382 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5383 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5384
5385 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5386 /* Make sure the VMX instructions don't cause #UD faults. */
5387 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5388
5389 /* Re-enter VMX Root Mode */
5390 int rc2 = VMXEnable(HCPhysCpuPage);
5391 if (RT_FAILURE(rc2))
5392 {
5393 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5394 ASMSetFlags(fOldEFlags);
5395 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5396 return rc2;
5397 }
5398
5399 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5400 AssertRC(rc2);
5401 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5402 Assert(!(ASMGetFlags() & X86_EFL_IF));
5403 ASMSetFlags(fOldEFlags);
5404 return rc;
5405}
5406
5407
5408/**
5409 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5410 * supporting 64-bit guests.
5411 *
5412 * @returns VBox status code.
5413 * @param fResume Whether to VMLAUNCH or VMRESUME.
5414 * @param pCtx Pointer to the guest-CPU context.
5415 * @param pCache Pointer to the VMCS cache.
5416 * @param pVM The cross context VM structure.
5417 * @param pVCpu The cross context virtual CPU structure.
5418 */
5419DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5420{
5421 NOREF(fResume);
5422
5423 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5424 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5425
5426#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5427 pCache->uPos = 1;
5428 pCache->interPD = PGMGetInterPaeCR3(pVM);
5429 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5430#endif
5431
5432#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5433 pCache->TestIn.HCPhysCpuPage = 0;
5434 pCache->TestIn.HCPhysVmcs = 0;
5435 pCache->TestIn.pCache = 0;
5436 pCache->TestOut.HCPhysVmcs = 0;
5437 pCache->TestOut.pCache = 0;
5438 pCache->TestOut.pCtx = 0;
5439 pCache->TestOut.eflags = 0;
5440#else
5441 NOREF(pCache);
5442#endif
5443
5444 uint32_t aParam[10];
5445 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5446 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5447 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5448 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5449 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5450 aParam[5] = 0;
5451 aParam[6] = VM_RC_ADDR(pVM, pVM);
5452 aParam[7] = 0;
5453 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5454 aParam[9] = 0;
5455
5456#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5457 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5458 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5459#endif
5460 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5461
5462#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5463 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5464 Assert(pCtx->dr[4] == 10);
5465 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5466#endif
5467
5468#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5469 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5470 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5471 pVCpu->hm.s.vmx.HCPhysVmcs));
5472 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5473 pCache->TestOut.HCPhysVmcs));
5474 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5475 pCache->TestOut.pCache));
5476 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5477 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5478 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5479 pCache->TestOut.pCtx));
5480 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5481#endif
5482 return rc;
5483}
5484
5485
5486/**
5487 * Initialize the VMCS-Read cache.
5488 *
5489 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5490 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5491 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5492 * (those that have a 32-bit FULL & HIGH part).
5493 *
5494 * @returns VBox status code.
5495 * @param pVM The cross context VM structure.
5496 * @param pVCpu The cross context virtual CPU structure.
5497 */
5498static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5499{
5500#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5501{ \
5502 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5503 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5504 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5505 ++cReadFields; \
5506}
5507
5508 AssertPtr(pVM);
5509 AssertPtr(pVCpu);
5510 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5511 uint32_t cReadFields = 0;
5512
5513 /*
5514 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5515 * and serve to indicate exceptions to the rules.
5516 */
5517
5518 /* Guest-natural selector base fields. */
5519#if 0
5520 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5523#endif
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5529 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5534 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5536#if 0
5537 /* Unused natural width guest-state fields. */
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5540#endif
5541 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5542 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5543
5544 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5545#if 0
5546 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5547 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5548 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5549 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5550 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5551 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5552 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5553 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5554 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5555#endif
5556
5557 /* Natural width guest-state fields. */
5558 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5559#if 0
5560 /* Currently unused field. */
5561 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5562#endif
5563
5564 if (pVM->hm.s.fNestedPaging)
5565 {
5566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5567 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5568 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5569 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5570 }
5571 else
5572 {
5573 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5574 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5575 }
5576
5577#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5578 return VINF_SUCCESS;
5579}
5580
5581
5582/**
5583 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5584 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5585 * darwin, running 64-bit guests).
5586 *
5587 * @returns VBox status code.
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param idxField The VMCS field encoding.
5590 * @param u64Val 16, 32 or 64-bit value.
5591 */
5592VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593{
5594 int rc;
5595 switch (idxField)
5596 {
5597 /*
5598 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5599 */
5600 /* 64-bit Control fields. */
5601 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5602 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5603 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5604 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5605 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5606 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5607 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5608 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5609 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5610 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5611 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5612 case VMX_VMCS64_CTRL_EPTP_FULL:
5613 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5614 /* 64-bit Guest-state fields. */
5615 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5616 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5617 case VMX_VMCS64_GUEST_PAT_FULL:
5618 case VMX_VMCS64_GUEST_EFER_FULL:
5619 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5620 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5621 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5622 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5623 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5624 /* 64-bit Host-state fields. */
5625 case VMX_VMCS64_HOST_PAT_FULL:
5626 case VMX_VMCS64_HOST_EFER_FULL:
5627 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5628 {
5629 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5630 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5631 break;
5632 }
5633
5634 /*
5635 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5636 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5637 */
5638 /* Natural-width Guest-state fields. */
5639 case VMX_VMCS_GUEST_CR3:
5640 case VMX_VMCS_GUEST_ES_BASE:
5641 case VMX_VMCS_GUEST_CS_BASE:
5642 case VMX_VMCS_GUEST_SS_BASE:
5643 case VMX_VMCS_GUEST_DS_BASE:
5644 case VMX_VMCS_GUEST_FS_BASE:
5645 case VMX_VMCS_GUEST_GS_BASE:
5646 case VMX_VMCS_GUEST_LDTR_BASE:
5647 case VMX_VMCS_GUEST_TR_BASE:
5648 case VMX_VMCS_GUEST_GDTR_BASE:
5649 case VMX_VMCS_GUEST_IDTR_BASE:
5650 case VMX_VMCS_GUEST_RSP:
5651 case VMX_VMCS_GUEST_RIP:
5652 case VMX_VMCS_GUEST_SYSENTER_ESP:
5653 case VMX_VMCS_GUEST_SYSENTER_EIP:
5654 {
5655 if (!(RT_HI_U32(u64Val)))
5656 {
5657 /* If this field is 64-bit, VT-x will zero out the top bits. */
5658 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5659 }
5660 else
5661 {
5662 /* Assert that only the 32->64 switcher case should ever come here. */
5663 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5664 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5665 }
5666 break;
5667 }
5668
5669 default:
5670 {
5671 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5672 rc = VERR_INVALID_PARAMETER;
5673 break;
5674 }
5675 }
5676 AssertRCReturn(rc, rc);
5677 return rc;
5678}
5679
5680
5681/**
5682 * Queue up a VMWRITE by using the VMCS write cache.
5683 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5684 *
5685 * @param pVCpu The cross context virtual CPU structure.
5686 * @param idxField The VMCS field encoding.
5687 * @param u64Val 16, 32 or 64-bit value.
5688 */
5689VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5690{
5691 AssertPtr(pVCpu);
5692 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5693
5694 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5695 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5696
5697 /* Make sure there are no duplicates. */
5698 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5699 {
5700 if (pCache->Write.aField[i] == idxField)
5701 {
5702 pCache->Write.aFieldVal[i] = u64Val;
5703 return VINF_SUCCESS;
5704 }
5705 }
5706
5707 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5708 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5709 pCache->Write.cValidEntries++;
5710 return VINF_SUCCESS;
5711}
5712#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5713
5714
5715/**
5716 * Sets up the usage of TSC-offsetting and updates the VMCS.
5717 *
5718 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5719 * VMX preemption timer.
5720 *
5721 * @returns VBox status code.
5722 * @param pVM The cross context VM structure.
5723 * @param pVCpu The cross context virtual CPU structure.
5724 *
5725 * @remarks No-long-jump zone!!!
5726 */
5727static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5728{
5729 int rc;
5730 bool fOffsettedTsc;
5731 bool fParavirtTsc;
5732 if (pVM->hm.s.vmx.fUsePreemptTimer)
5733 {
5734 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5735 &fOffsettedTsc, &fParavirtTsc);
5736
5737 /* Make sure the returned values have sane upper and lower boundaries. */
5738 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5739 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5740 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5741 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5742
5743 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5744 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5745 }
5746 else
5747 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5748
5749 /** @todo later optimize this to be done elsewhere and not before every
5750 * VM-entry. */
5751 if (fParavirtTsc)
5752 {
5753 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5754 information before every VM-entry, hence disable it for performance sake. */
5755#if 0
5756 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5757 AssertRC(rc);
5758#endif
5759 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5760 }
5761
5762 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5763 {
5764 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5765 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5766
5767 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5768 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5769 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5770 }
5771 else
5772 {
5773 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5774 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5775 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5776 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5777 }
5778}
5779
5780
5781#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5782/**
5783 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5784 * VM-exit interruption info type.
5785 *
5786 * @returns The IEM exception flags.
5787 * @param uVector The event vector.
5788 * @param uVmxVectorType The VMX event type.
5789 *
5790 * @remarks This function currently only constructs flags required for
5791 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5792 * and CR2 aspects of an exception are not included).
5793 */
5794static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5795{
5796 uint32_t fIemXcptFlags;
5797 switch (uVmxVectorType)
5798 {
5799 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5800 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5801 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5802 break;
5803
5804 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5805 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5806 break;
5807
5808 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5809 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5810 break;
5811
5812 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5813 {
5814 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5815 if (uVector == X86_XCPT_BP)
5816 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5817 else if (uVector == X86_XCPT_OF)
5818 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5819 else
5820 {
5821 fIemXcptFlags = 0;
5822 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5823 }
5824 break;
5825 }
5826
5827 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5828 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5829 break;
5830
5831 default:
5832 fIemXcptFlags = 0;
5833 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5834 break;
5835 }
5836 return fIemXcptFlags;
5837}
5838
5839#else
5840/**
5841 * Determines if an exception is a contributory exception.
5842 *
5843 * Contributory exceptions are ones which can cause double-faults unless the
5844 * original exception was a benign exception. Page-fault is intentionally not
5845 * included here as it's a conditional contributory exception.
5846 *
5847 * @returns true if the exception is contributory, false otherwise.
5848 * @param uVector The exception vector.
5849 */
5850DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5851{
5852 switch (uVector)
5853 {
5854 case X86_XCPT_GP:
5855 case X86_XCPT_SS:
5856 case X86_XCPT_NP:
5857 case X86_XCPT_TS:
5858 case X86_XCPT_DE:
5859 return true;
5860 default:
5861 break;
5862 }
5863 return false;
5864}
5865#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5866
5867
5868/**
5869 * Sets an event as a pending event to be injected into the guest.
5870 *
5871 * @param pVCpu The cross context virtual CPU structure.
5872 * @param u32IntInfo The VM-entry interruption-information field.
5873 * @param cbInstr The VM-entry instruction length in bytes (for software
5874 * interrupts, exceptions and privileged software
5875 * exceptions).
5876 * @param u32ErrCode The VM-entry exception error code.
5877 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5878 * page-fault.
5879 *
5880 * @remarks Statistics counter assumes this is a guest event being injected or
5881 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5882 * always incremented.
5883 */
5884DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5885 RTGCUINTPTR GCPtrFaultAddress)
5886{
5887 Assert(!pVCpu->hm.s.Event.fPending);
5888 pVCpu->hm.s.Event.fPending = true;
5889 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5890 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5891 pVCpu->hm.s.Event.cbInstr = cbInstr;
5892 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5893}
5894
5895
5896/**
5897 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5898 *
5899 * @param pVCpu The cross context virtual CPU structure.
5900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5901 * out-of-sync. Make sure to update the required fields
5902 * before using them.
5903 */
5904DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5905{
5906 NOREF(pMixedCtx);
5907 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5908 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5909 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5910 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5911}
5912
5913
5914/**
5915 * Handle a condition that occurred while delivering an event through the guest
5916 * IDT.
5917 *
5918 * @returns Strict VBox status code (i.e. informational status codes too).
5919 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5920 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5921 * to continue execution of the guest which will delivery the \#DF.
5922 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5923 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5924 *
5925 * @param pVCpu The cross context virtual CPU structure.
5926 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5927 * out-of-sync. Make sure to update the required fields
5928 * before using them.
5929 * @param pVmxTransient Pointer to the VMX transient structure.
5930 *
5931 * @remarks No-long-jump zone!!!
5932 */
5933static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5934{
5935 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5936
5937 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5938 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5939
5940 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5941 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5942 {
5943 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5944 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5945#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5946 /*
5947 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5948 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5949 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5950 * exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5951 * currently has no way of storing that these exceptions were caused by these instructions
5952 * (ICEBP's #DB poses the problem).
5953 */
5954 IEMXCPTRAISE enmRaise;
5955 IEMXCPTRAISEINFO fRaiseInfo;
5956 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5957 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5958 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5959 {
5960 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5961 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5962 }
5963 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5964 {
5965 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5966 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5967 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5968 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5969 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5970 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5971 uExitVectorType), VERR_VMX_IPE_5);
5972 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5973
5974 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5975 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5976 {
5977 pVmxTransient->fVectoringPF = true;
5978 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5979 }
5980 }
5981 else
5982 {
5983 /*
5984 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5985 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5986 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5987 */
5988 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5989 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5990 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5991 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5992 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5993 }
5994
5995 /*
5996 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5997 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5998 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5999 * subsequent VM-entry would fail.
6000 *
6001 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6002 */
6003 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
6004 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6005 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6006 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6007 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6008 {
6009 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6010 }
6011
6012 switch (enmRaise)
6013 {
6014 case IEMXCPTRAISE_CURRENT_XCPT:
6015 {
6016 Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6017 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6018 Assert(rcStrict == VINF_SUCCESS);
6019 break;
6020 }
6021
6022 case IEMXCPTRAISE_PREV_EVENT:
6023 {
6024 uint32_t u32ErrCode;
6025 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6026 {
6027 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6028 AssertRCReturn(rc2, rc2);
6029 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6030 }
6031 else
6032 u32ErrCode = 0;
6033
6034 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6035 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6036 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6037 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6038
6039 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6040 pVCpu->hm.s.Event.u32ErrCode));
6041 Assert(rcStrict == VINF_SUCCESS);
6042 break;
6043 }
6044
6045 case IEMXCPTRAISE_REEXEC_INSTR:
6046 Assert(rcStrict == VINF_SUCCESS);
6047 break;
6048
6049 case IEMXCPTRAISE_DOUBLE_FAULT:
6050 {
6051 /*
6052 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6053 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6054 */
6055 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6056 {
6057 pVmxTransient->fVectoringDoublePF = true;
6058 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6059 pMixedCtx->cr2));
6060 rcStrict = VINF_SUCCESS;
6061 }
6062 else
6063 {
6064 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6065 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6066 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6067 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6068 rcStrict = VINF_HM_DOUBLE_FAULT;
6069 }
6070 break;
6071 }
6072
6073 case IEMXCPTRAISE_TRIPLE_FAULT:
6074 {
6075 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6076 uExitVector));
6077 rcStrict = VINF_EM_RESET;
6078 break;
6079 }
6080
6081 case IEMXCPTRAISE_CPU_HANG:
6082 {
6083 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6084 rcStrict = VERR_EM_GUEST_CPU_HANG;
6085 break;
6086 }
6087
6088 default:
6089 {
6090 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6091 rcStrict = VERR_VMX_IPE_2;
6092 break;
6093 }
6094 }
6095#else
6096 typedef enum
6097 {
6098 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6099 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6100 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6101 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6102 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6103 } VMXREFLECTXCPT;
6104
6105 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6106 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6107 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6108 {
6109 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6110 {
6111 enmReflect = VMXREFLECTXCPT_XCPT;
6112#ifdef VBOX_STRICT
6113 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6114 && uExitVector == X86_XCPT_PF)
6115 {
6116 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6117 }
6118#endif
6119 if ( uExitVector == X86_XCPT_PF
6120 && uIdtVector == X86_XCPT_PF)
6121 {
6122 pVmxTransient->fVectoringDoublePF = true;
6123 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6124 }
6125 else if ( uExitVector == X86_XCPT_AC
6126 && uIdtVector == X86_XCPT_AC)
6127 {
6128 enmReflect = VMXREFLECTXCPT_HANG;
6129 Log4(("IDT: Nested #AC - Bad guest\n"));
6130 }
6131 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6132 && hmR0VmxIsContributoryXcpt(uExitVector)
6133 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6134 || uIdtVector == X86_XCPT_PF))
6135 {
6136 enmReflect = VMXREFLECTXCPT_DF;
6137 }
6138 else if (uIdtVector == X86_XCPT_DF)
6139 enmReflect = VMXREFLECTXCPT_TF;
6140 }
6141 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6142 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6143 {
6144 /*
6145 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6146 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6147 */
6148 enmReflect = VMXREFLECTXCPT_XCPT;
6149
6150 if (uExitVector == X86_XCPT_PF)
6151 {
6152 pVmxTransient->fVectoringPF = true;
6153 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6154 }
6155 }
6156 }
6157 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6158 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6159 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6160 {
6161 /*
6162 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6163 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6164 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6165 */
6166 enmReflect = VMXREFLECTXCPT_XCPT;
6167 }
6168
6169 /*
6170 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6171 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6172 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6173 *
6174 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6175 */
6176 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6177 && enmReflect == VMXREFLECTXCPT_XCPT
6178 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6179 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6180 {
6181 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6182 }
6183
6184 switch (enmReflect)
6185 {
6186 case VMXREFLECTXCPT_XCPT:
6187 {
6188 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6189 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6190 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6191
6192 uint32_t u32ErrCode = 0;
6193 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6194 {
6195 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6196 AssertRCReturn(rc2, rc2);
6197 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6198 }
6199
6200 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6201 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6202 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6203 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6204 rcStrict = VINF_SUCCESS;
6205 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6206 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6207
6208 break;
6209 }
6210
6211 case VMXREFLECTXCPT_DF:
6212 {
6213 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6214 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6215 rcStrict = VINF_HM_DOUBLE_FAULT;
6216 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6217 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6218
6219 break;
6220 }
6221
6222 case VMXREFLECTXCPT_TF:
6223 {
6224 rcStrict = VINF_EM_RESET;
6225 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6226 uExitVector));
6227 break;
6228 }
6229
6230 case VMXREFLECTXCPT_HANG:
6231 {
6232 rcStrict = VERR_EM_GUEST_CPU_HANG;
6233 break;
6234 }
6235
6236 default:
6237 Assert(rcStrict == VINF_SUCCESS);
6238 break;
6239 }
6240#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6241 }
6242 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6243 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6244 && uExitVector != X86_XCPT_DF
6245 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6246 {
6247 /*
6248 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6249 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6250 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6251 */
6252 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6253 {
6254 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6255 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6256 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6257 }
6258 }
6259
6260 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6261 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6262 return rcStrict;
6263}
6264
6265
6266/**
6267 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6268 *
6269 * @returns VBox status code.
6270 * @param pVCpu The cross context virtual CPU structure.
6271 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6272 * out-of-sync. Make sure to update the required fields
6273 * before using them.
6274 *
6275 * @remarks No-long-jump zone!!!
6276 */
6277static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6278{
6279 NOREF(pMixedCtx);
6280
6281 /*
6282 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6283 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6284 */
6285 VMMRZCallRing3Disable(pVCpu);
6286 HM_DISABLE_PREEMPT();
6287
6288 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6289 {
6290#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6291 * and 'dbgc-init' containing:
6292 * sxe "xcpt_de"
6293 * sxe "xcpt_bp"
6294 * sxi "xcpt_gp"
6295 * sxi "xcpt_ss"
6296 * sxi "xcpt_np"
6297 */
6298 /** @todo r=ramshankar: Should be fixed after r119291. */
6299 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6300#endif
6301 uint32_t uVal = 0;
6302 uint32_t uShadow = 0;
6303 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6304 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6305 AssertRCReturn(rc, rc);
6306
6307 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6308 CPUMSetGuestCR0(pVCpu, uVal);
6309 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6310 }
6311
6312 HM_RESTORE_PREEMPT();
6313 VMMRZCallRing3Enable(pVCpu);
6314 return VINF_SUCCESS;
6315}
6316
6317
6318/**
6319 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6320 *
6321 * @returns VBox status code.
6322 * @param pVCpu The cross context virtual CPU structure.
6323 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6324 * out-of-sync. Make sure to update the required fields
6325 * before using them.
6326 *
6327 * @remarks No-long-jump zone!!!
6328 */
6329static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6330{
6331 NOREF(pMixedCtx);
6332
6333 int rc = VINF_SUCCESS;
6334 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6335 {
6336 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6337 uint32_t uVal = 0;
6338 uint32_t uShadow = 0;
6339 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6340 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6341 AssertRCReturn(rc, rc);
6342
6343 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6344 CPUMSetGuestCR4(pVCpu, uVal);
6345 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6346 }
6347 return rc;
6348}
6349
6350
6351/**
6352 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6353 *
6354 * @returns VBox status code.
6355 * @param pVCpu The cross context virtual CPU structure.
6356 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6357 * out-of-sync. Make sure to update the required fields
6358 * before using them.
6359 *
6360 * @remarks No-long-jump zone!!!
6361 */
6362static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6363{
6364 int rc = VINF_SUCCESS;
6365 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6366 {
6367 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6368 uint64_t u64Val = 0;
6369 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6370 AssertRCReturn(rc, rc);
6371
6372 pMixedCtx->rip = u64Val;
6373 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6374 }
6375 return rc;
6376}
6377
6378
6379/**
6380 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6381 *
6382 * @returns VBox status code.
6383 * @param pVCpu The cross context virtual CPU structure.
6384 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6385 * out-of-sync. Make sure to update the required fields
6386 * before using them.
6387 *
6388 * @remarks No-long-jump zone!!!
6389 */
6390static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6391{
6392 int rc = VINF_SUCCESS;
6393 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6394 {
6395 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6396 uint64_t u64Val = 0;
6397 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6398 AssertRCReturn(rc, rc);
6399
6400 pMixedCtx->rsp = u64Val;
6401 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6402 }
6403 return rc;
6404}
6405
6406
6407/**
6408 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6409 *
6410 * @returns VBox status code.
6411 * @param pVCpu The cross context virtual CPU structure.
6412 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6413 * out-of-sync. Make sure to update the required fields
6414 * before using them.
6415 *
6416 * @remarks No-long-jump zone!!!
6417 */
6418static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6419{
6420 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6421 {
6422 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6423 uint32_t uVal = 0;
6424 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6425 AssertRCReturn(rc, rc);
6426
6427 pMixedCtx->eflags.u32 = uVal;
6428 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6429 {
6430 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6431 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6432
6433 pMixedCtx->eflags.Bits.u1VM = 0;
6434 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6435 }
6436
6437 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6438 }
6439 return VINF_SUCCESS;
6440}
6441
6442
6443/**
6444 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6445 * guest-CPU context.
6446 */
6447DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6448{
6449 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6450 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6451 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6452 return rc;
6453}
6454
6455
6456/**
6457 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6458 * from the guest-state area in the VMCS.
6459 *
6460 * @param pVCpu The cross context virtual CPU structure.
6461 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6462 * out-of-sync. Make sure to update the required fields
6463 * before using them.
6464 *
6465 * @remarks No-long-jump zone!!!
6466 */
6467static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6468{
6469 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6470 {
6471 uint32_t uIntrState = 0;
6472 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6473 AssertRC(rc);
6474
6475 if (!uIntrState)
6476 {
6477 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6478 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6479
6480 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6482 }
6483 else
6484 {
6485 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6486 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6487 {
6488 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6489 AssertRC(rc);
6490 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6491 AssertRC(rc);
6492
6493 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6494 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6495 }
6496 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6497 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6498
6499 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6500 {
6501 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6502 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6503 }
6504 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6505 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6506 }
6507
6508 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6509 }
6510}
6511
6512
6513/**
6514 * Saves the guest's activity state.
6515 *
6516 * @returns VBox status code.
6517 * @param pVCpu The cross context virtual CPU structure.
6518 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6519 * out-of-sync. Make sure to update the required fields
6520 * before using them.
6521 *
6522 * @remarks No-long-jump zone!!!
6523 */
6524static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6525{
6526 NOREF(pMixedCtx);
6527 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6528 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6529 return VINF_SUCCESS;
6530}
6531
6532
6533/**
6534 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6535 * the current VMCS into the guest-CPU context.
6536 *
6537 * @returns VBox status code.
6538 * @param pVCpu The cross context virtual CPU structure.
6539 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6540 * out-of-sync. Make sure to update the required fields
6541 * before using them.
6542 *
6543 * @remarks No-long-jump zone!!!
6544 */
6545static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6546{
6547 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6548 {
6549 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6550 uint32_t u32Val = 0;
6551 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6552 pMixedCtx->SysEnter.cs = u32Val;
6553 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6554 }
6555
6556 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6557 {
6558 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6559 uint64_t u64Val = 0;
6560 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6561 pMixedCtx->SysEnter.eip = u64Val;
6562 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6563 }
6564 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6565 {
6566 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6567 uint64_t u64Val = 0;
6568 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6569 pMixedCtx->SysEnter.esp = u64Val;
6570 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6571 }
6572 return VINF_SUCCESS;
6573}
6574
6575
6576/**
6577 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6578 * the CPU back into the guest-CPU context.
6579 *
6580 * @returns VBox status code.
6581 * @param pVCpu The cross context virtual CPU structure.
6582 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6583 * out-of-sync. Make sure to update the required fields
6584 * before using them.
6585 *
6586 * @remarks No-long-jump zone!!!
6587 */
6588static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589{
6590 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6591 VMMRZCallRing3Disable(pVCpu);
6592 HM_DISABLE_PREEMPT();
6593
6594 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6595 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6596 {
6597 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6598 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6599 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6600 }
6601
6602 HM_RESTORE_PREEMPT();
6603 VMMRZCallRing3Enable(pVCpu);
6604
6605 return VINF_SUCCESS;
6606}
6607
6608
6609/**
6610 * Saves the auto load/store'd guest MSRs from the current VMCS into
6611 * the guest-CPU context.
6612 *
6613 * @returns VBox status code.
6614 * @param pVCpu The cross context virtual CPU structure.
6615 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6616 * out-of-sync. Make sure to update the required fields
6617 * before using them.
6618 *
6619 * @remarks No-long-jump zone!!!
6620 */
6621static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622{
6623 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6624 return VINF_SUCCESS;
6625
6626 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6627 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6628 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6629 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6630 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6631 {
6632 switch (pMsr->u32Msr)
6633 {
6634 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6635 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6636 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6637 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6638 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6639 case MSR_IA32_SPEC_CTRL: CPUMR0SetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6640 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6641 break;
6642
6643 default:
6644 {
6645 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6646 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6647 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6648 }
6649 }
6650 }
6651
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6653 return VINF_SUCCESS;
6654}
6655
6656
6657/**
6658 * Saves the guest control registers from the current VMCS into the guest-CPU
6659 * context.
6660 *
6661 * @returns VBox status code.
6662 * @param pVCpu The cross context virtual CPU structure.
6663 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6664 * out-of-sync. Make sure to update the required fields
6665 * before using them.
6666 *
6667 * @remarks No-long-jump zone!!!
6668 */
6669static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6670{
6671 /* Guest CR0. Guest FPU. */
6672 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6673 AssertRCReturn(rc, rc);
6674
6675 /* Guest CR4. */
6676 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6677 AssertRCReturn(rc, rc);
6678
6679 /* Guest CR2 - updated always during the world-switch or in #PF. */
6680 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6681 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6682 {
6683 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6684 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6685 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6686
6687 PVM pVM = pVCpu->CTX_SUFF(pVM);
6688 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6689 || ( pVM->hm.s.fNestedPaging
6690 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6691 {
6692 uint64_t u64Val = 0;
6693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6694 if (pMixedCtx->cr3 != u64Val)
6695 {
6696 CPUMSetGuestCR3(pVCpu, u64Val);
6697 if (VMMRZCallRing3IsEnabled(pVCpu))
6698 {
6699 PGMUpdateCR3(pVCpu, u64Val);
6700 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6701 }
6702 else
6703 {
6704 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6705 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6706 }
6707 }
6708
6709 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6710 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6711 {
6712 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6713 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6714 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6715 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6716 AssertRCReturn(rc, rc);
6717
6718 if (VMMRZCallRing3IsEnabled(pVCpu))
6719 {
6720 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6721 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6722 }
6723 else
6724 {
6725 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6726 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6727 }
6728 }
6729 }
6730
6731 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6732 }
6733
6734 /*
6735 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6736 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6737 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6738 *
6739 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6740 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6741 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6742 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6743 *
6744 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6745 */
6746 if (VMMRZCallRing3IsEnabled(pVCpu))
6747 {
6748 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6749 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6750
6751 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6752 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6753
6754 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6755 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6756 }
6757
6758 return rc;
6759}
6760
6761
6762/**
6763 * Reads a guest segment register from the current VMCS into the guest-CPU
6764 * context.
6765 *
6766 * @returns VBox status code.
6767 * @param pVCpu The cross context virtual CPU structure.
6768 * @param idxSel Index of the selector in the VMCS.
6769 * @param idxLimit Index of the segment limit in the VMCS.
6770 * @param idxBase Index of the segment base in the VMCS.
6771 * @param idxAccess Index of the access rights of the segment in the VMCS.
6772 * @param pSelReg Pointer to the segment selector.
6773 *
6774 * @remarks No-long-jump zone!!!
6775 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6776 * macro as that takes care of whether to read from the VMCS cache or
6777 * not.
6778 */
6779DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6780 PCPUMSELREG pSelReg)
6781{
6782 NOREF(pVCpu);
6783
6784 uint32_t u32Val = 0;
6785 int rc = VMXReadVmcs32(idxSel, &u32Val);
6786 AssertRCReturn(rc, rc);
6787 pSelReg->Sel = (uint16_t)u32Val;
6788 pSelReg->ValidSel = (uint16_t)u32Val;
6789 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6790
6791 rc = VMXReadVmcs32(idxLimit, &u32Val);
6792 AssertRCReturn(rc, rc);
6793 pSelReg->u32Limit = u32Val;
6794
6795 uint64_t u64Val = 0;
6796 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6797 AssertRCReturn(rc, rc);
6798 pSelReg->u64Base = u64Val;
6799
6800 rc = VMXReadVmcs32(idxAccess, &u32Val);
6801 AssertRCReturn(rc, rc);
6802 pSelReg->Attr.u = u32Val;
6803
6804 /*
6805 * If VT-x marks the segment as unusable, most other bits remain undefined:
6806 * - For CS the L, D and G bits have meaning.
6807 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6808 * - For the remaining data segments no bits are defined.
6809 *
6810 * The present bit and the unusable bit has been observed to be set at the
6811 * same time (the selector was supposed to be invalid as we started executing
6812 * a V8086 interrupt in ring-0).
6813 *
6814 * What should be important for the rest of the VBox code, is that the P bit is
6815 * cleared. Some of the other VBox code recognizes the unusable bit, but
6816 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6817 * safe side here, we'll strip off P and other bits we don't care about. If
6818 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6819 *
6820 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6821 */
6822 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6823 {
6824 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6825
6826 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6827 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6828 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6829
6830 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6831#ifdef DEBUG_bird
6832 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6833 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6834 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6835#endif
6836 }
6837 return VINF_SUCCESS;
6838}
6839
6840
6841#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6842# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6843 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6844 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6845#else
6846# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6847 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6848 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6849#endif
6850
6851
6852/**
6853 * Saves the guest segment registers from the current VMCS into the guest-CPU
6854 * context.
6855 *
6856 * @returns VBox status code.
6857 * @param pVCpu The cross context virtual CPU structure.
6858 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6859 * out-of-sync. Make sure to update the required fields
6860 * before using them.
6861 *
6862 * @remarks No-long-jump zone!!!
6863 */
6864static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6865{
6866 /* Guest segment registers. */
6867 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6868 {
6869 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6870 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6871 AssertRCReturn(rc, rc);
6872
6873 rc = VMXLOCAL_READ_SEG(CS, cs);
6874 rc |= VMXLOCAL_READ_SEG(SS, ss);
6875 rc |= VMXLOCAL_READ_SEG(DS, ds);
6876 rc |= VMXLOCAL_READ_SEG(ES, es);
6877 rc |= VMXLOCAL_READ_SEG(FS, fs);
6878 rc |= VMXLOCAL_READ_SEG(GS, gs);
6879 AssertRCReturn(rc, rc);
6880
6881 /* Restore segment attributes for real-on-v86 mode hack. */
6882 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6883 {
6884 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6885 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6886 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6887 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6888 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6889 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6890 }
6891 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6892 }
6893
6894 return VINF_SUCCESS;
6895}
6896
6897
6898/**
6899 * Saves the guest descriptor table registers and task register from the current
6900 * VMCS into the guest-CPU context.
6901 *
6902 * @returns VBox status code.
6903 * @param pVCpu The cross context virtual CPU structure.
6904 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6905 * out-of-sync. Make sure to update the required fields
6906 * before using them.
6907 *
6908 * @remarks No-long-jump zone!!!
6909 */
6910static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6911{
6912 int rc = VINF_SUCCESS;
6913
6914 /* Guest LDTR. */
6915 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6916 {
6917 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6918 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6919 AssertRCReturn(rc, rc);
6920 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6921 }
6922
6923 /* Guest GDTR. */
6924 uint64_t u64Val = 0;
6925 uint32_t u32Val = 0;
6926 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6927 {
6928 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6929 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6930 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6931 pMixedCtx->gdtr.pGdt = u64Val;
6932 pMixedCtx->gdtr.cbGdt = u32Val;
6933 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6934 }
6935
6936 /* Guest IDTR. */
6937 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6938 {
6939 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6940 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6941 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6942 pMixedCtx->idtr.pIdt = u64Val;
6943 pMixedCtx->idtr.cbIdt = u32Val;
6944 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6945 }
6946
6947 /* Guest TR. */
6948 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6949 {
6950 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6951 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6952 AssertRCReturn(rc, rc);
6953
6954 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6955 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6956 {
6957 rc = VMXLOCAL_READ_SEG(TR, tr);
6958 AssertRCReturn(rc, rc);
6959 }
6960 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6961 }
6962 return rc;
6963}
6964
6965#undef VMXLOCAL_READ_SEG
6966
6967
6968/**
6969 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6970 * context.
6971 *
6972 * @returns VBox status code.
6973 * @param pVCpu The cross context virtual CPU structure.
6974 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6975 * out-of-sync. Make sure to update the required fields
6976 * before using them.
6977 *
6978 * @remarks No-long-jump zone!!!
6979 */
6980static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6981{
6982 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6983 {
6984 if (!pVCpu->hm.s.fUsingHyperDR7)
6985 {
6986 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6987 uint32_t u32Val;
6988 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6989 pMixedCtx->dr[7] = u32Val;
6990 }
6991
6992 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6993 }
6994 return VINF_SUCCESS;
6995}
6996
6997
6998/**
6999 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
7000 *
7001 * @returns VBox status code.
7002 * @param pVCpu The cross context virtual CPU structure.
7003 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7004 * out-of-sync. Make sure to update the required fields
7005 * before using them.
7006 *
7007 * @remarks No-long-jump zone!!!
7008 */
7009static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7010{
7011 NOREF(pMixedCtx);
7012
7013 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7014 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7015 return VINF_SUCCESS;
7016}
7017
7018
7019/**
7020 * Saves the entire guest state from the currently active VMCS into the
7021 * guest-CPU context.
7022 *
7023 * This essentially VMREADs all guest-data.
7024 *
7025 * @returns VBox status code.
7026 * @param pVCpu The cross context virtual CPU structure.
7027 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028 * out-of-sync. Make sure to update the required fields
7029 * before using them.
7030 */
7031static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7032{
7033 Assert(pVCpu);
7034 Assert(pMixedCtx);
7035
7036 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7037 return VINF_SUCCESS;
7038
7039 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7040 again on the ring-3 callback path, there is no real need to. */
7041 if (VMMRZCallRing3IsEnabled(pVCpu))
7042 VMMR0LogFlushDisable(pVCpu);
7043 else
7044 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7045 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7046
7047 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7057 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7060 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7063 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7066 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7069 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7072 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7073
7074 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7075 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7076
7077 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7078 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7079 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7080
7081 if (VMMRZCallRing3IsEnabled(pVCpu))
7082 VMMR0LogFlushEnable(pVCpu);
7083
7084 return VINF_SUCCESS;
7085}
7086
7087
7088/**
7089 * Saves basic guest registers needed for IEM instruction execution.
7090 *
7091 * @returns VBox status code (OR-able).
7092 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7093 * @param pMixedCtx Pointer to the CPU context of the guest.
7094 * @param fMemory Whether the instruction being executed operates on
7095 * memory or not. Only CR0 is synced up if clear.
7096 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7097 */
7098static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7099{
7100 /*
7101 * We assume all general purpose registers other than RSP are available.
7102 *
7103 * - RIP is a must, as it will be incremented or otherwise changed.
7104 * - RFLAGS are always required to figure the CPL.
7105 * - RSP isn't always required, however it's a GPR, so frequently required.
7106 * - SS and CS are the only segment register needed if IEM doesn't do memory
7107 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7108 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7109 * be required for memory accesses.
7110 *
7111 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7112 */
7113 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7114 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7115 if (fNeedRsp)
7116 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7117 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7118 if (!fMemory)
7119 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7120 else
7121 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7122 AssertRCReturn(rc, rc);
7123 return rc;
7124}
7125
7126
7127/**
7128 * Ensures that we've got a complete basic guest-context.
7129 *
7130 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7131 * is for the interpreter.
7132 *
7133 * @returns VBox status code.
7134 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7135 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7136 * needing to be synced in.
7137 * @thread EMT(pVCpu)
7138 */
7139VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7140{
7141 /* Note! Since this is only applicable to VT-x, the implementation is placed
7142 in the VT-x part of the sources instead of the generic stuff. */
7143 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7144 {
7145 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7146 /*
7147 * For now, imply that the caller might change everything too. Do this after
7148 * saving the guest state so as to not trigger assertions.
7149 */
7150 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7151 return rc;
7152 }
7153 return VINF_SUCCESS;
7154}
7155
7156
7157/**
7158 * Check per-VM and per-VCPU force flag actions that require us to go back to
7159 * ring-3 for one reason or another.
7160 *
7161 * @returns Strict VBox status code (i.e. informational status codes too)
7162 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7163 * ring-3.
7164 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7165 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7166 * interrupts)
7167 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7168 * all EMTs to be in ring-3.
7169 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7170 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7171 * to the EM loop.
7172 *
7173 * @param pVM The cross context VM structure.
7174 * @param pVCpu The cross context virtual CPU structure.
7175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7176 * out-of-sync. Make sure to update the required fields
7177 * before using them.
7178 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7179 */
7180static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7181{
7182 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7183
7184 /*
7185 * Anything pending? Should be more likely than not if we're doing a good job.
7186 */
7187 if ( !fStepping
7188 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7189 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7190 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7191 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7192 return VINF_SUCCESS;
7193
7194 /* We need the control registers now, make sure the guest-CPU context is updated. */
7195 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7196 AssertRCReturn(rc3, rc3);
7197
7198 /* Pending HM CR3 sync. */
7199 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7200 {
7201 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7202 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7203 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7204 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7205 }
7206
7207 /* Pending HM PAE PDPEs. */
7208 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7209 {
7210 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7211 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7212 }
7213
7214 /* Pending PGM C3 sync. */
7215 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7216 {
7217 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7218 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7219 if (rcStrict2 != VINF_SUCCESS)
7220 {
7221 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7222 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7223 return rcStrict2;
7224 }
7225 }
7226
7227 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7228 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7229 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7230 {
7231 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7232 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7233 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7234 return rc2;
7235 }
7236
7237 /* Pending VM request packets, such as hardware interrupts. */
7238 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7239 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7240 {
7241 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7242 return VINF_EM_PENDING_REQUEST;
7243 }
7244
7245 /* Pending PGM pool flushes. */
7246 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7247 {
7248 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7249 return VINF_PGM_POOL_FLUSH_PENDING;
7250 }
7251
7252 /* Pending DMA requests. */
7253 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7254 {
7255 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7256 return VINF_EM_RAW_TO_R3;
7257 }
7258
7259 return VINF_SUCCESS;
7260}
7261
7262
7263/**
7264 * Converts any TRPM trap into a pending HM event. This is typically used when
7265 * entering from ring-3 (not longjmp returns).
7266 *
7267 * @param pVCpu The cross context virtual CPU structure.
7268 */
7269static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7270{
7271 Assert(TRPMHasTrap(pVCpu));
7272 Assert(!pVCpu->hm.s.Event.fPending);
7273
7274 uint8_t uVector;
7275 TRPMEVENT enmTrpmEvent;
7276 RTGCUINT uErrCode;
7277 RTGCUINTPTR GCPtrFaultAddress;
7278 uint8_t cbInstr;
7279
7280 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7281 AssertRC(rc);
7282
7283 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7284 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7285 if (enmTrpmEvent == TRPM_TRAP)
7286 {
7287 switch (uVector)
7288 {
7289 case X86_XCPT_NMI:
7290 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7291 break;
7292
7293 case X86_XCPT_BP:
7294 case X86_XCPT_OF:
7295 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7296 break;
7297
7298 case X86_XCPT_PF:
7299 case X86_XCPT_DF:
7300 case X86_XCPT_TS:
7301 case X86_XCPT_NP:
7302 case X86_XCPT_SS:
7303 case X86_XCPT_GP:
7304 case X86_XCPT_AC:
7305 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7306 RT_FALL_THRU();
7307 default:
7308 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7309 break;
7310 }
7311 }
7312 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7313 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7314 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7315 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7316 else
7317 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7318
7319 rc = TRPMResetTrap(pVCpu);
7320 AssertRC(rc);
7321 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7322 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7323
7324 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7325}
7326
7327
7328/**
7329 * Converts the pending HM event into a TRPM trap.
7330 *
7331 * @param pVCpu The cross context virtual CPU structure.
7332 */
7333static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7334{
7335 Assert(pVCpu->hm.s.Event.fPending);
7336
7337 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7338 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7339 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7340 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7341
7342 /* If a trap was already pending, we did something wrong! */
7343 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7344
7345 TRPMEVENT enmTrapType;
7346 switch (uVectorType)
7347 {
7348 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7349 enmTrapType = TRPM_HARDWARE_INT;
7350 break;
7351
7352 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7353 enmTrapType = TRPM_SOFTWARE_INT;
7354 break;
7355
7356 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7357 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7358 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7359 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7360 enmTrapType = TRPM_TRAP;
7361 break;
7362
7363 default:
7364 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7365 enmTrapType = TRPM_32BIT_HACK;
7366 break;
7367 }
7368
7369 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7370
7371 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7372 AssertRC(rc);
7373
7374 if (fErrorCodeValid)
7375 TRPMSetErrorCode(pVCpu, uErrorCode);
7376
7377 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7378 && uVector == X86_XCPT_PF)
7379 {
7380 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7381 }
7382 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7383 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7384 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7385 {
7386 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7387 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7388 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7389 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7390 }
7391
7392 /* Clear any pending events from the VMCS. */
7393 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7394 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7395
7396 /* We're now done converting the pending event. */
7397 pVCpu->hm.s.Event.fPending = false;
7398}
7399
7400
7401/**
7402 * Does the necessary state syncing before returning to ring-3 for any reason
7403 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7404 *
7405 * @returns VBox status code.
7406 * @param pVCpu The cross context virtual CPU structure.
7407 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7408 * be out-of-sync. Make sure to update the required
7409 * fields before using them.
7410 * @param fSaveGuestState Whether to save the guest state or not.
7411 *
7412 * @remarks No-long-jmp zone!!!
7413 */
7414static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7415{
7416 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7417 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7418
7419 RTCPUID idCpu = RTMpCpuId();
7420 Log4Func(("HostCpuId=%u\n", idCpu));
7421
7422 /*
7423 * !!! IMPORTANT !!!
7424 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7425 */
7426
7427 /* Save the guest state if necessary. */
7428 if ( fSaveGuestState
7429 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7430 {
7431 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7432 AssertRCReturn(rc, rc);
7433 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7434 }
7435
7436 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7437 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7438 {
7439 /* We shouldn't reload CR0 without saving it first. */
7440 if (!fSaveGuestState)
7441 {
7442 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7443 AssertRCReturn(rc, rc);
7444 }
7445 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7446 }
7447
7448 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7449#ifdef VBOX_STRICT
7450 if (CPUMIsHyperDebugStateActive(pVCpu))
7451 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7452#endif
7453 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7454 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7455 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7456 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7457
7458#if HC_ARCH_BITS == 64
7459 /* Restore host-state bits that VT-x only restores partially. */
7460 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7461 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7462 {
7463 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7464 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7465 }
7466 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7467#endif
7468
7469 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7470 if (pVCpu->hm.s.vmx.fLazyMsrs)
7471 {
7472 /* We shouldn't reload the guest MSRs without saving it first. */
7473 if (!fSaveGuestState)
7474 {
7475 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7476 AssertRCReturn(rc, rc);
7477 }
7478 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7479 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7480 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7481 }
7482
7483 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7484 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7485
7486 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7487 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7488 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7489 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7490 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7491 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7492 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7493 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7494
7495 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7496
7497 /** @todo This partially defeats the purpose of having preemption hooks.
7498 * The problem is, deregistering the hooks should be moved to a place that
7499 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7500 * context.
7501 */
7502 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7503 {
7504 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7505 AssertRCReturn(rc, rc);
7506
7507 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7508 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7509 }
7510 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7511 NOREF(idCpu);
7512
7513 return VINF_SUCCESS;
7514}
7515
7516
7517/**
7518 * Leaves the VT-x session.
7519 *
7520 * @returns VBox status code.
7521 * @param pVCpu The cross context virtual CPU structure.
7522 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7523 * out-of-sync. Make sure to update the required fields
7524 * before using them.
7525 *
7526 * @remarks No-long-jmp zone!!!
7527 */
7528DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7529{
7530 HM_DISABLE_PREEMPT();
7531 HMVMX_ASSERT_CPU_SAFE();
7532 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7533 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7534
7535 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7536 and done this from the VMXR0ThreadCtxCallback(). */
7537 if (!pVCpu->hm.s.fLeaveDone)
7538 {
7539 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7540 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7541 pVCpu->hm.s.fLeaveDone = true;
7542 }
7543 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7544
7545 /*
7546 * !!! IMPORTANT !!!
7547 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7548 */
7549
7550 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7551 /** @todo Deregistering here means we need to VMCLEAR always
7552 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7553 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7554 VMMR0ThreadCtxHookDisable(pVCpu);
7555
7556 /* Leave HM context. This takes care of local init (term). */
7557 int rc = HMR0LeaveCpu(pVCpu);
7558
7559 HM_RESTORE_PREEMPT();
7560 return rc;
7561}
7562
7563
7564/**
7565 * Does the necessary state syncing before doing a longjmp to ring-3.
7566 *
7567 * @returns VBox status code.
7568 * @param pVCpu The cross context virtual CPU structure.
7569 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7570 * out-of-sync. Make sure to update the required fields
7571 * before using them.
7572 *
7573 * @remarks No-long-jmp zone!!!
7574 */
7575DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7576{
7577 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7578}
7579
7580
7581/**
7582 * Take necessary actions before going back to ring-3.
7583 *
7584 * An action requires us to go back to ring-3. This function does the necessary
7585 * steps before we can safely return to ring-3. This is not the same as longjmps
7586 * to ring-3, this is voluntary and prepares the guest so it may continue
7587 * executing outside HM (recompiler/IEM).
7588 *
7589 * @returns VBox status code.
7590 * @param pVM The cross context VM structure.
7591 * @param pVCpu The cross context virtual CPU structure.
7592 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7593 * out-of-sync. Make sure to update the required fields
7594 * before using them.
7595 * @param rcExit The reason for exiting to ring-3. Can be
7596 * VINF_VMM_UNKNOWN_RING3_CALL.
7597 */
7598static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7599{
7600 Assert(pVM);
7601 Assert(pVCpu);
7602 Assert(pMixedCtx);
7603 HMVMX_ASSERT_PREEMPT_SAFE();
7604
7605 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7606 {
7607 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7608 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7609 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7610 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7611 }
7612
7613 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7614 VMMRZCallRing3Disable(pVCpu);
7615 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7616
7617 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7618 if (pVCpu->hm.s.Event.fPending)
7619 {
7620 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7621 Assert(!pVCpu->hm.s.Event.fPending);
7622 }
7623
7624 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7625 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7626
7627 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7628 and if we're injecting an event we should have a TRPM trap pending. */
7629 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7630#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7631 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7632#endif
7633
7634 /* Save guest state and restore host state bits. */
7635 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7636 AssertRCReturn(rc, rc);
7637 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7638 /* Thread-context hooks are unregistered at this point!!! */
7639
7640 /* Sync recompiler state. */
7641 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7642 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7643 | CPUM_CHANGED_LDTR
7644 | CPUM_CHANGED_GDTR
7645 | CPUM_CHANGED_IDTR
7646 | CPUM_CHANGED_TR
7647 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7648 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7649 if ( pVM->hm.s.fNestedPaging
7650 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7651 {
7652 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7653 }
7654
7655 Assert(!pVCpu->hm.s.fClearTrapFlag);
7656
7657 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7658 if (rcExit != VINF_EM_RAW_INTERRUPT)
7659 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7660
7661 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7662
7663 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7664 VMMRZCallRing3RemoveNotification(pVCpu);
7665 VMMRZCallRing3Enable(pVCpu);
7666
7667 return rc;
7668}
7669
7670
7671/**
7672 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7673 * longjump to ring-3 and possibly get preempted.
7674 *
7675 * @returns VBox status code.
7676 * @param pVCpu The cross context virtual CPU structure.
7677 * @param enmOperation The operation causing the ring-3 longjump.
7678 * @param pvUser Opaque pointer to the guest-CPU context. The data
7679 * may be out-of-sync. Make sure to update the required
7680 * fields before using them.
7681 */
7682static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7683{
7684 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7685 {
7686 /*
7687 * !!! IMPORTANT !!!
7688 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7689 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7690 */
7691 VMMRZCallRing3RemoveNotification(pVCpu);
7692 VMMRZCallRing3Disable(pVCpu);
7693 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7694 RTThreadPreemptDisable(&PreemptState);
7695
7696 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7697 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7698
7699#if HC_ARCH_BITS == 64
7700 /* Restore host-state bits that VT-x only restores partially. */
7701 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7702 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7703 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7704 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7705#endif
7706 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7707 if (pVCpu->hm.s.vmx.fLazyMsrs)
7708 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7709
7710 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7711 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7712 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7713 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7714 {
7715 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7716 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7717 }
7718
7719 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7720 VMMR0ThreadCtxHookDisable(pVCpu);
7721 HMR0LeaveCpu(pVCpu);
7722 RTThreadPreemptRestore(&PreemptState);
7723 return VINF_SUCCESS;
7724 }
7725
7726 Assert(pVCpu);
7727 Assert(pvUser);
7728 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7729 HMVMX_ASSERT_PREEMPT_SAFE();
7730
7731 VMMRZCallRing3Disable(pVCpu);
7732 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7733
7734 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7735 enmOperation));
7736
7737 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7738 AssertRCReturn(rc, rc);
7739
7740 VMMRZCallRing3Enable(pVCpu);
7741 return VINF_SUCCESS;
7742}
7743
7744
7745/**
7746 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7747 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7748 *
7749 * @param pVCpu The cross context virtual CPU structure.
7750 */
7751DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7752{
7753 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7754 {
7755 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7756 {
7757 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7758 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7759 AssertRC(rc);
7760 Log4(("Setup interrupt-window exiting\n"));
7761 }
7762 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7763}
7764
7765
7766/**
7767 * Clears the interrupt-window exiting control in the VMCS.
7768 *
7769 * @param pVCpu The cross context virtual CPU structure.
7770 */
7771DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7772{
7773 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7774 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7775 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7776 AssertRC(rc);
7777 Log4(("Cleared interrupt-window exiting\n"));
7778}
7779
7780
7781/**
7782 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7783 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7784 *
7785 * @param pVCpu The cross context virtual CPU structure.
7786 */
7787DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7788{
7789 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7790 {
7791 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7792 {
7793 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7794 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7795 AssertRC(rc);
7796 Log4(("Setup NMI-window exiting\n"));
7797 }
7798 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7799}
7800
7801
7802/**
7803 * Clears the NMI-window exiting control in the VMCS.
7804 *
7805 * @param pVCpu The cross context virtual CPU structure.
7806 */
7807DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7808{
7809 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7810 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7811 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7812 AssertRC(rc);
7813 Log4(("Cleared NMI-window exiting\n"));
7814}
7815
7816
7817/**
7818 * Evaluates the event to be delivered to the guest and sets it as the pending
7819 * event.
7820 *
7821 * @returns The VT-x guest-interruptibility state.
7822 * @param pVCpu The cross context virtual CPU structure.
7823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7824 * out-of-sync. Make sure to update the required fields
7825 * before using them.
7826 */
7827static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7828{
7829 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7830 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7831 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7832 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7833 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7834
7835 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7836 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7837 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7838 Assert(!TRPMHasTrap(pVCpu));
7839
7840 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7841 APICUpdatePendingInterrupts(pVCpu);
7842
7843 /*
7844 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7845 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7846 */
7847 /** @todo SMI. SMIs take priority over NMIs. */
7848 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7849 {
7850 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7851 if ( !pVCpu->hm.s.Event.fPending
7852 && !fBlockNmi
7853 && !fBlockSti
7854 && !fBlockMovSS)
7855 {
7856 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7857 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7858 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7859
7860 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7861 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7862 }
7863 else
7864 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7865 }
7866 /*
7867 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7868 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7869 */
7870 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7871 && !pVCpu->hm.s.fSingleInstruction)
7872 {
7873 Assert(!DBGFIsStepping(pVCpu));
7874 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7875 AssertRC(rc);
7876 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7877 if ( !pVCpu->hm.s.Event.fPending
7878 && !fBlockInt
7879 && !fBlockSti
7880 && !fBlockMovSS)
7881 {
7882 uint8_t u8Interrupt;
7883 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7884 if (RT_SUCCESS(rc))
7885 {
7886 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7887 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7888 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7889
7890 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7891 }
7892 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7893 {
7894 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7895 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7896 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7897
7898 /*
7899 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7900 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7901 * need to re-set this force-flag here.
7902 */
7903 }
7904 else
7905 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7906 }
7907 else
7908 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7909 }
7910
7911 return uIntrState;
7912}
7913
7914
7915/**
7916 * Sets a pending-debug exception to be delivered to the guest if the guest is
7917 * single-stepping in the VMCS.
7918 *
7919 * @param pVCpu The cross context virtual CPU structure.
7920 */
7921DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7922{
7923 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7924 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7925 AssertRC(rc);
7926}
7927
7928
7929/**
7930 * Injects any pending events into the guest if the guest is in a state to
7931 * receive them.
7932 *
7933 * @returns Strict VBox status code (i.e. informational status codes too).
7934 * @param pVCpu The cross context virtual CPU structure.
7935 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7936 * out-of-sync. Make sure to update the required fields
7937 * before using them.
7938 * @param uIntrState The VT-x guest-interruptibility state.
7939 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7940 * return VINF_EM_DBG_STEPPED if the event was
7941 * dispatched directly.
7942 */
7943static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7944{
7945 HMVMX_ASSERT_PREEMPT_SAFE();
7946 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7947
7948 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7949 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7950
7951 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7952 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7953 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7954 Assert(!TRPMHasTrap(pVCpu));
7955
7956 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7957 if (pVCpu->hm.s.Event.fPending)
7958 {
7959 /*
7960 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7961 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7962 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7963 *
7964 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7965 */
7966 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7967#ifdef VBOX_STRICT
7968 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7969 {
7970 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7971 Assert(!fBlockInt);
7972 Assert(!fBlockSti);
7973 Assert(!fBlockMovSS);
7974 }
7975 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7976 {
7977 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7978 Assert(!fBlockSti);
7979 Assert(!fBlockMovSS);
7980 Assert(!fBlockNmi);
7981 }
7982#endif
7983 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7984 (uint8_t)uIntType));
7985 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7986 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7987 fStepping, &uIntrState);
7988 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7989
7990 /* Update the interruptibility-state as it could have been changed by
7991 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7992 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7993 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7994
7995 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7996 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7997 else
7998 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7999 }
8000
8001 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
8002 if ( fBlockSti
8003 || fBlockMovSS)
8004 {
8005 if (!pVCpu->hm.s.fSingleInstruction)
8006 {
8007 /*
8008 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8009 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8010 * See Intel spec. 27.3.4 "Saving Non-Register State".
8011 */
8012 Assert(!DBGFIsStepping(pVCpu));
8013 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8014 AssertRCReturn(rc2, rc2);
8015 if (pMixedCtx->eflags.Bits.u1TF)
8016 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8017 }
8018 else if (pMixedCtx->eflags.Bits.u1TF)
8019 {
8020 /*
8021 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8022 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8023 */
8024 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8025 uIntrState = 0;
8026 }
8027 }
8028
8029 /*
8030 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8031 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8032 */
8033 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8034 AssertRC(rc2);
8035
8036 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8037 NOREF(fBlockMovSS); NOREF(fBlockSti);
8038 return rcStrict;
8039}
8040
8041
8042/**
8043 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8044 *
8045 * @param pVCpu The cross context virtual CPU structure.
8046 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8047 * out-of-sync. Make sure to update the required fields
8048 * before using them.
8049 */
8050DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8051{
8052 NOREF(pMixedCtx);
8053 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8054 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8055}
8056
8057
8058/**
8059 * Injects a double-fault (\#DF) exception into the VM.
8060 *
8061 * @returns Strict VBox status code (i.e. informational status codes too).
8062 * @param pVCpu The cross context virtual CPU structure.
8063 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8064 * out-of-sync. Make sure to update the required fields
8065 * before using them.
8066 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8067 * and should return VINF_EM_DBG_STEPPED if the event
8068 * is injected directly (register modified by us, not
8069 * by hardware on VM-entry).
8070 * @param puIntrState Pointer to the current guest interruptibility-state.
8071 * This interruptibility-state will be updated if
8072 * necessary. This cannot not be NULL.
8073 */
8074DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8075{
8076 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8077 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8078 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8079 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8080 fStepping, puIntrState);
8081}
8082
8083
8084/**
8085 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8086 *
8087 * @param pVCpu The cross context virtual CPU structure.
8088 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8089 * out-of-sync. Make sure to update the required fields
8090 * before using them.
8091 */
8092DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8093{
8094 NOREF(pMixedCtx);
8095 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8096 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8097 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8098}
8099
8100
8101/**
8102 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8103 *
8104 * @param pVCpu The cross context virtual CPU structure.
8105 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8106 * out-of-sync. Make sure to update the required fields
8107 * before using them.
8108 * @param cbInstr The value of RIP that is to be pushed on the guest
8109 * stack.
8110 */
8111DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8112{
8113 NOREF(pMixedCtx);
8114 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8115 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8116 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8117}
8118
8119
8120/**
8121 * Injects a general-protection (\#GP) fault into the VM.
8122 *
8123 * @returns Strict VBox status code (i.e. informational status codes too).
8124 * @param pVCpu The cross context virtual CPU structure.
8125 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8126 * out-of-sync. Make sure to update the required fields
8127 * before using them.
8128 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8129 * mode, i.e. in real-mode it's not valid).
8130 * @param u32ErrorCode The error code associated with the \#GP.
8131 * @param fStepping Whether we're running in
8132 * hmR0VmxRunGuestCodeStep() and should return
8133 * VINF_EM_DBG_STEPPED if the event is injected
8134 * directly (register modified by us, not by
8135 * hardware on VM-entry).
8136 * @param puIntrState Pointer to the current guest interruptibility-state.
8137 * This interruptibility-state will be updated if
8138 * necessary. This cannot not be NULL.
8139 */
8140DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8141 bool fStepping, uint32_t *puIntrState)
8142{
8143 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8144 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8145 if (fErrorCodeValid)
8146 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8147 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8148 fStepping, puIntrState);
8149}
8150
8151
8152#if 0 /* unused */
8153/**
8154 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8155 * VM.
8156 *
8157 * @param pVCpu The cross context virtual CPU structure.
8158 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8159 * out-of-sync. Make sure to update the required fields
8160 * before using them.
8161 * @param u32ErrorCode The error code associated with the \#GP.
8162 */
8163DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8164{
8165 NOREF(pMixedCtx);
8166 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8167 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8168 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8169 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8170}
8171#endif /* unused */
8172
8173
8174/**
8175 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8176 *
8177 * @param pVCpu The cross context virtual CPU structure.
8178 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8179 * out-of-sync. Make sure to update the required fields
8180 * before using them.
8181 * @param uVector The software interrupt vector number.
8182 * @param cbInstr The value of RIP that is to be pushed on the guest
8183 * stack.
8184 */
8185DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8186{
8187 NOREF(pMixedCtx);
8188 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8189 if ( uVector == X86_XCPT_BP
8190 || uVector == X86_XCPT_OF)
8191 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8192 else
8193 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8194 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8195}
8196
8197
8198/**
8199 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8200 * stack.
8201 *
8202 * @returns Strict VBox status code (i.e. informational status codes too).
8203 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8204 * @param pVM The cross context VM structure.
8205 * @param pMixedCtx Pointer to the guest-CPU context.
8206 * @param uValue The value to push to the guest stack.
8207 */
8208DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8209{
8210 /*
8211 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8212 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8213 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8214 */
8215 if (pMixedCtx->sp == 1)
8216 return VINF_EM_RESET;
8217 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8218 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8219 AssertRC(rc);
8220 return rc;
8221}
8222
8223
8224/**
8225 * Injects an event into the guest upon VM-entry by updating the relevant fields
8226 * in the VM-entry area in the VMCS.
8227 *
8228 * @returns Strict VBox status code (i.e. informational status codes too).
8229 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8230 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8231 *
8232 * @param pVCpu The cross context virtual CPU structure.
8233 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8234 * be out-of-sync. Make sure to update the required
8235 * fields before using them.
8236 * @param u64IntInfo The VM-entry interruption-information field.
8237 * @param cbInstr The VM-entry instruction length in bytes (for
8238 * software interrupts, exceptions and privileged
8239 * software exceptions).
8240 * @param u32ErrCode The VM-entry exception error code.
8241 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8242 * @param puIntrState Pointer to the current guest interruptibility-state.
8243 * This interruptibility-state will be updated if
8244 * necessary. This cannot not be NULL.
8245 * @param fStepping Whether we're running in
8246 * hmR0VmxRunGuestCodeStep() and should return
8247 * VINF_EM_DBG_STEPPED if the event is injected
8248 * directly (register modified by us, not by
8249 * hardware on VM-entry).
8250 *
8251 * @remarks Requires CR0!
8252 */
8253static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8254 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8255 uint32_t *puIntrState)
8256{
8257 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8258 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8259 Assert(puIntrState);
8260 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8261
8262 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8263 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8264
8265#ifdef VBOX_STRICT
8266 /*
8267 * Validate the error-code-valid bit for hardware exceptions.
8268 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8269 */
8270 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8271 && !CPUMIsGuestInRealModeEx(pMixedCtx))
8272 {
8273 switch (uVector)
8274 {
8275 case X86_XCPT_PF:
8276 case X86_XCPT_DF:
8277 case X86_XCPT_TS:
8278 case X86_XCPT_NP:
8279 case X86_XCPT_SS:
8280 case X86_XCPT_GP:
8281 case X86_XCPT_AC:
8282 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8283 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8284 RT_FALL_THRU();
8285 default:
8286 break;
8287 }
8288 }
8289#endif
8290
8291 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8292 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8293 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8294
8295 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8296
8297 /* We require CR0 to check if the guest is in real-mode. */
8298 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8299 AssertRCReturn(rc, rc);
8300
8301 /*
8302 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8303 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8304 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8305 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8306 */
8307 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8308 {
8309 PVM pVM = pVCpu->CTX_SUFF(pVM);
8310 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8311 {
8312 Assert(PDMVmmDevHeapIsEnabled(pVM));
8313 Assert(pVM->hm.s.vmx.pRealModeTSS);
8314
8315 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8316 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8317 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8318 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8319 AssertRCReturn(rc, rc);
8320 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8321
8322 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8323 size_t const cbIdtEntry = sizeof(X86IDTR16);
8324 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8325 {
8326 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8327 if (uVector == X86_XCPT_DF)
8328 return VINF_EM_RESET;
8329
8330 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8331 if (uVector == X86_XCPT_GP)
8332 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8333
8334 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8335 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8336 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8337 fStepping, puIntrState);
8338 }
8339
8340 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8341 uint16_t uGuestIp = pMixedCtx->ip;
8342 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8343 {
8344 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8345 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8346 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8347 }
8348 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8349 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8350
8351 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8352 X86IDTR16 IdtEntry;
8353 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8354 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8355 AssertRCReturn(rc, rc);
8356
8357 /* Construct the stack frame for the interrupt/exception handler. */
8358 VBOXSTRICTRC rcStrict;
8359 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8360 if (rcStrict == VINF_SUCCESS)
8361 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8362 if (rcStrict == VINF_SUCCESS)
8363 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8364
8365 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8366 if (rcStrict == VINF_SUCCESS)
8367 {
8368 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8369 pMixedCtx->rip = IdtEntry.offSel;
8370 pMixedCtx->cs.Sel = IdtEntry.uSel;
8371 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8372 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8373 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8374 && uVector == X86_XCPT_PF)
8375 pMixedCtx->cr2 = GCPtrFaultAddress;
8376
8377 /* If any other guest-state bits are changed here, make sure to update
8378 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8379 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8380 | HM_CHANGED_GUEST_RIP
8381 | HM_CHANGED_GUEST_RFLAGS
8382 | HM_CHANGED_GUEST_RSP);
8383
8384 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8385 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8386 {
8387 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8388 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8389 Log4(("Clearing inhibition due to STI.\n"));
8390 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8391 }
8392 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8393 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8394
8395 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8396 it, if we are returning to ring-3 before executing guest code. */
8397 pVCpu->hm.s.Event.fPending = false;
8398
8399 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8400 if (fStepping)
8401 rcStrict = VINF_EM_DBG_STEPPED;
8402 }
8403 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8404 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8405 return rcStrict;
8406 }
8407
8408 /*
8409 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8410 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8411 */
8412 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8413 }
8414
8415 /* Validate. */
8416 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8417 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8418 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8419
8420 /* Inject. */
8421 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8422 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8423 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8424 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8425
8426 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8427 && uVector == X86_XCPT_PF)
8428 pMixedCtx->cr2 = GCPtrFaultAddress;
8429
8430 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8431 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8432
8433 AssertRCReturn(rc, rc);
8434 return VINF_SUCCESS;
8435}
8436
8437
8438/**
8439 * Clears the interrupt-window exiting control in the VMCS and if necessary
8440 * clears the current event in the VMCS as well.
8441 *
8442 * @returns VBox status code.
8443 * @param pVCpu The cross context virtual CPU structure.
8444 *
8445 * @remarks Use this function only to clear events that have not yet been
8446 * delivered to the guest but are injected in the VMCS!
8447 * @remarks No-long-jump zone!!!
8448 */
8449static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8450{
8451 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8452
8453 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8454 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8455
8456 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8457 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8458}
8459
8460
8461/**
8462 * Enters the VT-x session.
8463 *
8464 * @returns VBox status code.
8465 * @param pVM The cross context VM structure.
8466 * @param pVCpu The cross context virtual CPU structure.
8467 * @param pCpu Pointer to the CPU info struct.
8468 */
8469VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8470{
8471 AssertPtr(pVM);
8472 AssertPtr(pVCpu);
8473 Assert(pVM->hm.s.vmx.fSupported);
8474 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8475 NOREF(pCpu); NOREF(pVM);
8476
8477 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8478 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8479
8480#ifdef VBOX_STRICT
8481 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8482 RTCCUINTREG uHostCR4 = ASMGetCR4();
8483 if (!(uHostCR4 & X86_CR4_VMXE))
8484 {
8485 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8486 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8487 }
8488#endif
8489
8490 /*
8491 * Load the VCPU's VMCS as the current (and active) one.
8492 */
8493 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8494 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8495 if (RT_FAILURE(rc))
8496 return rc;
8497
8498 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8499 pVCpu->hm.s.fLeaveDone = false;
8500 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8501
8502 return VINF_SUCCESS;
8503}
8504
8505
8506/**
8507 * The thread-context callback (only on platforms which support it).
8508 *
8509 * @param enmEvent The thread-context event.
8510 * @param pVCpu The cross context virtual CPU structure.
8511 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8512 * @thread EMT(pVCpu)
8513 */
8514VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8515{
8516 NOREF(fGlobalInit);
8517
8518 switch (enmEvent)
8519 {
8520 case RTTHREADCTXEVENT_OUT:
8521 {
8522 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8523 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8524 VMCPU_ASSERT_EMT(pVCpu);
8525
8526 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8527
8528 /* No longjmps (logger flushes, locks) in this fragile context. */
8529 VMMRZCallRing3Disable(pVCpu);
8530 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8531
8532 /*
8533 * Restore host-state (FPU, debug etc.)
8534 */
8535 if (!pVCpu->hm.s.fLeaveDone)
8536 {
8537 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8538 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8539 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8540 pVCpu->hm.s.fLeaveDone = true;
8541 }
8542
8543 /* Leave HM context, takes care of local init (term). */
8544 int rc = HMR0LeaveCpu(pVCpu);
8545 AssertRC(rc); NOREF(rc);
8546
8547 /* Restore longjmp state. */
8548 VMMRZCallRing3Enable(pVCpu);
8549 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8550 break;
8551 }
8552
8553 case RTTHREADCTXEVENT_IN:
8554 {
8555 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8556 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8557 VMCPU_ASSERT_EMT(pVCpu);
8558
8559 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8560 VMMRZCallRing3Disable(pVCpu);
8561 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8562
8563 /* Initialize the bare minimum state required for HM. This takes care of
8564 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8565 int rc = HMR0EnterCpu(pVCpu);
8566 AssertRC(rc);
8567 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8568
8569 /* Load the active VMCS as the current one. */
8570 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8571 {
8572 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8573 AssertRC(rc); NOREF(rc);
8574 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8575 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8576 }
8577 pVCpu->hm.s.fLeaveDone = false;
8578
8579 /* Restore longjmp state. */
8580 VMMRZCallRing3Enable(pVCpu);
8581 break;
8582 }
8583
8584 default:
8585 break;
8586 }
8587}
8588
8589
8590/**
8591 * Saves the host state in the VMCS host-state.
8592 * Sets up the VM-exit MSR-load area.
8593 *
8594 * The CPU state will be loaded from these fields on every successful VM-exit.
8595 *
8596 * @returns VBox status code.
8597 * @param pVM The cross context VM structure.
8598 * @param pVCpu The cross context virtual CPU structure.
8599 *
8600 * @remarks No-long-jump zone!!!
8601 */
8602static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8603{
8604 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8605
8606 int rc = VINF_SUCCESS;
8607 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8608 {
8609 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8610 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8611
8612 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8613 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8614
8615 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8616 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8617
8618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8619 }
8620 return rc;
8621}
8622
8623
8624/**
8625 * Saves the host state in the VMCS host-state.
8626 *
8627 * @returns VBox status code.
8628 * @param pVM The cross context VM structure.
8629 * @param pVCpu The cross context virtual CPU structure.
8630 *
8631 * @remarks No-long-jump zone!!!
8632 */
8633VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8634{
8635 AssertPtr(pVM);
8636 AssertPtr(pVCpu);
8637
8638 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8639
8640 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8641 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8642 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8643 return hmR0VmxSaveHostState(pVM, pVCpu);
8644}
8645
8646
8647/**
8648 * Loads the guest state into the VMCS guest-state area.
8649 *
8650 * The will typically be done before VM-entry when the guest-CPU state and the
8651 * VMCS state may potentially be out of sync.
8652 *
8653 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8654 * VM-entry controls.
8655 * Sets up the appropriate VMX non-root function to execute guest code based on
8656 * the guest CPU mode.
8657 *
8658 * @returns VBox strict status code.
8659 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8660 * without unrestricted guest access and the VMMDev is not presently
8661 * mapped (e.g. EFI32).
8662 *
8663 * @param pVM The cross context VM structure.
8664 * @param pVCpu The cross context virtual CPU structure.
8665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8666 * out-of-sync. Make sure to update the required fields
8667 * before using them.
8668 *
8669 * @remarks No-long-jump zone!!!
8670 */
8671static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8672{
8673 AssertPtr(pVM);
8674 AssertPtr(pVCpu);
8675 AssertPtr(pMixedCtx);
8676 HMVMX_ASSERT_PREEMPT_SAFE();
8677
8678 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8679
8680 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8681
8682 /* Determine real-on-v86 mode. */
8683 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8684 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8685 && CPUMIsGuestInRealModeEx(pMixedCtx))
8686 {
8687 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8688 }
8689
8690 /*
8691 * Load the guest-state into the VMCS.
8692 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8693 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8694 */
8695 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8697
8698 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8699 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8701
8702 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8703 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8704 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8705
8706 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8707 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8708
8709 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8710 if (rcStrict == VINF_SUCCESS)
8711 { /* likely */ }
8712 else
8713 {
8714 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8715 return rcStrict;
8716 }
8717
8718 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8719 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8720 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8721
8722 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8723 determine we don't have to swap EFER after all. */
8724 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8725 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8726
8727 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8728 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8729
8730 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8731 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8732
8733 /*
8734 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8735 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8736 */
8737 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8738 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8739
8740 /* Clear any unused and reserved bits. */
8741 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8742
8743 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8744 return rc;
8745}
8746
8747
8748/**
8749 * Loads the state shared between the host and guest into the VMCS.
8750 *
8751 * @param pVM The cross context VM structure.
8752 * @param pVCpu The cross context virtual CPU structure.
8753 * @param pCtx Pointer to the guest-CPU context.
8754 *
8755 * @remarks No-long-jump zone!!!
8756 */
8757static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8758{
8759 NOREF(pVM);
8760
8761 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8762 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8763
8764 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8765 {
8766 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8767 AssertRC(rc);
8768 }
8769
8770 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8771 {
8772 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8773 AssertRC(rc);
8774
8775 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8776 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8777 {
8778 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8779 AssertRC(rc);
8780 }
8781 }
8782
8783 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8784 {
8785 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8787 }
8788
8789 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8790 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8791 {
8792 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8793 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8794 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8795 AssertRC(rc);
8796 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8797 }
8798
8799 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8800 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8801}
8802
8803
8804/**
8805 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8806 *
8807 * @returns Strict VBox status code (i.e. informational status codes too).
8808 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8809 * without unrestricted guest access and the VMMDev is not presently
8810 * mapped (e.g. EFI32).
8811 *
8812 * @param pVM The cross context VM structure.
8813 * @param pVCpu The cross context virtual CPU structure.
8814 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8815 * out-of-sync. Make sure to update the required fields
8816 * before using them.
8817 *
8818 * @remarks No-long-jump zone!!!
8819 */
8820static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8821{
8822 HMVMX_ASSERT_PREEMPT_SAFE();
8823 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8824 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8825
8826 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8827#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8828 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8829#endif
8830
8831 /*
8832 * RIP is what changes the most often and hence if it's the only bit needing to be
8833 * updated, we shall handle it early for performance reasons.
8834 */
8835 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8836 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8837 {
8838 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8839 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8840 { /* likely */}
8841 else
8842 {
8843 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8844 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8845 }
8846 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8847 }
8848 else if (HMCPU_CF_VALUE(pVCpu))
8849 {
8850 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8851 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8852 { /* likely */}
8853 else
8854 {
8855 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8856 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8857 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8858 return rcStrict;
8859 }
8860 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8861 }
8862
8863 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8864 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8865 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8866 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8867 return rcStrict;
8868}
8869
8870
8871/**
8872 * Does the preparations before executing guest code in VT-x.
8873 *
8874 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8875 * recompiler/IEM. We must be cautious what we do here regarding committing
8876 * guest-state information into the VMCS assuming we assuredly execute the
8877 * guest in VT-x mode.
8878 *
8879 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8880 * the common-state (TRPM/forceflags), we must undo those changes so that the
8881 * recompiler/IEM can (and should) use them when it resumes guest execution.
8882 * Otherwise such operations must be done when we can no longer exit to ring-3.
8883 *
8884 * @returns Strict VBox status code (i.e. informational status codes too).
8885 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8886 * have been disabled.
8887 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8888 * double-fault into the guest.
8889 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8890 * dispatched directly.
8891 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8892 *
8893 * @param pVM The cross context VM structure.
8894 * @param pVCpu The cross context virtual CPU structure.
8895 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8896 * out-of-sync. Make sure to update the required fields
8897 * before using them.
8898 * @param pVmxTransient Pointer to the VMX transient structure.
8899 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8900 * us ignore some of the reasons for returning to
8901 * ring-3, and return VINF_EM_DBG_STEPPED if event
8902 * dispatching took place.
8903 */
8904static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8905{
8906 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8907
8908#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8909 PGMRZDynMapFlushAutoSet(pVCpu);
8910#endif
8911
8912 /* Check force flag actions that might require us to go back to ring-3. */
8913 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8914 if (rcStrict == VINF_SUCCESS)
8915 { /* FFs doesn't get set all the time. */ }
8916 else
8917 return rcStrict;
8918
8919#ifndef IEM_VERIFICATION_MODE_FULL
8920 /*
8921 * Setup the virtualized-APIC accesses.
8922 *
8923 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8924 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8925 *
8926 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8927 */
8928 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8929 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8930 && PDMHasApic(pVM))
8931 {
8932 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8933 Assert(u64MsrApicBase);
8934 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8935
8936 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8937
8938 /* Unalias any existing mapping. */
8939 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8940 AssertRCReturn(rc, rc);
8941
8942 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8943 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8944 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8945 AssertRCReturn(rc, rc);
8946
8947 /* Update the per-VCPU cache of the APIC base MSR. */
8948 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8949 }
8950#endif /* !IEM_VERIFICATION_MODE_FULL */
8951
8952 if (TRPMHasTrap(pVCpu))
8953 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8954 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8955
8956 /*
8957 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8958 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8959 */
8960 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8961 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8962 { /* likely */ }
8963 else
8964 {
8965 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8966 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8967 return rcStrict;
8968 }
8969
8970 /*
8971 * No longjmps to ring-3 from this point on!!!
8972 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8973 * This also disables flushing of the R0-logger instance (if any).
8974 */
8975 VMMRZCallRing3Disable(pVCpu);
8976
8977 /*
8978 * Load the guest state bits.
8979 *
8980 * We cannot perform longjmps while loading the guest state because we do not preserve the
8981 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8982 * CPU migration.
8983 *
8984 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8985 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8986 * Hence, loading of the guest state needs to be done -after- injection of events.
8987 */
8988 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8989 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8990 { /* likely */ }
8991 else
8992 {
8993 VMMRZCallRing3Enable(pVCpu);
8994 return rcStrict;
8995 }
8996
8997 /*
8998 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8999 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
9000 *
9001 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
9002 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
9003 *
9004 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
9005 * executing guest code.
9006 */
9007 pVmxTransient->fEFlags = ASMIntDisableFlags();
9008
9009 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
9010 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9011 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9012 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
9013 {
9014 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9015 {
9016 pVCpu->hm.s.Event.fPending = false;
9017
9018 /*
9019 * We've injected any pending events. This is really the point of no return (to ring-3).
9020 *
9021 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
9022 * returns from this function, so don't enable them here.
9023 */
9024 return VINF_SUCCESS;
9025 }
9026
9027 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9028 rcStrict = VINF_EM_RAW_INTERRUPT;
9029 }
9030 else
9031 {
9032 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9033 rcStrict = VINF_EM_RAW_TO_R3;
9034 }
9035
9036 ASMSetFlags(pVmxTransient->fEFlags);
9037 VMMRZCallRing3Enable(pVCpu);
9038
9039 return rcStrict;
9040}
9041
9042
9043/**
9044 * Prepares to run guest code in VT-x and we've committed to doing so. This
9045 * means there is no backing out to ring-3 or anywhere else at this
9046 * point.
9047 *
9048 * @param pVM The cross context VM structure.
9049 * @param pVCpu The cross context virtual CPU structure.
9050 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9051 * out-of-sync. Make sure to update the required fields
9052 * before using them.
9053 * @param pVmxTransient Pointer to the VMX transient structure.
9054 *
9055 * @remarks Called with preemption disabled.
9056 * @remarks No-long-jump zone!!!
9057 */
9058static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9059{
9060 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9061 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9062 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9063
9064 /*
9065 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9066 */
9067 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9068 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9069
9070#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9071 if (!CPUMIsGuestFPUStateActive(pVCpu))
9072 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9073 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9074 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9075#endif
9076
9077 if ( pVCpu->hm.s.fPreloadGuestFpu
9078 && !CPUMIsGuestFPUStateActive(pVCpu))
9079 {
9080 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9081 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9082 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9083 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9084 }
9085
9086 /*
9087 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9088 */
9089 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9090 && pVCpu->hm.s.vmx.cMsrs > 0)
9091 {
9092 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9093 }
9094
9095 /*
9096 * Load the host state bits as we may've been preempted (only happens when
9097 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9098 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9099 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9100 * See @bugref{8432}.
9101 */
9102 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9103 {
9104 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9105 AssertRC(rc);
9106 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9107 }
9108 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9109
9110 /*
9111 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9112 */
9113 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9114 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9115 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9116
9117 /* Store status of the shared guest-host state at the time of VM-entry. */
9118#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9119 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9120 {
9121 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9122 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9123 }
9124 else
9125#endif
9126 {
9127 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9128 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9129 }
9130 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9131
9132 /*
9133 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9134 */
9135 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9136 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9137
9138 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9139 RTCPUID idCurrentCpu = pCpu->idCpu;
9140 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9141 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9142 {
9143 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9144 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9145 }
9146
9147 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9148 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9149 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9150 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9151
9152 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9153
9154 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9155 to start executing. */
9156
9157 /*
9158 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9159 */
9160 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9161 {
9162 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9163 {
9164 bool fMsrUpdated;
9165 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9166 AssertRC(rc2);
9167 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9168
9169 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9170 &fMsrUpdated);
9171 AssertRC(rc2);
9172 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9173
9174 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9175 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9176 }
9177 else
9178 {
9179 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9180 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9181 }
9182 }
9183
9184 if (pVM->cpum.ro.GuestFeatures.fIbrs)
9185 {
9186 bool fMsrUpdated;
9187 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9188 AssertRC(rc2);
9189 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9190
9191 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMR0GetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9192 &fMsrUpdated);
9193 AssertRC(rc2);
9194 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9195 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9196 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9197 }
9198
9199#ifdef VBOX_STRICT
9200 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9201 hmR0VmxCheckHostEferMsr(pVCpu);
9202 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9203#endif
9204#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9205 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9206 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9207 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9208#endif
9209}
9210
9211
9212/**
9213 * Performs some essential restoration of state after running guest code in
9214 * VT-x.
9215 *
9216 * @param pVM The cross context VM structure.
9217 * @param pVCpu The cross context virtual CPU structure.
9218 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9219 * out-of-sync. Make sure to update the required fields
9220 * before using them.
9221 * @param pVmxTransient Pointer to the VMX transient structure.
9222 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9223 *
9224 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9225 *
9226 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9227 * unconditionally when it is safe to do so.
9228 */
9229static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9230{
9231 NOREF(pVM);
9232
9233 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9234
9235 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9236 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9237 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9238 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9239 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9240 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9241
9242 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9243 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9244
9245 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9246 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9247 Assert(!ASMIntAreEnabled());
9248 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9249
9250#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9251 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9252 {
9253 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9254 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9255 }
9256#endif
9257
9258#if HC_ARCH_BITS == 64
9259 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9260#endif
9261#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9262 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9263 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9264 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9265#else
9266 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9267#endif
9268#ifdef VBOX_STRICT
9269 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9270#endif
9271 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9272
9273 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9274 uint32_t uExitReason;
9275 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9276 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9277 AssertRC(rc);
9278 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9279 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9280
9281 if (rcVMRun == VINF_SUCCESS)
9282 {
9283 /*
9284 * Update the VM-exit history array here even if the VM-entry failed due to:
9285 * - Invalid guest state.
9286 * - MSR loading.
9287 * - Machine-check event.
9288 *
9289 * In any of the above cases we will still have a "valid" VM-exit reason
9290 * despite @a fVMEntryFailed being false.
9291 *
9292 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9293 */
9294 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9295
9296 if (!pVmxTransient->fVMEntryFailed)
9297 {
9298 /** @todo We can optimize this by only syncing with our force-flags when
9299 * really needed and keeping the VMCS state as it is for most
9300 * VM-exits. */
9301 /* Update the guest interruptibility-state from the VMCS. */
9302 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9303
9304 /*
9305 * Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9306 * as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9307 * on the longjmp path to ring-3 while saving the (rest of) the guest state,
9308 * see @bugref{6208#c63}.
9309 */
9310 VMMRZCallRing3Enable(pVCpu);
9311
9312#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9313 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9314 AssertRC(rc);
9315#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9316 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9317 AssertRC(rc);
9318#endif
9319
9320 /*
9321 * Sync the TPR shadow with our APIC state.
9322 */
9323 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9324 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9325 {
9326 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9327 AssertRC(rc);
9328 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9329 }
9330
9331 return;
9332 }
9333 }
9334 else
9335 {
9336 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9337 pVmxTransient->fVMEntryFailed));
9338 }
9339
9340 VMMRZCallRing3Enable(pVCpu);
9341}
9342
9343
9344/**
9345 * Runs the guest code using VT-x the normal way.
9346 *
9347 * @returns VBox status code.
9348 * @param pVM The cross context VM structure.
9349 * @param pVCpu The cross context virtual CPU structure.
9350 * @param pCtx Pointer to the guest-CPU context.
9351 *
9352 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9353 */
9354static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9355{
9356 VMXTRANSIENT VmxTransient;
9357 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9358 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9359 uint32_t cLoops = 0;
9360
9361 for (;; cLoops++)
9362 {
9363 Assert(!HMR0SuspendPending());
9364 HMVMX_ASSERT_CPU_SAFE();
9365
9366 /* Preparatory work for running guest code, this may force us to return
9367 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9368 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9369 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9370 if (rcStrict != VINF_SUCCESS)
9371 break;
9372
9373 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9374 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9375 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9376
9377 /* Restore any residual host-state and save any bits shared between host
9378 and guest into the guest-CPU state. Re-enables interrupts! */
9379 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9380
9381 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9382 if (RT_SUCCESS(rcRun))
9383 { /* very likely */ }
9384 else
9385 {
9386 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9387 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9388 return rcRun;
9389 }
9390
9391 /* Profile the VM-exit. */
9392 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9394 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9395 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9396 HMVMX_START_EXIT_DISPATCH_PROF();
9397
9398 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9399
9400 /* Handle the VM-exit. */
9401#ifdef HMVMX_USE_FUNCTION_TABLE
9402 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9403#else
9404 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9405#endif
9406 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9407 if (rcStrict == VINF_SUCCESS)
9408 {
9409 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9410 continue; /* likely */
9411 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9412 rcStrict = VINF_EM_RAW_INTERRUPT;
9413 }
9414 break;
9415 }
9416
9417 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9418 return rcStrict;
9419}
9420
9421
9422
9423/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9424 * probes.
9425 *
9426 * The following few functions and associated structure contains the bloat
9427 * necessary for providing detailed debug events and dtrace probes as well as
9428 * reliable host side single stepping. This works on the principle of
9429 * "subclassing" the normal execution loop and workers. We replace the loop
9430 * method completely and override selected helpers to add necessary adjustments
9431 * to their core operation.
9432 *
9433 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9434 * any performance for debug and analysis features.
9435 *
9436 * @{
9437 */
9438
9439/**
9440 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9441 * the debug run loop.
9442 */
9443typedef struct VMXRUNDBGSTATE
9444{
9445 /** The RIP we started executing at. This is for detecting that we stepped. */
9446 uint64_t uRipStart;
9447 /** The CS we started executing with. */
9448 uint16_t uCsStart;
9449
9450 /** Whether we've actually modified the 1st execution control field. */
9451 bool fModifiedProcCtls : 1;
9452 /** Whether we've actually modified the 2nd execution control field. */
9453 bool fModifiedProcCtls2 : 1;
9454 /** Whether we've actually modified the exception bitmap. */
9455 bool fModifiedXcptBitmap : 1;
9456
9457 /** We desire the modified the CR0 mask to be cleared. */
9458 bool fClearCr0Mask : 1;
9459 /** We desire the modified the CR4 mask to be cleared. */
9460 bool fClearCr4Mask : 1;
9461 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9462 uint32_t fCpe1Extra;
9463 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9464 uint32_t fCpe1Unwanted;
9465 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9466 uint32_t fCpe2Extra;
9467 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9468 uint32_t bmXcptExtra;
9469 /** The sequence number of the Dtrace provider settings the state was
9470 * configured against. */
9471 uint32_t uDtraceSettingsSeqNo;
9472 /** VM-exits to check (one bit per VM-exit). */
9473 uint32_t bmExitsToCheck[3];
9474
9475 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9476 uint32_t fProcCtlsInitial;
9477 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9478 uint32_t fProcCtls2Initial;
9479 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9480 uint32_t bmXcptInitial;
9481} VMXRUNDBGSTATE;
9482AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9483typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9484
9485
9486/**
9487 * Initializes the VMXRUNDBGSTATE structure.
9488 *
9489 * @param pVCpu The cross context virtual CPU structure of the
9490 * calling EMT.
9491 * @param pCtx The CPU register context to go with @a pVCpu.
9492 * @param pDbgState The structure to initialize.
9493 */
9494DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9495{
9496 pDbgState->uRipStart = pCtx->rip;
9497 pDbgState->uCsStart = pCtx->cs.Sel;
9498
9499 pDbgState->fModifiedProcCtls = false;
9500 pDbgState->fModifiedProcCtls2 = false;
9501 pDbgState->fModifiedXcptBitmap = false;
9502 pDbgState->fClearCr0Mask = false;
9503 pDbgState->fClearCr4Mask = false;
9504 pDbgState->fCpe1Extra = 0;
9505 pDbgState->fCpe1Unwanted = 0;
9506 pDbgState->fCpe2Extra = 0;
9507 pDbgState->bmXcptExtra = 0;
9508 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9509 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9510 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9511}
9512
9513
9514/**
9515 * Updates the VMSC fields with changes requested by @a pDbgState.
9516 *
9517 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9518 * immediately before executing guest code, i.e. when interrupts are disabled.
9519 * We don't check status codes here as we cannot easily assert or return in the
9520 * latter case.
9521 *
9522 * @param pVCpu The cross context virtual CPU structure.
9523 * @param pDbgState The debug state.
9524 */
9525DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9526{
9527 /*
9528 * Ensure desired flags in VMCS control fields are set.
9529 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9530 *
9531 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9532 * there should be no stale data in pCtx at this point.
9533 */
9534 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9535 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9536 {
9537 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9538 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9539 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9540 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9541 pDbgState->fModifiedProcCtls = true;
9542 }
9543
9544 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9545 {
9546 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9547 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9548 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9549 pDbgState->fModifiedProcCtls2 = true;
9550 }
9551
9552 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9553 {
9554 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9555 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9556 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9557 pDbgState->fModifiedXcptBitmap = true;
9558 }
9559
9560 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9561 {
9562 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9563 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9564 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9565 }
9566
9567 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9568 {
9569 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9570 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9571 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9572 }
9573}
9574
9575
9576DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9577{
9578 /*
9579 * Restore VM-exit control settings as we may not reenter this function the
9580 * next time around.
9581 */
9582 /* We reload the initial value, trigger what we can of recalculations the
9583 next time around. From the looks of things, that's all that's required atm. */
9584 if (pDbgState->fModifiedProcCtls)
9585 {
9586 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9587 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9588 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9589 AssertRCReturn(rc2, rc2);
9590 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9591 }
9592
9593 /* We're currently the only ones messing with this one, so just restore the
9594 cached value and reload the field. */
9595 if ( pDbgState->fModifiedProcCtls2
9596 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9597 {
9598 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9599 AssertRCReturn(rc2, rc2);
9600 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9601 }
9602
9603 /* If we've modified the exception bitmap, we restore it and trigger
9604 reloading and partial recalculation the next time around. */
9605 if (pDbgState->fModifiedXcptBitmap)
9606 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9607
9608 return rcStrict;
9609}
9610
9611
9612/**
9613 * Configures VM-exit controls for current DBGF and DTrace settings.
9614 *
9615 * This updates @a pDbgState and the VMCS execution control fields to reflect
9616 * the necessary VM-exits demanded by DBGF and DTrace.
9617 *
9618 * @param pVM The cross context VM structure.
9619 * @param pVCpu The cross context virtual CPU structure.
9620 * @param pCtx Pointer to the guest-CPU context.
9621 * @param pDbgState The debug state.
9622 * @param pVmxTransient Pointer to the VMX transient structure. May update
9623 * fUpdateTscOffsettingAndPreemptTimer.
9624 */
9625static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9626 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9627{
9628 /*
9629 * Take down the dtrace serial number so we can spot changes.
9630 */
9631 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9632 ASMCompilerBarrier();
9633
9634 /*
9635 * We'll rebuild most of the middle block of data members (holding the
9636 * current settings) as we go along here, so start by clearing it all.
9637 */
9638 pDbgState->bmXcptExtra = 0;
9639 pDbgState->fCpe1Extra = 0;
9640 pDbgState->fCpe1Unwanted = 0;
9641 pDbgState->fCpe2Extra = 0;
9642 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9643 pDbgState->bmExitsToCheck[i] = 0;
9644
9645 /*
9646 * Software interrupts (INT XXh) - no idea how to trigger these...
9647 */
9648 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9649 || VBOXVMM_INT_SOFTWARE_ENABLED())
9650 {
9651 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9652 }
9653
9654 /*
9655 * INT3 breakpoints - triggered by #BP exceptions.
9656 */
9657 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9658 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9659
9660 /*
9661 * Exception bitmap and XCPT events+probes.
9662 */
9663 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9664 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9665 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9666
9667 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9668 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9669 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9670 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9671 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9672 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9673 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9674 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9675 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9676 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9677 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9678 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9679 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9680 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9681 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9682 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9683 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9684 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9685
9686 if (pDbgState->bmXcptExtra)
9687 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9688
9689 /*
9690 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9691 *
9692 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9693 * So, when adding/changing/removing please don't forget to update it.
9694 *
9695 * Some of the macros are picking up local variables to save horizontal space,
9696 * (being able to see it in a table is the lesser evil here).
9697 */
9698#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9699 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9700 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9701#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9702 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9703 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9704 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9705 } else do { } while (0)
9706#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9707 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9708 { \
9709 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9710 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9711 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9712 } else do { } while (0)
9713#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9714 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9715 { \
9716 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9717 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9718 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9719 } else do { } while (0)
9720#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9721 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9722 { \
9723 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9724 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9725 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9726 } else do { } while (0)
9727
9728 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9729 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9730 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9731 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9732 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9733
9734 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9735 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9736 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9737 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9738 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9739 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9740 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9741 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9742 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9743 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9744 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9745 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9746 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9747 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9748 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9749 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9750 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9751 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9752 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9753 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9754 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9755 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9756 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9757 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9758 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9759 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9760 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9761 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9762 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9763 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9764 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9765 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9766 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9767 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9768 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9769 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9770
9771 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9772 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9773 {
9774 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9775 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9776 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9777 AssertRC(rc2);
9778
9779#if 0 /** @todo fix me */
9780 pDbgState->fClearCr0Mask = true;
9781 pDbgState->fClearCr4Mask = true;
9782#endif
9783 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9784 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9785 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9786 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9787 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9788 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9789 require clearing here and in the loop if we start using it. */
9790 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9791 }
9792 else
9793 {
9794 if (pDbgState->fClearCr0Mask)
9795 {
9796 pDbgState->fClearCr0Mask = false;
9797 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9798 }
9799 if (pDbgState->fClearCr4Mask)
9800 {
9801 pDbgState->fClearCr4Mask = false;
9802 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9803 }
9804 }
9805 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9806 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9807
9808 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9809 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9810 {
9811 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9812 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9813 }
9814 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9815 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9816
9817 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9818 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9819 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9820 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9821 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9822 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9823 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9824 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9825#if 0 /** @todo too slow, fix handler. */
9826 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9827#endif
9828 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9829
9830 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9831 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9832 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9833 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9834 {
9835 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9836 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9837 }
9838 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9839 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9840 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9841 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9842
9843 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9844 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9845 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9846 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9847 {
9848 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9849 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9850 }
9851 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9852 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9853 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9854 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9855
9856 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9857 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9858 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9859 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9860 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9861 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9862 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9863 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9864 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9865 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9866 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9867 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9868 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9869 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9870 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9871 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9872 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9873 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9874 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9875 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9876 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9877 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9878
9879#undef IS_EITHER_ENABLED
9880#undef SET_ONLY_XBM_IF_EITHER_EN
9881#undef SET_CPE1_XBM_IF_EITHER_EN
9882#undef SET_CPEU_XBM_IF_EITHER_EN
9883#undef SET_CPE2_XBM_IF_EITHER_EN
9884
9885 /*
9886 * Sanitize the control stuff.
9887 */
9888 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9889 if (pDbgState->fCpe2Extra)
9890 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9891 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9892 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9893 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9894 {
9895 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9896 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9897 }
9898
9899 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9900 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9901 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9902 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9903}
9904
9905
9906/**
9907 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9908 * appropriate.
9909 *
9910 * The caller has checked the VM-exit against the
9911 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9912 * already, so we don't have to do that either.
9913 *
9914 * @returns Strict VBox status code (i.e. informational status codes too).
9915 * @param pVM The cross context VM structure.
9916 * @param pVCpu The cross context virtual CPU structure.
9917 * @param pMixedCtx Pointer to the guest-CPU context.
9918 * @param pVmxTransient Pointer to the VMX-transient structure.
9919 * @param uExitReason The VM-exit reason.
9920 *
9921 * @remarks The name of this function is displayed by dtrace, so keep it short
9922 * and to the point. No longer than 33 chars long, please.
9923 */
9924static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9925 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9926{
9927 /*
9928 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9929 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9930 *
9931 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9932 * does. Must add/change/remove both places. Same ordering, please.
9933 *
9934 * Added/removed events must also be reflected in the next section
9935 * where we dispatch dtrace events.
9936 */
9937 bool fDtrace1 = false;
9938 bool fDtrace2 = false;
9939 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9940 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9941 uint32_t uEventArg = 0;
9942#define SET_EXIT(a_EventSubName) \
9943 do { \
9944 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9945 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9946 } while (0)
9947#define SET_BOTH(a_EventSubName) \
9948 do { \
9949 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9950 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9951 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9952 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9953 } while (0)
9954 switch (uExitReason)
9955 {
9956 case VMX_EXIT_MTF:
9957 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9958
9959 case VMX_EXIT_XCPT_OR_NMI:
9960 {
9961 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9962 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9963 {
9964 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9965 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9966 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9967 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9968 {
9969 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9970 {
9971 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9972 uEventArg = pVmxTransient->uExitIntErrorCode;
9973 }
9974 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9975 switch (enmEvent1)
9976 {
9977 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9978 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9979 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9980 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9981 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9982 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9983 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9984 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9985 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9986 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9987 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9988 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9989 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9990 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9991 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9992 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9993 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9994 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9995 default: break;
9996 }
9997 }
9998 else
9999 AssertFailed();
10000 break;
10001
10002 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
10003 uEventArg = idxVector;
10004 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
10005 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
10006 break;
10007 }
10008 break;
10009 }
10010
10011 case VMX_EXIT_TRIPLE_FAULT:
10012 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
10013 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
10014 break;
10015 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
10016 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
10017 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
10018 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10019 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10020
10021 /* Instruction specific VM-exits: */
10022 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10023 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10024 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10025 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10026 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10027 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10028 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10029 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10030 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10031 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10032 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10033 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10034 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10035 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10036 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10037 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10038 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10039 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10040 case VMX_EXIT_MOV_CRX:
10041 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10042/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10043* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10044 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10045 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10046 SET_BOTH(CRX_READ);
10047 else
10048 SET_BOTH(CRX_WRITE);
10049 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10050 break;
10051 case VMX_EXIT_MOV_DRX:
10052 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10053 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10054 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10055 SET_BOTH(DRX_READ);
10056 else
10057 SET_BOTH(DRX_WRITE);
10058 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10059 break;
10060 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10061 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10062 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10063 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10064 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10065 case VMX_EXIT_XDTR_ACCESS:
10066 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10067 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10068 {
10069 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10070 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10071 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10072 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10073 }
10074 break;
10075
10076 case VMX_EXIT_TR_ACCESS:
10077 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10078 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10079 {
10080 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10081 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10082 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10083 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10084 }
10085 break;
10086
10087 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10088 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10089 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10090 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10091 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10092 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10093 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10094 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10095 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10096 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10097 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10098
10099 /* Events that aren't relevant at this point. */
10100 case VMX_EXIT_EXT_INT:
10101 case VMX_EXIT_INT_WINDOW:
10102 case VMX_EXIT_NMI_WINDOW:
10103 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10104 case VMX_EXIT_PREEMPT_TIMER:
10105 case VMX_EXIT_IO_INSTR:
10106 break;
10107
10108 /* Errors and unexpected events. */
10109 case VMX_EXIT_INIT_SIGNAL:
10110 case VMX_EXIT_SIPI:
10111 case VMX_EXIT_IO_SMI:
10112 case VMX_EXIT_SMI:
10113 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10114 case VMX_EXIT_ERR_MSR_LOAD:
10115 case VMX_EXIT_ERR_MACHINE_CHECK:
10116 break;
10117
10118 default:
10119 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10120 break;
10121 }
10122#undef SET_BOTH
10123#undef SET_EXIT
10124
10125 /*
10126 * Dtrace tracepoints go first. We do them here at once so we don't
10127 * have to copy the guest state saving and stuff a few dozen times.
10128 * Down side is that we've got to repeat the switch, though this time
10129 * we use enmEvent since the probes are a subset of what DBGF does.
10130 */
10131 if (fDtrace1 || fDtrace2)
10132 {
10133 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10134 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10135 switch (enmEvent1)
10136 {
10137 /** @todo consider which extra parameters would be helpful for each probe. */
10138 case DBGFEVENT_END: break;
10139 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10140 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10141 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10142 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10143 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10144 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10145 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10146 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10147 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10148 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10149 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10150 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10151 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10152 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10153 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10154 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10155 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10156 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10157 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10158 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10159 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10160 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10161 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10162 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10163 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10164 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10165 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10166 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10167 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10168 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10169 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10170 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10171 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10172 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10173 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10178 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10179 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10180 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10181 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10182 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10183 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10184 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10185 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10186 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10187 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10188 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10189 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10190 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10191 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10192 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10194 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10197 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10198 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10199 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10200 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10201 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10202 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10203 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10204 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10205 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10206 }
10207 switch (enmEvent2)
10208 {
10209 /** @todo consider which extra parameters would be helpful for each probe. */
10210 case DBGFEVENT_END: break;
10211 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10212 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10213 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10214 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10215 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10216 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10217 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10218 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10219 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10220 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10221 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10222 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10223 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10224 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10225 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10226 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10227 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10228 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10229 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10230 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10231 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10232 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10233 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10234 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10235 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10236 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10237 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10238 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10239 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10240 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10241 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10242 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10243 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10244 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10245 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10246 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10247 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10248 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10249 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10250 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10251 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10252 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10253 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10254 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10255 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10256 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10257 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10258 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10259 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10260 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10261 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10262 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10263 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10264 }
10265 }
10266
10267 /*
10268 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10269 * the DBGF call will do a full check).
10270 *
10271 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10272 * Note! If we have to events, we prioritize the first, i.e. the instruction
10273 * one, in order to avoid event nesting.
10274 */
10275 if ( enmEvent1 != DBGFEVENT_END
10276 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10277 {
10278 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10279 if (rcStrict != VINF_SUCCESS)
10280 return rcStrict;
10281 }
10282 else if ( enmEvent2 != DBGFEVENT_END
10283 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10284 {
10285 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10286 if (rcStrict != VINF_SUCCESS)
10287 return rcStrict;
10288 }
10289
10290 return VINF_SUCCESS;
10291}
10292
10293
10294/**
10295 * Single-stepping VM-exit filtering.
10296 *
10297 * This is preprocessing the VM-exits and deciding whether we've gotten far
10298 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10299 * handling is performed.
10300 *
10301 * @returns Strict VBox status code (i.e. informational status codes too).
10302 * @param pVM The cross context VM structure.
10303 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10304 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10305 * out-of-sync. Make sure to update the required
10306 * fields before using them.
10307 * @param pVmxTransient Pointer to the VMX-transient structure.
10308 * @param uExitReason The VM-exit reason.
10309 * @param pDbgState The debug state.
10310 */
10311DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10312 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10313{
10314 /*
10315 * Expensive (saves context) generic dtrace VM-exit probe.
10316 */
10317 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10318 { /* more likely */ }
10319 else
10320 {
10321 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10322 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10323 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10324 }
10325
10326 /*
10327 * Check for host NMI, just to get that out of the way.
10328 */
10329 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10330 { /* normally likely */ }
10331 else
10332 {
10333 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10334 AssertRCReturn(rc2, rc2);
10335 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10336 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10337 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10338 }
10339
10340 /*
10341 * Check for single stepping event if we're stepping.
10342 */
10343 if (pVCpu->hm.s.fSingleInstruction)
10344 {
10345 switch (uExitReason)
10346 {
10347 case VMX_EXIT_MTF:
10348 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10349
10350 /* Various events: */
10351 case VMX_EXIT_XCPT_OR_NMI:
10352 case VMX_EXIT_EXT_INT:
10353 case VMX_EXIT_TRIPLE_FAULT:
10354 case VMX_EXIT_INT_WINDOW:
10355 case VMX_EXIT_NMI_WINDOW:
10356 case VMX_EXIT_TASK_SWITCH:
10357 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10358 case VMX_EXIT_APIC_ACCESS:
10359 case VMX_EXIT_EPT_VIOLATION:
10360 case VMX_EXIT_EPT_MISCONFIG:
10361 case VMX_EXIT_PREEMPT_TIMER:
10362
10363 /* Instruction specific VM-exits: */
10364 case VMX_EXIT_CPUID:
10365 case VMX_EXIT_GETSEC:
10366 case VMX_EXIT_HLT:
10367 case VMX_EXIT_INVD:
10368 case VMX_EXIT_INVLPG:
10369 case VMX_EXIT_RDPMC:
10370 case VMX_EXIT_RDTSC:
10371 case VMX_EXIT_RSM:
10372 case VMX_EXIT_VMCALL:
10373 case VMX_EXIT_VMCLEAR:
10374 case VMX_EXIT_VMLAUNCH:
10375 case VMX_EXIT_VMPTRLD:
10376 case VMX_EXIT_VMPTRST:
10377 case VMX_EXIT_VMREAD:
10378 case VMX_EXIT_VMRESUME:
10379 case VMX_EXIT_VMWRITE:
10380 case VMX_EXIT_VMXOFF:
10381 case VMX_EXIT_VMXON:
10382 case VMX_EXIT_MOV_CRX:
10383 case VMX_EXIT_MOV_DRX:
10384 case VMX_EXIT_IO_INSTR:
10385 case VMX_EXIT_RDMSR:
10386 case VMX_EXIT_WRMSR:
10387 case VMX_EXIT_MWAIT:
10388 case VMX_EXIT_MONITOR:
10389 case VMX_EXIT_PAUSE:
10390 case VMX_EXIT_XDTR_ACCESS:
10391 case VMX_EXIT_TR_ACCESS:
10392 case VMX_EXIT_INVEPT:
10393 case VMX_EXIT_RDTSCP:
10394 case VMX_EXIT_INVVPID:
10395 case VMX_EXIT_WBINVD:
10396 case VMX_EXIT_XSETBV:
10397 case VMX_EXIT_RDRAND:
10398 case VMX_EXIT_INVPCID:
10399 case VMX_EXIT_VMFUNC:
10400 case VMX_EXIT_RDSEED:
10401 case VMX_EXIT_XSAVES:
10402 case VMX_EXIT_XRSTORS:
10403 {
10404 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10405 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10406 AssertRCReturn(rc2, rc2);
10407 if ( pMixedCtx->rip != pDbgState->uRipStart
10408 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10409 return VINF_EM_DBG_STEPPED;
10410 break;
10411 }
10412
10413 /* Errors and unexpected events: */
10414 case VMX_EXIT_INIT_SIGNAL:
10415 case VMX_EXIT_SIPI:
10416 case VMX_EXIT_IO_SMI:
10417 case VMX_EXIT_SMI:
10418 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10419 case VMX_EXIT_ERR_MSR_LOAD:
10420 case VMX_EXIT_ERR_MACHINE_CHECK:
10421 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10422 break;
10423
10424 default:
10425 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10426 break;
10427 }
10428 }
10429
10430 /*
10431 * Check for debugger event breakpoints and dtrace probes.
10432 */
10433 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10434 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10435 {
10436 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10437 if (rcStrict != VINF_SUCCESS)
10438 return rcStrict;
10439 }
10440
10441 /*
10442 * Normal processing.
10443 */
10444#ifdef HMVMX_USE_FUNCTION_TABLE
10445 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10446#else
10447 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10448#endif
10449}
10450
10451
10452/**
10453 * Single steps guest code using VT-x.
10454 *
10455 * @returns Strict VBox status code (i.e. informational status codes too).
10456 * @param pVM The cross context VM structure.
10457 * @param pVCpu The cross context virtual CPU structure.
10458 * @param pCtx Pointer to the guest-CPU context.
10459 *
10460 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10461 */
10462static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10463{
10464 VMXTRANSIENT VmxTransient;
10465 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10466
10467 /* Set HMCPU indicators. */
10468 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10469 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10470 pVCpu->hm.s.fDebugWantRdTscExit = false;
10471 pVCpu->hm.s.fUsingDebugLoop = true;
10472
10473 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10474 VMXRUNDBGSTATE DbgState;
10475 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10476 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10477
10478 /*
10479 * The loop.
10480 */
10481 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10482 for (uint32_t cLoops = 0; ; cLoops++)
10483 {
10484 Assert(!HMR0SuspendPending());
10485 HMVMX_ASSERT_CPU_SAFE();
10486 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10487
10488 /*
10489 * Preparatory work for running guest code, this may force us to return
10490 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10491 */
10492 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10493 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10494 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10495 if (rcStrict != VINF_SUCCESS)
10496 break;
10497
10498 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10499 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10500
10501 /*
10502 * Now we can run the guest code.
10503 */
10504 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10505
10506 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10507
10508 /*
10509 * Restore any residual host-state and save any bits shared between host
10510 * and guest into the guest-CPU state. Re-enables interrupts!
10511 */
10512 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10513
10514 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10515 if (RT_SUCCESS(rcRun))
10516 { /* very likely */ }
10517 else
10518 {
10519 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10520 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10521 return rcRun;
10522 }
10523
10524 /* Profile the VM-exit. */
10525 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10527 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10528 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10529 HMVMX_START_EXIT_DISPATCH_PROF();
10530
10531 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10532
10533 /*
10534 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10535 */
10536 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10537 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10538 if (rcStrict != VINF_SUCCESS)
10539 break;
10540 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10541 {
10542 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10543 rcStrict = VINF_EM_RAW_INTERRUPT;
10544 break;
10545 }
10546
10547 /*
10548 * Stepping: Did the RIP change, if so, consider it a single step.
10549 * Otherwise, make sure one of the TFs gets set.
10550 */
10551 if (fStepping)
10552 {
10553 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10554 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10555 AssertRCReturn(rc2, rc2);
10556 if ( pCtx->rip != DbgState.uRipStart
10557 || pCtx->cs.Sel != DbgState.uCsStart)
10558 {
10559 rcStrict = VINF_EM_DBG_STEPPED;
10560 break;
10561 }
10562 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10563 }
10564
10565 /*
10566 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10567 */
10568 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10569 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10570 }
10571
10572 /*
10573 * Clear the X86_EFL_TF if necessary.
10574 */
10575 if (pVCpu->hm.s.fClearTrapFlag)
10576 {
10577 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10578 AssertRCReturn(rc2, rc2);
10579 pVCpu->hm.s.fClearTrapFlag = false;
10580 pCtx->eflags.Bits.u1TF = 0;
10581 }
10582 /** @todo there seems to be issues with the resume flag when the monitor trap
10583 * flag is pending without being used. Seen early in bios init when
10584 * accessing APIC page in protected mode. */
10585
10586 /*
10587 * Restore VM-exit control settings as we may not reenter this function the
10588 * next time around.
10589 */
10590 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10591
10592 /* Restore HMCPU indicators. */
10593 pVCpu->hm.s.fUsingDebugLoop = false;
10594 pVCpu->hm.s.fDebugWantRdTscExit = false;
10595 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10596
10597 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10598 return rcStrict;
10599}
10600
10601
10602/** @} */
10603
10604
10605/**
10606 * Checks if any expensive dtrace probes are enabled and we should go to the
10607 * debug loop.
10608 *
10609 * @returns true if we should use debug loop, false if not.
10610 */
10611static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10612{
10613 /* It's probably faster to OR the raw 32-bit counter variables together.
10614 Since the variables are in an array and the probes are next to one
10615 another (more or less), we have good locality. So, better read
10616 eight-nine cache lines ever time and only have one conditional, than
10617 128+ conditionals, right? */
10618 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10619 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10620 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10621 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10622 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10623 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10624 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10625 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10626 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10627 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10628 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10629 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10630 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10631 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10632 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10633 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10634 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10635 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10636 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10637 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10638 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10639 ) != 0
10640 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10641 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10642 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10643 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10644 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10645 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10646 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10647 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10648 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10649 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10650 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10651 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10652 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10653 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10654 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10655 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10656 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10657 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10658 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10659 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10660 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10661 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10662 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10663 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10664 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10665 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10666 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10667 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10668 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10669 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10670 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10671 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10672 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10673 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10674 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10675 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10676 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10677 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10678 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10679 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10680 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10681 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10682 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10683 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10684 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10685 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10686 ) != 0
10687 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10688 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10689 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10690 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10691 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10692 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10693 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10694 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10695 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10696 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10697 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10698 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10699 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10700 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10701 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10702 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10703 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10704 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10705 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10706 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10707 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10708 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10709 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10710 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10711 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10712 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10713 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10714 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10715 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10716 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10717 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10718 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10719 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10720 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10721 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10722 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10723 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10724 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10725 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10726 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10727 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10728 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10729 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10730 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10731 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10732 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10733 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10734 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10735 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10736 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10737 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10738 ) != 0;
10739}
10740
10741
10742/**
10743 * Runs the guest code using VT-x.
10744 *
10745 * @returns Strict VBox status code (i.e. informational status codes too).
10746 * @param pVM The cross context VM structure.
10747 * @param pVCpu The cross context virtual CPU structure.
10748 * @param pCtx Pointer to the guest-CPU context.
10749 */
10750VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10751{
10752 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10753 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10754 HMVMX_ASSERT_PREEMPT_SAFE();
10755
10756 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10757
10758 VBOXSTRICTRC rcStrict;
10759 if ( !pVCpu->hm.s.fUseDebugLoop
10760 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10761 && !DBGFIsStepping(pVCpu)
10762 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10763 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10764 else
10765 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10766
10767 if (rcStrict == VERR_EM_INTERPRETER)
10768 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10769 else if (rcStrict == VINF_EM_RESET)
10770 rcStrict = VINF_EM_TRIPLE_FAULT;
10771
10772 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10773 if (RT_FAILURE(rc2))
10774 {
10775 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10776 rcStrict = rc2;
10777 }
10778 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10779 return rcStrict;
10780}
10781
10782
10783#ifndef HMVMX_USE_FUNCTION_TABLE
10784DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10785{
10786# ifdef DEBUG_ramshankar
10787# define RETURN_EXIT_CALL(a_CallExpr) \
10788 do { \
10789 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10790 VBOXSTRICTRC rcStrict = a_CallExpr; \
10791 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10792 return rcStrict; \
10793 } while (0)
10794# else
10795# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10796# endif
10797 switch (rcReason)
10798 {
10799 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10800 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10801 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10802 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10803 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10804 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10805 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10806 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10807 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10808 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10809 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10810 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10811 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10812 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10813 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10814 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10815 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10816 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10817 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10818 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10819 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10820 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10821 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10822 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10823 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10824 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10825 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10826 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10827 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10828 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10829 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10830 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10831 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10832 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10833
10834 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10835 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10836 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10837 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10838 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10839 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10840 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10841 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10842 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10843
10844 case VMX_EXIT_VMCLEAR:
10845 case VMX_EXIT_VMLAUNCH:
10846 case VMX_EXIT_VMPTRLD:
10847 case VMX_EXIT_VMPTRST:
10848 case VMX_EXIT_VMREAD:
10849 case VMX_EXIT_VMRESUME:
10850 case VMX_EXIT_VMWRITE:
10851 case VMX_EXIT_VMXOFF:
10852 case VMX_EXIT_VMXON:
10853 case VMX_EXIT_INVEPT:
10854 case VMX_EXIT_INVVPID:
10855 case VMX_EXIT_VMFUNC:
10856 case VMX_EXIT_XSAVES:
10857 case VMX_EXIT_XRSTORS:
10858 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10859 case VMX_EXIT_ENCLS:
10860 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10861 case VMX_EXIT_PML_FULL:
10862 default:
10863 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10864 }
10865#undef RETURN_EXIT_CALL
10866}
10867#endif /* !HMVMX_USE_FUNCTION_TABLE */
10868
10869
10870#ifdef VBOX_STRICT
10871/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10872# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10873 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10874
10875# define HMVMX_ASSERT_PREEMPT_CPUID() \
10876 do { \
10877 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10878 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10879 } while (0)
10880
10881# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10882 do { \
10883 AssertPtr(pVCpu); \
10884 AssertPtr(pMixedCtx); \
10885 AssertPtr(pVmxTransient); \
10886 Assert(pVmxTransient->fVMEntryFailed == false); \
10887 Assert(ASMIntAreEnabled()); \
10888 HMVMX_ASSERT_PREEMPT_SAFE(); \
10889 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10890 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10891 HMVMX_ASSERT_PREEMPT_SAFE(); \
10892 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10893 HMVMX_ASSERT_PREEMPT_CPUID(); \
10894 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10895 } while (0)
10896
10897# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10898 do { \
10899 Log4Func(("\n")); \
10900 } while (0)
10901#else /* nonstrict builds: */
10902# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10903 do { \
10904 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10905 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10906 } while (0)
10907# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10908#endif
10909
10910
10911/**
10912 * Advances the guest RIP by the specified number of bytes.
10913 *
10914 * @param pVCpu The cross context virtual CPU structure.
10915 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10916 * out-of-sync. Make sure to update the required fields
10917 * before using them.
10918 * @param cbInstr Number of bytes to advance the RIP by.
10919 *
10920 * @remarks No-long-jump zone!!!
10921 */
10922DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10923{
10924 /* Advance the RIP. */
10925 pMixedCtx->rip += cbInstr;
10926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10927
10928 /* Update interrupt inhibition. */
10929 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10930 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10931 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10932}
10933
10934
10935/**
10936 * Advances the guest RIP after reading it from the VMCS.
10937 *
10938 * @returns VBox status code, no informational status codes.
10939 * @param pVCpu The cross context virtual CPU structure.
10940 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10941 * out-of-sync. Make sure to update the required fields
10942 * before using them.
10943 * @param pVmxTransient Pointer to the VMX transient structure.
10944 *
10945 * @remarks No-long-jump zone!!!
10946 */
10947static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10948{
10949 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10950 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10951 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10952 AssertRCReturn(rc, rc);
10953
10954 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10955
10956 /*
10957 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10958 * pending debug exception field as it takes care of priority of events.
10959 *
10960 * See Intel spec. 32.2.1 "Debug Exceptions".
10961 */
10962 if ( !pVCpu->hm.s.fSingleInstruction
10963 && pMixedCtx->eflags.Bits.u1TF)
10964 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10965
10966 return VINF_SUCCESS;
10967}
10968
10969
10970/**
10971 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10972 * and update error record fields accordingly.
10973 *
10974 * @return VMX_IGS_* return codes.
10975 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10976 * wrong with the guest state.
10977 *
10978 * @param pVM The cross context VM structure.
10979 * @param pVCpu The cross context virtual CPU structure.
10980 * @param pCtx Pointer to the guest-CPU state.
10981 *
10982 * @remarks This function assumes our cache of the VMCS controls
10983 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10984 */
10985static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10986{
10987#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10988#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10989 uError = (err); \
10990 break; \
10991 } else do { } while (0)
10992
10993 int rc;
10994 uint32_t uError = VMX_IGS_ERROR;
10995 uint32_t u32Val;
10996 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10997
10998 do
10999 {
11000 /*
11001 * CR0.
11002 */
11003 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11004 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
11005 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
11006 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
11007 if (fUnrestrictedGuest)
11008 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
11009
11010 uint32_t u32GuestCR0;
11011 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
11012 AssertRCBreak(rc);
11013 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
11014 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
11015 if ( !fUnrestrictedGuest
11016 && (u32GuestCR0 & X86_CR0_PG)
11017 && !(u32GuestCR0 & X86_CR0_PE))
11018 {
11019 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11020 }
11021
11022 /*
11023 * CR4.
11024 */
11025 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11026 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11027
11028 uint32_t u32GuestCR4;
11029 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11030 AssertRCBreak(rc);
11031 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11032 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11033
11034 /*
11035 * IA32_DEBUGCTL MSR.
11036 */
11037 uint64_t u64Val;
11038 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11039 AssertRCBreak(rc);
11040 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11041 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11042 {
11043 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11044 }
11045 uint64_t u64DebugCtlMsr = u64Val;
11046
11047#ifdef VBOX_STRICT
11048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11049 AssertRCBreak(rc);
11050 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11051#endif
11052 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11053
11054 /*
11055 * RIP and RFLAGS.
11056 */
11057 uint32_t u32Eflags;
11058#if HC_ARCH_BITS == 64
11059 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11060 AssertRCBreak(rc);
11061 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11062 if ( !fLongModeGuest
11063 || !pCtx->cs.Attr.n.u1Long)
11064 {
11065 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11066 }
11067 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11068 * must be identical if the "IA-32e mode guest" VM-entry
11069 * control is 1 and CS.L is 1. No check applies if the
11070 * CPU supports 64 linear-address bits. */
11071
11072 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11073 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11074 AssertRCBreak(rc);
11075 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11076 VMX_IGS_RFLAGS_RESERVED);
11077 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11078 u32Eflags = u64Val;
11079#else
11080 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11081 AssertRCBreak(rc);
11082 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11083 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11084#endif
11085
11086 if ( fLongModeGuest
11087 || ( fUnrestrictedGuest
11088 && !(u32GuestCR0 & X86_CR0_PE)))
11089 {
11090 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11091 }
11092
11093 uint32_t u32EntryInfo;
11094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11095 AssertRCBreak(rc);
11096 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11097 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11098 {
11099 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11100 }
11101
11102 /*
11103 * 64-bit checks.
11104 */
11105#if HC_ARCH_BITS == 64
11106 if (fLongModeGuest)
11107 {
11108 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11109 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11110 }
11111
11112 if ( !fLongModeGuest
11113 && (u32GuestCR4 & X86_CR4_PCIDE))
11114 {
11115 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11116 }
11117
11118 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11119 * 51:32 beyond the processor's physical-address width are 0. */
11120
11121 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11122 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11123 {
11124 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11125 }
11126
11127 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11128 AssertRCBreak(rc);
11129 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11130
11131 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11132 AssertRCBreak(rc);
11133 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11134#endif
11135
11136 /*
11137 * PERF_GLOBAL MSR.
11138 */
11139 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11140 {
11141 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11142 AssertRCBreak(rc);
11143 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11144 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11145 }
11146
11147 /*
11148 * PAT MSR.
11149 */
11150 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11151 {
11152 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11153 AssertRCBreak(rc);
11154 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11155 for (unsigned i = 0; i < 8; i++)
11156 {
11157 uint8_t u8Val = (u64Val & 0xff);
11158 if ( u8Val != 0 /* UC */
11159 && u8Val != 1 /* WC */
11160 && u8Val != 4 /* WT */
11161 && u8Val != 5 /* WP */
11162 && u8Val != 6 /* WB */
11163 && u8Val != 7 /* UC- */)
11164 {
11165 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11166 }
11167 u64Val >>= 8;
11168 }
11169 }
11170
11171 /*
11172 * EFER MSR.
11173 */
11174 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11175 {
11176 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11177 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11178 AssertRCBreak(rc);
11179 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11180 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11181 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11182 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11183 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11184 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11185 || !(u32GuestCR0 & X86_CR0_PG)
11186 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11187 VMX_IGS_EFER_LMA_LME_MISMATCH);
11188 }
11189
11190 /*
11191 * Segment registers.
11192 */
11193 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11194 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11195 if (!(u32Eflags & X86_EFL_VM))
11196 {
11197 /* CS */
11198 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11199 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11200 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11201 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11202 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11203 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11204 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11205 /* CS cannot be loaded with NULL in protected mode. */
11206 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11207 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11208 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11209 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11210 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11211 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11212 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11213 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11214 else
11215 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11216
11217 /* SS */
11218 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11219 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11220 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11221 if ( !(pCtx->cr0 & X86_CR0_PE)
11222 || pCtx->cs.Attr.n.u4Type == 3)
11223 {
11224 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11225 }
11226 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11227 {
11228 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11229 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11230 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11231 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11232 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11233 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11234 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11235 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11236 }
11237
11238 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11239 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11240 {
11241 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11242 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11243 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11244 || pCtx->ds.Attr.n.u4Type > 11
11245 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11246 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11247 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11248 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11249 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11250 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11251 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11252 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11253 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11254 }
11255 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11256 {
11257 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11258 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11259 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11260 || pCtx->es.Attr.n.u4Type > 11
11261 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11262 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11263 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11264 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11265 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11266 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11267 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11268 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11269 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11270 }
11271 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11272 {
11273 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11274 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11275 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11276 || pCtx->fs.Attr.n.u4Type > 11
11277 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11278 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11279 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11280 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11281 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11282 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11283 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11284 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11285 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11286 }
11287 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11288 {
11289 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11290 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11291 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11292 || pCtx->gs.Attr.n.u4Type > 11
11293 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11294 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11295 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11296 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11297 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11298 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11299 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11300 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11301 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11302 }
11303 /* 64-bit capable CPUs. */
11304#if HC_ARCH_BITS == 64
11305 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11306 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11307 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11308 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11309 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11310 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11311 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11312 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11313 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11314 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11315 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11316#endif
11317 }
11318 else
11319 {
11320 /* V86 mode checks. */
11321 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11322 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11323 {
11324 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11325 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11326 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11327 }
11328 else
11329 {
11330 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11331 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11332 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11333 }
11334
11335 /* CS */
11336 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11337 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11338 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11339 /* SS */
11340 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11341 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11342 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11343 /* DS */
11344 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11345 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11346 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11347 /* ES */
11348 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11349 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11350 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11351 /* FS */
11352 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11353 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11354 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11355 /* GS */
11356 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11357 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11358 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11359 /* 64-bit capable CPUs. */
11360#if HC_ARCH_BITS == 64
11361 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11362 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11363 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11364 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11365 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11366 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11367 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11368 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11369 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11370 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11371 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11372#endif
11373 }
11374
11375 /*
11376 * TR.
11377 */
11378 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11379 /* 64-bit capable CPUs. */
11380#if HC_ARCH_BITS == 64
11381 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11382#endif
11383 if (fLongModeGuest)
11384 {
11385 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11386 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11387 }
11388 else
11389 {
11390 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11391 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11392 VMX_IGS_TR_ATTR_TYPE_INVALID);
11393 }
11394 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11395 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11396 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11397 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11398 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11399 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11400 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11401 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11402
11403 /*
11404 * GDTR and IDTR.
11405 */
11406#if HC_ARCH_BITS == 64
11407 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11408 AssertRCBreak(rc);
11409 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11410
11411 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11412 AssertRCBreak(rc);
11413 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11414#endif
11415
11416 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11417 AssertRCBreak(rc);
11418 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11419
11420 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11421 AssertRCBreak(rc);
11422 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11423
11424 /*
11425 * Guest Non-Register State.
11426 */
11427 /* Activity State. */
11428 uint32_t u32ActivityState;
11429 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11430 AssertRCBreak(rc);
11431 HMVMX_CHECK_BREAK( !u32ActivityState
11432 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11433 VMX_IGS_ACTIVITY_STATE_INVALID);
11434 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11435 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11436 uint32_t u32IntrState;
11437 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11438 AssertRCBreak(rc);
11439 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11440 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11441 {
11442 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11443 }
11444
11445 /** @todo Activity state and injecting interrupts. Left as a todo since we
11446 * currently don't use activity states but ACTIVE. */
11447
11448 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11449 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11450
11451 /* Guest interruptibility-state. */
11452 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11453 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11454 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11455 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11456 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11457 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11458 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11459 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11460 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11461 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11462 {
11463 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11464 {
11465 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11466 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11467 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11468 }
11469 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11470 {
11471 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11472 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11473 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11474 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11475 }
11476 }
11477 /** @todo Assumes the processor is not in SMM. */
11478 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11479 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11480 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11481 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11482 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11483 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11484 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11485 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11486 {
11487 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11488 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11489 }
11490
11491 /* Pending debug exceptions. */
11492#if HC_ARCH_BITS == 64
11493 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11494 AssertRCBreak(rc);
11495 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11496 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11497 u32Val = u64Val; /* For pending debug exceptions checks below. */
11498#else
11499 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11500 AssertRCBreak(rc);
11501 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11502 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11503#endif
11504
11505 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11506 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11507 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11508 {
11509 if ( (u32Eflags & X86_EFL_TF)
11510 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11511 {
11512 /* Bit 14 is PendingDebug.BS. */
11513 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11514 }
11515 if ( !(u32Eflags & X86_EFL_TF)
11516 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11517 {
11518 /* Bit 14 is PendingDebug.BS. */
11519 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11520 }
11521 }
11522
11523 /* VMCS link pointer. */
11524 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11525 AssertRCBreak(rc);
11526 if (u64Val != UINT64_C(0xffffffffffffffff))
11527 {
11528 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11529 /** @todo Bits beyond the processor's physical-address width MBZ. */
11530 /** @todo 32-bit located in memory referenced by value of this field (as a
11531 * physical address) must contain the processor's VMCS revision ID. */
11532 /** @todo SMM checks. */
11533 }
11534
11535 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11536 * not using Nested Paging? */
11537 if ( pVM->hm.s.fNestedPaging
11538 && !fLongModeGuest
11539 && CPUMIsGuestInPAEModeEx(pCtx))
11540 {
11541 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11542 AssertRCBreak(rc);
11543 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11544
11545 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11546 AssertRCBreak(rc);
11547 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11548
11549 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11550 AssertRCBreak(rc);
11551 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11552
11553 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11554 AssertRCBreak(rc);
11555 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11556 }
11557
11558 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11559 if (uError == VMX_IGS_ERROR)
11560 uError = VMX_IGS_REASON_NOT_FOUND;
11561 } while (0);
11562
11563 pVCpu->hm.s.u32HMError = uError;
11564 return uError;
11565
11566#undef HMVMX_ERROR_BREAK
11567#undef HMVMX_CHECK_BREAK
11568}
11569
11570/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11571/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11572/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11573
11574/** @name VM-exit handlers.
11575 * @{
11576 */
11577
11578/**
11579 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11580 */
11581HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11582{
11583 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11584 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11585 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11586 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11587 return VINF_SUCCESS;
11588 return VINF_EM_RAW_INTERRUPT;
11589}
11590
11591
11592/**
11593 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11594 */
11595HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11596{
11597 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11598 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11599
11600 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11601 AssertRCReturn(rc, rc);
11602
11603 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11604 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11605 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11606 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11607
11608 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11609 {
11610 /*
11611 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11612 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11613 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11614 *
11615 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11616 */
11617 VMXDispatchHostNmi();
11618 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11619 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11620 return VINF_SUCCESS;
11621 }
11622
11623 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11624 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11625 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11626 { /* likely */ }
11627 else
11628 {
11629 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11630 rcStrictRc1 = VINF_SUCCESS;
11631 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11632 return rcStrictRc1;
11633 }
11634
11635 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11636 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11637 switch (uIntType)
11638 {
11639 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11640 Assert(uVector == X86_XCPT_DB);
11641 RT_FALL_THRU();
11642 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11643 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11644 RT_FALL_THRU();
11645 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11646 {
11647 /*
11648 * If there's any exception caused as a result of event injection, the resulting
11649 * secondary/final execption will be pending, we shall continue guest execution
11650 * after injecting the event. The page-fault case is complicated and we manually
11651 * handle any currently pending event in hmR0VmxExitXcptPF.
11652 */
11653 if (!pVCpu->hm.s.Event.fPending)
11654 { /* likely */ }
11655 else if (uVector != X86_XCPT_PF)
11656 {
11657 rc = VINF_SUCCESS;
11658 break;
11659 }
11660
11661 switch (uVector)
11662 {
11663 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11664 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11665 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11666 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11667 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11668 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11669 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11670
11671 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11672 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11673 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11674 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11675 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11676 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11677 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11678 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11679 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11680 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11681 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11682 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11683 default:
11684 {
11685 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11686 AssertRCReturn(rc, rc);
11687
11688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11689 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11690 {
11691 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11692 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11693 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11694
11695 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11696 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11697 AssertRCReturn(rc, rc);
11698 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11699 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11700 0 /* GCPtrFaultAddress */);
11701 AssertRCReturn(rc, rc);
11702 }
11703 else
11704 {
11705 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11706 pVCpu->hm.s.u32HMError = uVector;
11707 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11708 }
11709 break;
11710 }
11711 }
11712 break;
11713 }
11714
11715 default:
11716 {
11717 pVCpu->hm.s.u32HMError = uExitIntInfo;
11718 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11719 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11720 break;
11721 }
11722 }
11723 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11724 return rc;
11725}
11726
11727
11728/**
11729 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11730 */
11731HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11732{
11733 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11734
11735 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11736 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11737
11738 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11740 return VINF_SUCCESS;
11741}
11742
11743
11744/**
11745 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11746 */
11747HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11748{
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11750 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11751 {
11752 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11753 HMVMX_RETURN_UNEXPECTED_EXIT();
11754 }
11755
11756 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11757
11758 /*
11759 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11760 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11761 */
11762 uint32_t uIntrState = 0;
11763 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11764 AssertRCReturn(rc, rc);
11765
11766 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11767 if ( fBlockSti
11768 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11769 {
11770 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11771 }
11772
11773 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11774 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11775
11776 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11777 return VINF_SUCCESS;
11778}
11779
11780
11781/**
11782 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11783 */
11784HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11785{
11786 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11787 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11788 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11789}
11790
11791
11792/**
11793 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11794 */
11795HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11796{
11797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11799 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11800}
11801
11802
11803/**
11804 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11805 */
11806HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11807{
11808 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11809 PVM pVM = pVCpu->CTX_SUFF(pVM);
11810 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11811 if (RT_LIKELY(rc == VINF_SUCCESS))
11812 {
11813 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11814 Assert(pVmxTransient->cbInstr == 2);
11815 }
11816 else
11817 {
11818 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11819 rc = VERR_EM_INTERPRETER;
11820 }
11821 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11822 return rc;
11823}
11824
11825
11826/**
11827 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11828 */
11829HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11830{
11831 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11832 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11833 AssertRCReturn(rc, rc);
11834
11835 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11836 return VINF_EM_RAW_EMULATE_INSTR;
11837
11838 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11839 HMVMX_RETURN_UNEXPECTED_EXIT();
11840}
11841
11842
11843/**
11844 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11845 */
11846HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11847{
11848 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11849 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11850 AssertRCReturn(rc, rc);
11851
11852 PVM pVM = pVCpu->CTX_SUFF(pVM);
11853 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11854 if (RT_LIKELY(rc == VINF_SUCCESS))
11855 {
11856 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11857 Assert(pVmxTransient->cbInstr == 2);
11858 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11859 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11860 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11861 }
11862 else
11863 rc = VERR_EM_INTERPRETER;
11864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11865 return rc;
11866}
11867
11868
11869/**
11870 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11871 */
11872HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11873{
11874 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11875 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11876 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11877 AssertRCReturn(rc, rc);
11878
11879 PVM pVM = pVCpu->CTX_SUFF(pVM);
11880 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11881 if (RT_SUCCESS(rc))
11882 {
11883 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11884 Assert(pVmxTransient->cbInstr == 3);
11885 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11886 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11887 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11888 }
11889 else
11890 {
11891 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11892 rc = VERR_EM_INTERPRETER;
11893 }
11894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11895 return rc;
11896}
11897
11898
11899/**
11900 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11901 */
11902HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11903{
11904 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11905 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11906 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11907 AssertRCReturn(rc, rc);
11908
11909 PVM pVM = pVCpu->CTX_SUFF(pVM);
11910 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11911 if (RT_LIKELY(rc == VINF_SUCCESS))
11912 {
11913 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11914 Assert(pVmxTransient->cbInstr == 2);
11915 }
11916 else
11917 {
11918 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11919 rc = VERR_EM_INTERPRETER;
11920 }
11921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11922 return rc;
11923}
11924
11925
11926/**
11927 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11928 */
11929HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11930{
11931 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11933
11934 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11935 if (pVCpu->hm.s.fHypercallsEnabled)
11936 {
11937#if 0
11938 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11939#else
11940 /* Aggressive state sync. for now. */
11941 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11942 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11943 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11944 AssertRCReturn(rc, rc);
11945#endif
11946
11947 /* Perform the hypercall. */
11948 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11949 if (rcStrict == VINF_SUCCESS)
11950 {
11951 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11952 AssertRCReturn(rc, rc);
11953 }
11954 else
11955 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11956 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11957 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11958
11959 /* If the hypercall changes anything other than guest's general-purpose registers,
11960 we would need to reload the guest changed bits here before VM-entry. */
11961 }
11962 else
11963 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11964
11965 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11966 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11967 {
11968 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11969 rcStrict = VINF_SUCCESS;
11970 }
11971
11972 return rcStrict;
11973}
11974
11975
11976/**
11977 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11978 */
11979HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11980{
11981 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11982 PVM pVM = pVCpu->CTX_SUFF(pVM);
11983 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11984
11985 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11986 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11987 AssertRCReturn(rc, rc);
11988
11989 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11990 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11991 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11992 else
11993 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11994 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11996 return rcStrict;
11997}
11998
11999
12000/**
12001 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
12002 */
12003HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12004{
12005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12006 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12007 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12008 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12009 AssertRCReturn(rc, rc);
12010
12011 PVM pVM = pVCpu->CTX_SUFF(pVM);
12012 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12013 if (RT_LIKELY(rc == VINF_SUCCESS))
12014 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12015 else
12016 {
12017 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
12018 rc = VERR_EM_INTERPRETER;
12019 }
12020 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12021 return rc;
12022}
12023
12024
12025/**
12026 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12027 */
12028HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12029{
12030 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12031 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12032 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12033 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12034 AssertRCReturn(rc, rc);
12035
12036 PVM pVM = pVCpu->CTX_SUFF(pVM);
12037 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12038 rc = VBOXSTRICTRC_VAL(rc2);
12039 if (RT_LIKELY( rc == VINF_SUCCESS
12040 || rc == VINF_EM_HALT))
12041 {
12042 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12043 AssertRCReturn(rc3, rc3);
12044
12045 if ( rc == VINF_EM_HALT
12046 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12047 {
12048 rc = VINF_SUCCESS;
12049 }
12050 }
12051 else
12052 {
12053 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12054 rc = VERR_EM_INTERPRETER;
12055 }
12056 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
12057 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12059 return rc;
12060}
12061
12062
12063/**
12064 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12065 */
12066HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12067{
12068 /*
12069 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12070 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12071 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12072 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12073 */
12074 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12075 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12076 HMVMX_RETURN_UNEXPECTED_EXIT();
12077}
12078
12079
12080/**
12081 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12082 */
12083HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12084{
12085 /*
12086 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12087 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12088 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12089 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12090 */
12091 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12092 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12093 HMVMX_RETURN_UNEXPECTED_EXIT();
12094}
12095
12096
12097/**
12098 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12099 */
12100HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12101{
12102 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12103 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12104 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12105 HMVMX_RETURN_UNEXPECTED_EXIT();
12106}
12107
12108
12109/**
12110 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12111 */
12112HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12113{
12114 /*
12115 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12116 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12117 * See Intel spec. 25.3 "Other Causes of VM-exits".
12118 */
12119 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12120 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12121 HMVMX_RETURN_UNEXPECTED_EXIT();
12122}
12123
12124
12125/**
12126 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12127 * VM-exit.
12128 */
12129HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12130{
12131 /*
12132 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12133 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12134 *
12135 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12136 * See Intel spec. "23.8 Restrictions on VMX operation".
12137 */
12138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12139 return VINF_SUCCESS;
12140}
12141
12142
12143/**
12144 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12145 * VM-exit.
12146 */
12147HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12148{
12149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12150 return VINF_EM_RESET;
12151}
12152
12153
12154/**
12155 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12156 */
12157HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12158{
12159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12160 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12161
12162 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12163 AssertRCReturn(rc, rc);
12164
12165 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12166 rc = VINF_SUCCESS;
12167 else
12168 rc = VINF_EM_HALT;
12169
12170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12171 if (rc != VINF_SUCCESS)
12172 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12173 return rc;
12174}
12175
12176
12177/**
12178 * VM-exit handler for instructions that result in a \#UD exception delivered to
12179 * the guest.
12180 */
12181HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12182{
12183 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12184 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12185 return VINF_SUCCESS;
12186}
12187
12188
12189/**
12190 * VM-exit handler for expiry of the VMX preemption timer.
12191 */
12192HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12193{
12194 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12195
12196 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12197 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12198
12199 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12200 PVM pVM = pVCpu->CTX_SUFF(pVM);
12201 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12202 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12203 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12204}
12205
12206
12207/**
12208 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12209 */
12210HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12211{
12212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12213
12214 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12215 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12216 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12217 AssertRCReturn(rc, rc);
12218
12219 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12220 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12221
12222 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12223
12224 return rcStrict;
12225}
12226
12227
12228/**
12229 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12230 */
12231HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12232{
12233 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12234 /** @todo Use VM-exit instruction information. */
12235 return VERR_EM_INTERPRETER;
12236}
12237
12238
12239/**
12240 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12241 * Error VM-exit.
12242 */
12243HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12244{
12245 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12246 AssertRCReturn(rc, rc);
12247
12248 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12249 AssertRCReturn(rc, rc);
12250
12251 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12252 NOREF(uInvalidReason);
12253
12254#ifdef VBOX_STRICT
12255 uint32_t uIntrState;
12256 RTHCUINTREG uHCReg;
12257 uint64_t u64Val;
12258 uint32_t u32Val;
12259
12260 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12261 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12262 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12263 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12264 AssertRCReturn(rc, rc);
12265
12266 Log4(("uInvalidReason %u\n", uInvalidReason));
12267 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12268 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12269 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12270 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12271
12272 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12273 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12274 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12275 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12276 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12277 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12278 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12279 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12280 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12281 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12282 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12283 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12284#else
12285 NOREF(pVmxTransient);
12286#endif
12287
12288 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12289 return VERR_VMX_INVALID_GUEST_STATE;
12290}
12291
12292
12293/**
12294 * VM-exit handler for VM-entry failure due to an MSR-load
12295 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12296 */
12297HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12298{
12299 NOREF(pVmxTransient);
12300 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12301 HMVMX_RETURN_UNEXPECTED_EXIT();
12302}
12303
12304
12305/**
12306 * VM-exit handler for VM-entry failure due to a machine-check event
12307 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12308 */
12309HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12310{
12311 NOREF(pVmxTransient);
12312 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12313 HMVMX_RETURN_UNEXPECTED_EXIT();
12314}
12315
12316
12317/**
12318 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12319 * theory.
12320 */
12321HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12322{
12323 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12324 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12325 return VERR_VMX_UNDEFINED_EXIT_CODE;
12326}
12327
12328
12329/**
12330 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12331 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12332 * Conditional VM-exit.
12333 */
12334HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12335{
12336 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12337
12338 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12340 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12341 return VERR_EM_INTERPRETER;
12342 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12343 HMVMX_RETURN_UNEXPECTED_EXIT();
12344}
12345
12346
12347/**
12348 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12349 */
12350HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12351{
12352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12353
12354 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12356 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12357 return VERR_EM_INTERPRETER;
12358 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12359 HMVMX_RETURN_UNEXPECTED_EXIT();
12360}
12361
12362
12363/**
12364 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12365 */
12366HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12367{
12368 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12369
12370 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12371 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12372 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12373 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12374 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12375 {
12376 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12377 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12378 }
12379 AssertRCReturn(rc, rc);
12380 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12381
12382#ifdef VBOX_STRICT
12383 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12384 {
12385 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12386 && pMixedCtx->ecx != MSR_K6_EFER)
12387 {
12388 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12389 pMixedCtx->ecx));
12390 HMVMX_RETURN_UNEXPECTED_EXIT();
12391 }
12392 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12393 {
12394 VMXMSREXITREAD enmRead;
12395 VMXMSREXITWRITE enmWrite;
12396 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12397 AssertRCReturn(rc2, rc2);
12398 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12399 {
12400 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12401 HMVMX_RETURN_UNEXPECTED_EXIT();
12402 }
12403 }
12404 }
12405#endif
12406
12407 PVM pVM = pVCpu->CTX_SUFF(pVM);
12408 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12409 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12410 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12411 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12412 if (RT_SUCCESS(rc))
12413 {
12414 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12415 Assert(pVmxTransient->cbInstr == 2);
12416 }
12417 return rc;
12418}
12419
12420
12421/**
12422 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12423 */
12424HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12425{
12426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12427 PVM pVM = pVCpu->CTX_SUFF(pVM);
12428 int rc = VINF_SUCCESS;
12429
12430 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12431 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12432 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12433 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12434 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12435 {
12436 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12437 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12438 }
12439 AssertRCReturn(rc, rc);
12440 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12441
12442 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12443 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12444 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12445
12446 if (RT_SUCCESS(rc))
12447 {
12448 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12449
12450 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12451 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12452 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12453 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12454 {
12455 /*
12456 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12457 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12458 * EMInterpretWrmsr() changes it.
12459 */
12460 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12461 }
12462 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12463 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12464 else if (pMixedCtx->ecx == MSR_K6_EFER)
12465 {
12466 /*
12467 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12468 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12469 * the other bits as well, SCE and NXE. See @bugref{7368}.
12470 */
12471 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12472 }
12473
12474 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12475 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12476 {
12477 switch (pMixedCtx->ecx)
12478 {
12479 /*
12480 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12481 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12482 */
12483 case MSR_IA32_SYSENTER_CS:
12484 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12485 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12486 break;
12487 case MSR_IA32_SYSENTER_EIP:
12488 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12489 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12490 break;
12491 case MSR_IA32_SYSENTER_ESP:
12492 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12493 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12494 break;
12495 case MSR_K8_FS_BASE: RT_FALL_THRU();
12496 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12497 case MSR_K6_EFER: /* already handled above */ break;
12498 default:
12499 {
12500 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12501 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12502 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12503 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12504 break;
12505 }
12506 }
12507 }
12508#ifdef VBOX_STRICT
12509 else
12510 {
12511 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12512 switch (pMixedCtx->ecx)
12513 {
12514 case MSR_IA32_SYSENTER_CS:
12515 case MSR_IA32_SYSENTER_EIP:
12516 case MSR_IA32_SYSENTER_ESP:
12517 case MSR_K8_FS_BASE:
12518 case MSR_K8_GS_BASE:
12519 {
12520 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12521 HMVMX_RETURN_UNEXPECTED_EXIT();
12522 }
12523
12524 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12525 default:
12526 {
12527 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12528 {
12529 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12530 if (pMixedCtx->ecx != MSR_K6_EFER)
12531 {
12532 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12533 pMixedCtx->ecx));
12534 HMVMX_RETURN_UNEXPECTED_EXIT();
12535 }
12536 }
12537
12538 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12539 {
12540 VMXMSREXITREAD enmRead;
12541 VMXMSREXITWRITE enmWrite;
12542 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12543 AssertRCReturn(rc2, rc2);
12544 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12545 {
12546 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12547 HMVMX_RETURN_UNEXPECTED_EXIT();
12548 }
12549 }
12550 break;
12551 }
12552 }
12553 }
12554#endif /* VBOX_STRICT */
12555 }
12556 return rc;
12557}
12558
12559
12560/**
12561 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12562 */
12563HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12564{
12565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12566
12567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12568 return VINF_EM_RAW_INTERRUPT;
12569}
12570
12571
12572/**
12573 * VM-exit handler for when the TPR value is lowered below the specified
12574 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12575 */
12576HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12577{
12578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12579 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12580
12581 /*
12582 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12583 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12584 */
12585 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12586 return VINF_SUCCESS;
12587}
12588
12589
12590/**
12591 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12592 * VM-exit.
12593 *
12594 * @retval VINF_SUCCESS when guest execution can continue.
12595 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12596 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12597 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12598 * interpreter.
12599 */
12600HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12601{
12602 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12603 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12604 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12605 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12606 AssertRCReturn(rc, rc);
12607
12608 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12609 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12610 PVM pVM = pVCpu->CTX_SUFF(pVM);
12611 VBOXSTRICTRC rcStrict;
12612 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12613 switch (uAccessType)
12614 {
12615 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12616 {
12617 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12618 AssertRCReturn(rc, rc);
12619
12620 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12621 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12622 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12623 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12624 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12625 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12626 {
12627 case 0: /* CR0 */
12628 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12629 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12630 break;
12631 case 2: /* CR2 */
12632 /* Nothing to do here, CR2 it's not part of the VMCS. */
12633 break;
12634 case 3: /* CR3 */
12635 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12636 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12637 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12638 break;
12639 case 4: /* CR4 */
12640 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12641 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12642 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12643 break;
12644 case 8: /* CR8 */
12645 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12646 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12647 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12648 break;
12649 default:
12650 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12651 break;
12652 }
12653
12654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12655 break;
12656 }
12657
12658 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12659 {
12660 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12661 AssertRCReturn(rc, rc);
12662
12663 Assert( !pVM->hm.s.fNestedPaging
12664 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12665 || pVCpu->hm.s.fUsingDebugLoop
12666 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12667
12668 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12669 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12670 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12671
12672 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12673 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12674 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12675 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12677 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12678 VBOXSTRICTRC_VAL(rcStrict)));
12679 if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12680 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12681 break;
12682 }
12683
12684 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12685 {
12686 AssertRCReturn(rc, rc);
12687 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12688 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12689 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12690 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12691 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12692 break;
12693 }
12694
12695 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12696 {
12697 AssertRCReturn(rc, rc);
12698 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12699 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12700 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12701 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12702 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12704 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12705 break;
12706 }
12707
12708 default:
12709 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12710 VERR_VMX_UNEXPECTED_EXCEPTION);
12711 }
12712
12713 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12714 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12715 NOREF(pVM);
12716 return rcStrict;
12717}
12718
12719
12720/**
12721 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12722 * VM-exit.
12723 */
12724HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12725{
12726 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12727 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12728
12729 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12730 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12731 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12732 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12733 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12734 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12735 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12736 AssertRCReturn(rc, rc);
12737
12738 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12739 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12740 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12741 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12742 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12743 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12744 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12745 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12746 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12747
12748 /* I/O operation lookup arrays. */
12749 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12750 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12751
12752 VBOXSTRICTRC rcStrict;
12753 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12754 uint32_t const cbInstr = pVmxTransient->cbInstr;
12755 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12756 PVM pVM = pVCpu->CTX_SUFF(pVM);
12757 if (fIOString)
12758 {
12759#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12760 See @bugref{5752#c158}. Should work now. */
12761 /*
12762 * INS/OUTS - I/O String instruction.
12763 *
12764 * Use instruction-information if available, otherwise fall back on
12765 * interpreting the instruction.
12766 */
12767 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12768 fIOWrite ? 'w' : 'r'));
12769 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12770 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12771 {
12772 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12773 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12774 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12775 AssertRCReturn(rc2, rc2);
12776 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12777 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12778 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12779 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12780 if (fIOWrite)
12781 {
12782 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12783 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12784 }
12785 else
12786 {
12787 /*
12788 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12789 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12790 * See Intel Instruction spec. for "INS".
12791 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12792 */
12793 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12794 }
12795 }
12796 else
12797 {
12798 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12799 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12800 AssertRCReturn(rc2, rc2);
12801 rcStrict = IEMExecOne(pVCpu);
12802 }
12803 /** @todo IEM needs to be setting these flags somehow. */
12804 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12805 fUpdateRipAlready = true;
12806#else
12807 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12808 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12809 if (RT_SUCCESS(rcStrict))
12810 {
12811 if (fIOWrite)
12812 {
12813 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12814 (DISCPUMODE)pDis->uAddrMode, cbValue);
12815 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12816 }
12817 else
12818 {
12819 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12820 (DISCPUMODE)pDis->uAddrMode, cbValue);
12821 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12822 }
12823 }
12824 else
12825 {
12826 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12827 pMixedCtx->rip));
12828 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12829 }
12830#endif
12831 }
12832 else
12833 {
12834 /*
12835 * IN/OUT - I/O instruction.
12836 */
12837 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12838 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12839 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12840 if (fIOWrite)
12841 {
12842 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12844 }
12845 else
12846 {
12847 uint32_t u32Result = 0;
12848 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12849 if (IOM_SUCCESS(rcStrict))
12850 {
12851 /* Save result of I/O IN instr. in AL/AX/EAX. */
12852 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12853 }
12854 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12855 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12857 }
12858 }
12859
12860 if (IOM_SUCCESS(rcStrict))
12861 {
12862 if (!fUpdateRipAlready)
12863 {
12864 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12865 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12866 }
12867
12868 /*
12869 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12870 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12871 */
12872 if (fIOString)
12873 {
12874 /** @todo Single-step for INS/OUTS with REP prefix? */
12875 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12876 }
12877 else if ( !fDbgStepping
12878 && fGstStepping)
12879 {
12880 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12881 }
12882
12883 /*
12884 * If any I/O breakpoints are armed, we need to check if one triggered
12885 * and take appropriate action.
12886 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12887 */
12888 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12889 AssertRCReturn(rc2, rc2);
12890
12891 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12892 * execution engines about whether hyper BPs and such are pending. */
12893 uint32_t const uDr7 = pMixedCtx->dr[7];
12894 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12895 && X86_DR7_ANY_RW_IO(uDr7)
12896 && (pMixedCtx->cr4 & X86_CR4_DE))
12897 || DBGFBpIsHwIoArmed(pVM)))
12898 {
12899 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12900
12901 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12902 VMMRZCallRing3Disable(pVCpu);
12903 HM_DISABLE_PREEMPT();
12904
12905 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12906
12907 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12908 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12909 {
12910 /* Raise #DB. */
12911 if (fIsGuestDbgActive)
12912 ASMSetDR6(pMixedCtx->dr[6]);
12913 if (pMixedCtx->dr[7] != uDr7)
12914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12915
12916 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12917 }
12918 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12919 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12920 else if ( rcStrict2 != VINF_SUCCESS
12921 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12922 rcStrict = rcStrict2;
12923 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12924
12925 HM_RESTORE_PREEMPT();
12926 VMMRZCallRing3Enable(pVCpu);
12927 }
12928 }
12929
12930#ifdef VBOX_STRICT
12931 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12932 Assert(!fIOWrite);
12933 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12934 Assert(fIOWrite);
12935 else
12936 {
12937#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12938 * statuses, that the VMM device and some others may return. See
12939 * IOM_SUCCESS() for guidance. */
12940 AssertMsg( RT_FAILURE(rcStrict)
12941 || rcStrict == VINF_SUCCESS
12942 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12943 || rcStrict == VINF_EM_DBG_BREAKPOINT
12944 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12945 || rcStrict == VINF_EM_RAW_TO_R3
12946 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12947#endif
12948 }
12949#endif
12950
12951 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12952 return rcStrict;
12953}
12954
12955
12956/**
12957 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12958 * VM-exit.
12959 */
12960HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12961{
12962 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12963
12964 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12965 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12966 AssertRCReturn(rc, rc);
12967 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12968 {
12969 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12970 AssertRCReturn(rc, rc);
12971 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12972 {
12973 uint32_t uErrCode;
12974 RTGCUINTPTR GCPtrFaultAddress;
12975 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12976 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12977 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12978 if (fErrorCodeValid)
12979 {
12980 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12981 AssertRCReturn(rc, rc);
12982 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12983 }
12984 else
12985 uErrCode = 0;
12986
12987 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12988 && uVector == X86_XCPT_PF)
12989 GCPtrFaultAddress = pMixedCtx->cr2;
12990 else
12991 GCPtrFaultAddress = 0;
12992
12993 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12994 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12995
12996 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12998 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12999 }
13000 }
13001
13002 /* Fall back to the interpreter to emulate the task-switch. */
13003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
13004 return VERR_EM_INTERPRETER;
13005}
13006
13007
13008/**
13009 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
13010 */
13011HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13012{
13013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13014 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
13015 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
13016 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13017 AssertRCReturn(rc, rc);
13018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
13019 return VINF_EM_DBG_STEPPED;
13020}
13021
13022
13023/**
13024 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13025 */
13026HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13027{
13028 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13029
13030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13031
13032 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13033 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13034 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13035 {
13036 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13037 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13038 {
13039 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13040 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13041 }
13042 }
13043 else
13044 {
13045 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13046 rcStrict1 = VINF_SUCCESS;
13047 return rcStrict1;
13048 }
13049
13050#if 0
13051 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13052 * just sync the whole thing. */
13053 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13054#else
13055 /* Aggressive state sync. for now. */
13056 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13057 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13058 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13059#endif
13060 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13061 AssertRCReturn(rc, rc);
13062
13063 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13064 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13065 VBOXSTRICTRC rcStrict2;
13066 switch (uAccessType)
13067 {
13068 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13069 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13070 {
13071 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13072 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13073 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13074
13075 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13076 GCPhys &= PAGE_BASE_GC_MASK;
13077 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13078 PVM pVM = pVCpu->CTX_SUFF(pVM);
13079 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13080 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13081
13082 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13083 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13084 CPUMCTX2CORE(pMixedCtx), GCPhys);
13085 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13086 if ( rcStrict2 == VINF_SUCCESS
13087 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13088 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13089 {
13090 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13091 | HM_CHANGED_GUEST_RSP
13092 | HM_CHANGED_GUEST_RFLAGS
13093 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13094 rcStrict2 = VINF_SUCCESS;
13095 }
13096 break;
13097 }
13098
13099 default:
13100 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13101 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13102 break;
13103 }
13104
13105 if (rcStrict2 != VINF_SUCCESS)
13106 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13107 return rcStrict2;
13108}
13109
13110
13111/**
13112 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13113 * VM-exit.
13114 */
13115HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13116{
13117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13118
13119 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13120 if (pVmxTransient->fWasGuestDebugStateActive)
13121 {
13122 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13123 HMVMX_RETURN_UNEXPECTED_EXIT();
13124 }
13125
13126 if ( !pVCpu->hm.s.fSingleInstruction
13127 && !pVmxTransient->fWasHyperDebugStateActive)
13128 {
13129 Assert(!DBGFIsStepping(pVCpu));
13130 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13131
13132 /* Don't intercept MOV DRx any more. */
13133 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13134 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13135 AssertRCReturn(rc, rc);
13136
13137 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13138 VMMRZCallRing3Disable(pVCpu);
13139 HM_DISABLE_PREEMPT();
13140
13141 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13143 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13144
13145 HM_RESTORE_PREEMPT();
13146 VMMRZCallRing3Enable(pVCpu);
13147
13148#ifdef VBOX_WITH_STATISTICS
13149 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13150 AssertRCReturn(rc, rc);
13151 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13153 else
13154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13155#endif
13156 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13157 return VINF_SUCCESS;
13158 }
13159
13160 /*
13161 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13162 * Update the segment registers and DR7 from the CPU.
13163 */
13164 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13165 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13166 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13167 AssertRCReturn(rc, rc);
13168 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13169
13170 PVM pVM = pVCpu->CTX_SUFF(pVM);
13171 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13172 {
13173 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13174 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13175 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13176 if (RT_SUCCESS(rc))
13177 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13179 }
13180 else
13181 {
13182 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13183 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13184 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13186 }
13187
13188 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13189 if (RT_SUCCESS(rc))
13190 {
13191 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13192 AssertRCReturn(rc2, rc2);
13193 return VINF_SUCCESS;
13194 }
13195 return rc;
13196}
13197
13198
13199/**
13200 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13201 * Conditional VM-exit.
13202 */
13203HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13204{
13205 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13206 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13207
13208 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13209 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13210 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13211 {
13212 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13213 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13214 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13215 {
13216 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13217 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13218 }
13219 }
13220 else
13221 {
13222 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13223 rcStrict1 = VINF_SUCCESS;
13224 return rcStrict1;
13225 }
13226
13227 RTGCPHYS GCPhys = 0;
13228 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13229
13230#if 0
13231 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13232#else
13233 /* Aggressive state sync. for now. */
13234 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13235 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13236 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13237#endif
13238 AssertRCReturn(rc, rc);
13239
13240 /*
13241 * If we succeed, resume guest execution.
13242 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13243 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13244 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13245 * weird case. See @bugref{6043}.
13246 */
13247 PVM pVM = pVCpu->CTX_SUFF(pVM);
13248 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13249 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13250 if ( rcStrict2 == VINF_SUCCESS
13251 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13252 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13253 {
13254 /* Successfully handled MMIO operation. */
13255 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13256 | HM_CHANGED_GUEST_RSP
13257 | HM_CHANGED_GUEST_RFLAGS
13258 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13259 return VINF_SUCCESS;
13260 }
13261 return rcStrict2;
13262}
13263
13264
13265/**
13266 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13267 * VM-exit.
13268 */
13269HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13270{
13271 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13272 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13273
13274 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13275 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13276 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13277 {
13278 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13279 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13280 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13281 }
13282 else
13283 {
13284 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13285 rcStrict1 = VINF_SUCCESS;
13286 return rcStrict1;
13287 }
13288
13289 RTGCPHYS GCPhys = 0;
13290 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13291 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13292#if 0
13293 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13294#else
13295 /* Aggressive state sync. for now. */
13296 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13297 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13298 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13299#endif
13300 AssertRCReturn(rc, rc);
13301
13302 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13303 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13304
13305 RTGCUINT uErrorCode = 0;
13306 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13307 uErrorCode |= X86_TRAP_PF_ID;
13308 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13309 uErrorCode |= X86_TRAP_PF_RW;
13310 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13311 uErrorCode |= X86_TRAP_PF_P;
13312
13313 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13314
13315 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13316 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13317
13318 /* Handle the pagefault trap for the nested shadow table. */
13319 PVM pVM = pVCpu->CTX_SUFF(pVM);
13320 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13321 TRPMResetTrap(pVCpu);
13322
13323 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13324 if ( rcStrict2 == VINF_SUCCESS
13325 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13326 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13327 {
13328 /* Successfully synced our nested page tables. */
13329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13330 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13331 | HM_CHANGED_GUEST_RSP
13332 | HM_CHANGED_GUEST_RFLAGS);
13333 return VINF_SUCCESS;
13334 }
13335
13336 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13337 return rcStrict2;
13338}
13339
13340/** @} */
13341
13342/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13343/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13344/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13345
13346/** @name VM-exit exception handlers.
13347 * @{
13348 */
13349
13350/**
13351 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13352 */
13353static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13354{
13355 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13357
13358 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13359 AssertRCReturn(rc, rc);
13360
13361 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13362 {
13363 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13364 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13365
13366 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13367 * provides VM-exit instruction length. If this causes problem later,
13368 * disassemble the instruction like it's done on AMD-V. */
13369 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13370 AssertRCReturn(rc2, rc2);
13371 return rc;
13372 }
13373
13374 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13375 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13376 return rc;
13377}
13378
13379
13380/**
13381 * VM-exit exception handler for \#BP (Breakpoint exception).
13382 */
13383static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13384{
13385 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13387
13388 /** @todo Try optimize this by not saving the entire guest state unless
13389 * really needed. */
13390 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13391 AssertRCReturn(rc, rc);
13392
13393 PVM pVM = pVCpu->CTX_SUFF(pVM);
13394 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13395 if (rc == VINF_EM_RAW_GUEST_TRAP)
13396 {
13397 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13398 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13399 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13400 AssertRCReturn(rc, rc);
13401
13402 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13403 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13404 }
13405
13406 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13407 return rc;
13408}
13409
13410
13411/**
13412 * VM-exit exception handler for \#AC (alignment check exception).
13413 */
13414static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13415{
13416 RT_NOREF_PV(pMixedCtx);
13417 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13418
13419 /*
13420 * Re-inject it. We'll detect any nesting before getting here.
13421 */
13422 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13423 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13424 AssertRCReturn(rc, rc);
13425 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13426
13427 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13428 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13429 return VINF_SUCCESS;
13430}
13431
13432
13433/**
13434 * VM-exit exception handler for \#DB (Debug exception).
13435 */
13436static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13437{
13438 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13440 Log6(("XcptDB\n"));
13441
13442 /*
13443 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13444 * for processing.
13445 */
13446 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13447 AssertRCReturn(rc, rc);
13448
13449 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13450 uint64_t uDR6 = X86_DR6_INIT_VAL;
13451 uDR6 |= ( pVmxTransient->uExitQualification
13452 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13453
13454 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13455 if (rc == VINF_EM_RAW_GUEST_TRAP)
13456 {
13457 /*
13458 * The exception was for the guest. Update DR6, DR7.GD and
13459 * IA32_DEBUGCTL.LBR before forwarding it.
13460 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13461 */
13462 VMMRZCallRing3Disable(pVCpu);
13463 HM_DISABLE_PREEMPT();
13464
13465 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13466 pMixedCtx->dr[6] |= uDR6;
13467 if (CPUMIsGuestDebugStateActive(pVCpu))
13468 ASMSetDR6(pMixedCtx->dr[6]);
13469
13470 HM_RESTORE_PREEMPT();
13471 VMMRZCallRing3Enable(pVCpu);
13472
13473 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13474 AssertRCReturn(rc, rc);
13475
13476 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13477 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13478
13479 /* Paranoia. */
13480 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13481 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13482
13483 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13484 AssertRCReturn(rc, rc);
13485
13486 /*
13487 * Raise #DB in the guest.
13488 *
13489 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13490 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13491 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13492 *
13493 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13494 */
13495 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13496 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13497 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13498 AssertRCReturn(rc, rc);
13499 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13500 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13501 return VINF_SUCCESS;
13502 }
13503
13504 /*
13505 * Not a guest trap, must be a hypervisor related debug event then.
13506 * Update DR6 in case someone is interested in it.
13507 */
13508 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13509 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13510 CPUMSetHyperDR6(pVCpu, uDR6);
13511
13512 return rc;
13513}
13514
13515
13516/**
13517 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13518 * point exception).
13519 */
13520static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13521{
13522 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13523
13524 /* We require CR0 and EFER. EFER is always up-to-date. */
13525 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13526 AssertRCReturn(rc, rc);
13527
13528 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13529 VMMRZCallRing3Disable(pVCpu);
13530 HM_DISABLE_PREEMPT();
13531
13532 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13533 if (pVmxTransient->fWasGuestFPUStateActive)
13534 {
13535 rc = VINF_EM_RAW_GUEST_TRAP;
13536 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13537 }
13538 else
13539 {
13540#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13541 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13542#endif
13543 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13544 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13545 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13546 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13547 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13548 }
13549
13550 HM_RESTORE_PREEMPT();
13551 VMMRZCallRing3Enable(pVCpu);
13552
13553 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13554 {
13555 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13556 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13557 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13558 pVCpu->hm.s.fPreloadGuestFpu = true;
13559 }
13560 else
13561 {
13562 /* Forward #NM to the guest. */
13563 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13564 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13565 AssertRCReturn(rc, rc);
13566 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13567 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13568 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13569 }
13570
13571 return VINF_SUCCESS;
13572}
13573
13574
13575/**
13576 * VM-exit exception handler for \#GP (General-protection exception).
13577 *
13578 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13579 */
13580static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13581{
13582 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13584
13585 int rc;
13586 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13587 { /* likely */ }
13588 else
13589 {
13590#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13591 Assert(pVCpu->hm.s.fUsingDebugLoop);
13592#endif
13593 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13594 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13595 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13596 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13597 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13598 AssertRCReturn(rc, rc);
13599 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13600 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13601 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13602 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13603 return rc;
13604 }
13605
13606 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13607 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13608
13609 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13610 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13611 AssertRCReturn(rc, rc);
13612
13613 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13614 uint32_t cbOp = 0;
13615 PVM pVM = pVCpu->CTX_SUFF(pVM);
13616 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13617 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13618 if (RT_SUCCESS(rc))
13619 {
13620 rc = VINF_SUCCESS;
13621 Assert(cbOp == pDis->cbInstr);
13622 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13623 switch (pDis->pCurInstr->uOpcode)
13624 {
13625 case OP_CLI:
13626 {
13627 pMixedCtx->eflags.Bits.u1IF = 0;
13628 pMixedCtx->eflags.Bits.u1RF = 0;
13629 pMixedCtx->rip += pDis->cbInstr;
13630 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13631 if ( !fDbgStepping
13632 && pMixedCtx->eflags.Bits.u1TF)
13633 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13635 break;
13636 }
13637
13638 case OP_STI:
13639 {
13640 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13641 pMixedCtx->eflags.Bits.u1IF = 1;
13642 pMixedCtx->eflags.Bits.u1RF = 0;
13643 pMixedCtx->rip += pDis->cbInstr;
13644 if (!fOldIF)
13645 {
13646 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13647 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13648 }
13649 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13650 if ( !fDbgStepping
13651 && pMixedCtx->eflags.Bits.u1TF)
13652 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13653 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13654 break;
13655 }
13656
13657 case OP_HLT:
13658 {
13659 rc = VINF_EM_HALT;
13660 pMixedCtx->rip += pDis->cbInstr;
13661 pMixedCtx->eflags.Bits.u1RF = 0;
13662 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13664 break;
13665 }
13666
13667 case OP_POPF:
13668 {
13669 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13670 uint32_t cbParm;
13671 uint32_t uMask;
13672 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13673 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13674 {
13675 cbParm = 4;
13676 uMask = 0xffffffff;
13677 }
13678 else
13679 {
13680 cbParm = 2;
13681 uMask = 0xffff;
13682 }
13683
13684 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13685 RTGCPTR GCPtrStack = 0;
13686 X86EFLAGS Eflags;
13687 Eflags.u32 = 0;
13688 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13689 &GCPtrStack);
13690 if (RT_SUCCESS(rc))
13691 {
13692 Assert(sizeof(Eflags.u32) >= cbParm);
13693 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13694 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13695 }
13696 if (RT_FAILURE(rc))
13697 {
13698 rc = VERR_EM_INTERPRETER;
13699 break;
13700 }
13701 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13702 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13703 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13704 pMixedCtx->esp += cbParm;
13705 pMixedCtx->esp &= uMask;
13706 pMixedCtx->rip += pDis->cbInstr;
13707 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13708 | HM_CHANGED_GUEST_RSP
13709 | HM_CHANGED_GUEST_RFLAGS);
13710 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13711 POPF restores EFLAGS.TF. */
13712 if ( !fDbgStepping
13713 && fGstStepping)
13714 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13716 break;
13717 }
13718
13719 case OP_PUSHF:
13720 {
13721 uint32_t cbParm;
13722 uint32_t uMask;
13723 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13724 {
13725 cbParm = 4;
13726 uMask = 0xffffffff;
13727 }
13728 else
13729 {
13730 cbParm = 2;
13731 uMask = 0xffff;
13732 }
13733
13734 /* Get the stack pointer & push the contents of eflags onto the stack. */
13735 RTGCPTR GCPtrStack = 0;
13736 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13737 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13738 if (RT_FAILURE(rc))
13739 {
13740 rc = VERR_EM_INTERPRETER;
13741 break;
13742 }
13743 X86EFLAGS Eflags = pMixedCtx->eflags;
13744 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13745 Eflags.Bits.u1RF = 0;
13746 Eflags.Bits.u1VM = 0;
13747
13748 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13749 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13750 {
13751 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13752 rc = VERR_EM_INTERPRETER;
13753 break;
13754 }
13755 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13756 pMixedCtx->esp -= cbParm;
13757 pMixedCtx->esp &= uMask;
13758 pMixedCtx->rip += pDis->cbInstr;
13759 pMixedCtx->eflags.Bits.u1RF = 0;
13760 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13761 | HM_CHANGED_GUEST_RSP
13762 | HM_CHANGED_GUEST_RFLAGS);
13763 if ( !fDbgStepping
13764 && pMixedCtx->eflags.Bits.u1TF)
13765 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13766 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13767 break;
13768 }
13769
13770 case OP_IRET:
13771 {
13772 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13773 * instruction reference. */
13774 RTGCPTR GCPtrStack = 0;
13775 uint32_t uMask = 0xffff;
13776 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13777 uint16_t aIretFrame[3];
13778 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13779 {
13780 rc = VERR_EM_INTERPRETER;
13781 break;
13782 }
13783 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13784 &GCPtrStack);
13785 if (RT_SUCCESS(rc))
13786 {
13787 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13788 PGMACCESSORIGIN_HM));
13789 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13790 }
13791 if (RT_FAILURE(rc))
13792 {
13793 rc = VERR_EM_INTERPRETER;
13794 break;
13795 }
13796 pMixedCtx->eip = 0;
13797 pMixedCtx->ip = aIretFrame[0];
13798 pMixedCtx->cs.Sel = aIretFrame[1];
13799 pMixedCtx->cs.ValidSel = aIretFrame[1];
13800 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13801 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13802 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13803 pMixedCtx->sp += sizeof(aIretFrame);
13804 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13805 | HM_CHANGED_GUEST_SEGMENT_REGS
13806 | HM_CHANGED_GUEST_RSP
13807 | HM_CHANGED_GUEST_RFLAGS);
13808 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13809 if ( !fDbgStepping
13810 && fGstStepping)
13811 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13812 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13814 break;
13815 }
13816
13817 case OP_INT:
13818 {
13819 uint16_t uVector = pDis->Param1.uValue & 0xff;
13820 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13821 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13823 break;
13824 }
13825
13826 case OP_INTO:
13827 {
13828 if (pMixedCtx->eflags.Bits.u1OF)
13829 {
13830 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13831 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13833 }
13834 else
13835 {
13836 pMixedCtx->eflags.Bits.u1RF = 0;
13837 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13838 }
13839 break;
13840 }
13841
13842 default:
13843 {
13844 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13845 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13846 EMCODETYPE_SUPERVISOR);
13847 rc = VBOXSTRICTRC_VAL(rc2);
13848 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13849 /** @todo We have to set pending-debug exceptions here when the guest is
13850 * single-stepping depending on the instruction that was interpreted. */
13851 Log4(("#GP rc=%Rrc\n", rc));
13852 break;
13853 }
13854 }
13855 }
13856 else
13857 rc = VERR_EM_INTERPRETER;
13858
13859 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13860 ("#GP Unexpected rc=%Rrc\n", rc));
13861 return rc;
13862}
13863
13864
13865/**
13866 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13867 * the exception reported in the VMX transient structure back into the VM.
13868 *
13869 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13870 * up-to-date.
13871 */
13872static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13873{
13874 RT_NOREF_PV(pMixedCtx);
13875 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13876#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13877 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13878 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13879 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13880#endif
13881
13882 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13883 hmR0VmxCheckExitDueToEventDelivery(). */
13884 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13885 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13886 AssertRCReturn(rc, rc);
13887 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13888
13889#ifdef DEBUG_ramshankar
13890 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13891 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13892 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13893#endif
13894
13895 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13896 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13897 return VINF_SUCCESS;
13898}
13899
13900
13901/**
13902 * VM-exit exception handler for \#PF (Page-fault exception).
13903 */
13904static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13905{
13906 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13907 PVM pVM = pVCpu->CTX_SUFF(pVM);
13908 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13909 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13910 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13911 AssertRCReturn(rc, rc);
13912
13913 if (!pVM->hm.s.fNestedPaging)
13914 { /* likely */ }
13915 else
13916 {
13917#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13918 Assert(pVCpu->hm.s.fUsingDebugLoop);
13919#endif
13920 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13921 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13922 {
13923 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13924 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13925 }
13926 else
13927 {
13928 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13929 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13930 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13931 }
13932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13933 return rc;
13934 }
13935
13936 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13937 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13938 if (pVmxTransient->fVectoringPF)
13939 {
13940 Assert(pVCpu->hm.s.Event.fPending);
13941 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13942 }
13943
13944 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13945 AssertRCReturn(rc, rc);
13946
13947 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13948 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13949
13950 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13951 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13952 (RTGCPTR)pVmxTransient->uExitQualification);
13953
13954 Log4(("#PF: rc=%Rrc\n", rc));
13955 if (rc == VINF_SUCCESS)
13956 {
13957#if 0
13958 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13959 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13960 * memory? We don't update the whole state here... */
13961 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13962 | HM_CHANGED_GUEST_RSP
13963 | HM_CHANGED_GUEST_RFLAGS
13964 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13965#else
13966 /*
13967 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13968 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13969 */
13970 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13971 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13972#endif
13973 TRPMResetTrap(pVCpu);
13974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13975 return rc;
13976 }
13977
13978 if (rc == VINF_EM_RAW_GUEST_TRAP)
13979 {
13980 if (!pVmxTransient->fVectoringDoublePF)
13981 {
13982 /* It's a guest page fault and needs to be reflected to the guest. */
13983 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13984 TRPMResetTrap(pVCpu);
13985 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13986 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13987 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13988 }
13989 else
13990 {
13991 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13992 TRPMResetTrap(pVCpu);
13993 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13994 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13995 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13996 }
13997
13998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13999 return VINF_SUCCESS;
14000 }
14001
14002 TRPMResetTrap(pVCpu);
14003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
14004 return rc;
14005}
14006
14007/** @} */
14008
注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette