HMVMXR0.cpp@ 66701

最後變更在這個檔案從66701是 66701,由 vboxsync 提交於 8 年前
VMM/HMVMXR0: Use IEM's event reflection logic.
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 596.5 KB

行
1	/* $Id: HMVMXR0.cpp 66701 2017-04-27 16:49:12Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2016 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#ifdef DEBUG_ramshankar
44	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49	# define HMVMX_ALWAYS_TRAP_PF
50	# define HMVMX_ALWAYS_SWAP_FPU_STATE
51	# define HMVMX_ALWAYS_FLUSH_TLB
52	# define HMVMX_ALWAYS_SWAP_EFER
53	#endif
54
55
56	/*********************************************************************************************************************************
57	* Defined Constants And Macros *
58	*********************************************************************************************************************************/
59	/** Use the function table. */
60	#define HMVMX_USE_FUNCTION_TABLE
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** @name Updated-guest-state flags.
69	* @{ */
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DR7 \
102	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
107	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108	\| HMVMX_UPDATED_GUEST_INTR_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110	/** @} */
111
112	/** @name
113	* Flags to skip redundant reads of some common VMCS fields that are not part of
114	* the guest-CPU state but are in the transient structure.
115	*/
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123	/** @} */
124
125	/** @name
126	* States of the VMCS.
127	*
128	* This does not reflect all possible VMCS states but currently only those
129	* needed for maintaining the VMCS consistently even when thread-context hooks
130	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
131	*/
132	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135	/** @} */
136
137	/**
138	* Exception bitmap mask for real-mode guests (real-on-v86).
139	*
140	* We need to intercept all exceptions manually except:
141	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143	* due to bugs in Intel CPUs.
144	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
145	* support.
146	*/
147	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
148	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
149	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
150	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
151	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
153	\| RT_BIT(X86_XCPT_XF))
154
155	/**
156	* Exception bitmap mask for all contributory exceptions.
157	*
158	* Page fault is deliberately excluded here as it's conditional as to whether
159	* it's contributory or benign. Page faults are handled separately.
160	*/
161	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
162	\| RT_BIT(X86_XCPT_DE))
163
164	/** Maximum VM-instruction error number. */
165	#define HMVMX_INSTR_ERROR_MAX 28
166
167	/** Profiling macro. */
168	#ifdef HM_PROFILE_EXIT_DISPATCH
169	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171	#else
172	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174	#endif
175
176	/** Assert that preemption is disabled or covered by thread-context hooks. */
177	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180	/** Assert that we haven't migrated CPUs when thread-context hooks are not
181	* used. */
182	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184	("Illegal migration! Entered on CPU %u Current %u\n", \
185	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187	/** Helper macro for VM-exit handlers called unexpectedly. */
188	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189	do { \
190	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191	return VERR_VMX_UNEXPECTED_EXIT; \
192	} while (0)
193
194
195	/*********************************************************************************************************************************
196	* Structures and Typedefs *
197	*********************************************************************************************************************************/
198	/**
199	* VMX transient state.
200	*
201	* A state structure for holding miscellaneous information across
202	* VMX non-root operation and restored after the transition.
203	*/
204	typedef struct VMXTRANSIENT
205	{
206	/** The host's rflags/eflags. */
207	RTCCUINTREG fEFlags;
208	#if HC_ARCH_BITS == 32
209	uint32_t u32Alignment0;
210	#endif
211	/** The guest's TPR value used for TPR shadowing. */
212	uint8_t u8GuestTpr;
213	/** Alignment. */
214	uint8_t abAlignment0[7];
215
216	/** The basic VM-exit reason. */
217	uint16_t uExitReason;
218	/** Alignment. */
219	uint16_t u16Alignment0;
220	/** The VM-exit interruption error code. */
221	uint32_t uExitIntErrorCode;
222	/** The VM-exit exit code qualification. */
223	uint64_t uExitQualification;
224
225	/** The VM-exit interruption-information field. */
226	uint32_t uExitIntInfo;
227	/** The VM-exit instruction-length field. */
228	uint32_t cbInstr;
229	/** The VM-exit instruction-information field. */
230	union
231	{
232	/** Plain unsigned int representation. */
233	uint32_t u;
234	/** INS and OUTS information. */
235	struct
236	{
237	uint32_t u7Reserved0 : 7;
238	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239	uint32_t u3AddrSize : 3;
240	uint32_t u5Reserved1 : 5;
241	/** The segment register (X86_SREG_XXX). */
242	uint32_t iSegReg : 3;
243	uint32_t uReserved2 : 14;
244	} StrIo;
245	} ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Alignment. */
249	uint8_t abAlignment1[3];
250
251	/** The VM-entry interruption-information field. */
252	uint32_t uEntryIntInfo;
253	/** The VM-entry exception error code field. */
254	uint32_t uEntryXcptErrorCode;
255	/** The VM-entry instruction length field. */
256	uint32_t cbEntryInstr;
257
258	/** IDT-vectoring information field. */
259	uint32_t uIdtVectoringInfo;
260	/** IDT-vectoring error code. */
261	uint32_t uIdtVectoringErrorCode;
262
263	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
264	uint32_t fVmcsFieldsRead;
265
266	/** Whether the guest FPU was active at the time of VM-exit. */
267	bool fWasGuestFPUStateActive;
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting should be setup before VM-entry. */
273	bool fUpdateTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	} VMXTRANSIENT;
281	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286	/** Pointer to VMX transient state. */
287	typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290	/**
291	* MSR-bitmap read permissions.
292	*/
293	typedef enum VMXMSREXITREAD
294	{
295	/** Reading this MSR causes a VM-exit. */
296	VMXMSREXIT_INTERCEPT_READ = 0xb,
297	/** Reading this MSR does not cause a VM-exit. */
298	VMXMSREXIT_PASSTHRU_READ
299	} VMXMSREXITREAD;
300	/** Pointer to MSR-bitmap read permissions. */
301	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303	/**
304	* MSR-bitmap write permissions.
305	*/
306	typedef enum VMXMSREXITWRITE
307	{
308	/** Writing to this MSR causes a VM-exit. */
309	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310	/** Writing to this MSR does not cause a VM-exit. */
311	VMXMSREXIT_PASSTHRU_WRITE
312	} VMXMSREXITWRITE;
313	/** Pointer to MSR-bitmap write permissions. */
314	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317	/**
318	* VMX VM-exit handler.
319	*
320	* @returns Strict VBox status code (i.e. informational status codes too).
321	* @param pVCpu The cross context virtual CPU structure.
322	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
323	* out-of-sync. Make sure to update the required
324	* fields before using them.
325	* @param pVmxTransient Pointer to the VMX-transient structure.
326	*/
327	#ifndef HMVMX_USE_FUNCTION_TABLE
328	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	#else
330	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333	#endif
334
335	/**
336	* VMX VM-exit handler, non-strict status code.
337	*
338	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339	*
340	* @returns VBox status code, no informational status code returned.
341	* @param pVCpu The cross context virtual CPU structure.
342	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
343	* out-of-sync. Make sure to update the required
344	* fields before using them.
345	* @param pVmxTransient Pointer to the VMX-transient structure.
346	*
347	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348	* use of that status code will be replaced with VINF_EM_SOMETHING
349	* later when switching over to IEM.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355	#endif
356
357
358	/*********************************************************************************************************************************
359	* Internal Functions *
360	*********************************************************************************************************************************/
361	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366	bool fStepping, uint32_t *puIntState);
367	#if HC_ARCH_BITS == 32
368	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369	#endif
370	#ifndef HMVMX_USE_FUNCTION_TABLE
371	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374	#else
375	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377	#endif
378
379
380	/** @name VM-exit handlers.
381	* @{
382	*/
383	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395	static FNVMXEXITHANDLER hmR0VmxExitHlt;
396	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411	static FNVMXEXITHANDLER hmR0VmxExitMwait;
412	static FNVMXEXITHANDLER hmR0VmxExitMtf;
413	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414	static FNVMXEXITHANDLER hmR0VmxExitPause;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428	/** @} */
429
430	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441	/*********************************************************************************************************************************
442	* Global Variables *
443	*********************************************************************************************************************************/
444	#ifdef HMVMX_USE_FUNCTION_TABLE
445
446	/**
447	* VMX_EXIT dispatch table.
448	*/
449	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450	{
451	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491	/* 40 UNDEFINED */ hmR0VmxExitPause,
492	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
513	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516	};
517	#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519	#ifdef VBOX_STRICT
520	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521	{
522	/* 0 */ "(Not Used)",
523	/* 1 */ "VMCALL executed in VMX root operation.",
524	/* 2 */ "VMCLEAR with invalid physical address.",
525	/* 3 */ "VMCLEAR with VMXON pointer.",
526	/* 4 */ "VMLAUNCH with non-clear VMCS.",
527	/* 5 */ "VMRESUME with non-launched VMCS.",
528	/* 6 */ "VMRESUME after VMXOFF",
529	/* 7 */ "VM-entry with invalid control fields.",
530	/* 8 */ "VM-entry with invalid host state fields.",
531	/* 9 */ "VMPTRLD with invalid physical address.",
532	/* 10 */ "VMPTRLD with VMXON pointer.",
533	/* 11 */ "VMPTRLD with incorrect revision identifier.",
534	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535	/* 13 */ "VMWRITE to read-only VMCS component.",
536	/* 14 */ "(Not Used)",
537	/* 15 */ "VMXON executed in VMX root operation.",
538	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539	/* 17 */ "VM-entry with non-launched executing VMCS.",
540	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541	/* 19 */ "VMCALL with non-clear VMCS.",
542	/* 20 */ "VMCALL with invalid VM-exit control fields.",
543	/* 21 */ "(Not Used)",
544	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546	/* 24 */ "VMCALL with invalid SMM-monitor features.",
547	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548	/* 26 */ "VM-entry with events blocked by MOV SS.",
549	/* 27 */ "(Not Used)",
550	/* 28 */ "Invalid operand to INVEPT/INVVPID."
551	};
552	#endif /* VBOX_STRICT */
553
554
555
556	/**
557	* Updates the VM's last error record.
558	*
559	* If there was a VMX instruction error, reads the error data from the VMCS and
560	* updates VCPU's last error record as well.
561	*
562	* @param pVM The cross context VM structure.
563	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
564	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565	* VERR_VMX_INVALID_VMCS_FIELD.
566	* @param rc The error code.
567	*/
568	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569	{
570	AssertPtr(pVM);
571	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
573	{
574	AssertPtrReturnVoid(pVCpu);
575	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576	}
577	pVM->hm.s.lLastError = rc;
578	}
579
580
581	/**
582	* Reads the VM-entry interruption-information field from the VMCS into the VMX
583	* transient structure.
584	*
585	* @returns VBox status code.
586	* @param pVmxTransient Pointer to the VMX transient structure.
587	*
588	* @remarks No-long-jump zone!!!
589	*/
590	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591	{
592	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593	AssertRCReturn(rc, rc);
594	return VINF_SUCCESS;
595	}
596
597
598	#ifdef VBOX_STRICT
599	/**
600	* Reads the VM-entry exception error code field from the VMCS into
601	* the VMX transient structure.
602	*
603	* @returns VBox status code.
604	* @param pVmxTransient Pointer to the VMX transient structure.
605	*
606	* @remarks No-long-jump zone!!!
607	*/
608	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609	{
610	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611	AssertRCReturn(rc, rc);
612	return VINF_SUCCESS;
613	}
614	#endif /* VBOX_STRICT */
615
616
617	#ifdef VBOX_STRICT
618	/**
619	* Reads the VM-entry exception error code field from the VMCS into
620	* the VMX transient structure.
621	*
622	* @returns VBox status code.
623	* @param pVmxTransient Pointer to the VMX transient structure.
624	*
625	* @remarks No-long-jump zone!!!
626	*/
627	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628	{
629	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630	AssertRCReturn(rc, rc);
631	return VINF_SUCCESS;
632	}
633	#endif /* VBOX_STRICT */
634
635
636	/**
637	* Reads the VM-exit interruption-information field from the VMCS into the VMX
638	* transient structure.
639	*
640	* @returns VBox status code.
641	* @param pVmxTransient Pointer to the VMX transient structure.
642	*/
643	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644	{
645	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646	{
647	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648	AssertRCReturn(rc, rc);
649	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650	}
651	return VINF_SUCCESS;
652	}
653
654
655	/**
656	* Reads the VM-exit interruption error code from the VMCS into the VMX
657	* transient structure.
658	*
659	* @returns VBox status code.
660	* @param pVmxTransient Pointer to the VMX transient structure.
661	*/
662	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663	{
664	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665	{
666	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667	AssertRCReturn(rc, rc);
668	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669	}
670	return VINF_SUCCESS;
671	}
672
673
674	/**
675	* Reads the VM-exit instruction length field from the VMCS into the VMX
676	* transient structure.
677	*
678	* @returns VBox status code.
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684	{
685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the VM-exit instruction-information field from the VMCS into
695	* the VMX transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*/
700	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701	{
702	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703	{
704	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705	AssertRCReturn(rc, rc);
706	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707	}
708	return VINF_SUCCESS;
709	}
710
711
712	/**
713	* Reads the exit code qualification from the VMCS into the VMX transient
714	* structure.
715	*
716	* @returns VBox status code.
717	* @param pVCpu The cross context virtual CPU structure of the
718	* calling EMT. (Required for the VMCS cache case.)
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724	{
725	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Reads the IDT-vectoring information field from the VMCS into the VMX
735	* transient structure.
736	*
737	* @returns VBox status code.
738	* @param pVmxTransient Pointer to the VMX transient structure.
739	*
740	* @remarks No-long-jump zone!!!
741	*/
742	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743	{
744	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745	{
746	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747	AssertRCReturn(rc, rc);
748	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749	}
750	return VINF_SUCCESS;
751	}
752
753
754	/**
755	* Reads the IDT-vectoring error code from the VMCS into the VMX
756	* transient structure.
757	*
758	* @returns VBox status code.
759	* @param pVmxTransient Pointer to the VMX transient structure.
760	*/
761	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762	{
763	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764	{
765	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766	AssertRCReturn(rc, rc);
767	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768	}
769	return VINF_SUCCESS;
770	}
771
772
773	/**
774	* Enters VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	* @param pVM The cross context VM structure. Can be
778	* NULL, after a resume.
779	* @param HCPhysCpuPage Physical address of the VMXON region.
780	* @param pvCpuPage Pointer to the VMXON region.
781	*/
782	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783	{
784	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786	Assert(pvCpuPage);
787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789	if (pVM)
790	{
791	/* Write the VMCS revision dword to the VMXON region. */
792	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793	}
794
795	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796	RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798	/* Enable the VMX bit in CR4 if necessary. */
799	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801	/* Enter VMX root mode. */
802	int rc = VMXEnable(HCPhysCpuPage);
803	if (RT_FAILURE(rc))
804	{
805	if (!(uOldCr4 & X86_CR4_VMXE))
806	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808	if (pVM)
809	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810	}
811
812	/* Restore interrupts. */
813	ASMSetFlags(fEFlags);
814	return rc;
815	}
816
817
818	/**
819	* Exits VMX root mode operation on the current CPU.
820	*
821	* @returns VBox status code.
822	*/
823	static int hmR0VmxLeaveRootMode(void)
824	{
825	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828	RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830	/* If we're for some reason not in VMX root mode, then don't leave it. */
831	RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833	int rc;
834	if (uHostCR4 & X86_CR4_VMXE)
835	{
836	/* Exit VMX root mode and clear the VMX bit in CR4. */
837	VMXDisable();
838	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839	rc = VINF_SUCCESS;
840	}
841	else
842	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844	/* Restore interrupts. */
845	ASMSetFlags(fEFlags);
846	return rc;
847	}
848
849
850	/**
851	* Allocates and maps one physically contiguous page. The allocated page is
852	* zero'd out. (Used by various VT-x structures).
853	*
854	* @returns IPRT status code.
855	* @param pMemObj Pointer to the ring-0 memory object.
856	* @param ppVirt Where to store the virtual address of the
857	* allocation.
858	* @param pHCPhys Where to store the physical address of the
859	* allocation.
860	*/
861	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862	{
863	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868	if (RT_FAILURE(rc))
869	return rc;
870	ppVirt = RTR0MemObjAddress(pMemObj);
871	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
872	ASMMemZero32(*ppVirt, PAGE_SIZE);
873	return VINF_SUCCESS;
874	}
875
876
877	/**
878	* Frees and unmaps an allocated physical page.
879	*
880	* @param pMemObj Pointer to the ring-0 memory object.
881	* @param ppVirt Where to re-initialize the virtual address of
882	* allocation as 0.
883	* @param pHCPhys Where to re-initialize the physical address of the
884	* allocation as 0.
885	*/
886	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887	{
888	AssertPtr(pMemObj);
889	AssertPtr(ppVirt);
890	AssertPtr(pHCPhys);
891	if (*pMemObj != NIL_RTR0MEMOBJ)
892	{
893	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
894	AssertRC(rc);
895	*pMemObj = NIL_RTR0MEMOBJ;
896	*ppVirt = 0;
897	*pHCPhys = 0;
898	}
899	}
900
901
902	/**
903	* Worker function to free VT-x related structures.
904	*
905	* @returns IPRT status code.
906	* @param pVM The cross context VM structure.
907	*/
908	static void hmR0VmxStructsFree(PVM pVM)
909	{
910	for (VMCPUID i = 0; i < pVM->cCpus; i++)
911	{
912	PVMCPU pVCpu = &pVM->aCpus[i];
913	AssertPtr(pVCpu);
914
915	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922	}
923
924	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927	#endif
928	}
929
930
931	/**
932	* Worker function to allocate VT-x related VM structures.
933	*
934	* @returns IPRT status code.
935	* @param pVM The cross context VM structure.
936	*/
937	static int hmR0VmxStructsAlloc(PVM pVM)
938	{
939	/*
940	* Initialize members up-front so we can cleanup properly on allocation failure.
941	*/
942	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945	pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954	#endif
955	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958	for (VMCPUID i = 0; i < pVM->cCpus; i++)
959	{
960	PVMCPU pVCpu = &pVM->aCpus[i];
961	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965	}
966	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967	#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974	/*
975	* Allocate all the VT-x structures.
976	*/
977	int rc = VINF_SUCCESS;
978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980	if (RT_FAILURE(rc))
981	goto cleanup;
982	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984	#endif
985
986	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988	{
989	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990	&pVM->hm.s.vmx.HCPhysApicAccess);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	/*
996	* Initialize per-VCPU VT-x structures.
997	*/
998	for (VMCPUID i = 0; i < pVM->cCpus; i++)
999	{
1000	PVMCPU pVCpu = &pVM->aCpus[i];
1001	AssertPtr(pVCpu);
1002
1003	/* Allocate the VM control structure (VMCS). */
1004	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005	if (RT_FAILURE(rc))
1006	goto cleanup;
1007
1008	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010	{
1011	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012	NULL /* pR3Ptr /, NULL / pRCPtr */);
1013	if (RT_FAILURE(rc))
1014	goto cleanup;
1015	}
1016
1017	/*
1018	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019	* transparent accesses of specific MSRs.
1020	*
1021	* If the condition for enabling MSR bitmaps changes here, don't forget to
1022	* update HMAreMsrBitmapsAvailable().
1023	*/
1024	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025	{
1026	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028	if (RT_FAILURE(rc))
1029	goto cleanup;
1030	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031	}
1032
1033	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035	if (RT_FAILURE(rc))
1036	goto cleanup;
1037
1038	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1039	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040	if (RT_FAILURE(rc))
1041	goto cleanup;
1042	}
1043
1044	return VINF_SUCCESS;
1045
1046	cleanup:
1047	hmR0VmxStructsFree(pVM);
1048	return rc;
1049	}
1050
1051
1052	/**
1053	* Does global VT-x initialization (called during module initialization).
1054	*
1055	* @returns VBox status code.
1056	*/
1057	VMMR0DECL(int) VMXR0GlobalInit(void)
1058	{
1059	#ifdef HMVMX_USE_FUNCTION_TABLE
1060	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061	# ifdef VBOX_STRICT
1062	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063	Assert(g_apfnVMExitHandlers[i]);
1064	# endif
1065	#endif
1066	return VINF_SUCCESS;
1067	}
1068
1069
1070	/**
1071	* Does global VT-x termination (called during module termination).
1072	*/
1073	VMMR0DECL(void) VMXR0GlobalTerm()
1074	{
1075	/* Nothing to do currently. */
1076	}
1077
1078
1079	/**
1080	* Sets up and activates VT-x on the current CPU.
1081	*
1082	* @returns VBox status code.
1083	* @param pCpu Pointer to the global CPU info struct.
1084	* @param pVM The cross context VM structure. Can be
1085	* NULL after a host resume operation.
1086	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087	* fEnabledByHost is @c true).
1088	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089	* @a fEnabledByHost is @c true).
1090	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091	* enable VT-x on the host.
1092	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1093	*/
1094	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095	void *pvMsrs)
1096	{
1097	Assert(pCpu);
1098	Assert(pvMsrs);
1099	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101	/* Enable VT-x if it's not already enabled by the host. */
1102	if (!fEnabledByHost)
1103	{
1104	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105	if (RT_FAILURE(rc))
1106	return rc;
1107	}
1108
1109	/*
1110	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112	*/
1113	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115	{
1116	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117	pCpu->fFlushAsidBeforeUse = false;
1118	}
1119	else
1120	pCpu->fFlushAsidBeforeUse = true;
1121
1122	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123	++pCpu->cTlbFlushes;
1124
1125	return VINF_SUCCESS;
1126	}
1127
1128
1129	/**
1130	* Deactivates VT-x on the current CPU.
1131	*
1132	* @returns VBox status code.
1133	* @param pCpu Pointer to the global CPU info struct.
1134	* @param pvCpuPage Pointer to the VMXON region.
1135	* @param HCPhysCpuPage Physical address of the VMXON region.
1136	*
1137	* @remarks This function should never be called when SUPR0EnableVTx() or
1138	* similar was used to enable VT-x on the host.
1139	*/
1140	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141	{
1142	NOREF(pCpu);
1143	NOREF(pvCpuPage);
1144	NOREF(HCPhysCpuPage);
1145
1146	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147	return hmR0VmxLeaveRootMode();
1148	}
1149
1150
1151	/**
1152	* Sets the permission bits for the specified MSR in the MSR bitmap.
1153	*
1154	* @param pVCpu The cross context virtual CPU structure.
1155	* @param uMsr The MSR value.
1156	* @param enmRead Whether reading this MSR causes a VM-exit.
1157	* @param enmWrite Whether writing this MSR causes a VM-exit.
1158	*/
1159	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160	{
1161	int32_t iBit;
1162	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164	/*
1165	* Layout:
1166	* 0x000 - 0x3ff - Low MSR read bits
1167	* 0x400 - 0x7ff - High MSR read bits
1168	* 0x800 - 0xbff - Low MSR write bits
1169	* 0xc00 - 0xfff - High MSR write bits
1170	*/
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174	{
1175	iBit = uMsr - UINT32_C(0xC0000000);
1176	pbMsrBitmap += 0x400;
1177	}
1178	else
1179	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181	Assert(iBit <= 0x1fff);
1182	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183	ASMBitSet(pbMsrBitmap, iBit);
1184	else
1185	ASMBitClear(pbMsrBitmap, iBit);
1186
1187	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189	else
1190	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191	}
1192
1193
1194	#ifdef VBOX_STRICT
1195	/**
1196	* Gets the permission bits for the specified MSR in the MSR bitmap.
1197	*
1198	* @returns VBox status code.
1199	* @retval VINF_SUCCESS if the specified MSR is found.
1200	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1201	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202	*
1203	* @param pVCpu The cross context virtual CPU structure.
1204	* @param uMsr The MSR.
1205	* @param penmRead Where to store the read permissions.
1206	* @param penmWrite Where to store the write permissions.
1207	*/
1208	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209	{
1210	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212	int32_t iBit;
1213	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215	/* See hmR0VmxSetMsrPermission() for the layout. */
1216	if (uMsr <= 0x00001FFF)
1217	iBit = uMsr;
1218	else if ( uMsr >= 0xC0000000
1219	&& uMsr <= 0xC0001FFF)
1220	{
1221	iBit = (uMsr - 0xC0000000);
1222	pbMsrBitmap += 0x400;
1223	}
1224	else
1225	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227	Assert(iBit <= 0x1fff);
1228	if (ASMBitTest(pbMsrBitmap, iBit))
1229	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1230	else
1231	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235	else
1236	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237	return VINF_SUCCESS;
1238	}
1239	#endif /* VBOX_STRICT */
1240
1241
1242	/**
1243	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244	* area.
1245	*
1246	* @returns VBox status code.
1247	* @param pVCpu The cross context virtual CPU structure.
1248	* @param cMsrs The number of MSRs.
1249	*/
1250	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251	{
1252	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255	{
1256	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259	}
1260
1261	/* Update number of guest MSRs to load/store across the world-switch. */
1262	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267	AssertRCReturn(rc, rc);
1268
1269	/* Update the VCPU's copy of the MSR count. */
1270	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Adds a new (or updates the value of an existing) guest/host MSR
1278	* pair to be swapped during the world-switch as part of the
1279	* auto-load/store MSR area in the VMCS.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param uMsr The MSR.
1284	* @param uGuestMsrValue Value of the guest MSR.
1285	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1286	* necessary.
1287	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288	* its value was updated. Optional, can be NULL.
1289	*/
1290	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291	bool *pfAddedAndUpdated)
1292	{
1293	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295	uint32_t i;
1296	for (i = 0; i < cMsrs; i++)
1297	{
1298	if (pGuestMsr->u32Msr == uMsr)
1299	break;
1300	pGuestMsr++;
1301	}
1302
1303	bool fAdded = false;
1304	if (i == cMsrs)
1305	{
1306	++cMsrs;
1307	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314	fAdded = true;
1315	}
1316
1317	/* Update the MSR values in the auto-load/store MSR area. */
1318	pGuestMsr->u32Msr = uMsr;
1319	pGuestMsr->u64Value = uGuestMsrValue;
1320
1321	/* Create/update the MSR slot in the host MSR area. */
1322	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323	pHostMsr += i;
1324	pHostMsr->u32Msr = uMsr;
1325
1326	/*
1327	* Update the host MSR only when requested by the caller AND when we're
1328	* adding it to the auto-load/store area. Otherwise, it would have been
1329	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330	*/
1331	bool fUpdatedMsrValue = false;
1332	if ( fAdded
1333	&& fUpdateHostMsr)
1334	{
1335	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338	fUpdatedMsrValue = true;
1339	}
1340
1341	if (pfAddedAndUpdated)
1342	*pfAddedAndUpdated = fUpdatedMsrValue;
1343	return VINF_SUCCESS;
1344	}
1345
1346
1347	/**
1348	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1349	* auto-load/store MSR area in the VMCS.
1350	*
1351	* @returns VBox status code.
1352	* @param pVCpu The cross context virtual CPU structure.
1353	* @param uMsr The MSR.
1354	*/
1355	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356	{
1357	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359	for (uint32_t i = 0; i < cMsrs; i++)
1360	{
1361	/* Find the MSR. */
1362	if (pGuestMsr->u32Msr == uMsr)
1363	{
1364	/* If it's the last MSR, simply reduce the count. */
1365	if (i == cMsrs - 1)
1366	{
1367	--cMsrs;
1368	break;
1369	}
1370
1371	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373	pLastGuestMsr += cMsrs - 1;
1374	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379	pLastHostMsr += cMsrs - 1;
1380	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381	pHostMsr->u64Value = pLastHostMsr->u64Value;
1382	--cMsrs;
1383	break;
1384	}
1385	pGuestMsr++;
1386	}
1387
1388	/* Update the VMCS if the count changed (meaning the MSR was found). */
1389	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390	{
1391	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392	AssertRCReturn(rc, rc);
1393
1394	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399	return VINF_SUCCESS;
1400	}
1401
1402	return VERR_NOT_FOUND;
1403	}
1404
1405
1406	/**
1407	* Checks if the specified guest MSR is part of the auto-load/store area in
1408	* the VMCS.
1409	*
1410	* @returns true if found, false otherwise.
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param uMsr The MSR to find.
1413	*/
1414	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415	{
1416	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420	{
1421	if (pGuestMsr->u32Msr == uMsr)
1422	return true;
1423	}
1424	return false;
1425	}
1426
1427
1428	/**
1429	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430	*
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks No-long-jump zone!!!
1434	*/
1435	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436	{
1437	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443	{
1444	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446	/*
1447	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449	*/
1450	if (pHostMsr->u32Msr == MSR_K6_EFER)
1451	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452	else
1453	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454	}
1455
1456	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457	}
1458
1459
1460	/**
1461	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1462	* perform lazy restoration of the host MSRs while leaving VT-x.
1463	*
1464	* @param pVCpu The cross context virtual CPU structure.
1465	*
1466	* @remarks No-long-jump zone!!!
1467	*/
1468	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469	{
1470	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472	/*
1473	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474	*/
1475	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476	{
1477	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478	#if HC_ARCH_BITS == 64
1479	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480	{
1481	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485	}
1486	#endif
1487	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1488	}
1489	}
1490
1491
1492	/**
1493	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1494	* lazily while leaving VT-x.
1495	*
1496	* @returns true if it does, false otherwise.
1497	* @param pVCpu The cross context virtual CPU structure.
1498	* @param uMsr The MSR to check.
1499	*/
1500	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501	{
1502	NOREF(pVCpu);
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	switch (uMsr)
1507	{
1508	case MSR_K8_LSTAR:
1509	case MSR_K6_STAR:
1510	case MSR_K8_SF_MASK:
1511	case MSR_K8_KERNEL_GS_BASE:
1512	return true;
1513	}
1514	}
1515	#else
1516	RT_NOREF(pVCpu, uMsr);
1517	#endif
1518	return false;
1519	}
1520
1521
1522	/**
1523	* Saves a set of guest MSRs back into the guest-CPU context.
1524	*
1525	* @param pVCpu The cross context virtual CPU structure.
1526	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527	* out-of-sync. Make sure to update the required fields
1528	* before using them.
1529	*
1530	* @remarks No-long-jump zone!!!
1531	*/
1532	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533	{
1534	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538	{
1539	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540	#if HC_ARCH_BITS == 64
1541	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542	{
1543	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547	}
1548	#else
1549	NOREF(pMixedCtx);
1550	#endif
1551	}
1552	}
1553
1554
1555	/**
1556	* Loads a set of guests MSRs to allow read/passthru to the guest.
1557	*
1558	* The name of this function is slightly confusing. This function does NOT
1559	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560	* common prefix for functions dealing with "lazy restoration" of the shared
1561	* MSRs.
1562	*
1563	* @param pVCpu The cross context virtual CPU structure.
1564	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565	* out-of-sync. Make sure to update the required fields
1566	* before using them.
1567	*
1568	* @remarks No-long-jump zone!!!
1569	*/
1570	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571	{
1572	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1576	#if HC_ARCH_BITS == 64
1577	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1578	{
1579	/*
1580	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1581	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1582	* we can skip a few MSR writes.
1583	*
1584	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1585	* guest MSR values in the guest-CPU context might be different to what's currently
1586	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1587	* CPU, see @bugref{8728}.
1588	*/
1589	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1590	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1591	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1592	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1593	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1594	{
1595	#ifdef VBOX_STRICT
1596	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1597	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1598	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1599	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1600	#endif
1601	}
1602	else
1603	{
1604	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1605	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1606	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1607	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1608	}
1609	}
1610	#else
1611	RT_NOREF(pMixedCtx);
1612	#endif
1613	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1614	}
1615
1616
1617	/**
1618	* Performs lazy restoration of the set of host MSRs if they were previously
1619	* loaded with guest MSR values.
1620	*
1621	* @param pVCpu The cross context virtual CPU structure.
1622	*
1623	* @remarks No-long-jump zone!!!
1624	* @remarks The guest MSRs should have been saved back into the guest-CPU
1625	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1626	*/
1627	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1628	{
1629	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1630	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1631
1632	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1633	{
1634	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1635	#if HC_ARCH_BITS == 64
1636	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1637	{
1638	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1639	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1640	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1641	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1642	}
1643	#endif
1644	}
1645	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1646	}
1647
1648
1649	/**
1650	* Verifies that our cached values of the VMCS controls are all
1651	* consistent with what's actually present in the VMCS.
1652	*
1653	* @returns VBox status code.
1654	* @param pVCpu The cross context virtual CPU structure.
1655	*/
1656	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1657	{
1658	uint32_t u32Val;
1659	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1660	AssertRCReturn(rc, rc);
1661	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1662	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1663
1664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1665	AssertRCReturn(rc, rc);
1666	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1667	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1668
1669	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1670	AssertRCReturn(rc, rc);
1671	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1672	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1673
1674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1675	AssertRCReturn(rc, rc);
1676	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1677	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1678
1679	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1680	{
1681	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1682	AssertRCReturn(rc, rc);
1683	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1684	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1685	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1686	}
1687
1688	return VINF_SUCCESS;
1689	}
1690
1691
1692	#ifdef VBOX_STRICT
1693	/**
1694	* Verifies that our cached host EFER value has not changed
1695	* since we cached it.
1696	*
1697	* @param pVCpu The cross context virtual CPU structure.
1698	*/
1699	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1700	{
1701	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1702
1703	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1704	{
1705	uint64_t u64Val;
1706	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1707	AssertRC(rc);
1708
1709	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1710	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1711	}
1712	}
1713
1714
1715	/**
1716	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1717	* VMCS are correct.
1718	*
1719	* @param pVCpu The cross context virtual CPU structure.
1720	*/
1721	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1722	{
1723	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1724
1725	/* Verify MSR counts in the VMCS are what we think it should be. */
1726	uint32_t cMsrs;
1727	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1728	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1729
1730	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1731	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1732
1733	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1734	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1735
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1737	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1738	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1739	{
1740	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1741	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1742	pGuestMsr->u32Msr, cMsrs));
1743
1744	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1745	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1746	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1747
1748	/* Verify that the permissions are as expected in the MSR bitmap. */
1749	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1750	{
1751	VMXMSREXITREAD enmRead;
1752	VMXMSREXITWRITE enmWrite;
1753	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1754	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1755	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1756	{
1757	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1758	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1759	}
1760	else
1761	{
1762	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1763	pGuestMsr->u32Msr, cMsrs));
1764	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1765	pGuestMsr->u32Msr, cMsrs));
1766	}
1767	}
1768	}
1769	}
1770	#endif /* VBOX_STRICT */
1771
1772
1773	/**
1774	* Flushes the TLB using EPT.
1775	*
1776	* @returns VBox status code.
1777	* @param pVCpu The cross context virtual CPU structure of the calling
1778	* EMT. Can be NULL depending on @a enmFlush.
1779	* @param enmFlush Type of flush.
1780	*
1781	* @remarks Caller is responsible for making sure this function is called only
1782	* when NestedPaging is supported and providing @a enmFlush that is
1783	* supported by the CPU.
1784	* @remarks Can be called with interrupts disabled.
1785	*/
1786	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1787	{
1788	uint64_t au64Descriptor[2];
1789	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1790	au64Descriptor[0] = 0;
1791	else
1792	{
1793	Assert(pVCpu);
1794	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1795	}
1796	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1797
1798	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1799	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1800	rc));
1801	if ( RT_SUCCESS(rc)
1802	&& pVCpu)
1803	{
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1805	}
1806	}
1807
1808
1809	/**
1810	* Flushes the TLB using VPID.
1811	*
1812	* @returns VBox status code.
1813	* @param pVM The cross context VM structure.
1814	* @param pVCpu The cross context virtual CPU structure of the calling
1815	* EMT. Can be NULL depending on @a enmFlush.
1816	* @param enmFlush Type of flush.
1817	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1818	* on @a enmFlush).
1819	*
1820	* @remarks Can be called with interrupts disabled.
1821	*/
1822	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1823	{
1824	NOREF(pVM);
1825	AssertPtr(pVM);
1826	Assert(pVM->hm.s.vmx.fVpid);
1827
1828	uint64_t au64Descriptor[2];
1829	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1830	{
1831	au64Descriptor[0] = 0;
1832	au64Descriptor[1] = 0;
1833	}
1834	else
1835	{
1836	AssertPtr(pVCpu);
1837	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1838	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1839	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1840	au64Descriptor[1] = GCPtr;
1841	}
1842
1843	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1844	AssertMsg(rc == VINF_SUCCESS,
1845	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1846	if ( RT_SUCCESS(rc)
1847	&& pVCpu)
1848	{
1849	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1850	}
1851	}
1852
1853
1854	/**
1855	* Invalidates a guest page by guest virtual address. Only relevant for
1856	* EPT/VPID, otherwise there is nothing really to invalidate.
1857	*
1858	* @returns VBox status code.
1859	* @param pVM The cross context VM structure.
1860	* @param pVCpu The cross context virtual CPU structure.
1861	* @param GCVirt Guest virtual address of the page to invalidate.
1862	*/
1863	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1864	{
1865	AssertPtr(pVM);
1866	AssertPtr(pVCpu);
1867	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1868
1869	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1870	if (!fFlushPending)
1871	{
1872	/*
1873	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1874	* See @bugref{6043} and @bugref{6177}.
1875	*
1876	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1877	* function maybe called in a loop with individual addresses.
1878	*/
1879	if (pVM->hm.s.vmx.fVpid)
1880	{
1881	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1882	{
1883	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1884	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1885	}
1886	else
1887	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1888	}
1889	else if (pVM->hm.s.fNestedPaging)
1890	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1891	}
1892
1893	return VINF_SUCCESS;
1894	}
1895
1896
1897	/**
1898	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1899	* otherwise there is nothing really to invalidate.
1900	*
1901	* @returns VBox status code.
1902	* @param pVM The cross context VM structure.
1903	* @param pVCpu The cross context virtual CPU structure.
1904	* @param GCPhys Guest physical address of the page to invalidate.
1905	*/
1906	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1907	{
1908	NOREF(pVM); NOREF(GCPhys);
1909	LogFlowFunc(("%RGp\n", GCPhys));
1910
1911	/*
1912	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1913	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1914	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1915	*/
1916	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1917	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1924	* case where neither EPT nor VPID is supported by the CPU.
1925	*
1926	* @param pVM The cross context VM structure.
1927	* @param pVCpu The cross context virtual CPU structure.
1928	* @param pCpu Pointer to the global HM struct.
1929	*
1930	* @remarks Called with interrupts disabled.
1931	*/
1932	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1933	{
1934	AssertPtr(pVCpu);
1935	AssertPtr(pCpu);
1936	NOREF(pVM);
1937
1938	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1939
1940	Assert(pCpu->idCpu != NIL_RTCPUID);
1941	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1942	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1943	pVCpu->hm.s.fForceTLBFlush = false;
1944	return;
1945	}
1946
1947
1948	/**
1949	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1950	*
1951	* @param pVM The cross context VM structure.
1952	* @param pVCpu The cross context virtual CPU structure.
1953	* @param pCpu Pointer to the global HM CPU struct.
1954	* @remarks All references to "ASID" in this function pertains to "VPID" in
1955	* Intel's nomenclature. The reason is, to avoid confusion in compare
1956	* statements since the host-CPU copies are named "ASID".
1957	*
1958	* @remarks Called with interrupts disabled.
1959	*/
1960	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1961	{
1962	#ifdef VBOX_WITH_STATISTICS
1963	bool fTlbFlushed = false;
1964	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1965	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1966	if (!fTlbFlushed) \
1967	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1968	} while (0)
1969	#else
1970	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1971	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1972	#endif
1973
1974	AssertPtr(pVM);
1975	AssertPtr(pCpu);
1976	AssertPtr(pVCpu);
1977	Assert(pCpu->idCpu != NIL_RTCPUID);
1978
1979	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1980	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1981	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1982
1983	/*
1984	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1985	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1986	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1987	*/
1988	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1989	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1990	{
1991	++pCpu->uCurrentAsid;
1992	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1993	{
1994	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1995	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1996	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1997	}
1998
1999	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2000	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2001	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2002
2003	/*
2004	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2005	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2006	*/
2007	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2008	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2009	HMVMX_SET_TAGGED_TLB_FLUSHED();
2010	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2011	}
2012
2013	/* Check for explicit TLB flushes. */
2014	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2015	{
2016	/*
2017	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2018	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2019	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2020	* but not guest-physical mappings.
2021	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2022	*/
2023	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2024	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2025	HMVMX_SET_TAGGED_TLB_FLUSHED();
2026	}
2027
2028	pVCpu->hm.s.fForceTLBFlush = false;
2029	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2030
2031	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2032	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2033	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2034	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2035	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2036	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2037	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2038	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2039	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2040
2041	/* Update VMCS with the VPID. */
2042	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2043	AssertRC(rc);
2044
2045	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2046	}
2047
2048
2049	/**
2050	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2051	*
2052	* @returns VBox status code.
2053	* @param pVM The cross context VM structure.
2054	* @param pVCpu The cross context virtual CPU structure.
2055	* @param pCpu Pointer to the global HM CPU struct.
2056	*
2057	* @remarks Called with interrupts disabled.
2058	*/
2059	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2060	{
2061	AssertPtr(pVM);
2062	AssertPtr(pVCpu);
2063	AssertPtr(pCpu);
2064	Assert(pCpu->idCpu != NIL_RTCPUID);
2065	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2066	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2067
2068	/*
2069	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2070	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2071	*/
2072	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2073	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2074	{
2075	pVCpu->hm.s.fForceTLBFlush = true;
2076	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2077	}
2078
2079	/* Check for explicit TLB flushes. */
2080	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2081	{
2082	pVCpu->hm.s.fForceTLBFlush = true;
2083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2084	}
2085
2086	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2087	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2088
2089	if (pVCpu->hm.s.fForceTLBFlush)
2090	{
2091	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2092	pVCpu->hm.s.fForceTLBFlush = false;
2093	}
2094	}
2095
2096
2097	/**
2098	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2099	*
2100	* @returns VBox status code.
2101	* @param pVM The cross context VM structure.
2102	* @param pVCpu The cross context virtual CPU structure.
2103	* @param pCpu Pointer to the global HM CPU struct.
2104	*
2105	* @remarks Called with interrupts disabled.
2106	*/
2107	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2108	{
2109	AssertPtr(pVM);
2110	AssertPtr(pVCpu);
2111	AssertPtr(pCpu);
2112	Assert(pCpu->idCpu != NIL_RTCPUID);
2113	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2114	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2115
2116	/*
2117	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2118	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2119	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2120	*/
2121	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2122	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2123	{
2124	pVCpu->hm.s.fForceTLBFlush = true;
2125	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2126	}
2127
2128	/* Check for explicit TLB flushes. */
2129	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2130	{
2131	/*
2132	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2133	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2134	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2135	*/
2136	pVCpu->hm.s.fForceTLBFlush = true;
2137	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2138	}
2139
2140	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2141	if (pVCpu->hm.s.fForceTLBFlush)
2142	{
2143	++pCpu->uCurrentAsid;
2144	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2145	{
2146	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2147	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2148	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2149	}
2150
2151	pVCpu->hm.s.fForceTLBFlush = false;
2152	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2153	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2154	if (pCpu->fFlushAsidBeforeUse)
2155	{
2156	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2157	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2158	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2159	{
2160	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2161	pCpu->fFlushAsidBeforeUse = false;
2162	}
2163	else
2164	{
2165	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2166	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2167	}
2168	}
2169	}
2170
2171	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2172	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2173	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2174	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2175	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2176	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2177	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2178
2179	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2180	AssertRC(rc);
2181	}
2182
2183
2184	/**
2185	* Flushes the guest TLB entry based on CPU capabilities.
2186	*
2187	* @param pVCpu The cross context virtual CPU structure.
2188	* @param pCpu Pointer to the global HM CPU struct.
2189	*/
2190	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2191	{
2192	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2193	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2194	#endif
2195	PVM pVM = pVCpu->CTX_SUFF(pVM);
2196	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2197	{
2198	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2199	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2200	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2201	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2202	default:
2203	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2204	break;
2205	}
2206
2207	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2208	}
2209
2210
2211	/**
2212	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2213	* TLB entries from the host TLB before VM-entry.
2214	*
2215	* @returns VBox status code.
2216	* @param pVM The cross context VM structure.
2217	*/
2218	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2219	{
2220	/*
2221	* Determine optimal flush type for Nested Paging.
2222	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2223	* guest execution (see hmR3InitFinalizeR0()).
2224	*/
2225	if (pVM->hm.s.fNestedPaging)
2226	{
2227	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2228	{
2229	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2230	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2231	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2232	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2233	else
2234	{
2235	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2236	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2237	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2238	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2239	}
2240
2241	/* Make sure the write-back cacheable memory type for EPT is supported. */
2242	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2243	{
2244	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2246	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2247	}
2248
2249	/* EPT requires a page-walk length of 4. */
2250	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2251	{
2252	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2254	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2255	}
2256	}
2257	else
2258	{
2259	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2260	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2261	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2262	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2263	}
2264	}
2265
2266	/*
2267	* Determine optimal flush type for VPID.
2268	*/
2269	if (pVM->hm.s.vmx.fVpid)
2270	{
2271	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2272	{
2273	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2274	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2275	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2276	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2277	else
2278	{
2279	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2280	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2281	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2282	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2283	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2284	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2285	pVM->hm.s.vmx.fVpid = false;
2286	}
2287	}
2288	else
2289	{
2290	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2291	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2292	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2293	pVM->hm.s.vmx.fVpid = false;
2294	}
2295	}
2296
2297	/*
2298	* Setup the handler for flushing tagged-TLBs.
2299	*/
2300	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2301	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2302	else if (pVM->hm.s.fNestedPaging)
2303	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2304	else if (pVM->hm.s.vmx.fVpid)
2305	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2306	else
2307	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2308	return VINF_SUCCESS;
2309	}
2310
2311
2312	/**
2313	* Sets up pin-based VM-execution controls in the VMCS.
2314	*
2315	* @returns VBox status code.
2316	* @param pVM The cross context VM structure.
2317	* @param pVCpu The cross context virtual CPU structure.
2318	*/
2319	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2320	{
2321	AssertPtr(pVM);
2322	AssertPtr(pVCpu);
2323
2324	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2325	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2326
2327	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2328	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2329
2330	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2331	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2332
2333	/* Enable the VMX preemption timer. */
2334	if (pVM->hm.s.vmx.fUsePreemptTimer)
2335	{
2336	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2337	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2338	}
2339
2340	#if 0
2341	/* Enable posted-interrupt processing. */
2342	if (pVM->hm.s.fPostedIntrs)
2343	{
2344	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2345	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2346	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2347	}
2348	#endif
2349
2350	if ((val & zap) != val)
2351	{
2352	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2353	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2354	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2355	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2356	}
2357
2358	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2359	AssertRCReturn(rc, rc);
2360
2361	pVCpu->hm.s.vmx.u32PinCtls = val;
2362	return rc;
2363	}
2364
2365
2366	/**
2367	* Sets up processor-based VM-execution controls in the VMCS.
2368	*
2369	* @returns VBox status code.
2370	* @param pVM The cross context VM structure.
2371	* @param pVCpu The cross context virtual CPU structure.
2372	*/
2373	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2374	{
2375	AssertPtr(pVM);
2376	AssertPtr(pVCpu);
2377
2378	int rc = VERR_INTERNAL_ERROR_5;
2379	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2380	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2381
2382	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2383	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2384	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2385	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2386	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2387	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2388	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2389
2390	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2391	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2392	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2393	{
2394	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2395	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2396	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2397	}
2398
2399	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2400	if (!pVM->hm.s.fNestedPaging)
2401	{
2402	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2403	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2404	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2405	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2406	}
2407
2408	/* Use TPR shadowing if supported by the CPU. */
2409	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2410	{
2411	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2412	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2413	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2414	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2415	AssertRCReturn(rc, rc);
2416
2417	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2418	/* CR8 writes cause a VM-exit based on TPR threshold. */
2419	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2420	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2421	}
2422	else
2423	{
2424	/*
2425	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2426	* Set this control only for 64-bit guests.
2427	*/
2428	if (pVM->hm.s.fAllow64BitGuests)
2429	{
2430	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2431	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2432	}
2433	}
2434
2435	/* Use MSR-bitmaps if supported by the CPU. */
2436	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2437	{
2438	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2439
2440	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2441	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2442	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2443	AssertRCReturn(rc, rc);
2444
2445	/*
2446	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2447	* automatically using dedicated fields in the VMCS.
2448	*/
2449	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454
2455	#if HC_ARCH_BITS == 64
2456	/*
2457	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2458	*/
2459	if (pVM->hm.s.fAllow64BitGuests)
2460	{
2461	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2462	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2463	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2464	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465	}
2466	#endif
2467	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2468	}
2469
2470	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2471	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2472	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2473
2474	if ((val & zap) != val)
2475	{
2476	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2477	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2478	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2479	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2480	}
2481
2482	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2483	AssertRCReturn(rc, rc);
2484
2485	pVCpu->hm.s.vmx.u32ProcCtls = val;
2486
2487	/*
2488	* Secondary processor-based VM-execution controls.
2489	*/
2490	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2491	{
2492	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2493	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2494
2495	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2496	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2497
2498	if (pVM->hm.s.fNestedPaging)
2499	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2500	else
2501	{
2502	/*
2503	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2504	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2505	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2506	*/
2507	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2508	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2509	}
2510
2511	if (pVM->hm.s.vmx.fVpid)
2512	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2513
2514	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2515	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2516
2517	#if 0
2518	if (pVM->hm.s.fVirtApicRegs)
2519	{
2520	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2521	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2522
2523	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2524	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2525	}
2526	#endif
2527
2528	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2529	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2530	* done dynamically. */
2531	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2532	{
2533	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2534	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2535	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2536	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2537	AssertRCReturn(rc, rc);
2538	}
2539
2540	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2541	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2542
2543	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2544	&& pVM->hm.s.vmx.cPleGapTicks
2545	&& pVM->hm.s.vmx.cPleWindowTicks)
2546	{
2547	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2548
2549	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2550	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2551	AssertRCReturn(rc, rc);
2552	}
2553
2554	if ((val & zap) != val)
2555	{
2556	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2557	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2558	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2559	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560	}
2561
2562	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2563	AssertRCReturn(rc, rc);
2564
2565	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2566	}
2567	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2568	{
2569	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2570	"available\n"));
2571	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573	}
2574
2575	return VINF_SUCCESS;
2576	}
2577
2578
2579	/**
2580	* Sets up miscellaneous (everything other than Pin & Processor-based
2581	* VM-execution) control fields in the VMCS.
2582	*
2583	* @returns VBox status code.
2584	* @param pVM The cross context VM structure.
2585	* @param pVCpu The cross context virtual CPU structure.
2586	*/
2587	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2588	{
2589	NOREF(pVM);
2590	AssertPtr(pVM);
2591	AssertPtr(pVCpu);
2592
2593	int rc = VERR_GENERAL_FAILURE;
2594
2595	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2596	#if 0
2597	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2598	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2599	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2600
2601	/*
2602	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2603	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2604	* We thus use the exception bitmap to control it rather than use both.
2605	*/
2606	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2607	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2608
2609	/** @todo Explore possibility of using IO-bitmaps. */
2610	/* All IO & IOIO instructions cause VM-exits. */
2611	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2612	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2613
2614	/* Initialize the MSR-bitmap area. */
2615	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2616	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2617	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2618	AssertRCReturn(rc, rc);
2619	#endif
2620
2621	/* Setup MSR auto-load/store area. */
2622	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2624	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2625	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626	AssertRCReturn(rc, rc);
2627
2628	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2629	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2630	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2631	AssertRCReturn(rc, rc);
2632
2633	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2634	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2635	AssertRCReturn(rc, rc);
2636
2637	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2638	#if 0
2639	/* Setup debug controls */
2640	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2641	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2642	AssertRCReturn(rc, rc);
2643	#endif
2644
2645	return rc;
2646	}
2647
2648
2649	/**
2650	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2651	*
2652	* We shall setup those exception intercepts that don't change during the
2653	* lifetime of the VM here. The rest are done dynamically while loading the
2654	* guest state.
2655	*
2656	* @returns VBox status code.
2657	* @param pVM The cross context VM structure.
2658	* @param pVCpu The cross context virtual CPU structure.
2659	*/
2660	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2661	{
2662	AssertPtr(pVM);
2663	AssertPtr(pVCpu);
2664
2665	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2666
2667	uint32_t u32XcptBitmap = 0;
2668
2669	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2670	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2671
2672	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2673	and writes, and because recursive #DBs can cause the CPU hang, we must always
2674	intercept #DB. */
2675	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2676
2677	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2678	if (!pVM->hm.s.fNestedPaging)
2679	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2680
2681	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2682	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2683	AssertRCReturn(rc, rc);
2684	return rc;
2685	}
2686
2687
2688	/**
2689	* Sets up the initial guest-state mask. The guest-state mask is consulted
2690	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2691	* for the nested virtualization case (as it would cause a VM-exit).
2692	*
2693	* @param pVCpu The cross context virtual CPU structure.
2694	*/
2695	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2696	{
2697	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2698	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2699	return VINF_SUCCESS;
2700	}
2701
2702
2703	/**
2704	* Does per-VM VT-x initialization.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM The cross context VM structure.
2708	*/
2709	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2710	{
2711	LogFlowFunc(("pVM=%p\n", pVM));
2712
2713	int rc = hmR0VmxStructsAlloc(pVM);
2714	if (RT_FAILURE(rc))
2715	{
2716	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2717	return rc;
2718	}
2719
2720	return VINF_SUCCESS;
2721	}
2722
2723
2724	/**
2725	* Does per-VM VT-x termination.
2726	*
2727	* @returns VBox status code.
2728	* @param pVM The cross context VM structure.
2729	*/
2730	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2731	{
2732	LogFlowFunc(("pVM=%p\n", pVM));
2733
2734	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2735	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2736	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2737	#endif
2738	hmR0VmxStructsFree(pVM);
2739	return VINF_SUCCESS;
2740	}
2741
2742
2743	/**
2744	* Sets up the VM for execution under VT-x.
2745	* This function is only called once per-VM during initialization.
2746	*
2747	* @returns VBox status code.
2748	* @param pVM The cross context VM structure.
2749	*/
2750	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2751	{
2752	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2754
2755	LogFlowFunc(("pVM=%p\n", pVM));
2756
2757	/*
2758	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2759	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2760	*/
2761	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2762	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2763	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2764	{
2765	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2766	return VERR_INTERNAL_ERROR;
2767	}
2768
2769	/* Initialize these always, see hmR3InitFinalizeR0().*/
2770	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2771	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2772
2773	/* Setup the tagged-TLB flush handlers. */
2774	int rc = hmR0VmxSetupTaggedTlb(pVM);
2775	if (RT_FAILURE(rc))
2776	{
2777	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2778	return rc;
2779	}
2780
2781	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2782	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2783	#if HC_ARCH_BITS == 64
2784	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2785	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2786	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2787	{
2788	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2789	}
2790	#endif
2791
2792	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2793	RTCCUINTREG uHostCR4 = ASMGetCR4();
2794	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2795	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2796
2797	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2798	{
2799	PVMCPU pVCpu = &pVM->aCpus[i];
2800	AssertPtr(pVCpu);
2801	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2802
2803	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2804	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2805
2806	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2807	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2808	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2809
2810	/* Set revision dword at the beginning of the VMCS structure. */
2811	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2812
2813	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2814	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2815	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818	/* Load this VMCS as the current VMCS. */
2819	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2820	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822
2823	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2824	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2825	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2826
2827	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2828	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2829	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2830
2831	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2832	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2836	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2837	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2838
2839	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2840	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2841	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2842
2843	#if HC_ARCH_BITS == 32
2844	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2845	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2846	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2847	#endif
2848
2849	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2850	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2851	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2852	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2853
2854	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2855
2856	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2857	}
2858
2859	return VINF_SUCCESS;
2860	}
2861
2862
2863	/**
2864	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2865	* the VMCS.
2866	*
2867	* @returns VBox status code.
2868	* @param pVM The cross context VM structure.
2869	* @param pVCpu The cross context virtual CPU structure.
2870	*/
2871	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2872	{
2873	NOREF(pVM); NOREF(pVCpu);
2874
2875	RTCCUINTREG uReg = ASMGetCR0();
2876	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2877	AssertRCReturn(rc, rc);
2878
2879	uReg = ASMGetCR3();
2880	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2881	AssertRCReturn(rc, rc);
2882
2883	uReg = ASMGetCR4();
2884	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2885	AssertRCReturn(rc, rc);
2886	return rc;
2887	}
2888
2889
2890	#if HC_ARCH_BITS == 64
2891	/**
2892	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2893	* requirements. See hmR0VmxSaveHostSegmentRegs().
2894	*/
2895	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2896	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2897	{ \
2898	bool fValidSelector = true; \
2899	if ((selValue) & X86_SEL_LDT) \
2900	{ \
2901	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2902	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2903	} \
2904	if (fValidSelector) \
2905	{ \
2906	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2907	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2908	} \
2909	(selValue) = 0; \
2910	}
2911	#endif
2912
2913
2914	/**
2915	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2916	* the host-state area in the VMCS.
2917	*
2918	* @returns VBox status code.
2919	* @param pVM The cross context VM structure.
2920	* @param pVCpu The cross context virtual CPU structure.
2921	*/
2922	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2923	{
2924	int rc = VERR_INTERNAL_ERROR_5;
2925
2926	#if HC_ARCH_BITS == 64
2927	/*
2928	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2929	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2930	*
2931	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2932	* Was observed booting Solaris10u10 32-bit guest.
2933	*/
2934	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2935	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2936	{
2937	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2938	pVCpu->idCpu));
2939	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2940	}
2941	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2942	#else
2943	RT_NOREF(pVCpu);
2944	#endif
2945
2946	/*
2947	* Host DS, ES, FS and GS segment registers.
2948	*/
2949	#if HC_ARCH_BITS == 64
2950	RTSEL uSelDS = ASMGetDS();
2951	RTSEL uSelES = ASMGetES();
2952	RTSEL uSelFS = ASMGetFS();
2953	RTSEL uSelGS = ASMGetGS();
2954	#else
2955	RTSEL uSelDS = 0;
2956	RTSEL uSelES = 0;
2957	RTSEL uSelFS = 0;
2958	RTSEL uSelGS = 0;
2959	#endif
2960
2961	/*
2962	* Host CS and SS segment registers.
2963	*/
2964	RTSEL uSelCS = ASMGetCS();
2965	RTSEL uSelSS = ASMGetSS();
2966
2967	/*
2968	* Host TR segment register.
2969	*/
2970	RTSEL uSelTR = ASMGetTR();
2971
2972	#if HC_ARCH_BITS == 64
2973	/*
2974	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2975	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2976	*/
2977	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2978	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2979	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2980	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2981	# undef VMXLOCAL_ADJUST_HOST_SEG
2982	#endif
2983
2984	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2985	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2986	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2987	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2988	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2989	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2990	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2991	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2992	Assert(uSelCS);
2993	Assert(uSelTR);
2994
2995	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2996	#if 0
2997	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2998	Assert(uSelSS != 0);
2999	#endif
3000
3001	/* Write these host selector fields into the host-state area in the VMCS. */
3002	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3003	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3004	#if HC_ARCH_BITS == 64
3005	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3006	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3007	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3008	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3009	#else
3010	NOREF(uSelDS);
3011	NOREF(uSelES);
3012	NOREF(uSelFS);
3013	NOREF(uSelGS);
3014	#endif
3015	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3016	AssertRCReturn(rc, rc);
3017
3018	/*
3019	* Host GDTR and IDTR.
3020	*/
3021	RTGDTR Gdtr;
3022	RTIDTR Idtr;
3023	RT_ZERO(Gdtr);
3024	RT_ZERO(Idtr);
3025	ASMGetGDTR(&Gdtr);
3026	ASMGetIDTR(&Idtr);
3027	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3028	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3029	AssertRCReturn(rc, rc);
3030
3031	#if HC_ARCH_BITS == 64
3032	/*
3033	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3034	* maximum limit (0xffff) on every VM-exit.
3035	*/
3036	if (Gdtr.cbGdt != 0xffff)
3037	{
3038	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3039	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3040	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3041	}
3042
3043	/*
3044	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3045	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3046	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3047	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3048	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3049	* hosts where we are pretty sure it won't cause trouble.
3050	*/
3051	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3052	if (Idtr.cbIdt < 0x0fff)
3053	# else
3054	if (Idtr.cbIdt != 0xffff)
3055	# endif
3056	{
3057	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3058	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3059	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3060	}
3061	#endif
3062
3063	/*
3064	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3065	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3066	*/
3067	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3068	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3069	VERR_VMX_INVALID_HOST_STATE);
3070
3071	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3072	#if HC_ARCH_BITS == 64
3073	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3074
3075	/*
3076	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3077	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3078	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3079	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3080	*
3081	* [1] See Intel spec. 3.5 "System Descriptor Types".
3082	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3083	*/
3084	Assert(pDesc->System.u4Type == 11);
3085	if ( pDesc->System.u16LimitLow != 0x67
3086	\|\| pDesc->System.u4LimitHigh)
3087	{
3088	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3089	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3090	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3091	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3092	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3093
3094	/* Store the GDTR here as we need it while restoring TR. */
3095	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3096	}
3097	#else
3098	NOREF(pVM);
3099	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3100	#endif
3101	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3102	AssertRCReturn(rc, rc);
3103
3104	/*
3105	* Host FS base and GS base.
3106	*/
3107	#if HC_ARCH_BITS == 64
3108	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3109	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3110	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3111	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3112	AssertRCReturn(rc, rc);
3113
3114	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3115	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3116	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3117	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3118	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3119	#endif
3120	return rc;
3121	}
3122
3123
3124	/**
3125	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3126	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3127	* the host after every successful VM-exit.
3128	*
3129	* @returns VBox status code.
3130	* @param pVM The cross context VM structure.
3131	* @param pVCpu The cross context virtual CPU structure.
3132	*
3133	* @remarks No-long-jump zone!!!
3134	*/
3135	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3136	{
3137	NOREF(pVM);
3138
3139	AssertPtr(pVCpu);
3140	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3141
3142	/*
3143	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3144	* rather than swapping them on every VM-entry.
3145	*/
3146	hmR0VmxLazySaveHostMsrs(pVCpu);
3147
3148	/*
3149	* Host Sysenter MSRs.
3150	*/
3151	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3152	#if HC_ARCH_BITS == 32
3153	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3154	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3155	#else
3156	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3157	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3158	#endif
3159	AssertRCReturn(rc, rc);
3160
3161	/*
3162	* Host EFER MSR.
3163	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3164	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3165	*/
3166	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3167	{
3168	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3169	AssertRCReturn(rc, rc);
3170	}
3171
3172	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3173	* hmR0VmxLoadGuestExitCtls() !! */
3174
3175	return rc;
3176	}
3177
3178
3179	/**
3180	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3181	*
3182	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3183	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3184	* hmR0VMxLoadGuestEntryCtls().
3185	*
3186	* @returns true if we need to load guest EFER, false otherwise.
3187	* @param pVCpu The cross context virtual CPU structure.
3188	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3189	* out-of-sync. Make sure to update the required fields
3190	* before using them.
3191	*
3192	* @remarks Requires EFER, CR4.
3193	* @remarks No-long-jump zone!!!
3194	*/
3195	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3196	{
3197	#ifdef HMVMX_ALWAYS_SWAP_EFER
3198	return true;
3199	#endif
3200
3201	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3202	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3203	if (CPUMIsGuestInLongMode(pVCpu))
3204	return false;
3205	#endif
3206
3207	PVM pVM = pVCpu->CTX_SUFF(pVM);
3208	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3209	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3210
3211	/*
3212	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3213	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3214	*/
3215	if ( CPUMIsGuestInLongMode(pVCpu)
3216	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3217	{
3218	return true;
3219	}
3220
3221	/*
3222	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3223	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3224	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3225	*/
3226	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3227	&& (pMixedCtx->cr0 & X86_CR0_PG)
3228	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3229	{
3230	/* Assert that host is PAE capable. */
3231	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3232	return true;
3233	}
3234
3235	/** @todo Check the latest Intel spec. for any other bits,
3236	* like SMEP/SMAP? */
3237	return false;
3238	}
3239
3240
3241	/**
3242	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3243	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3244	* controls".
3245	*
3246	* @returns VBox status code.
3247	* @param pVCpu The cross context virtual CPU structure.
3248	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3249	* out-of-sync. Make sure to update the required fields
3250	* before using them.
3251	*
3252	* @remarks Requires EFER.
3253	* @remarks No-long-jump zone!!!
3254	*/
3255	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3256	{
3257	int rc = VINF_SUCCESS;
3258	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3259	{
3260	PVM pVM = pVCpu->CTX_SUFF(pVM);
3261	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3262	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3263
3264	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3265	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3266
3267	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3268	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3269	{
3270	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3271	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3272	}
3273	else
3274	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3275
3276	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3277	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3278	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3279	{
3280	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3281	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3282	}
3283
3284	/*
3285	* The following should -not- be set (since we're not in SMM mode):
3286	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3287	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3288	*/
3289
3290	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3291	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3292
3293	if ((val & zap) != val)
3294	{
3295	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3296	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3297	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3298	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3299	}
3300
3301	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3302	AssertRCReturn(rc, rc);
3303
3304	pVCpu->hm.s.vmx.u32EntryCtls = val;
3305	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3306	}
3307	return rc;
3308	}
3309
3310
3311	/**
3312	* Sets up the VM-exit controls in the VMCS.
3313	*
3314	* @returns VBox status code.
3315	* @param pVCpu The cross context virtual CPU structure.
3316	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3317	* out-of-sync. Make sure to update the required fields
3318	* before using them.
3319	*
3320	* @remarks Requires EFER.
3321	*/
3322	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3323	{
3324	NOREF(pMixedCtx);
3325
3326	int rc = VINF_SUCCESS;
3327	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3328	{
3329	PVM pVM = pVCpu->CTX_SUFF(pVM);
3330	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3331	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3332
3333	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3334	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3335
3336	/*
3337	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3338	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3339	*/
3340	#if HC_ARCH_BITS == 64
3341	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3342	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3343	#else
3344	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3345	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3346	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3347	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3348	{
3349	/* The switcher returns to long mode, EFER is managed by the switcher. */
3350	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3351	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3352	}
3353	else
3354	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3355	#endif
3356
3357	/* If the newer VMCS fields for managing EFER exists, use it. */
3358	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3359	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3360	{
3361	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3362	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3363	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3364	}
3365
3366	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3367	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3368
3369	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3370	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3371	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3372
3373	if ( pVM->hm.s.vmx.fUsePreemptTimer
3374	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3375	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3376
3377	if ((val & zap) != val)
3378	{
3379	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3380	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3381	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3382	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3383	}
3384
3385	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3386	AssertRCReturn(rc, rc);
3387
3388	pVCpu->hm.s.vmx.u32ExitCtls = val;
3389	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3390	}
3391	return rc;
3392	}
3393
3394
3395	/**
3396	* Sets the TPR threshold in the VMCS.
3397	*
3398	* @returns VBox status code.
3399	* @param pVCpu The cross context virtual CPU structure.
3400	* @param u32TprThreshold The TPR threshold (task-priority class only).
3401	*/
3402	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3403	{
3404	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3405	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3406	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3407	}
3408
3409
3410	/**
3411	* Loads the guest APIC and related state.
3412	*
3413	* @returns VBox status code.
3414	* @param pVCpu The cross context virtual CPU structure.
3415	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3416	* out-of-sync. Make sure to update the required fields
3417	* before using them.
3418	*
3419	* @remarks No-long-jump zone!!!
3420	*/
3421	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3422	{
3423	NOREF(pMixedCtx);
3424
3425	int rc = VINF_SUCCESS;
3426	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3427	{
3428	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3429	&& APICIsEnabled(pVCpu))
3430	{
3431	/*
3432	* Setup TPR shadowing.
3433	*/
3434	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3435	{
3436	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3437
3438	bool fPendingIntr = false;
3439	uint8_t u8Tpr = 0;
3440	uint8_t u8PendingIntr = 0;
3441	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3442	AssertRCReturn(rc, rc);
3443
3444	/*
3445	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3446	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3447	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3448	*/
3449	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3450	uint32_t u32TprThreshold = 0;
3451	if (fPendingIntr)
3452	{
3453	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3454	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3455	const uint8_t u8TprPriority = u8Tpr >> 4;
3456	if (u8PendingPriority <= u8TprPriority)
3457	u32TprThreshold = u8PendingPriority;
3458	}
3459
3460	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3461	AssertRCReturn(rc, rc);
3462	}
3463	}
3464	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3465	}
3466
3467	return rc;
3468	}
3469
3470
3471	/**
3472	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3473	*
3474	* @returns Guest's interruptibility-state.
3475	* @param pVCpu The cross context virtual CPU structure.
3476	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3477	* out-of-sync. Make sure to update the required fields
3478	* before using them.
3479	*
3480	* @remarks No-long-jump zone!!!
3481	*/
3482	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3483	{
3484	/*
3485	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3486	*/
3487	uint32_t uIntrState = 0;
3488	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3489	{
3490	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3491	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3492	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3493	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3494	{
3495	if (pMixedCtx->eflags.Bits.u1IF)
3496	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3497	else
3498	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3499	}
3500	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3501	{
3502	/*
3503	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3504	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3505	*/
3506	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3507	}
3508	}
3509
3510	/*
3511	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3512	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3513	* setting this would block host-NMIs and IRET will not clear the blocking.
3514	*
3515	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3516	*/
3517	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3518	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3519	{
3520	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3521	}
3522
3523	return uIntrState;
3524	}
3525
3526
3527	/**
3528	* Loads the guest's interruptibility-state into the guest-state area in the
3529	* VMCS.
3530	*
3531	* @returns VBox status code.
3532	* @param pVCpu The cross context virtual CPU structure.
3533	* @param uIntrState The interruptibility-state to set.
3534	*/
3535	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3536	{
3537	NOREF(pVCpu);
3538	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3539	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3540	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3541	AssertRC(rc);
3542	return rc;
3543	}
3544
3545
3546	/**
3547	* Loads the exception intercepts required for guest execution in the VMCS.
3548	*
3549	* @returns VBox status code.
3550	* @param pVCpu The cross context virtual CPU structure.
3551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3552	* out-of-sync. Make sure to update the required fields
3553	* before using them.
3554	*/
3555	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3556	{
3557	NOREF(pMixedCtx);
3558	int rc = VINF_SUCCESS;
3559	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3560	{
3561	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3562	if (pVCpu->hm.s.fGIMTrapXcptUD)
3563	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3564	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3565	else
3566	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3567	#endif
3568
3569	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3570	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3571
3572	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3573	AssertRCReturn(rc, rc);
3574
3575	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3576	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3577	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3578	}
3579	return rc;
3580	}
3581
3582
3583	/**
3584	* Loads the guest's RIP into the guest-state area in the VMCS.
3585	*
3586	* @returns VBox status code.
3587	* @param pVCpu The cross context virtual CPU structure.
3588	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589	* out-of-sync. Make sure to update the required fields
3590	* before using them.
3591	*
3592	* @remarks No-long-jump zone!!!
3593	*/
3594	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3595	{
3596	int rc = VINF_SUCCESS;
3597	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3598	{
3599	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3600	AssertRCReturn(rc, rc);
3601
3602	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3603	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3604	HMCPU_CF_VALUE(pVCpu)));
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest's RSP into the guest-state area in the VMCS.
3612	*
3613	* @returns VBox status code.
3614	* @param pVCpu The cross context virtual CPU structure.
3615	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616	* out-of-sync. Make sure to update the required fields
3617	* before using them.
3618	*
3619	* @remarks No-long-jump zone!!!
3620	*/
3621	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622	{
3623	int rc = VINF_SUCCESS;
3624	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3625	{
3626	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3627	AssertRCReturn(rc, rc);
3628
3629	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3630	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3631	}
3632	return rc;
3633	}
3634
3635
3636	/**
3637	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3638	*
3639	* @returns VBox status code.
3640	* @param pVCpu The cross context virtual CPU structure.
3641	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3642	* out-of-sync. Make sure to update the required fields
3643	* before using them.
3644	*
3645	* @remarks No-long-jump zone!!!
3646	*/
3647	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3648	{
3649	int rc = VINF_SUCCESS;
3650	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3651	{
3652	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3653	Let us assert it as such and use 32-bit VMWRITE. */
3654	Assert(!(pMixedCtx->rflags.u64 >> 32));
3655	X86EFLAGS Eflags = pMixedCtx->eflags;
3656	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3657	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3658	* These will never be cleared/set, unless some other part of the VMM
3659	* code is buggy - in which case we're better of finding and fixing
3660	* those bugs than hiding them. */
3661	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3662	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3663	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3664	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3665
3666	/*
3667	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3668	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3669	*/
3670	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3671	{
3672	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3673	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3674	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3675	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3676	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3677	}
3678
3679	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3680	AssertRCReturn(rc, rc);
3681
3682	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3683	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3684	}
3685	return rc;
3686	}
3687
3688
3689	/**
3690	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3691	*
3692	* @returns VBox status code.
3693	* @param pVCpu The cross context virtual CPU structure.
3694	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3695	* out-of-sync. Make sure to update the required fields
3696	* before using them.
3697	*
3698	* @remarks No-long-jump zone!!!
3699	*/
3700	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3701	{
3702	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3703	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3704	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3705	AssertRCReturn(rc, rc);
3706	return rc;
3707	}
3708
3709
3710	/**
3711	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3712	* CR0 is partially shared with the host and we have to consider the FPU bits.
3713	*
3714	* @returns VBox status code.
3715	* @param pVCpu The cross context virtual CPU structure.
3716	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3717	* out-of-sync. Make sure to update the required fields
3718	* before using them.
3719	*
3720	* @remarks No-long-jump zone!!!
3721	*/
3722	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3723	{
3724	/*
3725	* Guest CR0.
3726	* Guest FPU.
3727	*/
3728	int rc = VINF_SUCCESS;
3729	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3730	{
3731	Assert(!(pMixedCtx->cr0 >> 32));
3732	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3733	PVM pVM = pVCpu->CTX_SUFF(pVM);
3734
3735	/* The guest's view (read access) of its CR0 is unblemished. */
3736	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3737	AssertRCReturn(rc, rc);
3738	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3739
3740	/* Setup VT-x's view of the guest CR0. */
3741	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3742	if (pVM->hm.s.fNestedPaging)
3743	{
3744	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3745	{
3746	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3747	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3748	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3749	}
3750	else
3751	{
3752	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3753	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3754	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3755	}
3756
3757	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3758	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3759	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3760
3761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3762	AssertRCReturn(rc, rc);
3763	}
3764	else
3765	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3766
3767	/*
3768	* Guest FPU bits.
3769	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3770	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3771	*/
3772	u32GuestCR0 \|= X86_CR0_NE;
3773	bool fInterceptNM = false;
3774	if (CPUMIsGuestFPUStateActive(pVCpu))
3775	{
3776	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3777	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3778	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3779	}
3780	else
3781	{
3782	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3783	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3784	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3785	}
3786
3787	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3788	bool fInterceptMF = false;
3789	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3790	fInterceptMF = true;
3791
3792	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3793	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3794	{
3795	Assert(PDMVmmDevHeapIsEnabled(pVM));
3796	Assert(pVM->hm.s.vmx.pRealModeTSS);
3797	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3798	fInterceptNM = true;
3799	fInterceptMF = true;
3800	}
3801	else
3802	{
3803	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3804	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3805	}
3806	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3807
3808	if (fInterceptNM)
3809	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3810	else
3811	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3812
3813	if (fInterceptMF)
3814	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3815	else
3816	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3817
3818	/* Additional intercepts for debugging, define these yourself explicitly. */
3819	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3820	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3821	\| RT_BIT(X86_XCPT_BP)
3822	\| RT_BIT(X86_XCPT_DE)
3823	\| RT_BIT(X86_XCPT_NM)
3824	\| RT_BIT(X86_XCPT_TS)
3825	\| RT_BIT(X86_XCPT_UD)
3826	\| RT_BIT(X86_XCPT_NP)
3827	\| RT_BIT(X86_XCPT_SS)
3828	\| RT_BIT(X86_XCPT_GP)
3829	\| RT_BIT(X86_XCPT_PF)
3830	\| RT_BIT(X86_XCPT_MF)
3831	;
3832	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3833	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3834	#endif
3835
3836	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3837
3838	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3839	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3840	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3841	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3842	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3843	else
3844	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3845
3846	u32GuestCR0 \|= uSetCR0;
3847	u32GuestCR0 &= uZapCR0;
3848	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3849
3850	/* Write VT-x's view of the guest CR0 into the VMCS. */
3851	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3852	AssertRCReturn(rc, rc);
3853	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3854	uZapCR0));
3855
3856	/*
3857	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3858	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3859	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3860	*/
3861	uint32_t u32CR0Mask = 0;
3862	u32CR0Mask = X86_CR0_PE
3863	\| X86_CR0_NE
3864	\| X86_CR0_WP
3865	\| X86_CR0_PG
3866	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3867	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3868	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3869
3870	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3871	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3872	* and @bugref{6944}. */
3873	#if 0
3874	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3875	u32CR0Mask &= ~X86_CR0_PE;
3876	#endif
3877	if (pVM->hm.s.fNestedPaging)
3878	u32CR0Mask &= ~X86_CR0_WP;
3879
3880	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3881	if (fInterceptNM)
3882	{
3883	u32CR0Mask \|= X86_CR0_TS
3884	\| X86_CR0_MP;
3885	}
3886
3887	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3888	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3889	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3890	AssertRCReturn(rc, rc);
3891	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3892
3893	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3894	}
3895	return rc;
3896	}
3897
3898
3899	/**
3900	* Loads the guest control registers (CR3, CR4) into the guest-state area
3901	* in the VMCS.
3902	*
3903	* @returns VBox strict status code.
3904	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3905	* without unrestricted guest access and the VMMDev is not presently
3906	* mapped (e.g. EFI32).
3907	*
3908	* @param pVCpu The cross context virtual CPU structure.
3909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3910	* out-of-sync. Make sure to update the required fields
3911	* before using them.
3912	*
3913	* @remarks No-long-jump zone!!!
3914	*/
3915	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3916	{
3917	int rc = VINF_SUCCESS;
3918	PVM pVM = pVCpu->CTX_SUFF(pVM);
3919
3920	/*
3921	* Guest CR2.
3922	* It's always loaded in the assembler code. Nothing to do here.
3923	*/
3924
3925	/*
3926	* Guest CR3.
3927	*/
3928	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3929	{
3930	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3931	if (pVM->hm.s.fNestedPaging)
3932	{
3933	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3934
3935	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3936	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3937	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3938	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3939
3940	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3941	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3942	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3943
3944	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3945	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3946	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3947	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3948	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3949	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3950	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3951
3952	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3953	AssertRCReturn(rc, rc);
3954	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3955
3956	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3957	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3958	{
3959	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3960	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3961	{
3962	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3963	AssertRCReturn(rc, rc);
3964	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3965	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3966	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3967	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3968	AssertRCReturn(rc, rc);
3969	}
3970
3971	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3972	have Unrestricted Execution to handle the guest when it's not using paging. */
3973	GCPhysGuestCR3 = pMixedCtx->cr3;
3974	}
3975	else
3976	{
3977	/*
3978	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3979	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3980	* EPT takes care of translating it to host-physical addresses.
3981	*/
3982	RTGCPHYS GCPhys;
3983	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3984
3985	/* We obtain it here every time as the guest could have relocated this PCI region. */
3986	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3987	if (RT_SUCCESS(rc))
3988	{ /* likely */ }
3989	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3990	{
3991	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3992	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3993	}
3994	else
3995	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3996
3997	GCPhysGuestCR3 = GCPhys;
3998	}
3999
4000	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4001	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4002	}
4003	else
4004	{
4005	/* Non-nested paging case, just use the hypervisor's CR3. */
4006	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4007
4008	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4009	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4010	}
4011	AssertRCReturn(rc, rc);
4012
4013	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4014	}
4015
4016	/*
4017	* Guest CR4.
4018	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4019	*/
4020	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4021	{
4022	Assert(!(pMixedCtx->cr4 >> 32));
4023	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4024
4025	/* The guest's view of its CR4 is unblemished. */
4026	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4027	AssertRCReturn(rc, rc);
4028	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4029
4030	/* Setup VT-x's view of the guest CR4. */
4031	/*
4032	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4033	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4034	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4035	*/
4036	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4037	{
4038	Assert(pVM->hm.s.vmx.pRealModeTSS);
4039	Assert(PDMVmmDevHeapIsEnabled(pVM));
4040	u32GuestCR4 &= ~X86_CR4_VME;
4041	}
4042
4043	if (pVM->hm.s.fNestedPaging)
4044	{
4045	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4046	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4047	{
4048	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4049	u32GuestCR4 \|= X86_CR4_PSE;
4050	/* Our identity mapping is a 32-bit page directory. */
4051	u32GuestCR4 &= ~X86_CR4_PAE;
4052	}
4053	/* else use guest CR4.*/
4054	}
4055	else
4056	{
4057	/*
4058	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4059	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4060	*/
4061	switch (pVCpu->hm.s.enmShadowMode)
4062	{
4063	case PGMMODE_REAL: /* Real-mode. */
4064	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4065	case PGMMODE_32_BIT: /* 32-bit paging. */
4066	{
4067	u32GuestCR4 &= ~X86_CR4_PAE;
4068	break;
4069	}
4070
4071	case PGMMODE_PAE: /* PAE paging. */
4072	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4073	{
4074	u32GuestCR4 \|= X86_CR4_PAE;
4075	break;
4076	}
4077
4078	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4079	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4080	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4081	break;
4082	#endif
4083	default:
4084	AssertFailed();
4085	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4086	}
4087	}
4088
4089	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4090	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4091	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4092	u32GuestCR4 \|= uSetCR4;
4093	u32GuestCR4 &= uZapCR4;
4094
4095	/* Write VT-x's view of the guest CR4 into the VMCS. */
4096	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4097	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4098	AssertRCReturn(rc, rc);
4099
4100	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4101	uint32_t u32CR4Mask = X86_CR4_VME
4102	\| X86_CR4_PAE
4103	\| X86_CR4_PGE
4104	\| X86_CR4_PSE
4105	\| X86_CR4_VMXE;
4106	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4107	u32CR4Mask \|= X86_CR4_OSXSAVE;
4108	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4109	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4110	AssertRCReturn(rc, rc);
4111
4112	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4113	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4114
4115	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4116	}
4117	return rc;
4118	}
4119
4120
4121	/**
4122	* Loads the guest debug registers into the guest-state area in the VMCS.
4123	*
4124	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4125	*
4126	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4127	*
4128	* @returns VBox status code.
4129	* @param pVCpu The cross context virtual CPU structure.
4130	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4131	* out-of-sync. Make sure to update the required fields
4132	* before using them.
4133	*
4134	* @remarks No-long-jump zone!!!
4135	*/
4136	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4137	{
4138	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4139	return VINF_SUCCESS;
4140
4141	#ifdef VBOX_STRICT
4142	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4143	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4144	{
4145	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4146	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4147	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4148	}
4149	#endif
4150
4151	int rc;
4152	PVM pVM = pVCpu->CTX_SUFF(pVM);
4153	bool fSteppingDB = false;
4154	bool fInterceptMovDRx = false;
4155	if (pVCpu->hm.s.fSingleInstruction)
4156	{
4157	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4158	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4159	{
4160	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4161	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4162	AssertRCReturn(rc, rc);
4163	Assert(fSteppingDB == false);
4164	}
4165	else
4166	{
4167	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4168	pVCpu->hm.s.fClearTrapFlag = true;
4169	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4170	fSteppingDB = true;
4171	}
4172	}
4173
4174	if ( fSteppingDB
4175	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4176	{
4177	/*
4178	* Use the combined guest and host DRx values found in the hypervisor
4179	* register set because the debugger has breakpoints active or someone
4180	* is single stepping on the host side without a monitor trap flag.
4181	*
4182	* Note! DBGF expects a clean DR6 state before executing guest code.
4183	*/
4184	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4185	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4186	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4187	{
4188	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4189	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4190	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4191	}
4192	else
4193	#endif
4194	if (!CPUMIsHyperDebugStateActive(pVCpu))
4195	{
4196	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4197	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4198	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4199	}
4200
4201	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4202	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4203	AssertRCReturn(rc, rc);
4204
4205	pVCpu->hm.s.fUsingHyperDR7 = true;
4206	fInterceptMovDRx = true;
4207	}
4208	else
4209	{
4210	/*
4211	* If the guest has enabled debug registers, we need to load them prior to
4212	* executing guest code so they'll trigger at the right time.
4213	*/
4214	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4215	{
4216	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4218	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4219	{
4220	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4221	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4222	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4223	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4224	}
4225	else
4226	#endif
4227	if (!CPUMIsGuestDebugStateActive(pVCpu))
4228	{
4229	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4230	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4231	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4232	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4233	}
4234	Assert(!fInterceptMovDRx);
4235	}
4236	/*
4237	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4238	* must intercept #DB in order to maintain a correct DR6 guest value, and
4239	* because we need to intercept it to prevent nested #DBs from hanging the
4240	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4241	*/
4242	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4243	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4244	&& !CPUMIsGuestDebugStateActive(pVCpu))
4245	#else
4246	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4247	#endif
4248	{
4249	fInterceptMovDRx = true;
4250	}
4251
4252	/* Update guest DR7. */
4253	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4254	AssertRCReturn(rc, rc);
4255
4256	pVCpu->hm.s.fUsingHyperDR7 = false;
4257	}
4258
4259	/*
4260	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4261	*/
4262	if (fInterceptMovDRx)
4263	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4264	else
4265	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4266	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4267	AssertRCReturn(rc, rc);
4268
4269	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4270	return VINF_SUCCESS;
4271	}
4272
4273
4274	#ifdef VBOX_STRICT
4275	/**
4276	* Strict function to validate segment registers.
4277	*
4278	* @remarks ASSUMES CR0 is up to date.
4279	*/
4280	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4281	{
4282	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4283	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4284	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4285	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4286	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4287	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4288	{
4289	/* Protected mode checks */
4290	/* CS */
4291	Assert(pCtx->cs.Attr.n.u1Present);
4292	Assert(!(pCtx->cs.Attr.u & 0xf00));
4293	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4294	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4295	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4296	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4297	\|\| (pCtx->cs.Attr.n.u1Granularity));
4298	/* CS cannot be loaded with NULL in protected mode. */
4299	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4300	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4301	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4302	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4303	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4304	else
4305	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4306	/* SS */
4307	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4308	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4309	if ( !(pCtx->cr0 & X86_CR0_PE)
4310	\|\| pCtx->cs.Attr.n.u4Type == 3)
4311	{
4312	Assert(!pCtx->ss.Attr.n.u2Dpl);
4313	}
4314	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4315	{
4316	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4317	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4318	Assert(pCtx->ss.Attr.n.u1Present);
4319	Assert(!(pCtx->ss.Attr.u & 0xf00));
4320	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4321	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4322	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4323	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4324	\|\| (pCtx->ss.Attr.n.u1Granularity));
4325	}
4326	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4327	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4328	{
4329	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4330	Assert(pCtx->ds.Attr.n.u1Present);
4331	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4332	Assert(!(pCtx->ds.Attr.u & 0xf00));
4333	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4334	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4335	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4336	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4337	\|\| (pCtx->ds.Attr.n.u1Granularity));
4338	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4339	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4340	}
4341	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4342	{
4343	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4344	Assert(pCtx->es.Attr.n.u1Present);
4345	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4346	Assert(!(pCtx->es.Attr.u & 0xf00));
4347	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4348	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4349	\|\| !(pCtx->es.Attr.n.u1Granularity));
4350	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4351	\|\| (pCtx->es.Attr.n.u1Granularity));
4352	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4353	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4354	}
4355	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4356	{
4357	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4358	Assert(pCtx->fs.Attr.n.u1Present);
4359	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4360	Assert(!(pCtx->fs.Attr.u & 0xf00));
4361	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4362	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4363	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4364	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4365	\|\| (pCtx->fs.Attr.n.u1Granularity));
4366	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4367	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4368	}
4369	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4370	{
4371	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4372	Assert(pCtx->gs.Attr.n.u1Present);
4373	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4374	Assert(!(pCtx->gs.Attr.u & 0xf00));
4375	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4376	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4377	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4378	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4379	\|\| (pCtx->gs.Attr.n.u1Granularity));
4380	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4381	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4382	}
4383	/* 64-bit capable CPUs. */
4384	# if HC_ARCH_BITS == 64
4385	Assert(!(pCtx->cs.u64Base >> 32));
4386	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4387	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4388	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4389	# endif
4390	}
4391	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4392	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4393	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4394	{
4395	/* Real and v86 mode checks. */
4396	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4397	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4398	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4399	{
4400	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4401	}
4402	else
4403	{
4404	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4405	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4406	}
4407
4408	/* CS */
4409	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4410	Assert(pCtx->cs.u32Limit == 0xffff);
4411	Assert(u32CSAttr == 0xf3);
4412	/* SS */
4413	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4414	Assert(pCtx->ss.u32Limit == 0xffff);
4415	Assert(u32SSAttr == 0xf3);
4416	/* DS */
4417	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4418	Assert(pCtx->ds.u32Limit == 0xffff);
4419	Assert(u32DSAttr == 0xf3);
4420	/* ES */
4421	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4422	Assert(pCtx->es.u32Limit == 0xffff);
4423	Assert(u32ESAttr == 0xf3);
4424	/* FS */
4425	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4426	Assert(pCtx->fs.u32Limit == 0xffff);
4427	Assert(u32FSAttr == 0xf3);
4428	/* GS */
4429	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4430	Assert(pCtx->gs.u32Limit == 0xffff);
4431	Assert(u32GSAttr == 0xf3);
4432	/* 64-bit capable CPUs. */
4433	# if HC_ARCH_BITS == 64
4434	Assert(!(pCtx->cs.u64Base >> 32));
4435	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4436	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4437	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4438	# endif
4439	}
4440	}
4441	#endif /* VBOX_STRICT */
4442
4443
4444	/**
4445	* Writes a guest segment register into the guest-state area in the VMCS.
4446	*
4447	* @returns VBox status code.
4448	* @param pVCpu The cross context virtual CPU structure.
4449	* @param idxSel Index of the selector in the VMCS.
4450	* @param idxLimit Index of the segment limit in the VMCS.
4451	* @param idxBase Index of the segment base in the VMCS.
4452	* @param idxAccess Index of the access rights of the segment in the VMCS.
4453	* @param pSelReg Pointer to the segment selector.
4454	*
4455	* @remarks No-long-jump zone!!!
4456	*/
4457	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4458	uint32_t idxAccess, PCPUMSELREG pSelReg)
4459	{
4460	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4461	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4462	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4463	AssertRCReturn(rc, rc);
4464
4465	uint32_t u32Access = pSelReg->Attr.u;
4466	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4467	{
4468	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4469	u32Access = 0xf3;
4470	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4471	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4472	}
4473	else
4474	{
4475	/*
4476	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4477	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4478	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4479	* loaded in protected-mode have their attribute as 0.
4480	*/
4481	if (!u32Access)
4482	u32Access = X86DESCATTR_UNUSABLE;
4483	}
4484
4485	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4486	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4487	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4488
4489	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4490	AssertRCReturn(rc, rc);
4491	return rc;
4492	}
4493
4494
4495	/**
4496	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4497	* into the guest-state area in the VMCS.
4498	*
4499	* @returns VBox status code.
4500	* @param pVCpu The cross context virtual CPU structure.
4501	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4502	* out-of-sync. Make sure to update the required fields
4503	* before using them.
4504	*
4505	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4506	* @remarks No-long-jump zone!!!
4507	*/
4508	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4509	{
4510	int rc = VERR_INTERNAL_ERROR_5;
4511	PVM pVM = pVCpu->CTX_SUFF(pVM);
4512
4513	/*
4514	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4515	*/
4516	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4517	{
4518	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4519	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4520	{
4521	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4522	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4523	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4524	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4525	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4526	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4527	}
4528
4529	#ifdef VBOX_WITH_REM
4530	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4531	{
4532	Assert(pVM->hm.s.vmx.pRealModeTSS);
4533	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4534	if ( pVCpu->hm.s.vmx.fWasInRealMode
4535	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4536	{
4537	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4538	in real-mode (e.g. OpenBSD 4.0) */
4539	REMFlushTBs(pVM);
4540	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4541	pVCpu->hm.s.vmx.fWasInRealMode = false;
4542	}
4543	}
4544	#endif
4545	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4546	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4547	AssertRCReturn(rc, rc);
4548	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4549	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4550	AssertRCReturn(rc, rc);
4551	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4552	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4553	AssertRCReturn(rc, rc);
4554	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4555	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4556	AssertRCReturn(rc, rc);
4557	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4558	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4559	AssertRCReturn(rc, rc);
4560	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4561	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4562	AssertRCReturn(rc, rc);
4563
4564	#ifdef VBOX_STRICT
4565	/* Validate. */
4566	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4567	#endif
4568
4569	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4570	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4571	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4572	}
4573
4574	/*
4575	* Guest TR.
4576	*/
4577	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4578	{
4579	/*
4580	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4581	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4582	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4583	*/
4584	uint16_t u16Sel = 0;
4585	uint32_t u32Limit = 0;
4586	uint64_t u64Base = 0;
4587	uint32_t u32AccessRights = 0;
4588
4589	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4590	{
4591	u16Sel = pMixedCtx->tr.Sel;
4592	u32Limit = pMixedCtx->tr.u32Limit;
4593	u64Base = pMixedCtx->tr.u64Base;
4594	u32AccessRights = pMixedCtx->tr.Attr.u;
4595	}
4596	else
4597	{
4598	Assert(pVM->hm.s.vmx.pRealModeTSS);
4599	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4600
4601	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4602	RTGCPHYS GCPhys;
4603	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4604	AssertRCReturn(rc, rc);
4605
4606	X86DESCATTR DescAttr;
4607	DescAttr.u = 0;
4608	DescAttr.n.u1Present = 1;
4609	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4610
4611	u16Sel = 0;
4612	u32Limit = HM_VTX_TSS_SIZE;
4613	u64Base = GCPhys; /* in real-mode phys = virt. */
4614	u32AccessRights = DescAttr.u;
4615	}
4616
4617	/* Validate. */
4618	Assert(!(u16Sel & RT_BIT(2)));
4619	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4620	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4621	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4622	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4623	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4624	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4625	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4626	Assert( (u32Limit & 0xfff) == 0xfff
4627	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4628	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4629	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4630
4631	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4632	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4633	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4634	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4635	AssertRCReturn(rc, rc);
4636
4637	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4638	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4639	}
4640
4641	/*
4642	* Guest GDTR.
4643	*/
4644	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4645	{
4646	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4647	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4648	AssertRCReturn(rc, rc);
4649
4650	/* Validate. */
4651	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4652
4653	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4654	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4655	}
4656
4657	/*
4658	* Guest LDTR.
4659	*/
4660	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4661	{
4662	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4663	uint32_t u32Access = 0;
4664	if (!pMixedCtx->ldtr.Attr.u)
4665	u32Access = X86DESCATTR_UNUSABLE;
4666	else
4667	u32Access = pMixedCtx->ldtr.Attr.u;
4668
4669	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4670	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4671	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4673	AssertRCReturn(rc, rc);
4674
4675	/* Validate. */
4676	if (!(u32Access & X86DESCATTR_UNUSABLE))
4677	{
4678	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4679	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4680	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4681	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4682	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4683	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4684	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4685	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4686	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4687	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4688	}
4689
4690	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4691	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4692	}
4693
4694	/*
4695	* Guest IDTR.
4696	*/
4697	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4698	{
4699	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4700	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4701	AssertRCReturn(rc, rc);
4702
4703	/* Validate. */
4704	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4705
4706	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4707	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4708	}
4709
4710	return VINF_SUCCESS;
4711	}
4712
4713
4714	/**
4715	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4716	* areas.
4717	*
4718	* These MSRs will automatically be loaded to the host CPU on every successful
4719	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4720	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4721	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4722	*
4723	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4724	*
4725	* @returns VBox status code.
4726	* @param pVCpu The cross context virtual CPU structure.
4727	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4728	* out-of-sync. Make sure to update the required fields
4729	* before using them.
4730	*
4731	* @remarks No-long-jump zone!!!
4732	*/
4733	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4734	{
4735	AssertPtr(pVCpu);
4736	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4737
4738	/*
4739	* MSRs that we use the auto-load/store MSR area in the VMCS.
4740	*/
4741	PVM pVM = pVCpu->CTX_SUFF(pVM);
4742	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4743	{
4744	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4745	#if HC_ARCH_BITS == 32
4746	if (pVM->hm.s.fAllow64BitGuests)
4747	{
4748	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4749	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4750	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4751	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4752	AssertRCReturn(rc, rc);
4753	# ifdef LOG_ENABLED
4754	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4755	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4756	{
4757	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4758	pMsr->u64Value));
4759	}
4760	# endif
4761	}
4762	#endif
4763	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4764	}
4765
4766	/*
4767	* Guest Sysenter MSRs.
4768	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4769	* VM-exits on WRMSRs for these MSRs.
4770	*/
4771	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4772	{
4773	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4774	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4775	}
4776
4777	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4778	{
4779	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4780	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4781	}
4782
4783	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4784	{
4785	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4786	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4787	}
4788
4789	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4790	{
4791	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4792	{
4793	/*
4794	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4795	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4796	*/
4797	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4798	{
4799	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4800	AssertRCReturn(rc,rc);
4801	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4802	}
4803	else
4804	{
4805	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4806	NULL /* pfAddedAndUpdated */);
4807	AssertRCReturn(rc, rc);
4808
4809	/* We need to intercept reads too, see @bugref{7386#c16}. */
4810	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4811	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4812	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4813	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4814	}
4815	}
4816	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4817	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4818	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4819	}
4820
4821	return VINF_SUCCESS;
4822	}
4823
4824
4825	/**
4826	* Loads the guest activity state into the guest-state area in the VMCS.
4827	*
4828	* @returns VBox status code.
4829	* @param pVCpu The cross context virtual CPU structure.
4830	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4831	* out-of-sync. Make sure to update the required fields
4832	* before using them.
4833	*
4834	* @remarks No-long-jump zone!!!
4835	*/
4836	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837	{
4838	NOREF(pMixedCtx);
4839	/** @todo See if we can make use of other states, e.g.
4840	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4841	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4842	{
4843	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4844	AssertRCReturn(rc, rc);
4845
4846	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4847	}
4848	return VINF_SUCCESS;
4849	}
4850
4851
4852	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4853	/**
4854	* Check if guest state allows safe use of 32-bit switcher again.
4855	*
4856	* Segment bases and protected mode structures must be 32-bit addressable
4857	* because the 32-bit switcher will ignore high dword when writing these VMCS
4858	* fields. See @bugref{8432} for details.
4859	*
4860	* @returns true if safe, false if must continue to use the 64-bit switcher.
4861	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4862	* out-of-sync. Make sure to update the required fields
4863	* before using them.
4864	*
4865	* @remarks No-long-jump zone!!!
4866	*/
4867	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4868	{
4869	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4870	return false;
4871	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4872	return false;
4873	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4874	return false;
4875	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4876	return false;
4877	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4878	return false;
4879	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4880	return false;
4881	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4882	return false;
4883	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4884	return false;
4885	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4886	return false;
4887	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4888	return false;
4889	/* All good, bases are 32-bit. */
4890	return true;
4891	}
4892	#endif
4893
4894
4895	/**
4896	* Sets up the appropriate function to run guest code.
4897	*
4898	* @returns VBox status code.
4899	* @param pVCpu The cross context virtual CPU structure.
4900	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4901	* out-of-sync. Make sure to update the required fields
4902	* before using them.
4903	*
4904	* @remarks No-long-jump zone!!!
4905	*/
4906	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4907	{
4908	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4909	{
4910	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4911	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4912	#endif
4913	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4914	#if HC_ARCH_BITS == 32
4915	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4916	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4917	{
4918	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4919	{
4920	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4921	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4922	\| HM_CHANGED_VMX_ENTRY_CTLS
4923	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4924	}
4925	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4926
4927	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4928	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4929	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4930	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4931	}
4932	#else
4933	/* 64-bit host. */
4934	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4935	#endif
4936	}
4937	else
4938	{
4939	/* Guest is not in long mode, use the 32-bit handler. */
4940	#if HC_ARCH_BITS == 32
4941	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4942	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4943	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4944	{
4945	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4946	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4947	\| HM_CHANGED_VMX_ENTRY_CTLS
4948	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4949	}
4950	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4951	/*
4952	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4953	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4954	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4955	* the much faster 32-bit switcher again.
4956	*/
4957	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4958	{
4959	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4960	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4961	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4962	}
4963	else
4964	{
4965	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4966	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4967	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4968	{
4969	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4970	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4971	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4972	\| HM_CHANGED_VMX_ENTRY_CTLS
4973	\| HM_CHANGED_VMX_EXIT_CTLS
4974	\| HM_CHANGED_HOST_CONTEXT);
4975	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4976	}
4977	}
4978	# else
4979	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4980	# endif
4981	#else
4982	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4983	#endif
4984	}
4985	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4986	return VINF_SUCCESS;
4987	}
4988
4989
4990	/**
4991	* Wrapper for running the guest code in VT-x.
4992	*
4993	* @returns VBox status code, no informational status codes.
4994	* @param pVM The cross context VM structure.
4995	* @param pVCpu The cross context virtual CPU structure.
4996	* @param pCtx Pointer to the guest-CPU context.
4997	*
4998	* @remarks No-long-jump zone!!!
4999	*/
5000	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5001	{
5002	/*
5003	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5004	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5005	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5006	*/
5007	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5008	/** @todo Add stats for resume vs launch. */
5009	#ifdef VBOX_WITH_KERNEL_USING_XMM
5010	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5011	#else
5012	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5013	#endif
5014	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5015	return rc;
5016	}
5017
5018
5019	/**
5020	* Reports world-switch error and dumps some useful debug info.
5021	*
5022	* @param pVM The cross context VM structure.
5023	* @param pVCpu The cross context virtual CPU structure.
5024	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5025	* @param pCtx Pointer to the guest-CPU context.
5026	* @param pVmxTransient Pointer to the VMX transient structure (only
5027	* exitReason updated).
5028	*/
5029	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5030	{
5031	Assert(pVM);
5032	Assert(pVCpu);
5033	Assert(pCtx);
5034	Assert(pVmxTransient);
5035	HMVMX_ASSERT_PREEMPT_SAFE();
5036
5037	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5038	switch (rcVMRun)
5039	{
5040	case VERR_VMX_INVALID_VMXON_PTR:
5041	AssertFailed();
5042	break;
5043	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5044	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5045	{
5046	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5047	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5048	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5049	AssertRC(rc);
5050
5051	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5052	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5053	Cannot do it here as we may have been long preempted. */
5054
5055	#ifdef VBOX_STRICT
5056	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5057	pVmxTransient->uExitReason));
5058	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5059	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5060	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5061	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5062	else
5063	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5064	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5065	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5066
5067	/* VMX control bits. */
5068	uint32_t u32Val;
5069	uint64_t u64Val;
5070	RTHCUINTREG uHCReg;
5071	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5072	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5074	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5075	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5076	{
5077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5078	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5079	}
5080	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5081	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5082	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5083	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5084	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5085	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5087	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5088	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5089	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5090	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5091	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5092	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5093	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5095	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5097	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5099	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5101	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5103	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5105	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5106	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5107	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5108	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5109	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5110	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5111	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5112	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5113	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5114	if (pVM->hm.s.fNestedPaging)
5115	{
5116	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5117	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5118	}
5119
5120	/* Guest bits. */
5121	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5122	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5123	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5124	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5125	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5126	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5127	if (pVM->hm.s.vmx.fVpid)
5128	{
5129	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5130	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5131	}
5132
5133	/* Host bits. */
5134	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5135	Log4(("Host CR0 %#RHr\n", uHCReg));
5136	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5137	Log4(("Host CR3 %#RHr\n", uHCReg));
5138	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5139	Log4(("Host CR4 %#RHr\n", uHCReg));
5140
5141	RTGDTR HostGdtr;
5142	PCX86DESCHC pDesc;
5143	ASMGetGDTR(&HostGdtr);
5144	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5145	Log4(("Host CS %#08x\n", u32Val));
5146	if (u32Val < HostGdtr.cbGdt)
5147	{
5148	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5149	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5150	}
5151
5152	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5153	Log4(("Host DS %#08x\n", u32Val));
5154	if (u32Val < HostGdtr.cbGdt)
5155	{
5156	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5157	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5158	}
5159
5160	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5161	Log4(("Host ES %#08x\n", u32Val));
5162	if (u32Val < HostGdtr.cbGdt)
5163	{
5164	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5165	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5166	}
5167
5168	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5169	Log4(("Host FS %#08x\n", u32Val));
5170	if (u32Val < HostGdtr.cbGdt)
5171	{
5172	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5173	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5174	}
5175
5176	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5177	Log4(("Host GS %#08x\n", u32Val));
5178	if (u32Val < HostGdtr.cbGdt)
5179	{
5180	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5181	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5182	}
5183
5184	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5185	Log4(("Host SS %#08x\n", u32Val));
5186	if (u32Val < HostGdtr.cbGdt)
5187	{
5188	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5189	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5190	}
5191
5192	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5193	Log4(("Host TR %#08x\n", u32Val));
5194	if (u32Val < HostGdtr.cbGdt)
5195	{
5196	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5197	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5198	}
5199
5200	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5201	Log4(("Host TR Base %#RHv\n", uHCReg));
5202	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5203	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5204	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5205	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5206	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5207	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5208	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5209	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5210	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5211	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5212	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5213	Log4(("Host RSP %#RHv\n", uHCReg));
5214	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5215	Log4(("Host RIP %#RHv\n", uHCReg));
5216	# if HC_ARCH_BITS == 64
5217	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5218	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5219	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5220	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5221	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5222	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5223	# endif
5224	#endif /* VBOX_STRICT */
5225	break;
5226	}
5227
5228	default:
5229	/* Impossible */
5230	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5231	break;
5232	}
5233	NOREF(pVM); NOREF(pCtx);
5234	}
5235
5236
5237	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5238	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5239	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5240	#endif
5241	#ifdef VBOX_STRICT
5242	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5243	{
5244	switch (idxField)
5245	{
5246	case VMX_VMCS_GUEST_RIP:
5247	case VMX_VMCS_GUEST_RSP:
5248	case VMX_VMCS_GUEST_SYSENTER_EIP:
5249	case VMX_VMCS_GUEST_SYSENTER_ESP:
5250	case VMX_VMCS_GUEST_GDTR_BASE:
5251	case VMX_VMCS_GUEST_IDTR_BASE:
5252	case VMX_VMCS_GUEST_CS_BASE:
5253	case VMX_VMCS_GUEST_DS_BASE:
5254	case VMX_VMCS_GUEST_ES_BASE:
5255	case VMX_VMCS_GUEST_FS_BASE:
5256	case VMX_VMCS_GUEST_GS_BASE:
5257	case VMX_VMCS_GUEST_SS_BASE:
5258	case VMX_VMCS_GUEST_LDTR_BASE:
5259	case VMX_VMCS_GUEST_TR_BASE:
5260	case VMX_VMCS_GUEST_CR3:
5261	return true;
5262	}
5263	return false;
5264	}
5265
5266	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5267	{
5268	switch (idxField)
5269	{
5270	/* Read-only fields. */
5271	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5272	return true;
5273	}
5274	/* Remaining readable fields should also be writable. */
5275	return hmR0VmxIsValidWriteField(idxField);
5276	}
5277	#endif /* VBOX_STRICT */
5278
5279
5280	/**
5281	* Executes the specified handler in 64-bit mode.
5282	*
5283	* @returns VBox status code (no informational status codes).
5284	* @param pVM The cross context VM structure.
5285	* @param pVCpu The cross context virtual CPU structure.
5286	* @param pCtx Pointer to the guest CPU context.
5287	* @param enmOp The operation to perform.
5288	* @param cParams Number of parameters.
5289	* @param paParam Array of 32-bit parameters.
5290	*/
5291	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5292	uint32_t cParams, uint32_t *paParam)
5293	{
5294	NOREF(pCtx);
5295
5296	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5297	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5298	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5299	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5300
5301	#ifdef VBOX_STRICT
5302	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5303	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5304
5305	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5306	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5307	#endif
5308
5309	/* Disable interrupts. */
5310	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5311
5312	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5313	RTCPUID idHostCpu = RTMpCpuId();
5314	CPUMR0SetLApic(pVCpu, idHostCpu);
5315	#endif
5316
5317	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5318	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5319
5320	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5321	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5322	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5323
5324	/* Leave VMX Root Mode. */
5325	VMXDisable();
5326
5327	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5328
5329	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5330	CPUMSetHyperEIP(pVCpu, enmOp);
5331	for (int i = (int)cParams - 1; i >= 0; i--)
5332	CPUMPushHyper(pVCpu, paParam[i]);
5333
5334	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5335
5336	/* Call the switcher. */
5337	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5338	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5339
5340	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5341	/* Make sure the VMX instructions don't cause #UD faults. */
5342	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5343
5344	/* Re-enter VMX Root Mode */
5345	int rc2 = VMXEnable(HCPhysCpuPage);
5346	if (RT_FAILURE(rc2))
5347	{
5348	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5349	ASMSetFlags(fOldEFlags);
5350	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5351	return rc2;
5352	}
5353
5354	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5355	AssertRC(rc2);
5356	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5357	Assert(!(ASMGetFlags() & X86_EFL_IF));
5358	ASMSetFlags(fOldEFlags);
5359	return rc;
5360	}
5361
5362
5363	/**
5364	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5365	* supporting 64-bit guests.
5366	*
5367	* @returns VBox status code.
5368	* @param fResume Whether to VMLAUNCH or VMRESUME.
5369	* @param pCtx Pointer to the guest-CPU context.
5370	* @param pCache Pointer to the VMCS cache.
5371	* @param pVM The cross context VM structure.
5372	* @param pVCpu The cross context virtual CPU structure.
5373	*/
5374	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5375	{
5376	NOREF(fResume);
5377
5378	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5379	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5380
5381	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5382	pCache->uPos = 1;
5383	pCache->interPD = PGMGetInterPaeCR3(pVM);
5384	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5385	#endif
5386
5387	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5388	pCache->TestIn.HCPhysCpuPage = 0;
5389	pCache->TestIn.HCPhysVmcs = 0;
5390	pCache->TestIn.pCache = 0;
5391	pCache->TestOut.HCPhysVmcs = 0;
5392	pCache->TestOut.pCache = 0;
5393	pCache->TestOut.pCtx = 0;
5394	pCache->TestOut.eflags = 0;
5395	#else
5396	NOREF(pCache);
5397	#endif
5398
5399	uint32_t aParam[10];
5400	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5401	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5402	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5403	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5404	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5405	aParam[5] = 0;
5406	aParam[6] = VM_RC_ADDR(pVM, pVM);
5407	aParam[7] = 0;
5408	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5409	aParam[9] = 0;
5410
5411	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5412	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5413	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5414	#endif
5415	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5416
5417	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5418	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5419	Assert(pCtx->dr[4] == 10);
5420	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5421	#endif
5422
5423	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5424	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5425	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5426	pVCpu->hm.s.vmx.HCPhysVmcs));
5427	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5428	pCache->TestOut.HCPhysVmcs));
5429	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5430	pCache->TestOut.pCache));
5431	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5432	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5433	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5434	pCache->TestOut.pCtx));
5435	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5436	#endif
5437	return rc;
5438	}
5439
5440
5441	/**
5442	* Initialize the VMCS-Read cache.
5443	*
5444	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5445	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5446	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5447	* (those that have a 32-bit FULL & HIGH part).
5448	*
5449	* @returns VBox status code.
5450	* @param pVM The cross context VM structure.
5451	* @param pVCpu The cross context virtual CPU structure.
5452	*/
5453	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5454	{
5455	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5456	{ \
5457	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5458	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5459	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5460	++cReadFields; \
5461	}
5462
5463	AssertPtr(pVM);
5464	AssertPtr(pVCpu);
5465	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5466	uint32_t cReadFields = 0;
5467
5468	/*
5469	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5470	* and serve to indicate exceptions to the rules.
5471	*/
5472
5473	/* Guest-natural selector base fields. */
5474	#if 0
5475	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5476	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5477	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5478	#endif
5479	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5480	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5481	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5482	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5483	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5484	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5485	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5486	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5487	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5488	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5489	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5490	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5491	#if 0
5492	/* Unused natural width guest-state fields. */
5493	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5494	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5495	#endif
5496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5498
5499	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5500	#if 0
5501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5510	#endif
5511
5512	/* Natural width guest-state fields. */
5513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5514	#if 0
5515	/* Currently unused field. */
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5517	#endif
5518
5519	if (pVM->hm.s.fNestedPaging)
5520	{
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5522	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5523	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5524	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5525	}
5526	else
5527	{
5528	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5529	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5530	}
5531
5532	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5533	return VINF_SUCCESS;
5534	}
5535
5536
5537	/**
5538	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5539	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5540	* darwin, running 64-bit guests).
5541	*
5542	* @returns VBox status code.
5543	* @param pVCpu The cross context virtual CPU structure.
5544	* @param idxField The VMCS field encoding.
5545	* @param u64Val 16, 32 or 64-bit value.
5546	*/
5547	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5548	{
5549	int rc;
5550	switch (idxField)
5551	{
5552	/*
5553	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5554	*/
5555	/* 64-bit Control fields. */
5556	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5557	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5558	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5559	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5560	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5561	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5562	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5563	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5564	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5565	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5566	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5567	case VMX_VMCS64_CTRL_EPTP_FULL:
5568	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5569	/* 64-bit Guest-state fields. */
5570	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5571	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5572	case VMX_VMCS64_GUEST_PAT_FULL:
5573	case VMX_VMCS64_GUEST_EFER_FULL:
5574	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5575	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5576	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5577	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5578	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5579	/* 64-bit Host-state fields. */
5580	case VMX_VMCS64_HOST_PAT_FULL:
5581	case VMX_VMCS64_HOST_EFER_FULL:
5582	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5583	{
5584	rc = VMXWriteVmcs32(idxField, u64Val);
5585	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5586	break;
5587	}
5588
5589	/*
5590	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5591	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5592	*/
5593	/* Natural-width Guest-state fields. */
5594	case VMX_VMCS_GUEST_CR3:
5595	case VMX_VMCS_GUEST_ES_BASE:
5596	case VMX_VMCS_GUEST_CS_BASE:
5597	case VMX_VMCS_GUEST_SS_BASE:
5598	case VMX_VMCS_GUEST_DS_BASE:
5599	case VMX_VMCS_GUEST_FS_BASE:
5600	case VMX_VMCS_GUEST_GS_BASE:
5601	case VMX_VMCS_GUEST_LDTR_BASE:
5602	case VMX_VMCS_GUEST_TR_BASE:
5603	case VMX_VMCS_GUEST_GDTR_BASE:
5604	case VMX_VMCS_GUEST_IDTR_BASE:
5605	case VMX_VMCS_GUEST_RSP:
5606	case VMX_VMCS_GUEST_RIP:
5607	case VMX_VMCS_GUEST_SYSENTER_ESP:
5608	case VMX_VMCS_GUEST_SYSENTER_EIP:
5609	{
5610	if (!(u64Val >> 32))
5611	{
5612	/* If this field is 64-bit, VT-x will zero out the top bits. */
5613	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5614	}
5615	else
5616	{
5617	/* Assert that only the 32->64 switcher case should ever come here. */
5618	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5619	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5620	}
5621	break;
5622	}
5623
5624	default:
5625	{
5626	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5627	rc = VERR_INVALID_PARAMETER;
5628	break;
5629	}
5630	}
5631	AssertRCReturn(rc, rc);
5632	return rc;
5633	}
5634
5635
5636	/**
5637	* Queue up a VMWRITE by using the VMCS write cache.
5638	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5639	*
5640	* @param pVCpu The cross context virtual CPU structure.
5641	* @param idxField The VMCS field encoding.
5642	* @param u64Val 16, 32 or 64-bit value.
5643	*/
5644	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5645	{
5646	AssertPtr(pVCpu);
5647	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5648
5649	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5650	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5651
5652	/* Make sure there are no duplicates. */
5653	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5654	{
5655	if (pCache->Write.aField[i] == idxField)
5656	{
5657	pCache->Write.aFieldVal[i] = u64Val;
5658	return VINF_SUCCESS;
5659	}
5660	}
5661
5662	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5663	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5664	pCache->Write.cValidEntries++;
5665	return VINF_SUCCESS;
5666	}
5667	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5668
5669
5670	/**
5671	* Sets up the usage of TSC-offsetting and updates the VMCS.
5672	*
5673	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5674	* VMX preemption timer.
5675	*
5676	* @returns VBox status code.
5677	* @param pVM The cross context VM structure.
5678	* @param pVCpu The cross context virtual CPU structure.
5679	*
5680	* @remarks No-long-jump zone!!!
5681	*/
5682	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5683	{
5684	int rc;
5685	bool fOffsettedTsc;
5686	bool fParavirtTsc;
5687	if (pVM->hm.s.vmx.fUsePreemptTimer)
5688	{
5689	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5690	&fOffsettedTsc, &fParavirtTsc);
5691
5692	/* Make sure the returned values have sane upper and lower boundaries. */
5693	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5694	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5695	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5696	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5697
5698	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5699	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5700	}
5701	else
5702	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5703
5704	/** @todo later optimize this to be done elsewhere and not before every
5705	* VM-entry. */
5706	if (fParavirtTsc)
5707	{
5708	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5709	information before every VM-entry, hence disable it for performance sake. */
5710	#if 0
5711	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5712	AssertRC(rc);
5713	#endif
5714	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5715	}
5716
5717	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5718	{
5719	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5720	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5721
5722	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5723	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5724	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5725	}
5726	else
5727	{
5728	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5729	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5731	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5732	}
5733	}
5734
5735
5736	/**
5737	* Determines if an exception is a contributory exception.
5738	*
5739	* Contributory exceptions are ones which can cause double-faults unless the
5740	* original exception was a benign exception. Page-fault is intentionally not
5741	* included here as it's a conditional contributory exception.
5742	*
5743	* @returns true if the exception is contributory, false otherwise.
5744	* @param uVector The exception vector.
5745	*/
5746	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5747	{
5748	switch (uVector)
5749	{
5750	case X86_XCPT_GP:
5751	case X86_XCPT_SS:
5752	case X86_XCPT_NP:
5753	case X86_XCPT_TS:
5754	case X86_XCPT_DE:
5755	return true;
5756	default:
5757	break;
5758	}
5759	return false;
5760	}
5761
5762
5763	/**
5764	* Sets an event as a pending event to be injected into the guest.
5765	*
5766	* @param pVCpu The cross context virtual CPU structure.
5767	* @param u32IntInfo The VM-entry interruption-information field.
5768	* @param cbInstr The VM-entry instruction length in bytes (for software
5769	* interrupts, exceptions and privileged software
5770	* exceptions).
5771	* @param u32ErrCode The VM-entry exception error code.
5772	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5773	* page-fault.
5774	*
5775	* @remarks Statistics counter assumes this is a guest event being injected or
5776	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5777	* always incremented.
5778	*/
5779	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5780	RTGCUINTPTR GCPtrFaultAddress)
5781	{
5782	Assert(!pVCpu->hm.s.Event.fPending);
5783	pVCpu->hm.s.Event.fPending = true;
5784	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5785	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5786	pVCpu->hm.s.Event.cbInstr = cbInstr;
5787	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5788	}
5789
5790
5791	/**
5792	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5793	*
5794	* @param pVCpu The cross context virtual CPU structure.
5795	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5796	* out-of-sync. Make sure to update the required fields
5797	* before using them.
5798	*/
5799	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5800	{
5801	NOREF(pMixedCtx);
5802	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5803	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5804	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5805	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5806	}
5807
5808
5809	/**
5810	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5811	* VM-exit interruption info type.
5812	*
5813	* @returns The IEM exception flags.
5814	* @param uVector The event vector.
5815	* @param uVmxVectorType The VMX event type.
5816	*/
5817	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5818	{
5819	uint32_t fIemXcptFlags;
5820	switch (uVmxVectorType)
5821	{
5822	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5823	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5824	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5825	break;
5826
5827	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5828	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5829	break;
5830
5831	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5832	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5833	break;
5834
5835	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5836	{
5837	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5838	if (uVector == X86_XCPT_BP)
5839	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5840	else if (uVector == X86_XCPT_OF)
5841	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5842	else
5843	{
5844	fIemXcptFlags = 0;
5845	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5846	}
5847	break;
5848	}
5849
5850	default:
5851	fIemXcptFlags = 0;
5852	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5853	break;
5854	}
5855	return fIemXcptFlags;
5856	}
5857
5858
5859	/**
5860	* Handle a condition that occurred while delivering an event through the guest
5861	* IDT.
5862	*
5863	* @returns Strict VBox status code (i.e. informational status codes too).
5864	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5865	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5866	* to continue execution of the guest which will delivery the \#DF.
5867	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5868	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5869	*
5870	* @param pVCpu The cross context virtual CPU structure.
5871	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5872	* out-of-sync. Make sure to update the required fields
5873	* before using them.
5874	* @param pVmxTransient Pointer to the VMX transient structure.
5875	*
5876	* @remarks No-long-jump zone!!!
5877	*/
5878	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5879	{
5880	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5881
5882	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5883	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5884
5885	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5886	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5887	{
5888	uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5889	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5890	#if 1
5891	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5892	IEMXCPTRAISE enmRaise;
5893	IEMXCPTRAISEINFO fRaiseInfo;
5894	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5895	{
5896	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5897	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5898	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5899	AssertMsg( uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
5900	\|\| uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI,
5901	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. type %#x!\n", uExitVectorType));
5902	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector,
5903	&fRaiseInfo);
5904	}
5905	else
5906	{
5907	/*
5908	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5909	* interruption-information will not be valid as it's not an exception and we end up here.
5910	*
5911	* If the event was an external interrupt or hardare exception (incl. NMI) it is sufficient to
5912	* reflect this event to the guest after handling the VM-exit.
5913	*
5914	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5915	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5916	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5917	* event, as that can cause premature trips to ring-3 before injection and involve TRPM which
5918	* currently has no way of storing that the exceptions were caused by these special instructions.
5919	*/
5920	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5921	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5922	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT)
5923	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5924	else
5925	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5926	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5927	}
5928
5929	/*
5930	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5931	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5932	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5933	* subsequent VM-entry would fail.
5934	*
5935	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5936	*/
5937	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5938	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5939	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5940	\|\| fRaiseInfo == IEMXCPTRAISEINFO_NMI_PF)
5941	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5942	{
5943	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5944	}
5945
5946	switch (enmRaise)
5947	{
5948	case IEMXCPTRAISE_CURRENT_XCPT:
5949	{
5950	/*
5951	* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF().
5952	*/
5953	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5954	pVmxTransient->fVectoringPF = true;
5955
5956	/*
5957	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5958	* second #PF as a guest #PF (and not a nested #PF) and needs to be converted into a #DF.
5959	*/
5960	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5961	pVmxTransient->fVectoringDoublePF = true;
5962
5963	Assert(rcStrict == VINF_SUCCESS);
5964	break;
5965	}
5966
5967	case IEMXCPTRAISE_PREV_EVENT:
5968	{
5969	/*
5970	* Re-raise the previous (first) exception/interrupt as delivery caused a premature VM-exit.
5971	*/
5972	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5973	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5974	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5975
5976	uint32_t u32ErrCode;
5977	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5978	{
5979	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5980	AssertRCReturn(rc2, rc2);
5981	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5982	}
5983	else
5984	u32ErrCode = 0;
5985
5986	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5987	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5988	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5989	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5990
5991	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
5992	pVCpu->hm.s.Event.u32ErrCode));
5993	Assert(rcStrict == VINF_SUCCESS);
5994	break;
5995	}
5996
5997	case IEMXCPTRAISE_DOUBLE_FAULT:
5998	{
5999	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6000	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6001
6002	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6003	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6004	rcStrict = VINF_HM_DOUBLE_FAULT;
6005	break;
6006	}
6007
6008	case IEMXCPTRAISE_TRIPLE_FAULT:
6009	{
6010	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6011	uExitVector));
6012	rcStrict = VINF_EM_RESET;
6013	break;
6014	}
6015
6016	case IEMXCPTRAISE_CPU_HANG:
6017	{
6018	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6019	rcStrict = VERR_EM_GUEST_CPU_HANG;
6020	break;
6021	}
6022
6023	case IEMXCPTRAISE_REEXEC_INSTR:
6024	Assert(rcStrict == VINF_SUCCESS);
6025	break;
6026
6027	default:
6028	{
6029	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6030	rcStrict = VERR_VMX_IPE_2;
6031	break;
6032	}
6033	}
6034	#else
6035	typedef enum
6036	{
6037	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6038	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6039	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6040	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6041	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6042	} VMXREFLECTXCPT;
6043
6044	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6045	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6046	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6047	{
6048	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6049	{
6050	enmReflect = VMXREFLECTXCPT_XCPT;
6051	#ifdef VBOX_STRICT
6052	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6053	&& uExitVector == X86_XCPT_PF)
6054	{
6055	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6056	}
6057	#endif
6058	if ( uExitVector == X86_XCPT_PF
6059	&& uIdtVector == X86_XCPT_PF)
6060	{
6061	pVmxTransient->fVectoringDoublePF = true;
6062	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6063	}
6064	else if ( uExitVector == X86_XCPT_AC
6065	&& uIdtVector == X86_XCPT_AC)
6066	{
6067	enmReflect = VMXREFLECTXCPT_HANG;
6068	Log4(("IDT: Nested #AC - Bad guest\n"));
6069	}
6070	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6071	&& hmR0VmxIsContributoryXcpt(uExitVector)
6072	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6073	\|\| uIdtVector == X86_XCPT_PF))
6074	{
6075	enmReflect = VMXREFLECTXCPT_DF;
6076	}
6077	else if (uIdtVector == X86_XCPT_DF)
6078	enmReflect = VMXREFLECTXCPT_TF;
6079	}
6080	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6081	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6082	{
6083	/*
6084	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6085	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6086	*/
6087	enmReflect = VMXREFLECTXCPT_XCPT;
6088
6089	if (uExitVector == X86_XCPT_PF)
6090	{
6091	pVmxTransient->fVectoringPF = true;
6092	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6093	}
6094	}
6095	}
6096	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6097	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6098	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6099	{
6100	/*
6101	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6102	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6103	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6104	*/
6105	enmReflect = VMXREFLECTXCPT_XCPT;
6106	}
6107
6108	/*
6109	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6110	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6111	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6112	*
6113	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6114	*/
6115	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6116	&& enmReflect == VMXREFLECTXCPT_XCPT
6117	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6118	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6119	{
6120	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6121	}
6122
6123	switch (enmReflect)
6124	{
6125	case VMXREFLECTXCPT_XCPT:
6126	{
6127	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6128	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6129	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6130
6131	uint32_t u32ErrCode = 0;
6132	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6133	{
6134	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6135	AssertRCReturn(rc2, rc2);
6136	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6137	}
6138
6139	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6140	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6141	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6142	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6143	rcStrict = VINF_SUCCESS;
6144	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6145	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6146
6147	break;
6148	}
6149
6150	case VMXREFLECTXCPT_DF:
6151	{
6152	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6153	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6154	rcStrict = VINF_HM_DOUBLE_FAULT;
6155	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6156	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6157
6158	break;
6159	}
6160
6161	case VMXREFLECTXCPT_TF:
6162	{
6163	rcStrict = VINF_EM_RESET;
6164	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6165	uExitVector));
6166	break;
6167	}
6168
6169	case VMXREFLECTXCPT_HANG:
6170	{
6171	rcStrict = VERR_EM_GUEST_CPU_HANG;
6172	break;
6173	}
6174
6175	default:
6176	Assert(rcStrict == VINF_SUCCESS);
6177	break;
6178	}
6179	#endif
6180	}
6181	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6182	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6183	&& uExitVector != X86_XCPT_DF
6184	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6185	{
6186	/*
6187	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6188	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6189	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6190	*/
6191	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6192	{
6193	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6194	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6195	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6196	}
6197	}
6198
6199	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6200	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6201	return rcStrict;
6202	}
6203
6204
6205	/**
6206	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6207	*
6208	* @returns VBox status code.
6209	* @param pVCpu The cross context virtual CPU structure.
6210	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6211	* out-of-sync. Make sure to update the required fields
6212	* before using them.
6213	*
6214	* @remarks No-long-jump zone!!!
6215	*/
6216	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6217	{
6218	NOREF(pMixedCtx);
6219
6220	/*
6221	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6222	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6223	*/
6224	VMMRZCallRing3Disable(pVCpu);
6225	HM_DISABLE_PREEMPT();
6226
6227	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6228	{
6229	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6230	* and 'dbgc-init' containing:
6231	* sxe "xcpt_de"
6232	* sxe "xcpt_bp"
6233	* sxi "xcpt_gp"
6234	* sxi "xcpt_ss"
6235	* sxi "xcpt_np"
6236	*/
6237	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6238	#endif
6239	uint32_t uVal = 0;
6240	uint32_t uShadow = 0;
6241	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6242	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6243	AssertRCReturn(rc, rc);
6244
6245	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6246	CPUMSetGuestCR0(pVCpu, uVal);
6247	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6248	}
6249
6250	HM_RESTORE_PREEMPT();
6251	VMMRZCallRing3Enable(pVCpu);
6252	return VINF_SUCCESS;
6253	}
6254
6255
6256	/**
6257	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6258	*
6259	* @returns VBox status code.
6260	* @param pVCpu The cross context virtual CPU structure.
6261	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6262	* out-of-sync. Make sure to update the required fields
6263	* before using them.
6264	*
6265	* @remarks No-long-jump zone!!!
6266	*/
6267	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6268	{
6269	NOREF(pMixedCtx);
6270
6271	int rc = VINF_SUCCESS;
6272	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6273	{
6274	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6275	uint32_t uVal = 0;
6276	uint32_t uShadow = 0;
6277	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6278	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6279	AssertRCReturn(rc, rc);
6280
6281	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6282	CPUMSetGuestCR4(pVCpu, uVal);
6283	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6284	}
6285	return rc;
6286	}
6287
6288
6289	/**
6290	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6291	*
6292	* @returns VBox status code.
6293	* @param pVCpu The cross context virtual CPU structure.
6294	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6295	* out-of-sync. Make sure to update the required fields
6296	* before using them.
6297	*
6298	* @remarks No-long-jump zone!!!
6299	*/
6300	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6301	{
6302	int rc = VINF_SUCCESS;
6303	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6304	{
6305	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6306	uint64_t u64Val = 0;
6307	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6308	AssertRCReturn(rc, rc);
6309
6310	pMixedCtx->rip = u64Val;
6311	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6312	}
6313	return rc;
6314	}
6315
6316
6317	/**
6318	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6319	*
6320	* @returns VBox status code.
6321	* @param pVCpu The cross context virtual CPU structure.
6322	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6323	* out-of-sync. Make sure to update the required fields
6324	* before using them.
6325	*
6326	* @remarks No-long-jump zone!!!
6327	*/
6328	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6329	{
6330	int rc = VINF_SUCCESS;
6331	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6332	{
6333	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6334	uint64_t u64Val = 0;
6335	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6336	AssertRCReturn(rc, rc);
6337
6338	pMixedCtx->rsp = u64Val;
6339	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6340	}
6341	return rc;
6342	}
6343
6344
6345	/**
6346	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6347	*
6348	* @returns VBox status code.
6349	* @param pVCpu The cross context virtual CPU structure.
6350	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6351	* out-of-sync. Make sure to update the required fields
6352	* before using them.
6353	*
6354	* @remarks No-long-jump zone!!!
6355	*/
6356	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6357	{
6358	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6359	{
6360	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6361	uint32_t uVal = 0;
6362	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6363	AssertRCReturn(rc, rc);
6364
6365	pMixedCtx->eflags.u32 = uVal;
6366	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6367	{
6368	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6369	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6370
6371	pMixedCtx->eflags.Bits.u1VM = 0;
6372	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6373	}
6374
6375	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6376	}
6377	return VINF_SUCCESS;
6378	}
6379
6380
6381	/**
6382	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6383	* guest-CPU context.
6384	*/
6385	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6386	{
6387	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6388	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6389	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6390	return rc;
6391	}
6392
6393
6394	/**
6395	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6396	* from the guest-state area in the VMCS.
6397	*
6398	* @param pVCpu The cross context virtual CPU structure.
6399	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6400	* out-of-sync. Make sure to update the required fields
6401	* before using them.
6402	*
6403	* @remarks No-long-jump zone!!!
6404	*/
6405	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6406	{
6407	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6408	{
6409	uint32_t uIntrState = 0;
6410	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6411	AssertRC(rc);
6412
6413	if (!uIntrState)
6414	{
6415	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6416	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6417
6418	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6419	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6420	}
6421	else
6422	{
6423	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6424	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6425	{
6426	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6427	AssertRC(rc);
6428	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6429	AssertRC(rc);
6430
6431	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6432	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6433	}
6434	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6435	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6436
6437	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6438	{
6439	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6440	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6441	}
6442	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6443	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6444	}
6445
6446	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6447	}
6448	}
6449
6450
6451	/**
6452	* Saves the guest's activity state.
6453	*
6454	* @returns VBox status code.
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6457	* out-of-sync. Make sure to update the required fields
6458	* before using them.
6459	*
6460	* @remarks No-long-jump zone!!!
6461	*/
6462	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6463	{
6464	NOREF(pMixedCtx);
6465	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6466	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6467	return VINF_SUCCESS;
6468	}
6469
6470
6471	/**
6472	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6473	* the current VMCS into the guest-CPU context.
6474	*
6475	* @returns VBox status code.
6476	* @param pVCpu The cross context virtual CPU structure.
6477	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6478	* out-of-sync. Make sure to update the required fields
6479	* before using them.
6480	*
6481	* @remarks No-long-jump zone!!!
6482	*/
6483	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6484	{
6485	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6486	{
6487	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6488	uint32_t u32Val = 0;
6489	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6490	pMixedCtx->SysEnter.cs = u32Val;
6491	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6492	}
6493
6494	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6495	{
6496	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6497	uint64_t u64Val = 0;
6498	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6499	pMixedCtx->SysEnter.eip = u64Val;
6500	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6501	}
6502	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6503	{
6504	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6505	uint64_t u64Val = 0;
6506	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6507	pMixedCtx->SysEnter.esp = u64Val;
6508	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6509	}
6510	return VINF_SUCCESS;
6511	}
6512
6513
6514	/**
6515	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6516	* the CPU back into the guest-CPU context.
6517	*
6518	* @returns VBox status code.
6519	* @param pVCpu The cross context virtual CPU structure.
6520	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6521	* out-of-sync. Make sure to update the required fields
6522	* before using them.
6523	*
6524	* @remarks No-long-jump zone!!!
6525	*/
6526	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6527	{
6528	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6529	VMMRZCallRing3Disable(pVCpu);
6530	HM_DISABLE_PREEMPT();
6531
6532	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6533	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6534	{
6535	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6536	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6537	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6538	}
6539
6540	HM_RESTORE_PREEMPT();
6541	VMMRZCallRing3Enable(pVCpu);
6542
6543	return VINF_SUCCESS;
6544	}
6545
6546
6547	/**
6548	* Saves the auto load/store'd guest MSRs from the current VMCS into
6549	* the guest-CPU context.
6550	*
6551	* @returns VBox status code.
6552	* @param pVCpu The cross context virtual CPU structure.
6553	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6554	* out-of-sync. Make sure to update the required fields
6555	* before using them.
6556	*
6557	* @remarks No-long-jump zone!!!
6558	*/
6559	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6560	{
6561	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6562	return VINF_SUCCESS;
6563
6564	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6565	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6566	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6567	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6568	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6569	{
6570	switch (pMsr->u32Msr)
6571	{
6572	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6573	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6574	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6575	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6576	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6577	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6578	break;
6579
6580	default:
6581	{
6582	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6583	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6584	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6585	}
6586	}
6587	}
6588
6589	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6590	return VINF_SUCCESS;
6591	}
6592
6593
6594	/**
6595	* Saves the guest control registers from the current VMCS into the guest-CPU
6596	* context.
6597	*
6598	* @returns VBox status code.
6599	* @param pVCpu The cross context virtual CPU structure.
6600	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6601	* out-of-sync. Make sure to update the required fields
6602	* before using them.
6603	*
6604	* @remarks No-long-jump zone!!!
6605	*/
6606	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6607	{
6608	/* Guest CR0. Guest FPU. */
6609	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6610	AssertRCReturn(rc, rc);
6611
6612	/* Guest CR4. */
6613	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6614	AssertRCReturn(rc, rc);
6615
6616	/* Guest CR2 - updated always during the world-switch or in #PF. */
6617	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6618	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6619	{
6620	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6621	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6622	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6623
6624	PVM pVM = pVCpu->CTX_SUFF(pVM);
6625	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6626	\|\| ( pVM->hm.s.fNestedPaging
6627	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6628	{
6629	uint64_t u64Val = 0;
6630	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6631	if (pMixedCtx->cr3 != u64Val)
6632	{
6633	CPUMSetGuestCR3(pVCpu, u64Val);
6634	if (VMMRZCallRing3IsEnabled(pVCpu))
6635	{
6636	PGMUpdateCR3(pVCpu, u64Val);
6637	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6638	}
6639	else
6640	{
6641	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6642	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6643	}
6644	}
6645
6646	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6647	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6648	{
6649	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6650	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6651	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6652	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6653	AssertRCReturn(rc, rc);
6654
6655	if (VMMRZCallRing3IsEnabled(pVCpu))
6656	{
6657	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6658	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6659	}
6660	else
6661	{
6662	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6663	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6664	}
6665	}
6666	}
6667
6668	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6669	}
6670
6671	/*
6672	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6673	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6674	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6675	*
6676	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6677	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6678	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6679	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6680	*
6681	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6682	*/
6683	if (VMMRZCallRing3IsEnabled(pVCpu))
6684	{
6685	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6686	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6687
6688	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6689	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6690
6691	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6692	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6693	}
6694
6695	return rc;
6696	}
6697
6698
6699	/**
6700	* Reads a guest segment register from the current VMCS into the guest-CPU
6701	* context.
6702	*
6703	* @returns VBox status code.
6704	* @param pVCpu The cross context virtual CPU structure.
6705	* @param idxSel Index of the selector in the VMCS.
6706	* @param idxLimit Index of the segment limit in the VMCS.
6707	* @param idxBase Index of the segment base in the VMCS.
6708	* @param idxAccess Index of the access rights of the segment in the VMCS.
6709	* @param pSelReg Pointer to the segment selector.
6710	*
6711	* @remarks No-long-jump zone!!!
6712	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6713	* macro as that takes care of whether to read from the VMCS cache or
6714	* not.
6715	*/
6716	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6717	PCPUMSELREG pSelReg)
6718	{
6719	NOREF(pVCpu);
6720
6721	uint32_t u32Val = 0;
6722	int rc = VMXReadVmcs32(idxSel, &u32Val);
6723	AssertRCReturn(rc, rc);
6724	pSelReg->Sel = (uint16_t)u32Val;
6725	pSelReg->ValidSel = (uint16_t)u32Val;
6726	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6727
6728	rc = VMXReadVmcs32(idxLimit, &u32Val);
6729	AssertRCReturn(rc, rc);
6730	pSelReg->u32Limit = u32Val;
6731
6732	uint64_t u64Val = 0;
6733	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6734	AssertRCReturn(rc, rc);
6735	pSelReg->u64Base = u64Val;
6736
6737	rc = VMXReadVmcs32(idxAccess, &u32Val);
6738	AssertRCReturn(rc, rc);
6739	pSelReg->Attr.u = u32Val;
6740
6741	/*
6742	* If VT-x marks the segment as unusable, most other bits remain undefined:
6743	* - For CS the L, D and G bits have meaning.
6744	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6745	* - For the remaining data segments no bits are defined.
6746	*
6747	* The present bit and the unusable bit has been observed to be set at the
6748	* same time (the selector was supposed to be invalid as we started executing
6749	* a V8086 interrupt in ring-0).
6750	*
6751	* What should be important for the rest of the VBox code, is that the P bit is
6752	* cleared. Some of the other VBox code recognizes the unusable bit, but
6753	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6754	* safe side here, we'll strip off P and other bits we don't care about. If
6755	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6756	*
6757	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6758	*/
6759	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6760	{
6761	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6762
6763	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6764	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6765	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6766
6767	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6768	#ifdef DEBUG_bird
6769	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6770	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6771	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6772	#endif
6773	}
6774	return VINF_SUCCESS;
6775	}
6776
6777
6778	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6779	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6780	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6781	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6782	#else
6783	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6784	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6785	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6786	#endif
6787
6788
6789	/**
6790	* Saves the guest segment registers from the current VMCS into the guest-CPU
6791	* context.
6792	*
6793	* @returns VBox status code.
6794	* @param pVCpu The cross context virtual CPU structure.
6795	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6796	* out-of-sync. Make sure to update the required fields
6797	* before using them.
6798	*
6799	* @remarks No-long-jump zone!!!
6800	*/
6801	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6802	{
6803	/* Guest segment registers. */
6804	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6805	{
6806	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6807	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6808	AssertRCReturn(rc, rc);
6809
6810	rc = VMXLOCAL_READ_SEG(CS, cs);
6811	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6812	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6813	rc \|= VMXLOCAL_READ_SEG(ES, es);
6814	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6815	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6816	AssertRCReturn(rc, rc);
6817
6818	/* Restore segment attributes for real-on-v86 mode hack. */
6819	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6820	{
6821	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6822	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6823	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6824	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6825	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6826	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6827	}
6828	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6829	}
6830
6831	return VINF_SUCCESS;
6832	}
6833
6834
6835	/**
6836	* Saves the guest descriptor table registers and task register from the current
6837	* VMCS into the guest-CPU context.
6838	*
6839	* @returns VBox status code.
6840	* @param pVCpu The cross context virtual CPU structure.
6841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6842	* out-of-sync. Make sure to update the required fields
6843	* before using them.
6844	*
6845	* @remarks No-long-jump zone!!!
6846	*/
6847	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6848	{
6849	int rc = VINF_SUCCESS;
6850
6851	/* Guest LDTR. */
6852	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6853	{
6854	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6855	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6856	AssertRCReturn(rc, rc);
6857	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6858	}
6859
6860	/* Guest GDTR. */
6861	uint64_t u64Val = 0;
6862	uint32_t u32Val = 0;
6863	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6864	{
6865	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6866	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6867	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6868	pMixedCtx->gdtr.pGdt = u64Val;
6869	pMixedCtx->gdtr.cbGdt = u32Val;
6870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6871	}
6872
6873	/* Guest IDTR. */
6874	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6875	{
6876	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6877	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6878	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6879	pMixedCtx->idtr.pIdt = u64Val;
6880	pMixedCtx->idtr.cbIdt = u32Val;
6881	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6882	}
6883
6884	/* Guest TR. */
6885	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6886	{
6887	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6888	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6889	AssertRCReturn(rc, rc);
6890
6891	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6892	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6893	{
6894	rc = VMXLOCAL_READ_SEG(TR, tr);
6895	AssertRCReturn(rc, rc);
6896	}
6897	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6898	}
6899	return rc;
6900	}
6901
6902	#undef VMXLOCAL_READ_SEG
6903
6904
6905	/**
6906	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6907	* context.
6908	*
6909	* @returns VBox status code.
6910	* @param pVCpu The cross context virtual CPU structure.
6911	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6912	* out-of-sync. Make sure to update the required fields
6913	* before using them.
6914	*
6915	* @remarks No-long-jump zone!!!
6916	*/
6917	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6918	{
6919	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6920	{
6921	if (!pVCpu->hm.s.fUsingHyperDR7)
6922	{
6923	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6924	uint32_t u32Val;
6925	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6926	pMixedCtx->dr[7] = u32Val;
6927	}
6928
6929	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6930	}
6931	return VINF_SUCCESS;
6932	}
6933
6934
6935	/**
6936	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6937	*
6938	* @returns VBox status code.
6939	* @param pVCpu The cross context virtual CPU structure.
6940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6941	* out-of-sync. Make sure to update the required fields
6942	* before using them.
6943	*
6944	* @remarks No-long-jump zone!!!
6945	*/
6946	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6947	{
6948	NOREF(pMixedCtx);
6949
6950	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6951	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6952	return VINF_SUCCESS;
6953	}
6954
6955
6956	/**
6957	* Saves the entire guest state from the currently active VMCS into the
6958	* guest-CPU context.
6959	*
6960	* This essentially VMREADs all guest-data.
6961	*
6962	* @returns VBox status code.
6963	* @param pVCpu The cross context virtual CPU structure.
6964	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6965	* out-of-sync. Make sure to update the required fields
6966	* before using them.
6967	*/
6968	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6969	{
6970	Assert(pVCpu);
6971	Assert(pMixedCtx);
6972
6973	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6974	return VINF_SUCCESS;
6975
6976	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6977	again on the ring-3 callback path, there is no real need to. */
6978	if (VMMRZCallRing3IsEnabled(pVCpu))
6979	VMMR0LogFlushDisable(pVCpu);
6980	else
6981	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6982	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6983
6984	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6985	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6986
6987	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6988	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6989
6990	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6991	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6992
6993	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6994	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6995
6996	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6997	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6998
6999	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7000	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7001
7002	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7003	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7004
7005	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7006	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7007
7008	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7009	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7010
7011	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7012	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7013
7014	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7015	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7016	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7017
7018	if (VMMRZCallRing3IsEnabled(pVCpu))
7019	VMMR0LogFlushEnable(pVCpu);
7020
7021	return VINF_SUCCESS;
7022	}
7023
7024
7025	/**
7026	* Saves basic guest registers needed for IEM instruction execution.
7027	*
7028	* @returns VBox status code (OR-able).
7029	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7030	* @param pMixedCtx Pointer to the CPU context of the guest.
7031	* @param fMemory Whether the instruction being executed operates on
7032	* memory or not. Only CR0 is synced up if clear.
7033	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7034	*/
7035	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7036	{
7037	/*
7038	* We assume all general purpose registers other than RSP are available.
7039	*
7040	* - RIP is a must, as it will be incremented or otherwise changed.
7041	* - RFLAGS are always required to figure the CPL.
7042	* - RSP isn't always required, however it's a GPR, so frequently required.
7043	* - SS and CS are the only segment register needed if IEM doesn't do memory
7044	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7045	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7046	* be required for memory accesses.
7047	*
7048	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7049	*/
7050	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7051	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7052	if (fNeedRsp)
7053	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7054	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7055	if (!fMemory)
7056	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7057	else
7058	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7059	AssertRCReturn(rc, rc);
7060	return rc;
7061	}
7062
7063
7064	/**
7065	* Ensures that we've got a complete basic guest-context.
7066	*
7067	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7068	* is for the interpreter.
7069	*
7070	* @returns VBox status code.
7071	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7072	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7073	* needing to be synced in.
7074	* @thread EMT(pVCpu)
7075	*/
7076	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7077	{
7078	/* Note! Since this is only applicable to VT-x, the implementation is placed
7079	in the VT-x part of the sources instead of the generic stuff. */
7080	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7081	{
7082	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7083	/*
7084	* For now, imply that the caller might change everything too. Do this after
7085	* saving the guest state so as to not trigger assertions.
7086	*/
7087	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7088	return rc;
7089	}
7090	return VINF_SUCCESS;
7091	}
7092
7093
7094	/**
7095	* Check per-VM and per-VCPU force flag actions that require us to go back to
7096	* ring-3 for one reason or another.
7097	*
7098	* @returns Strict VBox status code (i.e. informational status codes too)
7099	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7100	* ring-3.
7101	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7102	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7103	* interrupts)
7104	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7105	* all EMTs to be in ring-3.
7106	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7107	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7108	* to the EM loop.
7109	*
7110	* @param pVM The cross context VM structure.
7111	* @param pVCpu The cross context virtual CPU structure.
7112	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7113	* out-of-sync. Make sure to update the required fields
7114	* before using them.
7115	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7116	*/
7117	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7118	{
7119	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7120
7121	/*
7122	* Anything pending? Should be more likely than not if we're doing a good job.
7123	*/
7124	if ( !fStepping
7125	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7126	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7127	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7128	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7129	return VINF_SUCCESS;
7130
7131	/* We need the control registers now, make sure the guest-CPU context is updated. */
7132	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7133	AssertRCReturn(rc3, rc3);
7134
7135	/* Pending HM CR3 sync. */
7136	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7137	{
7138	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7139	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7140	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7141	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7142	}
7143
7144	/* Pending HM PAE PDPEs. */
7145	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7146	{
7147	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7148	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7149	}
7150
7151	/* Pending PGM C3 sync. */
7152	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7153	{
7154	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7155	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7156	if (rcStrict2 != VINF_SUCCESS)
7157	{
7158	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7159	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7160	return rcStrict2;
7161	}
7162	}
7163
7164	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7165	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7166	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7167	{
7168	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7169	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7170	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7171	return rc2;
7172	}
7173
7174	/* Pending VM request packets, such as hardware interrupts. */
7175	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7176	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7177	{
7178	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7179	return VINF_EM_PENDING_REQUEST;
7180	}
7181
7182	/* Pending PGM pool flushes. */
7183	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7184	{
7185	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7186	return VINF_PGM_POOL_FLUSH_PENDING;
7187	}
7188
7189	/* Pending DMA requests. */
7190	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7191	{
7192	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7193	return VINF_EM_RAW_TO_R3;
7194	}
7195
7196	return VINF_SUCCESS;
7197	}
7198
7199
7200	/**
7201	* Converts any TRPM trap into a pending HM event. This is typically used when
7202	* entering from ring-3 (not longjmp returns).
7203	*
7204	* @param pVCpu The cross context virtual CPU structure.
7205	*/
7206	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7207	{
7208	Assert(TRPMHasTrap(pVCpu));
7209	Assert(!pVCpu->hm.s.Event.fPending);
7210
7211	uint8_t uVector;
7212	TRPMEVENT enmTrpmEvent;
7213	RTGCUINT uErrCode;
7214	RTGCUINTPTR GCPtrFaultAddress;
7215	uint8_t cbInstr;
7216
7217	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7218	AssertRC(rc);
7219
7220	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7221	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7222	if (enmTrpmEvent == TRPM_TRAP)
7223	{
7224	switch (uVector)
7225	{
7226	case X86_XCPT_NMI:
7227	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7228	break;
7229
7230	case X86_XCPT_BP:
7231	case X86_XCPT_OF:
7232	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7233	break;
7234
7235	case X86_XCPT_PF:
7236	case X86_XCPT_DF:
7237	case X86_XCPT_TS:
7238	case X86_XCPT_NP:
7239	case X86_XCPT_SS:
7240	case X86_XCPT_GP:
7241	case X86_XCPT_AC:
7242	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7243	/* fall thru */
7244	default:
7245	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7246	break;
7247	}
7248	}
7249	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7250	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7251	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7252	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7253	else
7254	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7255
7256	rc = TRPMResetTrap(pVCpu);
7257	AssertRC(rc);
7258	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7259	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7260
7261	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7262	}
7263
7264
7265	/**
7266	* Converts the pending HM event into a TRPM trap.
7267	*
7268	* @param pVCpu The cross context virtual CPU structure.
7269	*/
7270	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7271	{
7272	Assert(pVCpu->hm.s.Event.fPending);
7273
7274	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7275	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7276	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7277	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7278
7279	/* If a trap was already pending, we did something wrong! */
7280	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7281
7282	TRPMEVENT enmTrapType;
7283	switch (uVectorType)
7284	{
7285	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7286	enmTrapType = TRPM_HARDWARE_INT;
7287	break;
7288
7289	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7290	enmTrapType = TRPM_SOFTWARE_INT;
7291	break;
7292
7293	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7294	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7295	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7296	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7297	enmTrapType = TRPM_TRAP;
7298	break;
7299
7300	default:
7301	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7302	enmTrapType = TRPM_32BIT_HACK;
7303	break;
7304	}
7305
7306	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7307
7308	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7309	AssertRC(rc);
7310
7311	if (fErrorCodeValid)
7312	TRPMSetErrorCode(pVCpu, uErrorCode);
7313
7314	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7315	&& uVector == X86_XCPT_PF)
7316	{
7317	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7318	}
7319	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7320	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7321	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7322	{
7323	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7324	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7325	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7326	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7327	}
7328
7329	/* Clear any pending events from the VMCS. */
7330	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7331	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7332
7333	/* We're now done converting the pending event. */
7334	pVCpu->hm.s.Event.fPending = false;
7335	}
7336
7337
7338	/**
7339	* Does the necessary state syncing before returning to ring-3 for any reason
7340	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7341	*
7342	* @returns VBox status code.
7343	* @param pVCpu The cross context virtual CPU structure.
7344	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7345	* be out-of-sync. Make sure to update the required
7346	* fields before using them.
7347	* @param fSaveGuestState Whether to save the guest state or not.
7348	*
7349	* @remarks No-long-jmp zone!!!
7350	*/
7351	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7352	{
7353	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7354	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7355
7356	RTCPUID idCpu = RTMpCpuId();
7357	Log4Func(("HostCpuId=%u\n", idCpu));
7358
7359	/*
7360	* !!! IMPORTANT !!!
7361	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7362	*/
7363
7364	/* Save the guest state if necessary. */
7365	if ( fSaveGuestState
7366	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7367	{
7368	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7369	AssertRCReturn(rc, rc);
7370	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7371	}
7372
7373	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7374	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7375	{
7376	/* We shouldn't reload CR0 without saving it first. */
7377	if (!fSaveGuestState)
7378	{
7379	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7380	AssertRCReturn(rc, rc);
7381	}
7382	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7383	}
7384
7385	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7386	#ifdef VBOX_STRICT
7387	if (CPUMIsHyperDebugStateActive(pVCpu))
7388	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7389	#endif
7390	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7391	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7392	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7393	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7394
7395	#if HC_ARCH_BITS == 64
7396	/* Restore host-state bits that VT-x only restores partially. */
7397	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7398	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7399	{
7400	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7401	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7402	}
7403	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7404	#endif
7405
7406	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7407	if (pVCpu->hm.s.vmx.fLazyMsrs)
7408	{
7409	/* We shouldn't reload the guest MSRs without saving it first. */
7410	if (!fSaveGuestState)
7411	{
7412	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7413	AssertRCReturn(rc, rc);
7414	}
7415	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7416	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7417	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7418	}
7419
7420	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7421	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7422
7423	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7424	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7425	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7426	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7427	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7428	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7429	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7430	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7431
7432	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7433
7434	/** @todo This partially defeats the purpose of having preemption hooks.
7435	* The problem is, deregistering the hooks should be moved to a place that
7436	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7437	* context.
7438	*/
7439	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7440	{
7441	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7442	AssertRCReturn(rc, rc);
7443
7444	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7445	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7446	}
7447	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7448	NOREF(idCpu);
7449
7450	return VINF_SUCCESS;
7451	}
7452
7453
7454	/**
7455	* Leaves the VT-x session.
7456	*
7457	* @returns VBox status code.
7458	* @param pVCpu The cross context virtual CPU structure.
7459	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7460	* out-of-sync. Make sure to update the required fields
7461	* before using them.
7462	*
7463	* @remarks No-long-jmp zone!!!
7464	*/
7465	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7466	{
7467	HM_DISABLE_PREEMPT();
7468	HMVMX_ASSERT_CPU_SAFE();
7469	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7470	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7471
7472	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7473	and done this from the VMXR0ThreadCtxCallback(). */
7474	if (!pVCpu->hm.s.fLeaveDone)
7475	{
7476	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7477	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7478	pVCpu->hm.s.fLeaveDone = true;
7479	}
7480	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7481
7482	/*
7483	* !!! IMPORTANT !!!
7484	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7485	*/
7486
7487	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7488	/** @todo Deregistering here means we need to VMCLEAR always
7489	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7490	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7491	VMMR0ThreadCtxHookDisable(pVCpu);
7492
7493	/* Leave HM context. This takes care of local init (term). */
7494	int rc = HMR0LeaveCpu(pVCpu);
7495
7496	HM_RESTORE_PREEMPT();
7497	return rc;
7498	}
7499
7500
7501	/**
7502	* Does the necessary state syncing before doing a longjmp to ring-3.
7503	*
7504	* @returns VBox status code.
7505	* @param pVCpu The cross context virtual CPU structure.
7506	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7507	* out-of-sync. Make sure to update the required fields
7508	* before using them.
7509	*
7510	* @remarks No-long-jmp zone!!!
7511	*/
7512	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7513	{
7514	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7515	}
7516
7517
7518	/**
7519	* Take necessary actions before going back to ring-3.
7520	*
7521	* An action requires us to go back to ring-3. This function does the necessary
7522	* steps before we can safely return to ring-3. This is not the same as longjmps
7523	* to ring-3, this is voluntary and prepares the guest so it may continue
7524	* executing outside HM (recompiler/IEM).
7525	*
7526	* @returns VBox status code.
7527	* @param pVM The cross context VM structure.
7528	* @param pVCpu The cross context virtual CPU structure.
7529	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7530	* out-of-sync. Make sure to update the required fields
7531	* before using them.
7532	* @param rcExit The reason for exiting to ring-3. Can be
7533	* VINF_VMM_UNKNOWN_RING3_CALL.
7534	*/
7535	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7536	{
7537	Assert(pVM);
7538	Assert(pVCpu);
7539	Assert(pMixedCtx);
7540	HMVMX_ASSERT_PREEMPT_SAFE();
7541
7542	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7543	{
7544	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7545	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7546	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7547	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7548	}
7549
7550	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7551	VMMRZCallRing3Disable(pVCpu);
7552	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7553
7554	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7555	if (pVCpu->hm.s.Event.fPending)
7556	{
7557	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7558	Assert(!pVCpu->hm.s.Event.fPending);
7559	}
7560
7561	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7562	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7563
7564	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7565	and if we're injecting an event we should have a TRPM trap pending. */
7566	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7567	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7568	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7569	#endif
7570
7571	/* Save guest state and restore host state bits. */
7572	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7573	AssertRCReturn(rc, rc);
7574	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7575	/* Thread-context hooks are unregistered at this point!!! */
7576
7577	/* Sync recompiler state. */
7578	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7579	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7580	\| CPUM_CHANGED_LDTR
7581	\| CPUM_CHANGED_GDTR
7582	\| CPUM_CHANGED_IDTR
7583	\| CPUM_CHANGED_TR
7584	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7585	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7586	if ( pVM->hm.s.fNestedPaging
7587	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7588	{
7589	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7590	}
7591
7592	Assert(!pVCpu->hm.s.fClearTrapFlag);
7593
7594	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7595	if (rcExit != VINF_EM_RAW_INTERRUPT)
7596	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7597
7598	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7599
7600	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7601	VMMRZCallRing3RemoveNotification(pVCpu);
7602	VMMRZCallRing3Enable(pVCpu);
7603
7604	return rc;
7605	}
7606
7607
7608	/**
7609	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7610	* longjump to ring-3 and possibly get preempted.
7611	*
7612	* @returns VBox status code.
7613	* @param pVCpu The cross context virtual CPU structure.
7614	* @param enmOperation The operation causing the ring-3 longjump.
7615	* @param pvUser Opaque pointer to the guest-CPU context. The data
7616	* may be out-of-sync. Make sure to update the required
7617	* fields before using them.
7618	*/
7619	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7620	{
7621	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7622	{
7623	/*
7624	* !!! IMPORTANT !!!
7625	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7626	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7627	*/
7628	VMMRZCallRing3RemoveNotification(pVCpu);
7629	VMMRZCallRing3Disable(pVCpu);
7630	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7631	RTThreadPreemptDisable(&PreemptState);
7632
7633	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7634	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7635
7636	#if HC_ARCH_BITS == 64
7637	/* Restore host-state bits that VT-x only restores partially. */
7638	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7639	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7640	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7641	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7642	#endif
7643	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7644	if (pVCpu->hm.s.vmx.fLazyMsrs)
7645	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7646
7647	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7648	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7649	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7650	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7651	{
7652	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7653	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7654	}
7655
7656	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7657	VMMR0ThreadCtxHookDisable(pVCpu);
7658	HMR0LeaveCpu(pVCpu);
7659	RTThreadPreemptRestore(&PreemptState);
7660	return VINF_SUCCESS;
7661	}
7662
7663	Assert(pVCpu);
7664	Assert(pvUser);
7665	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7666	HMVMX_ASSERT_PREEMPT_SAFE();
7667
7668	VMMRZCallRing3Disable(pVCpu);
7669	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7670
7671	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7672	enmOperation));
7673
7674	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7675	AssertRCReturn(rc, rc);
7676
7677	VMMRZCallRing3Enable(pVCpu);
7678	return VINF_SUCCESS;
7679	}
7680
7681
7682	/**
7683	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7684	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7685	*
7686	* @param pVCpu The cross context virtual CPU structure.
7687	*/
7688	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7689	{
7690	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7691	{
7692	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7693	{
7694	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7695	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7696	AssertRC(rc);
7697	Log4(("Setup interrupt-window exiting\n"));
7698	}
7699	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7700	}
7701
7702
7703	/**
7704	* Clears the interrupt-window exiting control in the VMCS.
7705	*
7706	* @param pVCpu The cross context virtual CPU structure.
7707	*/
7708	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7709	{
7710	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7711	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7712	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7713	AssertRC(rc);
7714	Log4(("Cleared interrupt-window exiting\n"));
7715	}
7716
7717
7718	/**
7719	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7720	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7721	*
7722	* @param pVCpu The cross context virtual CPU structure.
7723	*/
7724	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7725	{
7726	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7727	{
7728	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7729	{
7730	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7731	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7732	AssertRC(rc);
7733	Log4(("Setup NMI-window exiting\n"));
7734	}
7735	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7736	}
7737
7738
7739	/**
7740	* Clears the NMI-window exiting control in the VMCS.
7741	*
7742	* @param pVCpu The cross context virtual CPU structure.
7743	*/
7744	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7745	{
7746	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7747	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7749	AssertRC(rc);
7750	Log4(("Cleared NMI-window exiting\n"));
7751	}
7752
7753
7754	/**
7755	* Evaluates the event to be delivered to the guest and sets it as the pending
7756	* event.
7757	*
7758	* @returns The VT-x guest-interruptibility state.
7759	* @param pVCpu The cross context virtual CPU structure.
7760	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7761	* out-of-sync. Make sure to update the required fields
7762	* before using them.
7763	*/
7764	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7765	{
7766	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7767	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7768	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7769	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7770	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7771
7772	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7773	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7774	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7775	Assert(!TRPMHasTrap(pVCpu));
7776
7777	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7778	APICUpdatePendingInterrupts(pVCpu);
7779
7780	/*
7781	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7782	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7783	*/
7784	/** @todo SMI. SMIs take priority over NMIs. */
7785	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7786	{
7787	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7788	if ( !pVCpu->hm.s.Event.fPending
7789	&& !fBlockNmi
7790	&& !fBlockSti
7791	&& !fBlockMovSS)
7792	{
7793	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7794	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7795	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7796
7797	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7798	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7799	}
7800	else
7801	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7802	}
7803	/*
7804	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7805	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7806	*/
7807	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7808	&& !pVCpu->hm.s.fSingleInstruction)
7809	{
7810	Assert(!DBGFIsStepping(pVCpu));
7811	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7812	AssertRC(rc);
7813	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7814	if ( !pVCpu->hm.s.Event.fPending
7815	&& !fBlockInt
7816	&& !fBlockSti
7817	&& !fBlockMovSS)
7818	{
7819	uint8_t u8Interrupt;
7820	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7821	if (RT_SUCCESS(rc))
7822	{
7823	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7824	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7825	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7826
7827	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7828	}
7829	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7830	{
7831	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7832	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7833	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7834
7835	/*
7836	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7837	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7838	* need to re-set this force-flag here.
7839	*/
7840	}
7841	else
7842	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7843	}
7844	else
7845	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7846	}
7847
7848	return uIntrState;
7849	}
7850
7851
7852	/**
7853	* Sets a pending-debug exception to be delivered to the guest if the guest is
7854	* single-stepping in the VMCS.
7855	*
7856	* @param pVCpu The cross context virtual CPU structure.
7857	*/
7858	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7859	{
7860	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7861	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7862	AssertRC(rc);
7863	}
7864
7865
7866	/**
7867	* Injects any pending events into the guest if the guest is in a state to
7868	* receive them.
7869	*
7870	* @returns Strict VBox status code (i.e. informational status codes too).
7871	* @param pVCpu The cross context virtual CPU structure.
7872	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7873	* out-of-sync. Make sure to update the required fields
7874	* before using them.
7875	* @param uIntrState The VT-x guest-interruptibility state.
7876	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7877	* return VINF_EM_DBG_STEPPED if the event was
7878	* dispatched directly.
7879	*/
7880	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7881	{
7882	HMVMX_ASSERT_PREEMPT_SAFE();
7883	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7884
7885	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7886	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7887
7888	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7889	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7890	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7891	Assert(!TRPMHasTrap(pVCpu));
7892
7893	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7894	if (pVCpu->hm.s.Event.fPending)
7895	{
7896	/*
7897	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7898	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7899	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7900	*
7901	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7902	*/
7903	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7904	#ifdef VBOX_STRICT
7905	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7906	{
7907	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7908	Assert(!fBlockInt);
7909	Assert(!fBlockSti);
7910	Assert(!fBlockMovSS);
7911	}
7912	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7913	{
7914	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7915	Assert(!fBlockSti);
7916	Assert(!fBlockMovSS);
7917	Assert(!fBlockNmi);
7918	}
7919	#endif
7920	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7921	(uint8_t)uIntType));
7922	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7923	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7924	fStepping, &uIntrState);
7925	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7926
7927	/* Update the interruptibility-state as it could have been changed by
7928	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7929	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7930	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7931
7932	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7933	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7934	else
7935	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7936	}
7937
7938	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7939	if ( fBlockSti
7940	\|\| fBlockMovSS)
7941	{
7942	if (!pVCpu->hm.s.fSingleInstruction)
7943	{
7944	/*
7945	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7946	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7947	* See Intel spec. 27.3.4 "Saving Non-Register State".
7948	*/
7949	Assert(!DBGFIsStepping(pVCpu));
7950	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7951	AssertRCReturn(rc2, rc2);
7952	if (pMixedCtx->eflags.Bits.u1TF)
7953	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7954	}
7955	else if (pMixedCtx->eflags.Bits.u1TF)
7956	{
7957	/*
7958	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7959	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7960	*/
7961	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7962	uIntrState = 0;
7963	}
7964	}
7965
7966	/*
7967	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7968	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7969	*/
7970	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7971	AssertRC(rc2);
7972
7973	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7974	NOREF(fBlockMovSS); NOREF(fBlockSti);
7975	return rcStrict;
7976	}
7977
7978
7979	/**
7980	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7981	*
7982	* @param pVCpu The cross context virtual CPU structure.
7983	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7984	* out-of-sync. Make sure to update the required fields
7985	* before using them.
7986	*/
7987	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7988	{
7989	NOREF(pMixedCtx);
7990	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7991	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7992	}
7993
7994
7995	/**
7996	* Injects a double-fault (\#DF) exception into the VM.
7997	*
7998	* @returns Strict VBox status code (i.e. informational status codes too).
7999	* @param pVCpu The cross context virtual CPU structure.
8000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8001	* out-of-sync. Make sure to update the required fields
8002	* before using them.
8003	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8004	* and should return VINF_EM_DBG_STEPPED if the event
8005	* is injected directly (register modified by us, not
8006	* by hardware on VM-entry).
8007	* @param puIntrState Pointer to the current guest interruptibility-state.
8008	* This interruptibility-state will be updated if
8009	* necessary. This cannot not be NULL.
8010	*/
8011	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8012	{
8013	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8014	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8015	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8016	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8017	fStepping, puIntrState);
8018	}
8019
8020
8021	/**
8022	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8023	*
8024	* @param pVCpu The cross context virtual CPU structure.
8025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8026	* out-of-sync. Make sure to update the required fields
8027	* before using them.
8028	*/
8029	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8030	{
8031	NOREF(pMixedCtx);
8032	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8033	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8034	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8035	}
8036
8037
8038	/**
8039	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8040	*
8041	* @param pVCpu The cross context virtual CPU structure.
8042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8043	* out-of-sync. Make sure to update the required fields
8044	* before using them.
8045	* @param cbInstr The value of RIP that is to be pushed on the guest
8046	* stack.
8047	*/
8048	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8049	{
8050	NOREF(pMixedCtx);
8051	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8052	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8053	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8054	}
8055
8056
8057	/**
8058	* Injects a general-protection (\#GP) fault into the VM.
8059	*
8060	* @returns Strict VBox status code (i.e. informational status codes too).
8061	* @param pVCpu The cross context virtual CPU structure.
8062	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8063	* out-of-sync. Make sure to update the required fields
8064	* before using them.
8065	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8066	* mode, i.e. in real-mode it's not valid).
8067	* @param u32ErrorCode The error code associated with the \#GP.
8068	* @param fStepping Whether we're running in
8069	* hmR0VmxRunGuestCodeStep() and should return
8070	* VINF_EM_DBG_STEPPED if the event is injected
8071	* directly (register modified by us, not by
8072	* hardware on VM-entry).
8073	* @param puIntrState Pointer to the current guest interruptibility-state.
8074	* This interruptibility-state will be updated if
8075	* necessary. This cannot not be NULL.
8076	*/
8077	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8078	bool fStepping, uint32_t *puIntrState)
8079	{
8080	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8081	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8082	if (fErrorCodeValid)
8083	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8084	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8085	fStepping, puIntrState);
8086	}
8087
8088
8089	#if 0 /* unused */
8090	/**
8091	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8092	* VM.
8093	*
8094	* @param pVCpu The cross context virtual CPU structure.
8095	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8096	* out-of-sync. Make sure to update the required fields
8097	* before using them.
8098	* @param u32ErrorCode The error code associated with the \#GP.
8099	*/
8100	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8101	{
8102	NOREF(pMixedCtx);
8103	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8104	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8105	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8106	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8107	}
8108	#endif /* unused */
8109
8110
8111	/**
8112	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8113	*
8114	* @param pVCpu The cross context virtual CPU structure.
8115	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8116	* out-of-sync. Make sure to update the required fields
8117	* before using them.
8118	* @param uVector The software interrupt vector number.
8119	* @param cbInstr The value of RIP that is to be pushed on the guest
8120	* stack.
8121	*/
8122	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8123	{
8124	NOREF(pMixedCtx);
8125	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8126	if ( uVector == X86_XCPT_BP
8127	\|\| uVector == X86_XCPT_OF)
8128	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8129	else
8130	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8131	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8132	}
8133
8134
8135	/**
8136	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8137	* stack.
8138	*
8139	* @returns Strict VBox status code (i.e. informational status codes too).
8140	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8141	* @param pVM The cross context VM structure.
8142	* @param pMixedCtx Pointer to the guest-CPU context.
8143	* @param uValue The value to push to the guest stack.
8144	*/
8145	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8146	{
8147	/*
8148	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8149	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8150	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8151	*/
8152	if (pMixedCtx->sp == 1)
8153	return VINF_EM_RESET;
8154	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8155	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8156	AssertRC(rc);
8157	return rc;
8158	}
8159
8160
8161	/**
8162	* Injects an event into the guest upon VM-entry by updating the relevant fields
8163	* in the VM-entry area in the VMCS.
8164	*
8165	* @returns Strict VBox status code (i.e. informational status codes too).
8166	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8167	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8168	*
8169	* @param pVCpu The cross context virtual CPU structure.
8170	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8171	* be out-of-sync. Make sure to update the required
8172	* fields before using them.
8173	* @param u64IntInfo The VM-entry interruption-information field.
8174	* @param cbInstr The VM-entry instruction length in bytes (for
8175	* software interrupts, exceptions and privileged
8176	* software exceptions).
8177	* @param u32ErrCode The VM-entry exception error code.
8178	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8179	* @param puIntrState Pointer to the current guest interruptibility-state.
8180	* This interruptibility-state will be updated if
8181	* necessary. This cannot not be NULL.
8182	* @param fStepping Whether we're running in
8183	* hmR0VmxRunGuestCodeStep() and should return
8184	* VINF_EM_DBG_STEPPED if the event is injected
8185	* directly (register modified by us, not by
8186	* hardware on VM-entry).
8187	*
8188	* @remarks Requires CR0!
8189	*/
8190	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8191	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8192	uint32_t *puIntrState)
8193	{
8194	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8195	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8196	Assert(puIntrState);
8197	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8198
8199	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8200	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8201
8202	#ifdef VBOX_STRICT
8203	/* Validate the error-code-valid bit for hardware exceptions. */
8204	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8205	{
8206	switch (uVector)
8207	{
8208	case X86_XCPT_PF:
8209	case X86_XCPT_DF:
8210	case X86_XCPT_TS:
8211	case X86_XCPT_NP:
8212	case X86_XCPT_SS:
8213	case X86_XCPT_GP:
8214	case X86_XCPT_AC:
8215	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8216	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8217	/* fall thru */
8218	default:
8219	break;
8220	}
8221	}
8222	#endif
8223
8224	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8225	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8226	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8227
8228	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8229
8230	/* We require CR0 to check if the guest is in real-mode. */
8231	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8232	AssertRCReturn(rc, rc);
8233
8234	/*
8235	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8236	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8237	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8238	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8239	*/
8240	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8241	{
8242	PVM pVM = pVCpu->CTX_SUFF(pVM);
8243	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8244	{
8245	Assert(PDMVmmDevHeapIsEnabled(pVM));
8246	Assert(pVM->hm.s.vmx.pRealModeTSS);
8247
8248	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8249	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8250	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8251	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8252	AssertRCReturn(rc, rc);
8253	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8254
8255	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8256	size_t const cbIdtEntry = sizeof(X86IDTR16);
8257	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8258	{
8259	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8260	if (uVector == X86_XCPT_DF)
8261	return VINF_EM_RESET;
8262
8263	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8264	if (uVector == X86_XCPT_GP)
8265	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8266
8267	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8268	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8269	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8270	fStepping, puIntrState);
8271	}
8272
8273	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8274	uint16_t uGuestIp = pMixedCtx->ip;
8275	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8276	{
8277	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8278	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8279	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8280	}
8281	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8282	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8283
8284	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8285	X86IDTR16 IdtEntry;
8286	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8287	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8288	AssertRCReturn(rc, rc);
8289
8290	/* Construct the stack frame for the interrupt/exception handler. */
8291	VBOXSTRICTRC rcStrict;
8292	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8293	if (rcStrict == VINF_SUCCESS)
8294	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8295	if (rcStrict == VINF_SUCCESS)
8296	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8297
8298	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8299	if (rcStrict == VINF_SUCCESS)
8300	{
8301	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8302	pMixedCtx->rip = IdtEntry.offSel;
8303	pMixedCtx->cs.Sel = IdtEntry.uSel;
8304	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8305	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8306	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8307	&& uVector == X86_XCPT_PF)
8308	pMixedCtx->cr2 = GCPtrFaultAddress;
8309
8310	/* If any other guest-state bits are changed here, make sure to update
8311	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8312	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8313	\| HM_CHANGED_GUEST_RIP
8314	\| HM_CHANGED_GUEST_RFLAGS
8315	\| HM_CHANGED_GUEST_RSP);
8316
8317	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8318	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8319	{
8320	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8321	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8322	Log4(("Clearing inhibition due to STI.\n"));
8323	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8324	}
8325	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8326	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8327
8328	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8329	it, if we are returning to ring-3 before executing guest code. */
8330	pVCpu->hm.s.Event.fPending = false;
8331
8332	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8333	if (fStepping)
8334	rcStrict = VINF_EM_DBG_STEPPED;
8335	}
8336	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8337	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8338	return rcStrict;
8339	}
8340
8341	/*
8342	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8343	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8344	*/
8345	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8346	}
8347
8348	/* Validate. */
8349	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8350	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8351	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8352
8353	/* Inject. */
8354	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8355	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8356	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8357	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8358
8359	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8360	&& uVector == X86_XCPT_PF)
8361	pMixedCtx->cr2 = GCPtrFaultAddress;
8362
8363	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8364	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8365
8366	AssertRCReturn(rc, rc);
8367	return VINF_SUCCESS;
8368	}
8369
8370
8371	/**
8372	* Clears the interrupt-window exiting control in the VMCS and if necessary
8373	* clears the current event in the VMCS as well.
8374	*
8375	* @returns VBox status code.
8376	* @param pVCpu The cross context virtual CPU structure.
8377	*
8378	* @remarks Use this function only to clear events that have not yet been
8379	* delivered to the guest but are injected in the VMCS!
8380	* @remarks No-long-jump zone!!!
8381	*/
8382	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8383	{
8384	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8385
8386	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8387	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8388
8389	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8390	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8391	}
8392
8393
8394	/**
8395	* Enters the VT-x session.
8396	*
8397	* @returns VBox status code.
8398	* @param pVM The cross context VM structure.
8399	* @param pVCpu The cross context virtual CPU structure.
8400	* @param pCpu Pointer to the CPU info struct.
8401	*/
8402	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8403	{
8404	AssertPtr(pVM);
8405	AssertPtr(pVCpu);
8406	Assert(pVM->hm.s.vmx.fSupported);
8407	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8408	NOREF(pCpu); NOREF(pVM);
8409
8410	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8411	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8412
8413	#ifdef VBOX_STRICT
8414	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8415	RTCCUINTREG uHostCR4 = ASMGetCR4();
8416	if (!(uHostCR4 & X86_CR4_VMXE))
8417	{
8418	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8419	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8420	}
8421	#endif
8422
8423	/*
8424	* Load the VCPU's VMCS as the current (and active) one.
8425	*/
8426	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8427	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8428	if (RT_FAILURE(rc))
8429	return rc;
8430
8431	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8432	pVCpu->hm.s.fLeaveDone = false;
8433	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8434
8435	return VINF_SUCCESS;
8436	}
8437
8438
8439	/**
8440	* The thread-context callback (only on platforms which support it).
8441	*
8442	* @param enmEvent The thread-context event.
8443	* @param pVCpu The cross context virtual CPU structure.
8444	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8445	* @thread EMT(pVCpu)
8446	*/
8447	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8448	{
8449	NOREF(fGlobalInit);
8450
8451	switch (enmEvent)
8452	{
8453	case RTTHREADCTXEVENT_OUT:
8454	{
8455	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8456	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8457	VMCPU_ASSERT_EMT(pVCpu);
8458
8459	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8460
8461	/* No longjmps (logger flushes, locks) in this fragile context. */
8462	VMMRZCallRing3Disable(pVCpu);
8463	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8464
8465	/*
8466	* Restore host-state (FPU, debug etc.)
8467	*/
8468	if (!pVCpu->hm.s.fLeaveDone)
8469	{
8470	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8471	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8472	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8473	pVCpu->hm.s.fLeaveDone = true;
8474	}
8475
8476	/* Leave HM context, takes care of local init (term). */
8477	int rc = HMR0LeaveCpu(pVCpu);
8478	AssertRC(rc); NOREF(rc);
8479
8480	/* Restore longjmp state. */
8481	VMMRZCallRing3Enable(pVCpu);
8482	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8483	break;
8484	}
8485
8486	case RTTHREADCTXEVENT_IN:
8487	{
8488	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8489	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8490	VMCPU_ASSERT_EMT(pVCpu);
8491
8492	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8493	VMMRZCallRing3Disable(pVCpu);
8494	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8495
8496	/* Initialize the bare minimum state required for HM. This takes care of
8497	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8498	int rc = HMR0EnterCpu(pVCpu);
8499	AssertRC(rc);
8500	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8501
8502	/* Load the active VMCS as the current one. */
8503	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8504	{
8505	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8506	AssertRC(rc); NOREF(rc);
8507	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8508	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8509	}
8510	pVCpu->hm.s.fLeaveDone = false;
8511
8512	/* Restore longjmp state. */
8513	VMMRZCallRing3Enable(pVCpu);
8514	break;
8515	}
8516
8517	default:
8518	break;
8519	}
8520	}
8521
8522
8523	/**
8524	* Saves the host state in the VMCS host-state.
8525	* Sets up the VM-exit MSR-load area.
8526	*
8527	* The CPU state will be loaded from these fields on every successful VM-exit.
8528	*
8529	* @returns VBox status code.
8530	* @param pVM The cross context VM structure.
8531	* @param pVCpu The cross context virtual CPU structure.
8532	*
8533	* @remarks No-long-jump zone!!!
8534	*/
8535	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8536	{
8537	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8538
8539	int rc = VINF_SUCCESS;
8540	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8541	{
8542	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8543	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8544
8545	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8546	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8547
8548	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8549	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8550
8551	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8552	}
8553	return rc;
8554	}
8555
8556
8557	/**
8558	* Saves the host state in the VMCS host-state.
8559	*
8560	* @returns VBox status code.
8561	* @param pVM The cross context VM structure.
8562	* @param pVCpu The cross context virtual CPU structure.
8563	*
8564	* @remarks No-long-jump zone!!!
8565	*/
8566	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8567	{
8568	AssertPtr(pVM);
8569	AssertPtr(pVCpu);
8570
8571	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8572
8573	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8574	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8575	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8576	return hmR0VmxSaveHostState(pVM, pVCpu);
8577	}
8578
8579
8580	/**
8581	* Loads the guest state into the VMCS guest-state area.
8582	*
8583	* The will typically be done before VM-entry when the guest-CPU state and the
8584	* VMCS state may potentially be out of sync.
8585	*
8586	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8587	* VM-entry controls.
8588	* Sets up the appropriate VMX non-root function to execute guest code based on
8589	* the guest CPU mode.
8590	*
8591	* @returns VBox strict status code.
8592	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8593	* without unrestricted guest access and the VMMDev is not presently
8594	* mapped (e.g. EFI32).
8595	*
8596	* @param pVM The cross context VM structure.
8597	* @param pVCpu The cross context virtual CPU structure.
8598	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8599	* out-of-sync. Make sure to update the required fields
8600	* before using them.
8601	*
8602	* @remarks No-long-jump zone!!!
8603	*/
8604	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8605	{
8606	AssertPtr(pVM);
8607	AssertPtr(pVCpu);
8608	AssertPtr(pMixedCtx);
8609	HMVMX_ASSERT_PREEMPT_SAFE();
8610
8611	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8612
8613	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8614
8615	/* Determine real-on-v86 mode. */
8616	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8617	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8618	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8619	{
8620	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8621	}
8622
8623	/*
8624	* Load the guest-state into the VMCS.
8625	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8626	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8627	*/
8628	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8629	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8630
8631	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8632	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8633	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8634
8635	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8636	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8637	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8638
8639	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8640	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8641
8642	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8643	if (rcStrict == VINF_SUCCESS)
8644	{ /* likely */ }
8645	else
8646	{
8647	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8648	return rcStrict;
8649	}
8650
8651	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8652	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8653	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8654
8655	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8656	determine we don't have to swap EFER after all. */
8657	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8659
8660	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8662
8663	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8665
8666	/*
8667	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8668	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8669	*/
8670	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8671	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8672
8673	/* Clear any unused and reserved bits. */
8674	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8675
8676	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8677	return rc;
8678	}
8679
8680
8681	/**
8682	* Loads the state shared between the host and guest into the VMCS.
8683	*
8684	* @param pVM The cross context VM structure.
8685	* @param pVCpu The cross context virtual CPU structure.
8686	* @param pCtx Pointer to the guest-CPU context.
8687	*
8688	* @remarks No-long-jump zone!!!
8689	*/
8690	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8691	{
8692	NOREF(pVM);
8693
8694	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8695	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8696
8697	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8698	{
8699	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8700	AssertRC(rc);
8701	}
8702
8703	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8704	{
8705	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8706	AssertRC(rc);
8707
8708	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8709	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8710	{
8711	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8712	AssertRC(rc);
8713	}
8714	}
8715
8716	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8717	{
8718	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8719	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8720	}
8721
8722	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8723	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8724	{
8725	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8726	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8727	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8728	AssertRC(rc);
8729	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8730	}
8731
8732	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8733	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8734	}
8735
8736
8737	/**
8738	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8739	*
8740	* @returns Strict VBox status code (i.e. informational status codes too).
8741	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8742	* without unrestricted guest access and the VMMDev is not presently
8743	* mapped (e.g. EFI32).
8744	*
8745	* @param pVM The cross context VM structure.
8746	* @param pVCpu The cross context virtual CPU structure.
8747	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8748	* out-of-sync. Make sure to update the required fields
8749	* before using them.
8750	*
8751	* @remarks No-long-jump zone!!!
8752	*/
8753	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8754	{
8755	HMVMX_ASSERT_PREEMPT_SAFE();
8756	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8757	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8758
8759	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8760	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8761	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8762	#endif
8763
8764	/*
8765	* RIP is what changes the most often and hence if it's the only bit needing to be
8766	* updated, we shall handle it early for performance reasons.
8767	*/
8768	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8769	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8770	{
8771	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8772	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8773	{ /* likely */}
8774	else
8775	{
8776	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8777	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8778	}
8779	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8780	}
8781	else if (HMCPU_CF_VALUE(pVCpu))
8782	{
8783	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8784	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8785	{ /* likely */}
8786	else
8787	{
8788	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8789	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8790	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8791	return rcStrict;
8792	}
8793	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8794	}
8795
8796	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8797	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8798	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8799	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8800	return rcStrict;
8801	}
8802
8803
8804	/**
8805	* Does the preparations before executing guest code in VT-x.
8806	*
8807	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8808	* recompiler/IEM. We must be cautious what we do here regarding committing
8809	* guest-state information into the VMCS assuming we assuredly execute the
8810	* guest in VT-x mode.
8811	*
8812	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8813	* the common-state (TRPM/forceflags), we must undo those changes so that the
8814	* recompiler/IEM can (and should) use them when it resumes guest execution.
8815	* Otherwise such operations must be done when we can no longer exit to ring-3.
8816	*
8817	* @returns Strict VBox status code (i.e. informational status codes too).
8818	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8819	* have been disabled.
8820	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8821	* double-fault into the guest.
8822	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8823	* dispatched directly.
8824	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8825	*
8826	* @param pVM The cross context VM structure.
8827	* @param pVCpu The cross context virtual CPU structure.
8828	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8829	* out-of-sync. Make sure to update the required fields
8830	* before using them.
8831	* @param pVmxTransient Pointer to the VMX transient structure.
8832	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8833	* us ignore some of the reasons for returning to
8834	* ring-3, and return VINF_EM_DBG_STEPPED if event
8835	* dispatching took place.
8836	*/
8837	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8838	{
8839	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8840
8841	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8842	PGMRZDynMapFlushAutoSet(pVCpu);
8843	#endif
8844
8845	/* Check force flag actions that might require us to go back to ring-3. */
8846	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8847	if (rcStrict == VINF_SUCCESS)
8848	{ /* FFs doesn't get set all the time. */ }
8849	else
8850	return rcStrict;
8851
8852	#ifndef IEM_VERIFICATION_MODE_FULL
8853	/*
8854	* Setup the virtualized-APIC accesses.
8855	*
8856	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8857	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8858	*
8859	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8860	*/
8861	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8862	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8863	&& PDMHasApic(pVM))
8864	{
8865	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8866	Assert(u64MsrApicBase);
8867	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8868
8869	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8870
8871	/* Unalias any existing mapping. */
8872	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8873	AssertRCReturn(rc, rc);
8874
8875	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8876	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8877	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8878	AssertRCReturn(rc, rc);
8879
8880	/* Update the per-VCPU cache of the APIC base MSR. */
8881	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8882	}
8883	#endif /* !IEM_VERIFICATION_MODE_FULL */
8884
8885	if (TRPMHasTrap(pVCpu))
8886	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8887	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8888
8889	/*
8890	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8891	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8892	*/
8893	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8894	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8895	{ /* likely */ }
8896	else
8897	{
8898	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8899	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8900	return rcStrict;
8901	}
8902
8903	/*
8904	* No longjmps to ring-3 from this point on!!!
8905	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8906	* This also disables flushing of the R0-logger instance (if any).
8907	*/
8908	VMMRZCallRing3Disable(pVCpu);
8909
8910	/*
8911	* Load the guest state bits.
8912	*
8913	* We cannot perform longjmps while loading the guest state because we do not preserve the
8914	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8915	* CPU migration.
8916	*
8917	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8918	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8919	* Hence, loading of the guest state needs to be done -after- injection of events.
8920	*/
8921	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8922	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8923	{ /* likely */ }
8924	else
8925	{
8926	VMMRZCallRing3Enable(pVCpu);
8927	return rcStrict;
8928	}
8929
8930	/*
8931	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8932	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8933	*
8934	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8935	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8936	*
8937	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8938	* executing guest code.
8939	*/
8940	pVmxTransient->fEFlags = ASMIntDisableFlags();
8941
8942	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8943	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8944	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8945	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
8946	{
8947	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8948	{
8949	pVCpu->hm.s.Event.fPending = false;
8950
8951	/*
8952	* We've injected any pending events. This is really the point of no return (to ring-3).
8953	*
8954	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
8955	* returns from this function, so don't enable them here.
8956	*/
8957	return VINF_SUCCESS;
8958	}
8959
8960	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8961	rcStrict = VINF_EM_RAW_INTERRUPT;
8962	}
8963	else
8964	{
8965	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8966	rcStrict = VINF_EM_RAW_TO_R3;
8967	}
8968
8969	ASMSetFlags(pVmxTransient->fEFlags);
8970	VMMRZCallRing3Enable(pVCpu);
8971
8972	return rcStrict;
8973	}
8974
8975
8976	/**
8977	* Prepares to run guest code in VT-x and we've committed to doing so. This
8978	* means there is no backing out to ring-3 or anywhere else at this
8979	* point.
8980	*
8981	* @param pVM The cross context VM structure.
8982	* @param pVCpu The cross context virtual CPU structure.
8983	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8984	* out-of-sync. Make sure to update the required fields
8985	* before using them.
8986	* @param pVmxTransient Pointer to the VMX transient structure.
8987	*
8988	* @remarks Called with preemption disabled.
8989	* @remarks No-long-jump zone!!!
8990	*/
8991	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8992	{
8993	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8994	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8995	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8996
8997	/*
8998	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8999	*/
9000	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9001	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9002
9003	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9004	if (!CPUMIsGuestFPUStateActive(pVCpu))
9005	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9006	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9007	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9008	#endif
9009
9010	if ( pVCpu->hm.s.fPreloadGuestFpu
9011	&& !CPUMIsGuestFPUStateActive(pVCpu))
9012	{
9013	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9014	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9015	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9016	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9017	}
9018
9019	/*
9020	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9021	*/
9022	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9023	&& pVCpu->hm.s.vmx.cMsrs > 0)
9024	{
9025	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9026	}
9027
9028	/*
9029	* Load the host state bits as we may've been preempted (only happens when
9030	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9031	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9032	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9033	* See @bugref{8432}.
9034	*/
9035	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9036	{
9037	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9038	AssertRC(rc);
9039	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9040	}
9041	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9042
9043	/*
9044	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9045	*/
9046	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9047	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9048	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9049
9050	/* Store status of the shared guest-host state at the time of VM-entry. */
9051	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9052	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9053	{
9054	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9055	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9056	}
9057	else
9058	#endif
9059	{
9060	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9061	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9062	}
9063	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9064
9065	/*
9066	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9067	*/
9068	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9069	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9070
9071	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9072	RTCPUID idCurrentCpu = pCpu->idCpu;
9073	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9074	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9075	{
9076	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9077	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9078	}
9079
9080	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9081	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9082	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9083	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9084
9085	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9086
9087	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9088	to start executing. */
9089
9090	/*
9091	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9092	*/
9093	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9094	{
9095	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9096	{
9097	bool fMsrUpdated;
9098	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9099	AssertRC(rc2);
9100	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9101
9102	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9103	&fMsrUpdated);
9104	AssertRC(rc2);
9105	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9106
9107	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9108	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9109	}
9110	else
9111	{
9112	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9113	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9114	}
9115	}
9116
9117	#ifdef VBOX_STRICT
9118	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9119	hmR0VmxCheckHostEferMsr(pVCpu);
9120	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9121	#endif
9122	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9123	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9124	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9125	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9126	#endif
9127	}
9128
9129
9130	/**
9131	* Performs some essential restoration of state after running guest code in
9132	* VT-x.
9133	*
9134	* @param pVM The cross context VM structure.
9135	* @param pVCpu The cross context virtual CPU structure.
9136	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9137	* out-of-sync. Make sure to update the required fields
9138	* before using them.
9139	* @param pVmxTransient Pointer to the VMX transient structure.
9140	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9141	*
9142	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9143	*
9144	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9145	* unconditionally when it is safe to do so.
9146	*/
9147	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9148	{
9149	NOREF(pVM);
9150
9151	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9152
9153	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9154	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9155	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9156	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9157	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9158	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9159
9160	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9161	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9162
9163	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9164	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9165	Assert(!ASMIntAreEnabled());
9166	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9167
9168	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9169	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9170	{
9171	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9172	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9173	}
9174	#endif
9175
9176	#if HC_ARCH_BITS == 64
9177	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9178	#endif
9179	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9180	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9181	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9182	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9183	#else
9184	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9185	#endif
9186	#ifdef VBOX_STRICT
9187	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9188	#endif
9189	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9190	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
9191
9192	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9193	uint32_t uExitReason;
9194	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9195	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9196	AssertRC(rc);
9197	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9198	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9199
9200	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
9201	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
9202	{
9203	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9204	pVmxTransient->fVMEntryFailed));
9205	return;
9206	}
9207
9208	/*
9209	* Update the VM-exit history array here even if the VM-entry failed due to:
9210	* - Invalid guest state.
9211	* - MSR loading.
9212	* - Machine-check event.
9213	*
9214	* In any of the above cases we will still have a "valid" VM-exit reason
9215	* despite @a fVMEntryFailed being false.
9216	*
9217	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9218	*/
9219	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9220
9221	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
9222	{
9223	/** @todo We can optimize this by only syncing with our force-flags when
9224	* really needed and keeping the VMCS state as it is for most
9225	* VM-exits. */
9226	/* Update the guest interruptibility-state from the VMCS. */
9227	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9228
9229	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9230	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9231	AssertRC(rc);
9232	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9233	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9234	AssertRC(rc);
9235	#endif
9236
9237	/*
9238	* Sync the TPR shadow with our APIC state.
9239	*/
9240	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9241	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9242	{
9243	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9244	AssertRC(rc);
9245	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9246	}
9247	}
9248	}
9249
9250
9251	/**
9252	* Runs the guest code using VT-x the normal way.
9253	*
9254	* @returns VBox status code.
9255	* @param pVM The cross context VM structure.
9256	* @param pVCpu The cross context virtual CPU structure.
9257	* @param pCtx Pointer to the guest-CPU context.
9258	*
9259	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9260	*/
9261	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9262	{
9263	VMXTRANSIENT VmxTransient;
9264	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9265	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9266	uint32_t cLoops = 0;
9267
9268	for (;; cLoops++)
9269	{
9270	Assert(!HMR0SuspendPending());
9271	HMVMX_ASSERT_CPU_SAFE();
9272
9273	/* Preparatory work for running guest code, this may force us to return
9274	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9275	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9276	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9277	if (rcStrict != VINF_SUCCESS)
9278	break;
9279
9280	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9281	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9282	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9283
9284	/* Restore any residual host-state and save any bits shared between host
9285	and guest into the guest-CPU state. Re-enables interrupts! */
9286	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9287
9288	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9289	if (RT_SUCCESS(rcRun))
9290	{ /* very likely */ }
9291	else
9292	{
9293	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9294	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9295	return rcRun;
9296	}
9297
9298	/* Profile the VM-exit. */
9299	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9301	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9302	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9303	HMVMX_START_EXIT_DISPATCH_PROF();
9304
9305	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9306
9307	/* Handle the VM-exit. */
9308	#ifdef HMVMX_USE_FUNCTION_TABLE
9309	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9310	#else
9311	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9312	#endif
9313	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9314	if (rcStrict == VINF_SUCCESS)
9315	{
9316	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9317	continue; /* likely */
9318	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9319	rcStrict = VINF_EM_RAW_INTERRUPT;
9320	}
9321	break;
9322	}
9323
9324	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9325	return rcStrict;
9326	}
9327
9328
9329
9330	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9331	* probes.
9332	*
9333	* The following few functions and associated structure contains the bloat
9334	* necessary for providing detailed debug events and dtrace probes as well as
9335	* reliable host side single stepping. This works on the principle of
9336	* "subclassing" the normal execution loop and workers. We replace the loop
9337	* method completely and override selected helpers to add necessary adjustments
9338	* to their core operation.
9339	*
9340	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9341	* any performance for debug and analysis features.
9342	*
9343	* @{
9344	*/
9345
9346	/**
9347	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9348	* the debug run loop.
9349	*/
9350	typedef struct VMXRUNDBGSTATE
9351	{
9352	/** The RIP we started executing at. This is for detecting that we stepped. */
9353	uint64_t uRipStart;
9354	/** The CS we started executing with. */
9355	uint16_t uCsStart;
9356
9357	/** Whether we've actually modified the 1st execution control field. */
9358	bool fModifiedProcCtls : 1;
9359	/** Whether we've actually modified the 2nd execution control field. */
9360	bool fModifiedProcCtls2 : 1;
9361	/** Whether we've actually modified the exception bitmap. */
9362	bool fModifiedXcptBitmap : 1;
9363
9364	/** We desire the modified the CR0 mask to be cleared. */
9365	bool fClearCr0Mask : 1;
9366	/** We desire the modified the CR4 mask to be cleared. */
9367	bool fClearCr4Mask : 1;
9368	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9369	uint32_t fCpe1Extra;
9370	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9371	uint32_t fCpe1Unwanted;
9372	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9373	uint32_t fCpe2Extra;
9374	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9375	uint32_t bmXcptExtra;
9376	/** The sequence number of the Dtrace provider settings the state was
9377	* configured against. */
9378	uint32_t uDtraceSettingsSeqNo;
9379	/** VM-exits to check (one bit per VM-exit). */
9380	uint32_t bmExitsToCheck[3];
9381
9382	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9383	uint32_t fProcCtlsInitial;
9384	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9385	uint32_t fProcCtls2Initial;
9386	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9387	uint32_t bmXcptInitial;
9388	} VMXRUNDBGSTATE;
9389	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9390	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9391
9392
9393	/**
9394	* Initializes the VMXRUNDBGSTATE structure.
9395	*
9396	* @param pVCpu The cross context virtual CPU structure of the
9397	* calling EMT.
9398	* @param pCtx The CPU register context to go with @a pVCpu.
9399	* @param pDbgState The structure to initialize.
9400	*/
9401	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9402	{
9403	pDbgState->uRipStart = pCtx->rip;
9404	pDbgState->uCsStart = pCtx->cs.Sel;
9405
9406	pDbgState->fModifiedProcCtls = false;
9407	pDbgState->fModifiedProcCtls2 = false;
9408	pDbgState->fModifiedXcptBitmap = false;
9409	pDbgState->fClearCr0Mask = false;
9410	pDbgState->fClearCr4Mask = false;
9411	pDbgState->fCpe1Extra = 0;
9412	pDbgState->fCpe1Unwanted = 0;
9413	pDbgState->fCpe2Extra = 0;
9414	pDbgState->bmXcptExtra = 0;
9415	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9416	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9417	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9418	}
9419
9420
9421	/**
9422	* Updates the VMSC fields with changes requested by @a pDbgState.
9423	*
9424	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9425	* immediately before executing guest code, i.e. when interrupts are disabled.
9426	* We don't check status codes here as we cannot easily assert or return in the
9427	* latter case.
9428	*
9429	* @param pVCpu The cross context virtual CPU structure.
9430	* @param pDbgState The debug state.
9431	*/
9432	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9433	{
9434	/*
9435	* Ensure desired flags in VMCS control fields are set.
9436	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9437	*
9438	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9439	* there should be no stale data in pCtx at this point.
9440	*/
9441	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9442	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9443	{
9444	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9445	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9446	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9447	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9448	pDbgState->fModifiedProcCtls = true;
9449	}
9450
9451	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9452	{
9453	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9454	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9455	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9456	pDbgState->fModifiedProcCtls2 = true;
9457	}
9458
9459	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9460	{
9461	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9462	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9463	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9464	pDbgState->fModifiedXcptBitmap = true;
9465	}
9466
9467	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9468	{
9469	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9470	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9471	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9472	}
9473
9474	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9475	{
9476	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9477	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9478	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9479	}
9480	}
9481
9482
9483	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9484	{
9485	/*
9486	* Restore VM-exit control settings as we may not reenter this function the
9487	* next time around.
9488	*/
9489	/* We reload the initial value, trigger what we can of recalculations the
9490	next time around. From the looks of things, that's all that's required atm. */
9491	if (pDbgState->fModifiedProcCtls)
9492	{
9493	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9494	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9495	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9496	AssertRCReturn(rc2, rc2);
9497	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9498	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_DEBUG);
9499	}
9500
9501	/* We're currently the only ones messing with this one, so just restore the
9502	cached value and reload the field. */
9503	if ( pDbgState->fModifiedProcCtls2
9504	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9505	{
9506	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9507	AssertRCReturn(rc2, rc2);
9508	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9509	}
9510
9511	/* If we've modified the exception bitmap, we restore it and trigger
9512	reloading and partial recalculation the next time around. */
9513	if (pDbgState->fModifiedXcptBitmap)
9514	{
9515	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9516	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS \| HM_CHANGED_GUEST_CR0);
9517	}
9518
9519	/* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9520	if (pDbgState->fClearCr0Mask)
9521	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9522
9523	/* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9524	if (pDbgState->fClearCr4Mask)
9525	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9526
9527	return rcStrict;
9528	}
9529
9530
9531	/**
9532	* Configures VM-exit controls for current DBGF and DTrace settings.
9533	*
9534	* This updates @a pDbgState and the VMCS execution control fields to reflect
9535	* the necessary VM-exits demanded by DBGF and DTrace.
9536	*
9537	* @param pVM The cross context VM structure.
9538	* @param pVCpu The cross context virtual CPU structure.
9539	* @param pCtx Pointer to the guest-CPU context.
9540	* @param pDbgState The debug state.
9541	* @param pVmxTransient Pointer to the VMX transient structure. May update
9542	* fUpdateTscOffsettingAndPreemptTimer.
9543	*/
9544	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9545	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9546	{
9547	/*
9548	* Take down the dtrace serial number so we can spot changes.
9549	*/
9550	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9551	ASMCompilerBarrier();
9552
9553	/*
9554	* We'll rebuild most of the middle block of data members (holding the
9555	* current settings) as we go along here, so start by clearing it all.
9556	*/
9557	pDbgState->bmXcptExtra = 0;
9558	pDbgState->fCpe1Extra = 0;
9559	pDbgState->fCpe1Unwanted = 0;
9560	pDbgState->fCpe2Extra = 0;
9561	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9562	pDbgState->bmExitsToCheck[i] = 0;
9563
9564	/*
9565	* Software interrupts (INT XXh) - no idea how to trigger these...
9566	*/
9567	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9568	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9569	{
9570	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9571	}
9572
9573	/*
9574	* INT3 breakpoints - triggered by #BP exceptions.
9575	*/
9576	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9577	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9578
9579	/*
9580	* Exception bitmap and XCPT events+probes.
9581	*/
9582	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9583	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9584	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9585
9586	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9587	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9588	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9589	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9590	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9591	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9592	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9593	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9594	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9595	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9596	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9597	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9598	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9599	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9600	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9601	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9602	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9603	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9604
9605	if (pDbgState->bmXcptExtra)
9606	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9607
9608	/*
9609	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9610	*
9611	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9612	* So, when adding/changing/removing please don't forget to update it.
9613	*
9614	* Some of the macros are picking up local variables to save horizontal space,
9615	* (being able to see it in a table is the lesser evil here).
9616	*/
9617	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9618	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9619	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9620	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9621	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9622	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9623	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9624	} else do { } while (0)
9625	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9626	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9627	{ \
9628	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9629	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9630	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9631	} else do { } while (0)
9632	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9633	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9634	{ \
9635	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9636	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9637	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9638	} else do { } while (0)
9639	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9640	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9641	{ \
9642	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9643	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9644	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9645	} else do { } while (0)
9646
9647	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9648	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9649	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9650	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9651	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9652
9653	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9654	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9655	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9656	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9657	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9658	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9659	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9660	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9661	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9662	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9663	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9664	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9665	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9666	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9667	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9668	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9669	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9670	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9671	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9672	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9673	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9674	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9675	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9676	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9677	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9678	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9679	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9680	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9681	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9682	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9683	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9684	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9685	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9686	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9687	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9688	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9689
9690	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9691	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9692	{
9693	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9694	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9695	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9696	AssertRC(rc2);
9697
9698	#if 0 /** @todo fix me */
9699	pDbgState->fClearCr0Mask = true;
9700	pDbgState->fClearCr4Mask = true;
9701	#endif
9702	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9703	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9704	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9705	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9706	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9707	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9708	require clearing here and in the loop if we start using it. */
9709	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9710	}
9711	else
9712	{
9713	if (pDbgState->fClearCr0Mask)
9714	{
9715	pDbgState->fClearCr0Mask = false;
9716	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9717	}
9718	if (pDbgState->fClearCr4Mask)
9719	{
9720	pDbgState->fClearCr4Mask = false;
9721	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9722	}
9723	}
9724	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9725	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9726
9727	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9728	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9729	{
9730	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9731	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9732	}
9733	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9735
9736	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9737	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9738	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9739	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9740	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9741	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9742	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9743	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9744	#if 0 /** @todo too slow, fix handler. */
9745	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9746	#endif
9747	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9748
9749	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9750	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9751	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9752	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9753	{
9754	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9755	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9756	}
9757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9759	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9761
9762	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9763	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9764	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9765	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9766	{
9767	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9768	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9769	}
9770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9774
9775	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9777	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9778	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9779	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9780	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9781	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9783	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9784	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9785	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9786	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9787	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9788	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9789	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9790	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9791	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9793	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9794	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9795	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9796	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9797
9798	#undef IS_EITHER_ENABLED
9799	#undef SET_ONLY_XBM_IF_EITHER_EN
9800	#undef SET_CPE1_XBM_IF_EITHER_EN
9801	#undef SET_CPEU_XBM_IF_EITHER_EN
9802	#undef SET_CPE2_XBM_IF_EITHER_EN
9803
9804	/*
9805	* Sanitize the control stuff.
9806	*/
9807	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9808	if (pDbgState->fCpe2Extra)
9809	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9810	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9811	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9812	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9813	{
9814	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9815	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9816	}
9817
9818	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9819	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9820	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9821	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9822	}
9823
9824
9825	/**
9826	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9827	* appropriate.
9828	*
9829	* The caller has checked the VM-exit against the
9830	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9831	* already, so we don't have to do that either.
9832	*
9833	* @returns Strict VBox status code (i.e. informational status codes too).
9834	* @param pVM The cross context VM structure.
9835	* @param pVCpu The cross context virtual CPU structure.
9836	* @param pMixedCtx Pointer to the guest-CPU context.
9837	* @param pVmxTransient Pointer to the VMX-transient structure.
9838	* @param uExitReason The VM-exit reason.
9839	*
9840	* @remarks The name of this function is displayed by dtrace, so keep it short
9841	* and to the point. No longer than 33 chars long, please.
9842	*/
9843	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9844	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9845	{
9846	/*
9847	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9848	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9849	*
9850	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9851	* does. Must add/change/remove both places. Same ordering, please.
9852	*
9853	* Added/removed events must also be reflected in the next section
9854	* where we dispatch dtrace events.
9855	*/
9856	bool fDtrace1 = false;
9857	bool fDtrace2 = false;
9858	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9859	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9860	uint32_t uEventArg = 0;
9861	#define SET_EXIT(a_EventSubName) \
9862	do { \
9863	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9864	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9865	} while (0)
9866	#define SET_BOTH(a_EventSubName) \
9867	do { \
9868	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9869	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9870	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9871	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9872	} while (0)
9873	switch (uExitReason)
9874	{
9875	case VMX_EXIT_MTF:
9876	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9877
9878	case VMX_EXIT_XCPT_OR_NMI:
9879	{
9880	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9881	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9882	{
9883	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9884	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9885	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9886	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9887	{
9888	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9889	{
9890	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9891	uEventArg = pVmxTransient->uExitIntErrorCode;
9892	}
9893	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9894	switch (enmEvent1)
9895	{
9896	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9897	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9898	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9899	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9900	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9901	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9902	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9903	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9904	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9905	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9906	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9907	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9908	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9909	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9910	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9911	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9912	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9913	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9914	default: break;
9915	}
9916	}
9917	else
9918	AssertFailed();
9919	break;
9920
9921	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9922	uEventArg = idxVector;
9923	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9924	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9925	break;
9926	}
9927	break;
9928	}
9929
9930	case VMX_EXIT_TRIPLE_FAULT:
9931	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9932	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9933	break;
9934	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9935	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9936	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9937	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9938	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9939
9940	/* Instruction specific VM-exits: */
9941	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9942	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9943	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9944	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9945	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9946	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9947	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9948	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9949	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9950	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9951	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9952	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9953	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9954	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9955	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9956	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9957	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9958	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9959	case VMX_EXIT_MOV_CRX:
9960	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9961	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9962	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9963	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9964	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9965	SET_BOTH(CRX_READ);
9966	else
9967	SET_BOTH(CRX_WRITE);
9968	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9969	break;
9970	case VMX_EXIT_MOV_DRX:
9971	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9972	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9973	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9974	SET_BOTH(DRX_READ);
9975	else
9976	SET_BOTH(DRX_WRITE);
9977	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9978	break;
9979	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9980	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9981	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9982	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9983	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9984	case VMX_EXIT_XDTR_ACCESS:
9985	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9986	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9987	{
9988	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9989	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9990	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9991	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9992	}
9993	break;
9994
9995	case VMX_EXIT_TR_ACCESS:
9996	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9997	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9998	{
9999	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10000	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10001	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10002	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10003	}
10004	break;
10005
10006	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10007	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10008	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10009	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10010	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10011	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10012	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10013	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10014	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10015	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10016	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10017
10018	/* Events that aren't relevant at this point. */
10019	case VMX_EXIT_EXT_INT:
10020	case VMX_EXIT_INT_WINDOW:
10021	case VMX_EXIT_NMI_WINDOW:
10022	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10023	case VMX_EXIT_PREEMPT_TIMER:
10024	case VMX_EXIT_IO_INSTR:
10025	break;
10026
10027	/* Errors and unexpected events. */
10028	case VMX_EXIT_INIT_SIGNAL:
10029	case VMX_EXIT_SIPI:
10030	case VMX_EXIT_IO_SMI:
10031	case VMX_EXIT_SMI:
10032	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10033	case VMX_EXIT_ERR_MSR_LOAD:
10034	case VMX_EXIT_ERR_MACHINE_CHECK:
10035	break;
10036
10037	default:
10038	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10039	break;
10040	}
10041	#undef SET_BOTH
10042	#undef SET_EXIT
10043
10044	/*
10045	* Dtrace tracepoints go first. We do them here at once so we don't
10046	* have to copy the guest state saving and stuff a few dozen times.
10047	* Down side is that we've got to repeat the switch, though this time
10048	* we use enmEvent since the probes are a subset of what DBGF does.
10049	*/
10050	if (fDtrace1 \|\| fDtrace2)
10051	{
10052	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10053	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10054	switch (enmEvent1)
10055	{
10056	/** @todo consider which extra parameters would be helpful for each probe. */
10057	case DBGFEVENT_END: break;
10058	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10059	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10060	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10061	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10062	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10063	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10064	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10065	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10066	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10067	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10068	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10069	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10070	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10071	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10072	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10073	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10074	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10075	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10076	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10077	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10078	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10079	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10080	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10081	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10082	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10083	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10084	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10085	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10086	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10087	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10088	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10089	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10090	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10091	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10092	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10093	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10094	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10095	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10096	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10097	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10098	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10099	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10100	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10101	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10102	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10103	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10104	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10105	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10106	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10107	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10108	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10109	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10110	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10111	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10112	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10113	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10114	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10115	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10116	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10118	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10120	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10124	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10125	}
10126	switch (enmEvent2)
10127	{
10128	/** @todo consider which extra parameters would be helpful for each probe. */
10129	case DBGFEVENT_END: break;
10130	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10131	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10132	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10135	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10136	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10140	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10141	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10142	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10143	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10144	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10145	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10146	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10147	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10148	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10149	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10150	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10157	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10158	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10159	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10160	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10161	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10162	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10163	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10182	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10183	}
10184	}
10185
10186	/*
10187	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10188	* the DBGF call will do a full check).
10189	*
10190	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10191	* Note! If we have to events, we prioritize the first, i.e. the instruction
10192	* one, in order to avoid event nesting.
10193	*/
10194	if ( enmEvent1 != DBGFEVENT_END
10195	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10196	{
10197	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10198	if (rcStrict != VINF_SUCCESS)
10199	return rcStrict;
10200	}
10201	else if ( enmEvent2 != DBGFEVENT_END
10202	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10203	{
10204	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10205	if (rcStrict != VINF_SUCCESS)
10206	return rcStrict;
10207	}
10208
10209	return VINF_SUCCESS;
10210	}
10211
10212
10213	/**
10214	* Single-stepping VM-exit filtering.
10215	*
10216	* This is preprocessing the VM-exits and deciding whether we've gotten far
10217	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10218	* handling is performed.
10219	*
10220	* @returns Strict VBox status code (i.e. informational status codes too).
10221	* @param pVM The cross context VM structure.
10222	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10223	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10224	* out-of-sync. Make sure to update the required
10225	* fields before using them.
10226	* @param pVmxTransient Pointer to the VMX-transient structure.
10227	* @param uExitReason The VM-exit reason.
10228	* @param pDbgState The debug state.
10229	*/
10230	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10231	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10232	{
10233	/*
10234	* Expensive (saves context) generic dtrace VM-exit probe.
10235	*/
10236	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10237	{ /* more likely */ }
10238	else
10239	{
10240	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10241	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10242	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10243	}
10244
10245	/*
10246	* Check for host NMI, just to get that out of the way.
10247	*/
10248	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10249	{ /* normally likely */ }
10250	else
10251	{
10252	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10253	AssertRCReturn(rc2, rc2);
10254	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10255	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10256	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10257	}
10258
10259	/*
10260	* Check for single stepping event if we're stepping.
10261	*/
10262	if (pVCpu->hm.s.fSingleInstruction)
10263	{
10264	switch (uExitReason)
10265	{
10266	case VMX_EXIT_MTF:
10267	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10268
10269	/* Various events: */
10270	case VMX_EXIT_XCPT_OR_NMI:
10271	case VMX_EXIT_EXT_INT:
10272	case VMX_EXIT_TRIPLE_FAULT:
10273	case VMX_EXIT_INT_WINDOW:
10274	case VMX_EXIT_NMI_WINDOW:
10275	case VMX_EXIT_TASK_SWITCH:
10276	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10277	case VMX_EXIT_APIC_ACCESS:
10278	case VMX_EXIT_EPT_VIOLATION:
10279	case VMX_EXIT_EPT_MISCONFIG:
10280	case VMX_EXIT_PREEMPT_TIMER:
10281
10282	/* Instruction specific VM-exits: */
10283	case VMX_EXIT_CPUID:
10284	case VMX_EXIT_GETSEC:
10285	case VMX_EXIT_HLT:
10286	case VMX_EXIT_INVD:
10287	case VMX_EXIT_INVLPG:
10288	case VMX_EXIT_RDPMC:
10289	case VMX_EXIT_RDTSC:
10290	case VMX_EXIT_RSM:
10291	case VMX_EXIT_VMCALL:
10292	case VMX_EXIT_VMCLEAR:
10293	case VMX_EXIT_VMLAUNCH:
10294	case VMX_EXIT_VMPTRLD:
10295	case VMX_EXIT_VMPTRST:
10296	case VMX_EXIT_VMREAD:
10297	case VMX_EXIT_VMRESUME:
10298	case VMX_EXIT_VMWRITE:
10299	case VMX_EXIT_VMXOFF:
10300	case VMX_EXIT_VMXON:
10301	case VMX_EXIT_MOV_CRX:
10302	case VMX_EXIT_MOV_DRX:
10303	case VMX_EXIT_IO_INSTR:
10304	case VMX_EXIT_RDMSR:
10305	case VMX_EXIT_WRMSR:
10306	case VMX_EXIT_MWAIT:
10307	case VMX_EXIT_MONITOR:
10308	case VMX_EXIT_PAUSE:
10309	case VMX_EXIT_XDTR_ACCESS:
10310	case VMX_EXIT_TR_ACCESS:
10311	case VMX_EXIT_INVEPT:
10312	case VMX_EXIT_RDTSCP:
10313	case VMX_EXIT_INVVPID:
10314	case VMX_EXIT_WBINVD:
10315	case VMX_EXIT_XSETBV:
10316	case VMX_EXIT_RDRAND:
10317	case VMX_EXIT_INVPCID:
10318	case VMX_EXIT_VMFUNC:
10319	case VMX_EXIT_RDSEED:
10320	case VMX_EXIT_XSAVES:
10321	case VMX_EXIT_XRSTORS:
10322	{
10323	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10324	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10325	AssertRCReturn(rc2, rc2);
10326	if ( pMixedCtx->rip != pDbgState->uRipStart
10327	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10328	return VINF_EM_DBG_STEPPED;
10329	break;
10330	}
10331
10332	/* Errors and unexpected events: */
10333	case VMX_EXIT_INIT_SIGNAL:
10334	case VMX_EXIT_SIPI:
10335	case VMX_EXIT_IO_SMI:
10336	case VMX_EXIT_SMI:
10337	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10338	case VMX_EXIT_ERR_MSR_LOAD:
10339	case VMX_EXIT_ERR_MACHINE_CHECK:
10340	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10341	break;
10342
10343	default:
10344	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10345	break;
10346	}
10347	}
10348
10349	/*
10350	* Check for debugger event breakpoints and dtrace probes.
10351	*/
10352	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10353	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10354	{
10355	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10356	if (rcStrict != VINF_SUCCESS)
10357	return rcStrict;
10358	}
10359
10360	/*
10361	* Normal processing.
10362	*/
10363	#ifdef HMVMX_USE_FUNCTION_TABLE
10364	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10365	#else
10366	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10367	#endif
10368	}
10369
10370
10371	/**
10372	* Single steps guest code using VT-x.
10373	*
10374	* @returns Strict VBox status code (i.e. informational status codes too).
10375	* @param pVM The cross context VM structure.
10376	* @param pVCpu The cross context virtual CPU structure.
10377	* @param pCtx Pointer to the guest-CPU context.
10378	*
10379	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10380	*/
10381	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10382	{
10383	VMXTRANSIENT VmxTransient;
10384	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10385
10386	/* Set HMCPU indicators. */
10387	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10388	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10389	pVCpu->hm.s.fDebugWantRdTscExit = false;
10390	pVCpu->hm.s.fUsingDebugLoop = true;
10391
10392	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10393	VMXRUNDBGSTATE DbgState;
10394	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10395	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10396
10397	/*
10398	* The loop.
10399	*/
10400	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10401	for (uint32_t cLoops = 0; ; cLoops++)
10402	{
10403	Assert(!HMR0SuspendPending());
10404	HMVMX_ASSERT_CPU_SAFE();
10405	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10406
10407	/*
10408	* Preparatory work for running guest code, this may force us to return
10409	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10410	*/
10411	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10412	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10413	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10414	if (rcStrict != VINF_SUCCESS)
10415	break;
10416
10417	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10418	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10419
10420	/*
10421	* Now we can run the guest code.
10422	*/
10423	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10424
10425	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10426
10427	/*
10428	* Restore any residual host-state and save any bits shared between host
10429	* and guest into the guest-CPU state. Re-enables interrupts!
10430	*/
10431	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10432
10433	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10434	if (RT_SUCCESS(rcRun))
10435	{ /* very likely */ }
10436	else
10437	{
10438	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10439	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10440	return rcRun;
10441	}
10442
10443	/* Profile the VM-exit. */
10444	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10446	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10447	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10448	HMVMX_START_EXIT_DISPATCH_PROF();
10449
10450	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10451
10452	/*
10453	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10454	*/
10455	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10456	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10457	if (rcStrict != VINF_SUCCESS)
10458	break;
10459	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10460	{
10461	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10462	rcStrict = VINF_EM_RAW_INTERRUPT;
10463	break;
10464	}
10465
10466	/*
10467	* Stepping: Did the RIP change, if so, consider it a single step.
10468	* Otherwise, make sure one of the TFs gets set.
10469	*/
10470	if (fStepping)
10471	{
10472	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10473	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10474	AssertRCReturn(rc2, rc2);
10475	if ( pCtx->rip != DbgState.uRipStart
10476	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10477	{
10478	rcStrict = VINF_EM_DBG_STEPPED;
10479	break;
10480	}
10481	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10482	}
10483
10484	/*
10485	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10486	*/
10487	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10488	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10489	}
10490
10491	/*
10492	* Clear the X86_EFL_TF if necessary.
10493	*/
10494	if (pVCpu->hm.s.fClearTrapFlag)
10495	{
10496	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10497	AssertRCReturn(rc2, rc2);
10498	pVCpu->hm.s.fClearTrapFlag = false;
10499	pCtx->eflags.Bits.u1TF = 0;
10500	}
10501	/** @todo there seems to be issues with the resume flag when the monitor trap
10502	* flag is pending without being used. Seen early in bios init when
10503	* accessing APIC page in protected mode. */
10504
10505	/*
10506	* Restore VM-exit control settings as we may not reenter this function the
10507	* next time around.
10508	*/
10509	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10510
10511	/* Restore HMCPU indicators. */
10512	pVCpu->hm.s.fUsingDebugLoop = false;
10513	pVCpu->hm.s.fDebugWantRdTscExit = false;
10514	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10515
10516	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10517	return rcStrict;
10518	}
10519
10520
10521	/** @} */
10522
10523
10524	/**
10525	* Checks if any expensive dtrace probes are enabled and we should go to the
10526	* debug loop.
10527	*
10528	* @returns true if we should use debug loop, false if not.
10529	*/
10530	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10531	{
10532	/* It's probably faster to OR the raw 32-bit counter variables together.
10533	Since the variables are in an array and the probes are next to one
10534	another (more or less), we have good locality. So, better read
10535	eight-nine cache lines ever time and only have one conditional, than
10536	128+ conditionals, right? */
10537	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10538	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10539	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10540	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10541	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10542	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10543	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10544	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10545	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10546	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10547	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10548	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10549	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10550	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10551	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10552	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10553	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10554	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10555	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10556	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10557	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10558	) != 0
10559	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10560	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10561	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10562	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10563	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10564	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10565	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10566	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10567	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10568	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10569	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10570	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10571	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10572	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10573	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10574	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10575	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10576	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10577	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10578	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10579	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10580	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10581	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10582	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10583	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10584	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10585	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10586	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10587	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10588	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10589	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10590	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10591	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10592	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10593	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10594	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10595	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10596	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10597	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10598	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10599	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10600	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10601	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10602	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10603	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10604	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10605	) != 0
10606	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10607	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10608	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10609	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10610	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10611	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10612	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10613	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10614	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10615	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10616	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10617	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10618	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10619	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10620	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10621	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10622	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10623	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10624	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10625	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10626	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10627	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10628	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10629	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10630	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10631	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10632	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10633	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10634	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10635	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10636	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10637	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10638	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10639	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10640	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10641	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10642	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10643	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10644	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10645	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10646	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10647	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10648	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10649	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10650	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10651	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10652	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10653	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10654	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10655	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10656	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10657	) != 0;
10658	}
10659
10660
10661	/**
10662	* Runs the guest code using VT-x.
10663	*
10664	* @returns Strict VBox status code (i.e. informational status codes too).
10665	* @param pVM The cross context VM structure.
10666	* @param pVCpu The cross context virtual CPU structure.
10667	* @param pCtx Pointer to the guest-CPU context.
10668	*/
10669	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10670	{
10671	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10672	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10673	HMVMX_ASSERT_PREEMPT_SAFE();
10674
10675	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10676
10677	VBOXSTRICTRC rcStrict;
10678	if ( !pVCpu->hm.s.fUseDebugLoop
10679	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10680	&& !DBGFIsStepping(pVCpu)
10681	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10682	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10683	else
10684	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10685
10686	if (rcStrict == VERR_EM_INTERPRETER)
10687	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10688	else if (rcStrict == VINF_EM_RESET)
10689	rcStrict = VINF_EM_TRIPLE_FAULT;
10690
10691	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10692	if (RT_FAILURE(rc2))
10693	{
10694	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10695	rcStrict = rc2;
10696	}
10697	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10698	return rcStrict;
10699	}
10700
10701
10702	#ifndef HMVMX_USE_FUNCTION_TABLE
10703	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10704	{
10705	# ifdef DEBUG_ramshankar
10706	# define RETURN_EXIT_CALL(a_CallExpr) \
10707	do { \
10708	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10709	VBOXSTRICTRC rcStrict = a_CallExpr; \
10710	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10711	return rcStrict; \
10712	} while (0)
10713	# else
10714	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10715	# endif
10716	switch (rcReason)
10717	{
10718	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10719	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10720	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10721	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10722	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10723	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10724	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10725	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10726	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10727	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10728	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10729	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10730	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10731	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10732	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10733	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10734	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10735	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10736	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10737	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10738	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10739	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10740	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10741	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10742	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10743	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10744	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10745	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10746	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10747	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10748	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10749	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10750	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10751	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10752
10753	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10754	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10755	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10756	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10757	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10758	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10759	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10760	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10761	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10762
10763	case VMX_EXIT_VMCLEAR:
10764	case VMX_EXIT_VMLAUNCH:
10765	case VMX_EXIT_VMPTRLD:
10766	case VMX_EXIT_VMPTRST:
10767	case VMX_EXIT_VMREAD:
10768	case VMX_EXIT_VMRESUME:
10769	case VMX_EXIT_VMWRITE:
10770	case VMX_EXIT_VMXOFF:
10771	case VMX_EXIT_VMXON:
10772	case VMX_EXIT_INVEPT:
10773	case VMX_EXIT_INVVPID:
10774	case VMX_EXIT_VMFUNC:
10775	case VMX_EXIT_XSAVES:
10776	case VMX_EXIT_XRSTORS:
10777	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10778	case VMX_EXIT_ENCLS:
10779	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10780	case VMX_EXIT_PML_FULL:
10781	default:
10782	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10783	}
10784	#undef RETURN_EXIT_CALL
10785	}
10786	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10787
10788
10789	#ifdef VBOX_STRICT
10790	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10791	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10792	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10793
10794	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10795	do { \
10796	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10797	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10798	} while (0)
10799
10800	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10801	do { \
10802	AssertPtr(pVCpu); \
10803	AssertPtr(pMixedCtx); \
10804	AssertPtr(pVmxTransient); \
10805	Assert(pVmxTransient->fVMEntryFailed == false); \
10806	Assert(ASMIntAreEnabled()); \
10807	HMVMX_ASSERT_PREEMPT_SAFE(); \
10808	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10809	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10810	HMVMX_ASSERT_PREEMPT_SAFE(); \
10811	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10812	HMVMX_ASSERT_PREEMPT_CPUID(); \
10813	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10814	} while (0)
10815
10816	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10817	do { \
10818	Log4Func(("\n")); \
10819	} while (0)
10820	#else /* nonstrict builds: */
10821	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10822	do { \
10823	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10824	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10825	} while (0)
10826	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10827	#endif
10828
10829
10830	/**
10831	* Advances the guest RIP by the specified number of bytes.
10832	*
10833	* @param pVCpu The cross context virtual CPU structure.
10834	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10835	* out-of-sync. Make sure to update the required fields
10836	* before using them.
10837	* @param cbInstr Number of bytes to advance the RIP by.
10838	*
10839	* @remarks No-long-jump zone!!!
10840	*/
10841	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10842	{
10843	/* Advance the RIP. */
10844	pMixedCtx->rip += cbInstr;
10845	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10846
10847	/* Update interrupt inhibition. */
10848	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10849	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10850	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10851	}
10852
10853
10854	/**
10855	* Advances the guest RIP after reading it from the VMCS.
10856	*
10857	* @returns VBox status code, no informational status codes.
10858	* @param pVCpu The cross context virtual CPU structure.
10859	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10860	* out-of-sync. Make sure to update the required fields
10861	* before using them.
10862	* @param pVmxTransient Pointer to the VMX transient structure.
10863	*
10864	* @remarks No-long-jump zone!!!
10865	*/
10866	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10867	{
10868	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10869	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10870	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10871	AssertRCReturn(rc, rc);
10872
10873	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10874
10875	/*
10876	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10877	* pending debug exception field as it takes care of priority of events.
10878	*
10879	* See Intel spec. 32.2.1 "Debug Exceptions".
10880	*/
10881	if ( !pVCpu->hm.s.fSingleInstruction
10882	&& pMixedCtx->eflags.Bits.u1TF)
10883	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10884
10885	return VINF_SUCCESS;
10886	}
10887
10888
10889	/**
10890	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10891	* and update error record fields accordingly.
10892	*
10893	* @return VMX_IGS_* return codes.
10894	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10895	* wrong with the guest state.
10896	*
10897	* @param pVM The cross context VM structure.
10898	* @param pVCpu The cross context virtual CPU structure.
10899	* @param pCtx Pointer to the guest-CPU state.
10900	*
10901	* @remarks This function assumes our cache of the VMCS controls
10902	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10903	*/
10904	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10905	{
10906	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10907	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10908	uError = (err); \
10909	break; \
10910	} else do { } while (0)
10911
10912	int rc;
10913	uint32_t uError = VMX_IGS_ERROR;
10914	uint32_t u32Val;
10915	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10916
10917	do
10918	{
10919	/*
10920	* CR0.
10921	*/
10922	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10923	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10924	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10925	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10926	if (fUnrestrictedGuest)
10927	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10928
10929	uint32_t u32GuestCR0;
10930	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10931	AssertRCBreak(rc);
10932	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10933	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10934	if ( !fUnrestrictedGuest
10935	&& (u32GuestCR0 & X86_CR0_PG)
10936	&& !(u32GuestCR0 & X86_CR0_PE))
10937	{
10938	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10939	}
10940
10941	/*
10942	* CR4.
10943	*/
10944	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10945	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10946
10947	uint32_t u32GuestCR4;
10948	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10949	AssertRCBreak(rc);
10950	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10951	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10952
10953	/*
10954	* IA32_DEBUGCTL MSR.
10955	*/
10956	uint64_t u64Val;
10957	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10958	AssertRCBreak(rc);
10959	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10960	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10961	{
10962	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10963	}
10964	uint64_t u64DebugCtlMsr = u64Val;
10965
10966	#ifdef VBOX_STRICT
10967	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10968	AssertRCBreak(rc);
10969	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10970	#endif
10971	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10972
10973	/*
10974	* RIP and RFLAGS.
10975	*/
10976	uint32_t u32Eflags;
10977	#if HC_ARCH_BITS == 64
10978	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10979	AssertRCBreak(rc);
10980	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10981	if ( !fLongModeGuest
10982	\|\| !pCtx->cs.Attr.n.u1Long)
10983	{
10984	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10985	}
10986	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10987	* must be identical if the "IA-32e mode guest" VM-entry
10988	* control is 1 and CS.L is 1. No check applies if the
10989	* CPU supports 64 linear-address bits. */
10990
10991	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10992	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10993	AssertRCBreak(rc);
10994	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10995	VMX_IGS_RFLAGS_RESERVED);
10996	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10997	u32Eflags = u64Val;
10998	#else
10999	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11000	AssertRCBreak(rc);
11001	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11002	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11003	#endif
11004
11005	if ( fLongModeGuest
11006	\|\| ( fUnrestrictedGuest
11007	&& !(u32GuestCR0 & X86_CR0_PE)))
11008	{
11009	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11010	}
11011
11012	uint32_t u32EntryInfo;
11013	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11014	AssertRCBreak(rc);
11015	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11016	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11017	{
11018	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11019	}
11020
11021	/*
11022	* 64-bit checks.
11023	*/
11024	#if HC_ARCH_BITS == 64
11025	if (fLongModeGuest)
11026	{
11027	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11028	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11029	}
11030
11031	if ( !fLongModeGuest
11032	&& (u32GuestCR4 & X86_CR4_PCIDE))
11033	{
11034	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11035	}
11036
11037	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11038	* 51:32 beyond the processor's physical-address width are 0. */
11039
11040	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11041	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11042	{
11043	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11044	}
11045
11046	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11047	AssertRCBreak(rc);
11048	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11049
11050	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11051	AssertRCBreak(rc);
11052	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11053	#endif
11054
11055	/*
11056	* PERF_GLOBAL MSR.
11057	*/
11058	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11059	{
11060	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11061	AssertRCBreak(rc);
11062	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11063	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11064	}
11065
11066	/*
11067	* PAT MSR.
11068	*/
11069	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11070	{
11071	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11072	AssertRCBreak(rc);
11073	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11074	for (unsigned i = 0; i < 8; i++)
11075	{
11076	uint8_t u8Val = (u64Val & 0xff);
11077	if ( u8Val != 0 /* UC */
11078	&& u8Val != 1 /* WC */
11079	&& u8Val != 4 /* WT */
11080	&& u8Val != 5 /* WP */
11081	&& u8Val != 6 /* WB */
11082	&& u8Val != 7 /* UC- */)
11083	{
11084	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11085	}
11086	u64Val >>= 8;
11087	}
11088	}
11089
11090	/*
11091	* EFER MSR.
11092	*/
11093	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11094	{
11095	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11096	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11097	AssertRCBreak(rc);
11098	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11099	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11100	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11101	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11102	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11103	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11104	\|\| !(u32GuestCR0 & X86_CR0_PG)
11105	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11106	VMX_IGS_EFER_LMA_LME_MISMATCH);
11107	}
11108
11109	/*
11110	* Segment registers.
11111	*/
11112	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11113	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11114	if (!(u32Eflags & X86_EFL_VM))
11115	{
11116	/* CS */
11117	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11118	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11119	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11120	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11121	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11122	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11123	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11124	/* CS cannot be loaded with NULL in protected mode. */
11125	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11126	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11127	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11128	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11129	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11130	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11131	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11132	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11133	else
11134	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11135
11136	/* SS */
11137	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11138	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11139	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11140	if ( !(pCtx->cr0 & X86_CR0_PE)
11141	\|\| pCtx->cs.Attr.n.u4Type == 3)
11142	{
11143	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11144	}
11145	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11146	{
11147	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11148	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11149	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11150	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11151	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11152	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11153	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11154	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11155	}
11156
11157	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11158	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11159	{
11160	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11161	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11162	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11163	\|\| pCtx->ds.Attr.n.u4Type > 11
11164	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11165	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11166	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11167	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11168	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11169	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11170	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11171	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11172	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11173	}
11174	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11175	{
11176	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11177	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11178	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11179	\|\| pCtx->es.Attr.n.u4Type > 11
11180	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11181	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11182	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11183	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11184	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11185	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11186	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11187	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11188	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11189	}
11190	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11191	{
11192	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11193	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11194	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11195	\|\| pCtx->fs.Attr.n.u4Type > 11
11196	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11197	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11198	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11199	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11200	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11201	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11202	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11203	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11204	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11205	}
11206	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11207	{
11208	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11209	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11210	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11211	\|\| pCtx->gs.Attr.n.u4Type > 11
11212	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11213	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11214	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11215	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11216	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11217	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11218	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11219	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11220	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11221	}
11222	/* 64-bit capable CPUs. */
11223	#if HC_ARCH_BITS == 64
11224	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11225	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11226	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11227	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11228	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11229	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11230	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11231	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11232	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11233	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11234	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11235	#endif
11236	}
11237	else
11238	{
11239	/* V86 mode checks. */
11240	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11241	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11242	{
11243	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11244	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11245	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11246	}
11247	else
11248	{
11249	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11250	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11251	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11252	}
11253
11254	/* CS */
11255	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11256	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11257	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11258	/* SS */
11259	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11260	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11261	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11262	/* DS */
11263	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11264	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11265	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11266	/* ES */
11267	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11268	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11269	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11270	/* FS */
11271	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11272	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11273	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11274	/* GS */
11275	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11276	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11277	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11278	/* 64-bit capable CPUs. */
11279	#if HC_ARCH_BITS == 64
11280	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11281	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11282	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11283	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11284	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11285	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11286	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11287	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11288	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11289	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11290	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11291	#endif
11292	}
11293
11294	/*
11295	* TR.
11296	*/
11297	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11298	/* 64-bit capable CPUs. */
11299	#if HC_ARCH_BITS == 64
11300	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11301	#endif
11302	if (fLongModeGuest)
11303	{
11304	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11305	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11306	}
11307	else
11308	{
11309	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11310	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11311	VMX_IGS_TR_ATTR_TYPE_INVALID);
11312	}
11313	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11314	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11315	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11316	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11317	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11318	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11319	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11320	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11321
11322	/*
11323	* GDTR and IDTR.
11324	*/
11325	#if HC_ARCH_BITS == 64
11326	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11327	AssertRCBreak(rc);
11328	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11329
11330	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11331	AssertRCBreak(rc);
11332	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11333	#endif
11334
11335	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11336	AssertRCBreak(rc);
11337	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11338
11339	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11340	AssertRCBreak(rc);
11341	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11342
11343	/*
11344	* Guest Non-Register State.
11345	*/
11346	/* Activity State. */
11347	uint32_t u32ActivityState;
11348	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11349	AssertRCBreak(rc);
11350	HMVMX_CHECK_BREAK( !u32ActivityState
11351	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11352	VMX_IGS_ACTIVITY_STATE_INVALID);
11353	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11354	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11355	uint32_t u32IntrState;
11356	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11357	AssertRCBreak(rc);
11358	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11359	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11360	{
11361	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11362	}
11363
11364	/** @todo Activity state and injecting interrupts. Left as a todo since we
11365	* currently don't use activity states but ACTIVE. */
11366
11367	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11368	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11369
11370	/* Guest interruptibility-state. */
11371	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11372	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11373	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11374	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11375	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11376	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11377	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11378	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11379	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11380	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11381	{
11382	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11383	{
11384	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11385	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11386	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11387	}
11388	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11389	{
11390	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11391	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11392	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11393	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11394	}
11395	}
11396	/** @todo Assumes the processor is not in SMM. */
11397	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11398	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11399	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11400	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11401	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11402	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11403	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11404	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11405	{
11406	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11407	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11408	}
11409
11410	/* Pending debug exceptions. */
11411	#if HC_ARCH_BITS == 64
11412	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11413	AssertRCBreak(rc);
11414	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11415	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11416	u32Val = u64Val; /* For pending debug exceptions checks below. */
11417	#else
11418	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11419	AssertRCBreak(rc);
11420	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11421	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11422	#endif
11423
11424	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11425	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11426	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11427	{
11428	if ( (u32Eflags & X86_EFL_TF)
11429	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11430	{
11431	/* Bit 14 is PendingDebug.BS. */
11432	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11433	}
11434	if ( !(u32Eflags & X86_EFL_TF)
11435	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11436	{
11437	/* Bit 14 is PendingDebug.BS. */
11438	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11439	}
11440	}
11441
11442	/* VMCS link pointer. */
11443	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11444	AssertRCBreak(rc);
11445	if (u64Val != UINT64_C(0xffffffffffffffff))
11446	{
11447	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11448	/** @todo Bits beyond the processor's physical-address width MBZ. */
11449	/** @todo 32-bit located in memory referenced by value of this field (as a
11450	* physical address) must contain the processor's VMCS revision ID. */
11451	/** @todo SMM checks. */
11452	}
11453
11454	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11455	* not using Nested Paging? */
11456	if ( pVM->hm.s.fNestedPaging
11457	&& !fLongModeGuest
11458	&& CPUMIsGuestInPAEModeEx(pCtx))
11459	{
11460	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11461	AssertRCBreak(rc);
11462	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11463
11464	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11465	AssertRCBreak(rc);
11466	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11467
11468	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11469	AssertRCBreak(rc);
11470	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11471
11472	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11473	AssertRCBreak(rc);
11474	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11475	}
11476
11477	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11478	if (uError == VMX_IGS_ERROR)
11479	uError = VMX_IGS_REASON_NOT_FOUND;
11480	} while (0);
11481
11482	pVCpu->hm.s.u32HMError = uError;
11483	return uError;
11484
11485	#undef HMVMX_ERROR_BREAK
11486	#undef HMVMX_CHECK_BREAK
11487	}
11488
11489	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11490	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11491	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11492
11493	/** @name VM-exit handlers.
11494	* @{
11495	*/
11496
11497	/**
11498	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11499	*/
11500	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11501	{
11502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11504	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11505	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11506	return VINF_SUCCESS;
11507	return VINF_EM_RAW_INTERRUPT;
11508	}
11509
11510
11511	/**
11512	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11513	*/
11514	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11515	{
11516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11518
11519	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11520	AssertRCReturn(rc, rc);
11521
11522	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11523	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11524	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11525	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11526
11527	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11528	{
11529	/*
11530	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11531	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11532	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11533	*
11534	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11535	*/
11536	VMXDispatchHostNmi();
11537	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11538	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11539	return VINF_SUCCESS;
11540	}
11541
11542	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11543	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11544	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11545	{ /* likely */ }
11546	else
11547	{
11548	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11549	rcStrictRc1 = VINF_SUCCESS;
11550	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11551	return rcStrictRc1;
11552	}
11553
11554	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11555	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11556	switch (uIntType)
11557	{
11558	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11559	Assert(uVector == X86_XCPT_DB);
11560	/* fall thru */
11561	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11562	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11563	/* fall thru */
11564	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11565	{
11566	/*
11567	* If there's any exception caused as a result of event injection, go back to
11568	* the interpreter. The page-fault case is complicated and we manually handle
11569	* any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11570	* handled in hmR0VmxCheckExitDueToEventDelivery.
11571	*/
11572	if (!pVCpu->hm.s.Event.fPending)
11573	{ /* likely */ }
11574	else if ( uVector != X86_XCPT_PF
11575	&& uVector != X86_XCPT_AC)
11576	{
11577	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11578	rc = VERR_EM_INTERPRETER;
11579	break;
11580	}
11581
11582	switch (uVector)
11583	{
11584	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11585	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11586	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11587	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11588	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11589	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11590	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11591
11592	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11593	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11594	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11595	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11596	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11597	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11598	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11599	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11600	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11601	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11602	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11603	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11604	default:
11605	{
11606	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11607	AssertRCReturn(rc, rc);
11608
11609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11610	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11611	{
11612	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11613	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11614	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11615
11616	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11617	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11618	AssertRCReturn(rc, rc);
11619	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11620	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11621	0 /* GCPtrFaultAddress */);
11622	AssertRCReturn(rc, rc);
11623	}
11624	else
11625	{
11626	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11627	pVCpu->hm.s.u32HMError = uVector;
11628	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11629	}
11630	break;
11631	}
11632	}
11633	break;
11634	}
11635
11636	default:
11637	{
11638	pVCpu->hm.s.u32HMError = uExitIntInfo;
11639	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11640	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11641	break;
11642	}
11643	}
11644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11645	return rc;
11646	}
11647
11648
11649	/**
11650	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11651	*/
11652	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11653	{
11654	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11655
11656	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11657	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11658
11659	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11661	return VINF_SUCCESS;
11662	}
11663
11664
11665	/**
11666	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11667	*/
11668	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11669	{
11670	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11671	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11672	{
11673	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11674	HMVMX_RETURN_UNEXPECTED_EXIT();
11675	}
11676
11677	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11678
11679	/*
11680	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11681	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11682	*/
11683	uint32_t uIntrState = 0;
11684	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11685	AssertRCReturn(rc, rc);
11686
11687	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11688	if ( fBlockSti
11689	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11690	{
11691	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11692	}
11693
11694	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11695	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11696
11697	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11698	return VINF_SUCCESS;
11699	}
11700
11701
11702	/**
11703	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11704	*/
11705	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11706	{
11707	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11709	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11710	}
11711
11712
11713	/**
11714	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11715	*/
11716	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11717	{
11718	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11720	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11721	}
11722
11723
11724	/**
11725	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11726	*/
11727	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11728	{
11729	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11730	PVM pVM = pVCpu->CTX_SUFF(pVM);
11731	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11732	if (RT_LIKELY(rc == VINF_SUCCESS))
11733	{
11734	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11735	Assert(pVmxTransient->cbInstr == 2);
11736	}
11737	else
11738	{
11739	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11740	rc = VERR_EM_INTERPRETER;
11741	}
11742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11743	return rc;
11744	}
11745
11746
11747	/**
11748	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11749	*/
11750	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11751	{
11752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11753	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11754	AssertRCReturn(rc, rc);
11755
11756	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11757	return VINF_EM_RAW_EMULATE_INSTR;
11758
11759	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11760	HMVMX_RETURN_UNEXPECTED_EXIT();
11761	}
11762
11763
11764	/**
11765	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11766	*/
11767	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11768	{
11769	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11770	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11771	AssertRCReturn(rc, rc);
11772
11773	PVM pVM = pVCpu->CTX_SUFF(pVM);
11774	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11775	if (RT_LIKELY(rc == VINF_SUCCESS))
11776	{
11777	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11778	Assert(pVmxTransient->cbInstr == 2);
11779	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11780	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11781	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11782	}
11783	else
11784	rc = VERR_EM_INTERPRETER;
11785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11786	return rc;
11787	}
11788
11789
11790	/**
11791	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11792	*/
11793	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11794	{
11795	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11796	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11797	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11798	AssertRCReturn(rc, rc);
11799
11800	PVM pVM = pVCpu->CTX_SUFF(pVM);
11801	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11802	if (RT_SUCCESS(rc))
11803	{
11804	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11805	Assert(pVmxTransient->cbInstr == 3);
11806	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11807	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11808	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11809	}
11810	else
11811	{
11812	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11813	rc = VERR_EM_INTERPRETER;
11814	}
11815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11816	return rc;
11817	}
11818
11819
11820	/**
11821	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11822	*/
11823	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11824	{
11825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11826	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11827	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11828	AssertRCReturn(rc, rc);
11829
11830	PVM pVM = pVCpu->CTX_SUFF(pVM);
11831	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11832	if (RT_LIKELY(rc == VINF_SUCCESS))
11833	{
11834	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11835	Assert(pVmxTransient->cbInstr == 2);
11836	}
11837	else
11838	{
11839	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11840	rc = VERR_EM_INTERPRETER;
11841	}
11842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11843	return rc;
11844	}
11845
11846
11847	/**
11848	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11849	*/
11850	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11851	{
11852	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11854
11855	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11856	if (pVCpu->hm.s.fHypercallsEnabled)
11857	{
11858	#if 0
11859	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11860	#else
11861	/* Aggressive state sync. for now. */
11862	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11863	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11864	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11865	AssertRCReturn(rc, rc);
11866	#endif
11867
11868	/* Perform the hypercall. */
11869	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11870	if (rcStrict == VINF_SUCCESS)
11871	{
11872	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11873	AssertRCReturn(rc, rc);
11874	}
11875	else
11876	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11877	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11878	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11879
11880	/* If the hypercall changes anything other than guest's general-purpose registers,
11881	we would need to reload the guest changed bits here before VM-entry. */
11882	}
11883	else
11884	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11885
11886	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11887	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11888	{
11889	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11890	rcStrict = VINF_SUCCESS;
11891	}
11892
11893	return rcStrict;
11894	}
11895
11896
11897	/**
11898	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11899	*/
11900	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11901	{
11902	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11903	PVM pVM = pVCpu->CTX_SUFF(pVM);
11904	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11905
11906	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11907	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11908	AssertRCReturn(rc, rc);
11909
11910	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11911	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11912	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11913	else
11914	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11915	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11917	return rcStrict;
11918	}
11919
11920
11921	/**
11922	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11923	*/
11924	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11925	{
11926	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11927	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11928	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11929	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11930	AssertRCReturn(rc, rc);
11931
11932	PVM pVM = pVCpu->CTX_SUFF(pVM);
11933	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11934	if (RT_LIKELY(rc == VINF_SUCCESS))
11935	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11936	else
11937	{
11938	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11939	rc = VERR_EM_INTERPRETER;
11940	}
11941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11942	return rc;
11943	}
11944
11945
11946	/**
11947	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11948	*/
11949	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11950	{
11951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11952	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11953	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11954	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11955	AssertRCReturn(rc, rc);
11956
11957	PVM pVM = pVCpu->CTX_SUFF(pVM);
11958	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11959	rc = VBOXSTRICTRC_VAL(rc2);
11960	if (RT_LIKELY( rc == VINF_SUCCESS
11961	\|\| rc == VINF_EM_HALT))
11962	{
11963	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11964	AssertRCReturn(rc3, rc3);
11965
11966	if ( rc == VINF_EM_HALT
11967	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11968	{
11969	rc = VINF_SUCCESS;
11970	}
11971	}
11972	else
11973	{
11974	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11975	rc = VERR_EM_INTERPRETER;
11976	}
11977	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
11978	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11980	return rc;
11981	}
11982
11983
11984	/**
11985	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11986	*/
11987	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11988	{
11989	/*
11990	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11991	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11992	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
11993	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11994	*/
11995	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11996	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11997	HMVMX_RETURN_UNEXPECTED_EXIT();
11998	}
11999
12000
12001	/**
12002	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12003	*/
12004	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12005	{
12006	/*
12007	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12008	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12009	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12010	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12011	*/
12012	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12013	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12014	HMVMX_RETURN_UNEXPECTED_EXIT();
12015	}
12016
12017
12018	/**
12019	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12020	*/
12021	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12022	{
12023	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12024	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12025	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12026	HMVMX_RETURN_UNEXPECTED_EXIT();
12027	}
12028
12029
12030	/**
12031	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12032	*/
12033	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12034	{
12035	/*
12036	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12037	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12038	* See Intel spec. 25.3 "Other Causes of VM-exits".
12039	*/
12040	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12041	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12042	HMVMX_RETURN_UNEXPECTED_EXIT();
12043	}
12044
12045
12046	/**
12047	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12048	* VM-exit.
12049	*/
12050	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12051	{
12052	/*
12053	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12054	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12055	*
12056	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12057	* See Intel spec. "23.8 Restrictions on VMX operation".
12058	*/
12059	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12060	return VINF_SUCCESS;
12061	}
12062
12063
12064	/**
12065	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12066	* VM-exit.
12067	*/
12068	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12069	{
12070	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12071	return VINF_EM_RESET;
12072	}
12073
12074
12075	/**
12076	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12077	*/
12078	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12079	{
12080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12081	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12082
12083	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12084	AssertRCReturn(rc, rc);
12085
12086	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12087	rc = VINF_SUCCESS;
12088	else
12089	rc = VINF_EM_HALT;
12090
12091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12092	if (rc != VINF_SUCCESS)
12093	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12094	return rc;
12095	}
12096
12097
12098	/**
12099	* VM-exit handler for instructions that result in a \#UD exception delivered to
12100	* the guest.
12101	*/
12102	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12103	{
12104	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12105	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12106	return VINF_SUCCESS;
12107	}
12108
12109
12110	/**
12111	* VM-exit handler for expiry of the VMX preemption timer.
12112	*/
12113	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12114	{
12115	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12116
12117	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12118	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12119
12120	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12121	PVM pVM = pVCpu->CTX_SUFF(pVM);
12122	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12124	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12125	}
12126
12127
12128	/**
12129	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12130	*/
12131	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12132	{
12133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12134
12135	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12136	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12137	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12138	AssertRCReturn(rc, rc);
12139
12140	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12141	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12142
12143	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12144
12145	return rcStrict;
12146	}
12147
12148
12149	/**
12150	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12151	*/
12152	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12153	{
12154	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12155
12156	/* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12157	/** @todo implement EMInterpretInvpcid() */
12158	return VERR_EM_INTERPRETER;
12159	}
12160
12161
12162	/**
12163	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12164	* Error VM-exit.
12165	*/
12166	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12167	{
12168	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12169	AssertRCReturn(rc, rc);
12170
12171	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12172	AssertRCReturn(rc, rc);
12173
12174	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12175	NOREF(uInvalidReason);
12176
12177	#ifdef VBOX_STRICT
12178	uint32_t uIntrState;
12179	RTHCUINTREG uHCReg;
12180	uint64_t u64Val;
12181	uint32_t u32Val;
12182
12183	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12184	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12185	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12186	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12187	AssertRCReturn(rc, rc);
12188
12189	Log4(("uInvalidReason %u\n", uInvalidReason));
12190	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12191	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12192	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12193	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12194
12195	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12196	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12197	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12198	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12199	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12200	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12201	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12202	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12203	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12204	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12205	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12206	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12207	#else
12208	NOREF(pVmxTransient);
12209	#endif
12210
12211	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12212	return VERR_VMX_INVALID_GUEST_STATE;
12213	}
12214
12215
12216	/**
12217	* VM-exit handler for VM-entry failure due to an MSR-load
12218	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12219	*/
12220	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12221	{
12222	NOREF(pVmxTransient);
12223	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12224	HMVMX_RETURN_UNEXPECTED_EXIT();
12225	}
12226
12227
12228	/**
12229	* VM-exit handler for VM-entry failure due to a machine-check event
12230	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12231	*/
12232	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12233	{
12234	NOREF(pVmxTransient);
12235	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12236	HMVMX_RETURN_UNEXPECTED_EXIT();
12237	}
12238
12239
12240	/**
12241	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12242	* theory.
12243	*/
12244	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12245	{
12246	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12247	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12248	return VERR_VMX_UNDEFINED_EXIT_CODE;
12249	}
12250
12251
12252	/**
12253	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12254	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12255	* Conditional VM-exit.
12256	*/
12257	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12258	{
12259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12260
12261	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12263	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12264	return VERR_EM_INTERPRETER;
12265	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12266	HMVMX_RETURN_UNEXPECTED_EXIT();
12267	}
12268
12269
12270	/**
12271	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12272	*/
12273	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12274	{
12275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12276
12277	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12278	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12279	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12280	return VERR_EM_INTERPRETER;
12281	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12282	HMVMX_RETURN_UNEXPECTED_EXIT();
12283	}
12284
12285
12286	/**
12287	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12288	*/
12289	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12290	{
12291	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12292
12293	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12294	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12295	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12296	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12297	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12298	{
12299	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12300	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12301	}
12302	AssertRCReturn(rc, rc);
12303	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12304
12305	#ifdef VBOX_STRICT
12306	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12307	{
12308	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12309	&& pMixedCtx->ecx != MSR_K6_EFER)
12310	{
12311	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12312	pMixedCtx->ecx));
12313	HMVMX_RETURN_UNEXPECTED_EXIT();
12314	}
12315	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12316	{
12317	VMXMSREXITREAD enmRead;
12318	VMXMSREXITWRITE enmWrite;
12319	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12320	AssertRCReturn(rc2, rc2);
12321	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12322	{
12323	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12324	HMVMX_RETURN_UNEXPECTED_EXIT();
12325	}
12326	}
12327	}
12328	#endif
12329
12330	PVM pVM = pVCpu->CTX_SUFF(pVM);
12331	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12332	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12333	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12335	if (RT_SUCCESS(rc))
12336	{
12337	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12338	Assert(pVmxTransient->cbInstr == 2);
12339	}
12340	return rc;
12341	}
12342
12343
12344	/**
12345	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12346	*/
12347	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12348	{
12349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12350	PVM pVM = pVCpu->CTX_SUFF(pVM);
12351	int rc = VINF_SUCCESS;
12352
12353	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12354	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12355	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12356	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12357	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12358	{
12359	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12360	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12361	}
12362	AssertRCReturn(rc, rc);
12363	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12364
12365	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12366	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12368
12369	if (RT_SUCCESS(rc))
12370	{
12371	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12372
12373	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12374	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12375	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12376	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12377	{
12378	/*
12379	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12380	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12381	* EMInterpretWrmsr() changes it.
12382	*/
12383	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12384	}
12385	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12386	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12387	else if (pMixedCtx->ecx == MSR_K6_EFER)
12388	{
12389	/*
12390	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12391	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12392	* the other bits as well, SCE and NXE. See @bugref{7368}.
12393	*/
12394	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12395	}
12396
12397	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12398	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12399	{
12400	switch (pMixedCtx->ecx)
12401	{
12402	/*
12403	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12404	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12405	*/
12406	case MSR_IA32_SYSENTER_CS:
12407	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12408	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12409	break;
12410	case MSR_IA32_SYSENTER_EIP:
12411	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12412	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12413	break;
12414	case MSR_IA32_SYSENTER_ESP:
12415	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12416	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12417	break;
12418	case MSR_K8_FS_BASE: /* fall thru */
12419	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12420	case MSR_K6_EFER: /* already handled above */ break;
12421	default:
12422	{
12423	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12424	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12425	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12426	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12427	break;
12428	}
12429	}
12430	}
12431	#ifdef VBOX_STRICT
12432	else
12433	{
12434	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12435	switch (pMixedCtx->ecx)
12436	{
12437	case MSR_IA32_SYSENTER_CS:
12438	case MSR_IA32_SYSENTER_EIP:
12439	case MSR_IA32_SYSENTER_ESP:
12440	case MSR_K8_FS_BASE:
12441	case MSR_K8_GS_BASE:
12442	{
12443	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12444	HMVMX_RETURN_UNEXPECTED_EXIT();
12445	}
12446
12447	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12448	default:
12449	{
12450	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12451	{
12452	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12453	if (pMixedCtx->ecx != MSR_K6_EFER)
12454	{
12455	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12456	pMixedCtx->ecx));
12457	HMVMX_RETURN_UNEXPECTED_EXIT();
12458	}
12459	}
12460
12461	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12462	{
12463	VMXMSREXITREAD enmRead;
12464	VMXMSREXITWRITE enmWrite;
12465	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12466	AssertRCReturn(rc2, rc2);
12467	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12468	{
12469	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12470	HMVMX_RETURN_UNEXPECTED_EXIT();
12471	}
12472	}
12473	break;
12474	}
12475	}
12476	}
12477	#endif /* VBOX_STRICT */
12478	}
12479	return rc;
12480	}
12481
12482
12483	/**
12484	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12485	*/
12486	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12487	{
12488	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12489
12490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12491	return VINF_EM_RAW_INTERRUPT;
12492	}
12493
12494
12495	/**
12496	* VM-exit handler for when the TPR value is lowered below the specified
12497	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12498	*/
12499	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12500	{
12501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12502	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12503
12504	/*
12505	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12506	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12507	*/
12508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12509	return VINF_SUCCESS;
12510	}
12511
12512
12513	/**
12514	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12515	* VM-exit.
12516	*
12517	* @retval VINF_SUCCESS when guest execution can continue.
12518	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12519	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12520	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12521	* interpreter.
12522	*/
12523	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12524	{
12525	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12526	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12527	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12528	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12529	AssertRCReturn(rc, rc);
12530
12531	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12532	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12533	PVM pVM = pVCpu->CTX_SUFF(pVM);
12534	VBOXSTRICTRC rcStrict;
12535	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12536	switch (uAccessType)
12537	{
12538	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12539	{
12540	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12541	AssertRCReturn(rc, rc);
12542
12543	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12544	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12545	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12546	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12547	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12548	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12549	{
12550	case 0: /* CR0 */
12551	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12552	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12553	break;
12554	case 2: /* CR2 */
12555	/* Nothing to do here, CR2 it's not part of the VMCS. */
12556	break;
12557	case 3: /* CR3 */
12558	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12559	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12560	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12561	break;
12562	case 4: /* CR4 */
12563	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12564	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12565	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12566	break;
12567	case 8: /* CR8 */
12568	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12569	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12570	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12571	break;
12572	default:
12573	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12574	break;
12575	}
12576
12577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12578	break;
12579	}
12580
12581	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12582	{
12583	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12584	AssertRCReturn(rc, rc);
12585
12586	Assert( !pVM->hm.s.fNestedPaging
12587	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12588	\|\| pVCpu->hm.s.fUsingDebugLoop
12589	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12590
12591	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12592	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12593	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12594
12595	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12596	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12597	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12598	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12600	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12601	VBOXSTRICTRC_VAL(rcStrict)));
12602	break;
12603	}
12604
12605	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12606	{
12607	AssertRCReturn(rc, rc);
12608	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12609	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12610	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12612	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12613	break;
12614	}
12615
12616	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12617	{
12618	AssertRCReturn(rc, rc);
12619	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12620	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12621	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12622	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12623	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12625	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12626	break;
12627	}
12628
12629	default:
12630	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12631	VERR_VMX_UNEXPECTED_EXCEPTION);
12632	}
12633
12634	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12635	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12636	NOREF(pVM);
12637	return rcStrict;
12638	}
12639
12640
12641	/**
12642	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12643	* VM-exit.
12644	*/
12645	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12646	{
12647	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12648	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12649
12650	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12651	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12652	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12653	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12654	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12655	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12656	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12657	AssertRCReturn(rc, rc);
12658
12659	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12660	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12661	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12662	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12663	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12664	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12665	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12666	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12667	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12668
12669	/* I/O operation lookup arrays. */
12670	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12671	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12672
12673	VBOXSTRICTRC rcStrict;
12674	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12675	uint32_t const cbInstr = pVmxTransient->cbInstr;
12676	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12677	PVM pVM = pVCpu->CTX_SUFF(pVM);
12678	if (fIOString)
12679	{
12680	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12681	See @bugref{5752#c158}. Should work now. */
12682	/*
12683	* INS/OUTS - I/O String instruction.
12684	*
12685	* Use instruction-information if available, otherwise fall back on
12686	* interpreting the instruction.
12687	*/
12688	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12689	fIOWrite ? 'w' : 'r'));
12690	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12691	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12692	{
12693	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12694	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12695	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12696	AssertRCReturn(rc2, rc2);
12697	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12698	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12699	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12700	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12701	if (fIOWrite)
12702	{
12703	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12704	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12705	}
12706	else
12707	{
12708	/*
12709	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12710	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12711	* See Intel Instruction spec. for "INS".
12712	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12713	*/
12714	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12715	}
12716	}
12717	else
12718	{
12719	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12720	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12721	AssertRCReturn(rc2, rc2);
12722	rcStrict = IEMExecOne(pVCpu);
12723	}
12724	/** @todo IEM needs to be setting these flags somehow. */
12725	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12726	fUpdateRipAlready = true;
12727	#else
12728	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12729	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12730	if (RT_SUCCESS(rcStrict))
12731	{
12732	if (fIOWrite)
12733	{
12734	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12735	(DISCPUMODE)pDis->uAddrMode, cbValue);
12736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12737	}
12738	else
12739	{
12740	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12741	(DISCPUMODE)pDis->uAddrMode, cbValue);
12742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12743	}
12744	}
12745	else
12746	{
12747	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12748	pMixedCtx->rip));
12749	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12750	}
12751	#endif
12752	}
12753	else
12754	{
12755	/*
12756	* IN/OUT - I/O instruction.
12757	*/
12758	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12759	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12760	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12761	if (fIOWrite)
12762	{
12763	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12765	}
12766	else
12767	{
12768	uint32_t u32Result = 0;
12769	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12770	if (IOM_SUCCESS(rcStrict))
12771	{
12772	/* Save result of I/O IN instr. in AL/AX/EAX. */
12773	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12774	}
12775	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12776	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12778	}
12779	}
12780
12781	if (IOM_SUCCESS(rcStrict))
12782	{
12783	if (!fUpdateRipAlready)
12784	{
12785	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12786	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12787	}
12788
12789	/*
12790	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12791	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12792	*/
12793	if (fIOString)
12794	{
12795	/** @todo Single-step for INS/OUTS with REP prefix? */
12796	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12797	}
12798	else if ( !fDbgStepping
12799	&& fGstStepping)
12800	{
12801	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12802	}
12803
12804	/*
12805	* If any I/O breakpoints are armed, we need to check if one triggered
12806	* and take appropriate action.
12807	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12808	*/
12809	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12810	AssertRCReturn(rc2, rc2);
12811
12812	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12813	* execution engines about whether hyper BPs and such are pending. */
12814	uint32_t const uDr7 = pMixedCtx->dr[7];
12815	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12816	&& X86_DR7_ANY_RW_IO(uDr7)
12817	&& (pMixedCtx->cr4 & X86_CR4_DE))
12818	\|\| DBGFBpIsHwIoArmed(pVM)))
12819	{
12820	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12821
12822	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12823	VMMRZCallRing3Disable(pVCpu);
12824	HM_DISABLE_PREEMPT();
12825
12826	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12827
12828	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12829	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12830	{
12831	/* Raise #DB. */
12832	if (fIsGuestDbgActive)
12833	ASMSetDR6(pMixedCtx->dr[6]);
12834	if (pMixedCtx->dr[7] != uDr7)
12835	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12836
12837	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12838	}
12839	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12840	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12841	else if ( rcStrict2 != VINF_SUCCESS
12842	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12843	rcStrict = rcStrict2;
12844	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12845
12846	HM_RESTORE_PREEMPT();
12847	VMMRZCallRing3Enable(pVCpu);
12848	}
12849	}
12850
12851	#ifdef VBOX_STRICT
12852	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12853	Assert(!fIOWrite);
12854	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12855	Assert(fIOWrite);
12856	else
12857	{
12858	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12859	* statuses, that the VMM device and some others may return. See
12860	* IOM_SUCCESS() for guidance. */
12861	AssertMsg( RT_FAILURE(rcStrict)
12862	\|\| rcStrict == VINF_SUCCESS
12863	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12864	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12865	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12866	\|\| rcStrict == VINF_EM_RAW_TO_R3
12867	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12868	#endif
12869	}
12870	#endif
12871
12872	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12873	return rcStrict;
12874	}
12875
12876
12877	/**
12878	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12879	* VM-exit.
12880	*/
12881	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12882	{
12883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12884
12885	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12886	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12887	AssertRCReturn(rc, rc);
12888	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12889	{
12890	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12891	AssertRCReturn(rc, rc);
12892	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12893	{
12894	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12895
12896	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12897	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12898
12899	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12900	Assert(!pVCpu->hm.s.Event.fPending);
12901	pVCpu->hm.s.Event.fPending = true;
12902	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12903	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12904	AssertRCReturn(rc, rc);
12905	if (fErrorCodeValid)
12906	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12907	else
12908	pVCpu->hm.s.Event.u32ErrCode = 0;
12909	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12910	&& uVector == X86_XCPT_PF)
12911	{
12912	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12913	}
12914
12915	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12917	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12918	}
12919	}
12920
12921	/* Fall back to the interpreter to emulate the task-switch. */
12922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12923	return VERR_EM_INTERPRETER;
12924	}
12925
12926
12927	/**
12928	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12929	*/
12930	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12931	{
12932	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12933	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12934	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12935	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12936	AssertRCReturn(rc, rc);
12937	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12938	return VINF_EM_DBG_STEPPED;
12939	}
12940
12941
12942	/**
12943	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12944	*/
12945	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12946	{
12947	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12948
12949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12950
12951	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12952	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12953	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12954	{
12955	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12956	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12957	{
12958	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12959	return VERR_EM_INTERPRETER;
12960	}
12961	}
12962	else
12963	{
12964	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12965	rcStrict1 = VINF_SUCCESS;
12966	return rcStrict1;
12967	}
12968
12969	#if 0
12970	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12971	* just sync the whole thing. */
12972	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12973	#else
12974	/* Aggressive state sync. for now. */
12975	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12976	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12977	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12978	#endif
12979	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12980	AssertRCReturn(rc, rc);
12981
12982	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12983	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12984	VBOXSTRICTRC rcStrict2;
12985	switch (uAccessType)
12986	{
12987	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12988	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12989	{
12990	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12991	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12992	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12993
12994	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12995	GCPhys &= PAGE_BASE_GC_MASK;
12996	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12997	PVM pVM = pVCpu->CTX_SUFF(pVM);
12998	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12999	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13000
13001	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13002	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13003	CPUMCTX2CORE(pMixedCtx), GCPhys);
13004	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13005	if ( rcStrict2 == VINF_SUCCESS
13006	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13007	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13008	{
13009	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13010	\| HM_CHANGED_GUEST_RSP
13011	\| HM_CHANGED_GUEST_RFLAGS
13012	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13013	rcStrict2 = VINF_SUCCESS;
13014	}
13015	break;
13016	}
13017
13018	default:
13019	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13020	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13021	break;
13022	}
13023
13024	if (rcStrict2 != VINF_SUCCESS)
13025	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13026	return rcStrict2;
13027	}
13028
13029
13030	/**
13031	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13032	* VM-exit.
13033	*/
13034	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13035	{
13036	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13037
13038	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13039	if (pVmxTransient->fWasGuestDebugStateActive)
13040	{
13041	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13042	HMVMX_RETURN_UNEXPECTED_EXIT();
13043	}
13044
13045	if ( !pVCpu->hm.s.fSingleInstruction
13046	&& !pVmxTransient->fWasHyperDebugStateActive)
13047	{
13048	Assert(!DBGFIsStepping(pVCpu));
13049	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13050
13051	/* Don't intercept MOV DRx any more. */
13052	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13053	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13054	AssertRCReturn(rc, rc);
13055
13056	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13057	VMMRZCallRing3Disable(pVCpu);
13058	HM_DISABLE_PREEMPT();
13059
13060	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13061	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13062	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13063
13064	HM_RESTORE_PREEMPT();
13065	VMMRZCallRing3Enable(pVCpu);
13066
13067	#ifdef VBOX_WITH_STATISTICS
13068	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13069	AssertRCReturn(rc, rc);
13070	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13072	else
13073	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13074	#endif
13075	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13076	return VINF_SUCCESS;
13077	}
13078
13079	/*
13080	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13081	* Update the segment registers and DR7 from the CPU.
13082	*/
13083	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13084	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13085	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13086	AssertRCReturn(rc, rc);
13087	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13088
13089	PVM pVM = pVCpu->CTX_SUFF(pVM);
13090	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13091	{
13092	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13093	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13094	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13095	if (RT_SUCCESS(rc))
13096	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13097	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13098	}
13099	else
13100	{
13101	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13102	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13103	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13105	}
13106
13107	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13108	if (RT_SUCCESS(rc))
13109	{
13110	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13111	AssertRCReturn(rc2, rc2);
13112	return VINF_SUCCESS;
13113	}
13114	return rc;
13115	}
13116
13117
13118	/**
13119	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13120	* Conditional VM-exit.
13121	*/
13122	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13123	{
13124	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13125	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13126
13127	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13128	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13129	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13130	{
13131	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13132	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13133	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13134	{
13135	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13136	return VERR_EM_INTERPRETER;
13137	}
13138	}
13139	else
13140	{
13141	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13142	rcStrict1 = VINF_SUCCESS;
13143	return rcStrict1;
13144	}
13145
13146	RTGCPHYS GCPhys = 0;
13147	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13148
13149	#if 0
13150	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13151	#else
13152	/* Aggressive state sync. for now. */
13153	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13154	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13155	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13156	#endif
13157	AssertRCReturn(rc, rc);
13158
13159	/*
13160	* If we succeed, resume guest execution.
13161	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13162	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13163	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13164	* weird case. See @bugref{6043}.
13165	*/
13166	PVM pVM = pVCpu->CTX_SUFF(pVM);
13167	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13168	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13169	if ( rcStrict2 == VINF_SUCCESS
13170	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13171	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13172	{
13173	/* Successfully handled MMIO operation. */
13174	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13175	\| HM_CHANGED_GUEST_RSP
13176	\| HM_CHANGED_GUEST_RFLAGS
13177	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13178	return VINF_SUCCESS;
13179	}
13180	return rcStrict2;
13181	}
13182
13183
13184	/**
13185	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13186	* VM-exit.
13187	*/
13188	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13189	{
13190	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13191	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13192
13193	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13194	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13195	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13196	{
13197	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13198	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13199	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13200	}
13201	else
13202	{
13203	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13204	rcStrict1 = VINF_SUCCESS;
13205	return rcStrict1;
13206	}
13207
13208	RTGCPHYS GCPhys = 0;
13209	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13210	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13211	#if 0
13212	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13213	#else
13214	/* Aggressive state sync. for now. */
13215	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13216	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13217	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13218	#endif
13219	AssertRCReturn(rc, rc);
13220
13221	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13222	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13223
13224	RTGCUINT uErrorCode = 0;
13225	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13226	uErrorCode \|= X86_TRAP_PF_ID;
13227	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13228	uErrorCode \|= X86_TRAP_PF_RW;
13229	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13230	uErrorCode \|= X86_TRAP_PF_P;
13231
13232	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13233
13234	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13235	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13236
13237	/* Handle the pagefault trap for the nested shadow table. */
13238	PVM pVM = pVCpu->CTX_SUFF(pVM);
13239	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13240	TRPMResetTrap(pVCpu);
13241
13242	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13243	if ( rcStrict2 == VINF_SUCCESS
13244	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13245	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13246	{
13247	/* Successfully synced our nested page tables. */
13248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13249	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13250	\| HM_CHANGED_GUEST_RSP
13251	\| HM_CHANGED_GUEST_RFLAGS);
13252	return VINF_SUCCESS;
13253	}
13254
13255	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13256	return rcStrict2;
13257	}
13258
13259	/** @} */
13260
13261	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13262	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13263	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13264
13265	/** @name VM-exit exception handlers.
13266	* @{
13267	*/
13268
13269	/**
13270	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13271	*/
13272	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13273	{
13274	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13275	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13276
13277	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13278	AssertRCReturn(rc, rc);
13279
13280	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13281	{
13282	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13283	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13284
13285	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13286	* provides VM-exit instruction length. If this causes problem later,
13287	* disassemble the instruction like it's done on AMD-V. */
13288	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13289	AssertRCReturn(rc2, rc2);
13290	return rc;
13291	}
13292
13293	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13294	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13295	return rc;
13296	}
13297
13298
13299	/**
13300	* VM-exit exception handler for \#BP (Breakpoint exception).
13301	*/
13302	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13303	{
13304	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13306
13307	/** @todo Try optimize this by not saving the entire guest state unless
13308	* really needed. */
13309	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13310	AssertRCReturn(rc, rc);
13311
13312	PVM pVM = pVCpu->CTX_SUFF(pVM);
13313	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13314	if (rc == VINF_EM_RAW_GUEST_TRAP)
13315	{
13316	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13317	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13318	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13319	AssertRCReturn(rc, rc);
13320
13321	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13322	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13323	}
13324
13325	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13326	return rc;
13327	}
13328
13329
13330	/**
13331	* VM-exit exception handler for \#AC (alignment check exception).
13332	*/
13333	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13334	{
13335	RT_NOREF_PV(pMixedCtx);
13336	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13337
13338	/*
13339	* Re-inject it. We'll detect any nesting before getting here.
13340	*/
13341	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13342	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13343	AssertRCReturn(rc, rc);
13344	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13345
13346	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13347	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13348	return VINF_SUCCESS;
13349	}
13350
13351
13352	/**
13353	* VM-exit exception handler for \#DB (Debug exception).
13354	*/
13355	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13356	{
13357	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13359	Log6(("XcptDB\n"));
13360
13361	/*
13362	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13363	* for processing.
13364	*/
13365	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13366	AssertRCReturn(rc, rc);
13367
13368	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13369	uint64_t uDR6 = X86_DR6_INIT_VAL;
13370	uDR6 \|= ( pVmxTransient->uExitQualification
13371	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13372
13373	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13374	if (rc == VINF_EM_RAW_GUEST_TRAP)
13375	{
13376	/*
13377	* The exception was for the guest. Update DR6, DR7.GD and
13378	* IA32_DEBUGCTL.LBR before forwarding it.
13379	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13380	*/
13381	VMMRZCallRing3Disable(pVCpu);
13382	HM_DISABLE_PREEMPT();
13383
13384	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13385	pMixedCtx->dr[6] \|= uDR6;
13386	if (CPUMIsGuestDebugStateActive(pVCpu))
13387	ASMSetDR6(pMixedCtx->dr[6]);
13388
13389	HM_RESTORE_PREEMPT();
13390	VMMRZCallRing3Enable(pVCpu);
13391
13392	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13393	AssertRCReturn(rc, rc);
13394
13395	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13396	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13397
13398	/* Paranoia. */
13399	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13400	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13401
13402	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13403	AssertRCReturn(rc, rc);
13404
13405	/*
13406	* Raise #DB in the guest.
13407	*
13408	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13409	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13410	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13411	*
13412	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13413	*/
13414	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13415	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13416	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13417	AssertRCReturn(rc, rc);
13418	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13419	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13420	return VINF_SUCCESS;
13421	}
13422
13423	/*
13424	* Not a guest trap, must be a hypervisor related debug event then.
13425	* Update DR6 in case someone is interested in it.
13426	*/
13427	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13428	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13429	CPUMSetHyperDR6(pVCpu, uDR6);
13430
13431	return rc;
13432	}
13433
13434
13435	/**
13436	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13437	* point exception).
13438	*/
13439	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13440	{
13441	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13442
13443	/* We require CR0 and EFER. EFER is always up-to-date. */
13444	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13445	AssertRCReturn(rc, rc);
13446
13447	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13448	VMMRZCallRing3Disable(pVCpu);
13449	HM_DISABLE_PREEMPT();
13450
13451	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13452	if (pVmxTransient->fWasGuestFPUStateActive)
13453	{
13454	rc = VINF_EM_RAW_GUEST_TRAP;
13455	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13456	}
13457	else
13458	{
13459	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13460	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13461	#endif
13462	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13463	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13464	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13465	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13466	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13467	}
13468
13469	HM_RESTORE_PREEMPT();
13470	VMMRZCallRing3Enable(pVCpu);
13471
13472	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13473	{
13474	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13475	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13477	pVCpu->hm.s.fPreloadGuestFpu = true;
13478	}
13479	else
13480	{
13481	/* Forward #NM to the guest. */
13482	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13483	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13484	AssertRCReturn(rc, rc);
13485	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13486	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13488	}
13489
13490	return VINF_SUCCESS;
13491	}
13492
13493
13494	/**
13495	* VM-exit exception handler for \#GP (General-protection exception).
13496	*
13497	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13498	*/
13499	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13500	{
13501	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13502	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13503
13504	int rc;
13505	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13506	{ /* likely */ }
13507	else
13508	{
13509	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13510	Assert(pVCpu->hm.s.fUsingDebugLoop);
13511	#endif
13512	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13513	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13514	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13515	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13516	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13517	AssertRCReturn(rc, rc);
13518	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13519	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13520	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13521	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13522	return rc;
13523	}
13524
13525	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13526	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13527
13528	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13529	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13530	AssertRCReturn(rc, rc);
13531
13532	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13533	uint32_t cbOp = 0;
13534	PVM pVM = pVCpu->CTX_SUFF(pVM);
13535	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13536	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13537	if (RT_SUCCESS(rc))
13538	{
13539	rc = VINF_SUCCESS;
13540	Assert(cbOp == pDis->cbInstr);
13541	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13542	switch (pDis->pCurInstr->uOpcode)
13543	{
13544	case OP_CLI:
13545	{
13546	pMixedCtx->eflags.Bits.u1IF = 0;
13547	pMixedCtx->eflags.Bits.u1RF = 0;
13548	pMixedCtx->rip += pDis->cbInstr;
13549	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13550	if ( !fDbgStepping
13551	&& pMixedCtx->eflags.Bits.u1TF)
13552	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13554	break;
13555	}
13556
13557	case OP_STI:
13558	{
13559	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13560	pMixedCtx->eflags.Bits.u1IF = 1;
13561	pMixedCtx->eflags.Bits.u1RF = 0;
13562	pMixedCtx->rip += pDis->cbInstr;
13563	if (!fOldIF)
13564	{
13565	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13566	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13567	}
13568	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13569	if ( !fDbgStepping
13570	&& pMixedCtx->eflags.Bits.u1TF)
13571	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13573	break;
13574	}
13575
13576	case OP_HLT:
13577	{
13578	rc = VINF_EM_HALT;
13579	pMixedCtx->rip += pDis->cbInstr;
13580	pMixedCtx->eflags.Bits.u1RF = 0;
13581	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13583	break;
13584	}
13585
13586	case OP_POPF:
13587	{
13588	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13589	uint32_t cbParm;
13590	uint32_t uMask;
13591	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13592	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13593	{
13594	cbParm = 4;
13595	uMask = 0xffffffff;
13596	}
13597	else
13598	{
13599	cbParm = 2;
13600	uMask = 0xffff;
13601	}
13602
13603	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13604	RTGCPTR GCPtrStack = 0;
13605	X86EFLAGS Eflags;
13606	Eflags.u32 = 0;
13607	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13608	&GCPtrStack);
13609	if (RT_SUCCESS(rc))
13610	{
13611	Assert(sizeof(Eflags.u32) >= cbParm);
13612	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13613	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13614	}
13615	if (RT_FAILURE(rc))
13616	{
13617	rc = VERR_EM_INTERPRETER;
13618	break;
13619	}
13620	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13621	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13622	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13623	pMixedCtx->esp += cbParm;
13624	pMixedCtx->esp &= uMask;
13625	pMixedCtx->rip += pDis->cbInstr;
13626	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13627	\| HM_CHANGED_GUEST_RSP
13628	\| HM_CHANGED_GUEST_RFLAGS);
13629	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13630	POPF restores EFLAGS.TF. */
13631	if ( !fDbgStepping
13632	&& fGstStepping)
13633	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13634	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13635	break;
13636	}
13637
13638	case OP_PUSHF:
13639	{
13640	uint32_t cbParm;
13641	uint32_t uMask;
13642	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13643	{
13644	cbParm = 4;
13645	uMask = 0xffffffff;
13646	}
13647	else
13648	{
13649	cbParm = 2;
13650	uMask = 0xffff;
13651	}
13652
13653	/* Get the stack pointer & push the contents of eflags onto the stack. */
13654	RTGCPTR GCPtrStack = 0;
13655	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13656	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13657	if (RT_FAILURE(rc))
13658	{
13659	rc = VERR_EM_INTERPRETER;
13660	break;
13661	}
13662	X86EFLAGS Eflags = pMixedCtx->eflags;
13663	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13664	Eflags.Bits.u1RF = 0;
13665	Eflags.Bits.u1VM = 0;
13666
13667	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13668	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13669	{
13670	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13671	rc = VERR_EM_INTERPRETER;
13672	break;
13673	}
13674	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13675	pMixedCtx->esp -= cbParm;
13676	pMixedCtx->esp &= uMask;
13677	pMixedCtx->rip += pDis->cbInstr;
13678	pMixedCtx->eflags.Bits.u1RF = 0;
13679	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13680	\| HM_CHANGED_GUEST_RSP
13681	\| HM_CHANGED_GUEST_RFLAGS);
13682	if ( !fDbgStepping
13683	&& pMixedCtx->eflags.Bits.u1TF)
13684	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13686	break;
13687	}
13688
13689	case OP_IRET:
13690	{
13691	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13692	* instruction reference. */
13693	RTGCPTR GCPtrStack = 0;
13694	uint32_t uMask = 0xffff;
13695	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13696	uint16_t aIretFrame[3];
13697	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13698	{
13699	rc = VERR_EM_INTERPRETER;
13700	break;
13701	}
13702	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13703	&GCPtrStack);
13704	if (RT_SUCCESS(rc))
13705	{
13706	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13707	PGMACCESSORIGIN_HM));
13708	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13709	}
13710	if (RT_FAILURE(rc))
13711	{
13712	rc = VERR_EM_INTERPRETER;
13713	break;
13714	}
13715	pMixedCtx->eip = 0;
13716	pMixedCtx->ip = aIretFrame[0];
13717	pMixedCtx->cs.Sel = aIretFrame[1];
13718	pMixedCtx->cs.ValidSel = aIretFrame[1];
13719	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13720	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13721	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13722	pMixedCtx->sp += sizeof(aIretFrame);
13723	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13724	\| HM_CHANGED_GUEST_SEGMENT_REGS
13725	\| HM_CHANGED_GUEST_RSP
13726	\| HM_CHANGED_GUEST_RFLAGS);
13727	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13728	if ( !fDbgStepping
13729	&& fGstStepping)
13730	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13731	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13732	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13733	break;
13734	}
13735
13736	case OP_INT:
13737	{
13738	uint16_t uVector = pDis->Param1.uValue & 0xff;
13739	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13740	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13742	break;
13743	}
13744
13745	case OP_INTO:
13746	{
13747	if (pMixedCtx->eflags.Bits.u1OF)
13748	{
13749	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13750	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13752	}
13753	else
13754	{
13755	pMixedCtx->eflags.Bits.u1RF = 0;
13756	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13757	}
13758	break;
13759	}
13760
13761	default:
13762	{
13763	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13764	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13765	EMCODETYPE_SUPERVISOR);
13766	rc = VBOXSTRICTRC_VAL(rc2);
13767	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13768	/** @todo We have to set pending-debug exceptions here when the guest is
13769	* single-stepping depending on the instruction that was interpreted. */
13770	Log4(("#GP rc=%Rrc\n", rc));
13771	break;
13772	}
13773	}
13774	}
13775	else
13776	rc = VERR_EM_INTERPRETER;
13777
13778	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13779	("#GP Unexpected rc=%Rrc\n", rc));
13780	return rc;
13781	}
13782
13783
13784	/**
13785	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13786	* the exception reported in the VMX transient structure back into the VM.
13787	*
13788	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13789	* up-to-date.
13790	*/
13791	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13792	{
13793	RT_NOREF_PV(pMixedCtx);
13794	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13795	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13796	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13797	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13798	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13799	#endif
13800
13801	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13802	hmR0VmxCheckExitDueToEventDelivery(). */
13803	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13804	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13805	AssertRCReturn(rc, rc);
13806	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13807
13808	#ifdef DEBUG_ramshankar
13809	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13810	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13811	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13812	#endif
13813
13814	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13815	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13816	return VINF_SUCCESS;
13817	}
13818
13819
13820	/**
13821	* VM-exit exception handler for \#PF (Page-fault exception).
13822	*/
13823	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13824	{
13825	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13826	PVM pVM = pVCpu->CTX_SUFF(pVM);
13827	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13828	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13829	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13830	AssertRCReturn(rc, rc);
13831
13832	if (!pVM->hm.s.fNestedPaging)
13833	{ /* likely */ }
13834	else
13835	{
13836	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13837	Assert(pVCpu->hm.s.fUsingDebugLoop);
13838	#endif
13839	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13840	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13841	{
13842	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13843	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13844	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13845	}
13846	else
13847	{
13848	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13849	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13850	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13851	}
13852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13853	return rc;
13854	}
13855
13856	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13857	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13858	if (pVmxTransient->fVectoringPF)
13859	{
13860	Assert(pVCpu->hm.s.Event.fPending);
13861	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13862	}
13863
13864	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13865	AssertRCReturn(rc, rc);
13866
13867	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13868	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13869
13870	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13871	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13872	(RTGCPTR)pVmxTransient->uExitQualification);
13873
13874	Log4(("#PF: rc=%Rrc\n", rc));
13875	if (rc == VINF_SUCCESS)
13876	{
13877	#if 0
13878	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13879	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13880	* memory? We don't update the whole state here... */
13881	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13882	\| HM_CHANGED_GUEST_RSP
13883	\| HM_CHANGED_GUEST_RFLAGS
13884	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13885	#else
13886	/*
13887	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13888	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13889	*/
13890	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13891	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13892	#endif
13893	TRPMResetTrap(pVCpu);
13894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13895	return rc;
13896	}
13897
13898	if (rc == VINF_EM_RAW_GUEST_TRAP)
13899	{
13900	if (!pVmxTransient->fVectoringDoublePF)
13901	{
13902	/* It's a guest page fault and needs to be reflected to the guest. */
13903	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13904	TRPMResetTrap(pVCpu);
13905	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13906	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13907	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13908	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13909	}
13910	else
13911	{
13912	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13913	TRPMResetTrap(pVCpu);
13914	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13915	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13916	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13917	}
13918
13919	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13920	return VINF_SUCCESS;
13921	}
13922
13923	TRPMResetTrap(pVCpu);
13924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13925	return rc;
13926	}
13927
13928	/** @} */
13929

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 66701

以其他格式下載: