HMVMXR0.cpp@ 47826

最後變更在這個檔案從47826是 47803,由 vboxsync 提交於 12 年前
VMM/HM: Preemption hoooks, work in progress.
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 434.3 KB

行
1	/* $Id: HMVMXR0.cpp 47803 2013-08-16 11:58:57Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#endif
45
46
47	/*******************************************************************************
48	* Defined Constants And Macros *
49	*******************************************************************************/
50	#if defined(RT_ARCH_AMD64)
51	# define HMVMX_IS_64BIT_HOST_MODE() (true)
52	typedef RTHCUINTREG HMVMXHCUINTREG;
53	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54	extern "C" uint32_t g_fVMXIs64bitHost;
55	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56	typedef uint64_t HMVMXHCUINTREG;
57	#else
58	# define HMVMX_IS_64BIT_HOST_MODE() (false)
59	typedef RTHCUINTREG HMVMXHCUINTREG;
60	#endif
61
62	/** Use the function table. */
63	#define HMVMX_USE_FUNCTION_TABLE
64
65	/** Determine which tagged-TLB flush handler to use. */
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71	/** @name Updated-guest-state flags.
72	* @{ */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113	/** @} */
114
115	/** @name
116	* Flags to skip redundant reads of some common VMCS fields that are not part of
117	* the guest-CPU state but are in the transient structure.
118	*/
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125	/** @} */
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86).
129	*
130	* We need to intercept all exceptions manually (except #PF). #NM is also
131	* handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
132	* intercepted even in real-mode if we have Nested Paging support.
133	*/
134	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
135	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
136	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
137	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
138	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
139	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
140	\| RT_BIT(X86_XCPT_XF))
141
142	/**
143	* Exception bitmap mask for all contributory exceptions.
144	*
145	* Page fault is deliberately excluded here as it's conditional as to whether
146	* it's contributory or benign. Page faults are handled separately.
147	*/
148	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
149	\| RT_BIT(X86_XCPT_DE))
150
151	/** Maximum VM-instruction error number. */
152	#define HMVMX_INSTR_ERROR_MAX 28
153
154	/** Profiling macro. */
155	#ifdef HM_PROFILE_EXIT_DISPATCH
156	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
157	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
158	#else
159	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
160	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
161	#endif
162
163
164	/*******************************************************************************
165	* Structures and Typedefs *
166	*******************************************************************************/
167	/**
168	* VMX transient state.
169	*
170	* A state structure for holding miscellaneous information across
171	* VMX non-root operation and restored after the transition.
172	*/
173	typedef struct VMXTRANSIENT
174	{
175	/** The host's rflags/eflags. */
176	RTCCUINTREG uEflags;
177	#if HC_ARCH_BITS == 32
178	uint32_t u32Alignment0;
179	#endif
180	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
181	uint64_t u64LStarMsr;
182	/** The guest's TPR value used for TPR shadowing. */
183	uint8_t u8GuestTpr;
184	/** Alignment. */
185	uint8_t abAlignment0[7];
186
187	/** The basic VM-exit reason. */
188	uint16_t uExitReason;
189	/** Alignment. */
190	uint16_t u16Alignment0;
191	/** The VM-exit interruption error code. */
192	uint32_t uExitIntrErrorCode;
193	/** The VM-exit exit qualification. */
194	uint64_t uExitQualification;
195
196	/** The VM-exit interruption-information field. */
197	uint32_t uExitIntrInfo;
198	/** The VM-exit instruction-length field. */
199	uint32_t cbInstr;
200	/** The VM-exit instruction-information field. */
201	union
202	{
203	/** Plain unsigned int representation. */
204	uint32_t u;
205	/** INS and OUTS information. */
206	struct
207	{
208	uint32_t u6Reserved0 : 6;
209	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
210	uint32_t u3AddrSize : 3;
211	uint32_t u5Reserved1 : 5;
212	/** The segment register (X86_SREG_XXX). */
213	uint32_t iSegReg : 3;
214	uint32_t uReserved2 : 14;
215	} StrIo;
216	} ExitInstrInfo;
217	/** Whether the VM-entry failed or not. */
218	bool fVMEntryFailed;
219	/** Alignment. */
220	uint8_t abAlignment1[3];
221
222	/** The VM-entry interruption-information field. */
223	uint32_t uEntryIntrInfo;
224	/** The VM-entry exception error code field. */
225	uint32_t uEntryXcptErrorCode;
226	/** The VM-entry instruction length field. */
227	uint32_t cbEntryInstr;
228
229	/** IDT-vectoring information field. */
230	uint32_t uIdtVectoringInfo;
231	/** IDT-vectoring error code. */
232	uint32_t uIdtVectoringErrorCode;
233
234	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
235	uint32_t fVmcsFieldsRead;
236	/** Whether TSC-offsetting should be setup before VM-entry. */
237	bool fUpdateTscOffsettingAndPreemptTimer;
238	/** Whether the VM-exit was caused by a page-fault during delivery of a
239	* contributory exception or a page-fault. */
240	bool fVectoringPF;
241	} VMXTRANSIENT;
242	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
243	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
244	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
245	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
246	/** Pointer to VMX transient state. */
247	typedef VMXTRANSIENT *PVMXTRANSIENT;
248
249
250	/**
251	* MSR-bitmap read permissions.
252	*/
253	typedef enum VMXMSREXITREAD
254	{
255	/** Reading this MSR causes a VM-exit. */
256	VMXMSREXIT_INTERCEPT_READ = 0xb,
257	/** Reading this MSR does not cause a VM-exit. */
258	VMXMSREXIT_PASSTHRU_READ
259	} VMXMSREXITREAD;
260
261	/**
262	* MSR-bitmap write permissions.
263	*/
264	typedef enum VMXMSREXITWRITE
265	{
266	/** Writing to this MSR causes a VM-exit. */
267	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
268	/** Writing to this MSR does not cause a VM-exit. */
269	VMXMSREXIT_PASSTHRU_WRITE
270	} VMXMSREXITWRITE;
271
272	/**
273	* VM-exit handler.
274	*
275	* @returns VBox status code.
276	* @param pVCpu Pointer to the VMCPU.
277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
278	* out-of-sync. Make sure to update the required
279	* fields before using them.
280	* @param pVmxTransient Pointer to the VMX-transient structure.
281	*/
282	#ifndef HMVMX_USE_FUNCTION_TABLE
283	typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	#else
285	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	/** Pointer to VM-exit handler. */
287	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
288	#endif
289
290
291	/*******************************************************************************
292	* Internal Functions *
293	*******************************************************************************/
294	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
295	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
296	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
297	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
298	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
299	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
300	#endif
301	#ifndef HMVMX_USE_FUNCTION_TABLE
302	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
303	# define HMVMX_EXIT_DECL static int
304	#else
305	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
306	#endif
307
308	/** @name VM-exit handlers.
309	* @{
310	*/
311	static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
312	static FNVMEXITHANDLER hmR0VmxExitExtInt;
313	static FNVMEXITHANDLER hmR0VmxExitTripleFault;
314	static FNVMEXITHANDLER hmR0VmxExitInitSignal;
315	static FNVMEXITHANDLER hmR0VmxExitSipi;
316	static FNVMEXITHANDLER hmR0VmxExitIoSmi;
317	static FNVMEXITHANDLER hmR0VmxExitSmi;
318	static FNVMEXITHANDLER hmR0VmxExitIntWindow;
319	static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
320	static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
321	static FNVMEXITHANDLER hmR0VmxExitCpuid;
322	static FNVMEXITHANDLER hmR0VmxExitGetsec;
323	static FNVMEXITHANDLER hmR0VmxExitHlt;
324	static FNVMEXITHANDLER hmR0VmxExitInvd;
325	static FNVMEXITHANDLER hmR0VmxExitInvlpg;
326	static FNVMEXITHANDLER hmR0VmxExitRdpmc;
327	static FNVMEXITHANDLER hmR0VmxExitRdtsc;
328	static FNVMEXITHANDLER hmR0VmxExitRsm;
329	static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
330	static FNVMEXITHANDLER hmR0VmxExitMovCRx;
331	static FNVMEXITHANDLER hmR0VmxExitMovDRx;
332	static FNVMEXITHANDLER hmR0VmxExitIoInstr;
333	static FNVMEXITHANDLER hmR0VmxExitRdmsr;
334	static FNVMEXITHANDLER hmR0VmxExitWrmsr;
335	static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
336	static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
337	static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
338	static FNVMEXITHANDLER hmR0VmxExitMwait;
339	static FNVMEXITHANDLER hmR0VmxExitMtf;
340	static FNVMEXITHANDLER hmR0VmxExitMonitor;
341	static FNVMEXITHANDLER hmR0VmxExitPause;
342	static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
343	static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
344	static FNVMEXITHANDLER hmR0VmxExitApicAccess;
345	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
346	static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
347	static FNVMEXITHANDLER hmR0VmxExitEptViolation;
348	static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
349	static FNVMEXITHANDLER hmR0VmxExitRdtscp;
350	static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
351	static FNVMEXITHANDLER hmR0VmxExitWbinvd;
352	static FNVMEXITHANDLER hmR0VmxExitXsetbv;
353	static FNVMEXITHANDLER hmR0VmxExitRdrand;
354	static FNVMEXITHANDLER hmR0VmxExitInvpcid;
355	/** @} */
356
357	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
358	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
359	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
360	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
361	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
362	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
363	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
365
366	/*******************************************************************************
367	* Global Variables *
368	*******************************************************************************/
369	#ifdef HMVMX_USE_FUNCTION_TABLE
370
371	/**
372	* VMX_EXIT dispatch table.
373	*/
374	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
375	{
376	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
377	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
378	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
379	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
380	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
381	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
382	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
383	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
384	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
385	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
386	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
387	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
388	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
389	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
390	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
391	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
392	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
393	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
394	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
395	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
396	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
397	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
398	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
399	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
400	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
401	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
402	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
403	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
404	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
405	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
406	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
407	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
408	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
409	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
410	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
411	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
412	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
413	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
414	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
415	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
416	/* 40 UNDEFINED */ hmR0VmxExitPause,
417	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
418	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
419	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
420	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
421	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
422	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
423	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
424	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
425	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
426	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
427	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
428	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
429	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
430	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
431	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
432	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
433	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
434	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
435	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
436	};
437	#endif /* HMVMX_USE_FUNCTION_TABLE */
438
439	#ifdef VBOX_STRICT
440	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
441	{
442	/* 0 */ "(Not Used)",
443	/* 1 */ "VMCALL executed in VMX root operation.",
444	/* 2 */ "VMCLEAR with invalid physical address.",
445	/* 3 */ "VMCLEAR with VMXON pointer.",
446	/* 4 */ "VMLAUNCH with non-clear VMCS.",
447	/* 5 */ "VMRESUME with non-launched VMCS.",
448	/* 6 */ "VMRESUME after VMXOFF",
449	/* 7 */ "VM entry with invalid control fields.",
450	/* 8 */ "VM entry with invalid host state fields.",
451	/* 9 */ "VMPTRLD with invalid physical address.",
452	/* 10 */ "VMPTRLD with VMXON pointer.",
453	/* 11 */ "VMPTRLD with incorrect revision identifier.",
454	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
455	/* 13 */ "VMWRITE to read-only VMCS component.",
456	/* 14 */ "(Not Used)",
457	/* 15 */ "VMXON executed in VMX root operation.",
458	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
459	/* 17 */ "VM entry with non-launched executing VMCS.",
460	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
461	/* 19 */ "VMCALL with non-clear VMCS.",
462	/* 20 */ "VMCALL with invalid VM-exit control fields.",
463	/* 21 */ "(Not Used)",
464	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
465	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
466	/* 24 */ "VMCALL with invalid SMM-monitor features.",
467	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
468	/* 26 */ "VM entry with events blocked by MOV SS.",
469	/* 27 */ "(Not Used)",
470	/* 28 */ "Invalid operand to INVEPT/INVVPID."
471	};
472	#endif /* VBOX_STRICT */
473
474
475
476	/**
477	* Updates the VM's last error record. If there was a VMX instruction error,
478	* reads the error data from the VMCS and updates VCPU's last error record as
479	* well.
480	*
481	* @param pVM Pointer to the VM.
482	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
483	* VERR_VMX_UNABLE_TO_START_VM or
484	* VERR_VMX_INVALID_VMCS_FIELD).
485	* @param rc The error code.
486	*/
487	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
488	{
489	AssertPtr(pVM);
490	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
491	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
492	{
493	AssertPtrReturnVoid(pVCpu);
494	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
495	}
496	pVM->hm.s.lLastError = rc;
497	}
498
499
500	/**
501	* Reads the VM-entry interruption-information field from the VMCS into the VMX
502	* transient structure.
503	*
504	* @returns VBox status code.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-entry exception error code field from the VMCS into
519	* the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*
524	* @remarks No-long-jump zone!!!
525	*/
526	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
529	AssertRCReturn(rc, rc);
530	return VINF_SUCCESS;
531	}
532
533
534	/**
535	* Reads the VM-entry exception error code field from the VMCS into
536	* the VMX transient structure.
537	*
538	* @returns VBox status code.
539	* @param pVCpu Pointer to the VMCPU.
540	* @param pVmxTransient Pointer to the VMX transient structure.
541	*
542	* @remarks No-long-jump zone!!!
543	*/
544	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
547	AssertRCReturn(rc, rc);
548	return VINF_SUCCESS;
549	}
550
551
552	/**
553	* Reads the VM-exit interruption-information field from the VMCS into the VMX
554	* transient structure.
555	*
556	* @returns VBox status code.
557	* @param pVCpu Pointer to the VMCPU.
558	* @param pVmxTransient Pointer to the VMX transient structure.
559	*/
560	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
561	{
562	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
563	{
564	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
565	AssertRCReturn(rc, rc);
566	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
567	}
568	return VINF_SUCCESS;
569	}
570
571
572	/**
573	* Reads the VM-exit interruption error code from the VMCS into the VMX
574	* transient structure.
575	*
576	* @returns VBox status code.
577	* @param pVCpu Pointer to the VMCPU.
578	* @param pVmxTransient Pointer to the VMX transient structure.
579	*/
580	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
581	{
582	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
583	{
584	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
585	AssertRCReturn(rc, rc);
586	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
587	}
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the VM-exit instruction length field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVCpu Pointer to the VMCPU.
598	* @param pVmxTransient Pointer to the VMX transient structure.
599	*/
600	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
601	{
602	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
603	{
604	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
605	AssertRCReturn(rc, rc);
606	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
607	}
608	return VINF_SUCCESS;
609	}
610
611
612	/**
613	* Reads the VM-exit instruction-information field from the VMCS into
614	* the VMX transient structure.
615	*
616	* @returns VBox status code.
617	* @param pVCpu The cross context per CPU structure.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Reads the exit qualification from the VMCS into the VMX transient structure.
634	*
635	* @returns VBox status code.
636	* @param pVCpu Pointer to the VMCPU.
637	* @param pVmxTransient Pointer to the VMX transient structure.
638	*/
639	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
640	{
641	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
642	{
643	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
644	AssertRCReturn(rc, rc);
645	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
646	}
647	return VINF_SUCCESS;
648	}
649
650
651	/**
652	* Reads the IDT-vectoring information field from the VMCS into the VMX
653	* transient structure.
654	*
655	* @returns VBox status code.
656	* @param pVmxTransient Pointer to the VMX transient structure.
657	*
658	* @remarks No-long-jump zone!!!
659	*/
660	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
661	{
662	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
663	{
664	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
665	AssertRCReturn(rc, rc);
666	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
667	}
668	return VINF_SUCCESS;
669	}
670
671
672	/**
673	* Reads the IDT-vectoring error code from the VMCS into the VMX
674	* transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVmxTransient Pointer to the VMX transient structure.
678	*/
679	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
680	{
681	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
682	{
683	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
684	AssertRCReturn(rc, rc);
685	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
686	}
687	return VINF_SUCCESS;
688	}
689
690
691	/**
692	* Enters VMX root mode operation on the current CPU.
693	*
694	* @returns VBox status code.
695	* @param pVM Pointer to the VM (optional, can be NULL, after
696	* a resume).
697	* @param HCPhysCpuPage Physical address of the VMXON region.
698	* @param pvCpuPage Pointer to the VMXON region.
699	*/
700	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
701	{
702	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
703	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
704	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
705
706	if (pVM)
707	{
708	/* Write the VMCS revision dword to the VMXON region. */
709	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
710	}
711
712	/* Enable the VMX bit in CR4 if necessary. */
713	RTCCUINTREG uCr4 = ASMGetCR4();
714	if (!(uCr4 & X86_CR4_VMXE))
715	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
716
717	/* Enter VMX root mode. */
718	int rc = VMXEnable(HCPhysCpuPage);
719	if (RT_FAILURE(rc))
720	ASMSetCR4(uCr4);
721
722	return rc;
723	}
724
725
726	/**
727	* Exits VMX root mode operation on the current CPU.
728	*
729	* @returns VBox status code.
730	*/
731	static int hmR0VmxLeaveRootMode(void)
732	{
733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
734
735	/* If we're for some reason not in VMX root mode, then don't leave it. */
736	RTCCUINTREG uHostCR4 = ASMGetCR4();
737	if (uHostCR4 & X86_CR4_VMXE)
738	{
739	/* Exit VMX root mode and clear the VMX bit in CR4. */
740	VMXDisable();
741	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
742	return VINF_SUCCESS;
743	}
744
745	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
746	}
747
748
749	/**
750	* Allocates and maps one physically contiguous page. The allocated page is
751	* zero'd out. (Used by various VT-x structures).
752	*
753	* @returns IPRT status code.
754	* @param pMemObj Pointer to the ring-0 memory object.
755	* @param ppVirt Where to store the virtual address of the
756	* allocation.
757	* @param pPhys Where to store the physical address of the
758	* allocation.
759	*/
760	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
761	{
762	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
763	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
764	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
765
766	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
767	if (RT_FAILURE(rc))
768	return rc;
769	ppVirt = RTR0MemObjAddress(pMemObj);
770	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
771	ASMMemZero32(*ppVirt, PAGE_SIZE);
772	return VINF_SUCCESS;
773	}
774
775
776	/**
777	* Frees and unmaps an allocated physical page.
778	*
779	* @param pMemObj Pointer to the ring-0 memory object.
780	* @param ppVirt Where to re-initialize the virtual address of
781	* allocation as 0.
782	* @param pHCPhys Where to re-initialize the physical address of the
783	* allocation as 0.
784	*/
785	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
786	{
787	AssertPtr(pMemObj);
788	AssertPtr(ppVirt);
789	AssertPtr(pHCPhys);
790	if (*pMemObj != NIL_RTR0MEMOBJ)
791	{
792	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
793	AssertRC(rc);
794	*pMemObj = NIL_RTR0MEMOBJ;
795	*ppVirt = 0;
796	*pHCPhys = 0;
797	}
798	}
799
800
801	/**
802	* Worker function to free VT-x related structures.
803	*
804	* @returns IPRT status code.
805	* @param pVM Pointer to the VM.
806	*/
807	static void hmR0VmxStructsFree(PVM pVM)
808	{
809	for (VMCPUID i = 0; i < pVM->cCpus; i++)
810	{
811	PVMCPU pVCpu = &pVM->aCpus[i];
812	AssertPtr(pVCpu);
813
814	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
815	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
816	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
817	#endif
818
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
820	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
821
822	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
823	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
824	}
825
826	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
827	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
828	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
829	#endif
830	}
831
832
833	/**
834	* Worker function to allocate VT-x related VM structures.
835	*
836	* @returns IPRT status code.
837	* @param pVM Pointer to the VM.
838	*/
839	static int hmR0VmxStructsAlloc(PVM pVM)
840	{
841	/*
842	* Initialize members up-front so we can cleanup properly on allocation failure.
843	*/
844	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
845	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
846	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
847	pVM->hm.s.vmx.HCPhys##a_Name = 0;
848
849	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
850	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
851	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
852	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
853
854	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
855	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
856	#endif
857	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
858
859	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
860	for (VMCPUID i = 0; i < pVM->cCpus; i++)
861	{
862	PVMCPU pVCpu = &pVM->aCpus[i];
863	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
864	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
865	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
866	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
867	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
868	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
869	#endif
870	}
871	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
872	#undef VMXLOCAL_INIT_VM_MEMOBJ
873
874	/*
875	* Allocate all the VT-x structures.
876	*/
877	int rc = VINF_SUCCESS;
878	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
883	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
884	#endif
885
886	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
887	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
888	{
889	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
890	&pVM->hm.s.vmx.HCPhysApicAccess);
891	if (RT_FAILURE(rc))
892	goto cleanup;
893	}
894
895	/*
896	* Initialize per-VCPU VT-x structures.
897	*/
898	for (VMCPUID i = 0; i < pVM->cCpus; i++)
899	{
900	PVMCPU pVCpu = &pVM->aCpus[i];
901	AssertPtr(pVCpu);
902
903	/* Allocate the VM control structure (VMCS). */
904	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
905	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
906	if (RT_FAILURE(rc))
907	goto cleanup;
908
909	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
910	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
911	{
912	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
913	&pVCpu->hm.s.vmx.HCPhysVirtApic);
914	if (RT_FAILURE(rc))
915	goto cleanup;
916	}
917
918	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
919	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920	{
921	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
922	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
923	if (RT_FAILURE(rc))
924	goto cleanup;
925	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
926	}
927
928	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
929	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
930	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
931	if (RT_FAILURE(rc))
932	goto cleanup;
933
934	/* Allocate the VM-exit MSR-load page for the host MSRs. */
935	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
936	if (RT_FAILURE(rc))
937	goto cleanup;
938	#endif
939	}
940
941	return VINF_SUCCESS;
942
943	cleanup:
944	hmR0VmxStructsFree(pVM);
945	return rc;
946	}
947
948
949	/**
950	* Does global VT-x initialization (called during module initialization).
951	*
952	* @returns VBox status code.
953	*/
954	VMMR0DECL(int) VMXR0GlobalInit(void)
955	{
956	#ifdef HMVMX_USE_FUNCTION_TABLE
957	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
958	# ifdef VBOX_STRICT
959	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
960	Assert(g_apfnVMExitHandlers[i]);
961	# endif
962	#endif
963	return VINF_SUCCESS;
964	}
965
966
967	/**
968	* Does global VT-x termination (called during module termination).
969	*/
970	VMMR0DECL(void) VMXR0GlobalTerm()
971	{
972	/* Nothing to do currently. */
973	}
974
975
976	/**
977	* Sets up and activates VT-x on the current CPU.
978	*
979	* @returns VBox status code.
980	* @param pCpu Pointer to the global CPU info struct.
981	* @param pVM Pointer to the VM (can be NULL after a host resume
982	* operation).
983	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
984	* fEnabledByHost is true).
985	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
986	* @a fEnabledByHost is true).
987	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
988	* enable VT-x on the host.
989	*/
990	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
991	{
992	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
993	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
994
995	if (!fEnabledByHost)
996	{
997	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
998	if (RT_FAILURE(rc))
999	return rc;
1000	}
1001
1002	/*
1003	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1004	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
1005	* each time while reusing a VPID after hitting the MaxASID limit once.
1006	*/
1007	if ( pVM
1008	&& pVM->hm.s.fNestedPaging)
1009	{
1010	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1011	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1012	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1013	pCpu->fFlushAsidBeforeUse = false;
1014	}
1015	else
1016	{
1017	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1018	* without Nested Paging triggered this function) we still have the risk
1019	* of potentially running with stale TLB-entries from other hypervisors
1020	* when later we use a VM with NestedPaging. To fix this properly we will
1021	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1022	* 'vmx_ept_vpid_caps' from it. Sigh. */
1023	pCpu->fFlushAsidBeforeUse = true;
1024	}
1025
1026	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1027	++pCpu->cTlbFlushes;
1028
1029	return VINF_SUCCESS;
1030	}
1031
1032
1033	/**
1034	* Deactivates VT-x on the current CPU.
1035	*
1036	* @returns VBox status code.
1037	* @param pCpu Pointer to the global CPU info struct.
1038	* @param pvCpuPage Pointer to the VMXON region.
1039	* @param HCPhysCpuPage Physical address of the VMXON region.
1040	*
1041	* @remarks This function should never be called when SUPR0EnableVTx() or
1042	* similar was used to enable VT-x on the host.
1043	*/
1044	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1045	{
1046	NOREF(pCpu);
1047	NOREF(pvCpuPage);
1048	NOREF(HCPhysCpuPage);
1049
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051	return hmR0VmxLeaveRootMode();
1052	}
1053
1054
1055	/**
1056	* Sets the permission bits for the specified MSR in the MSR bitmap.
1057	*
1058	* @param pVCpu Pointer to the VMCPU.
1059	* @param uMSR The MSR value.
1060	* @param enmRead Whether reading this MSR causes a VM-exit.
1061	* @param enmWrite Whether writing this MSR causes a VM-exit.
1062	*/
1063	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1064	{
1065	int32_t iBit;
1066	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1067
1068	/*
1069	* Layout:
1070	* 0x000 - 0x3ff - Low MSR read bits
1071	* 0x400 - 0x7ff - High MSR read bits
1072	* 0x800 - 0xbff - Low MSR write bits
1073	* 0xc00 - 0xfff - High MSR write bits
1074	*/
1075	if (uMsr <= 0x00001FFF)
1076	iBit = uMsr;
1077	else if ( uMsr >= 0xC0000000
1078	&& uMsr <= 0xC0001FFF)
1079	{
1080	iBit = (uMsr - 0xC0000000);
1081	pbMsrBitmap += 0x400;
1082	}
1083	else
1084	{
1085	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1086	return;
1087	}
1088
1089	Assert(iBit <= 0x1fff);
1090	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1091	ASMBitSet(pbMsrBitmap, iBit);
1092	else
1093	ASMBitClear(pbMsrBitmap, iBit);
1094
1095	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1096	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1097	else
1098	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1099	}
1100
1101
1102	/**
1103	* Flushes the TLB using EPT.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1108	* enmFlush).
1109	* @param enmFlush Type of flush.
1110	*/
1111	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1112	{
1113	AssertPtr(pVM);
1114	Assert(pVM->hm.s.fNestedPaging);
1115
1116	uint64_t descriptor[2];
1117	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1118	descriptor[0] = 0;
1119	else
1120	{
1121	Assert(pVCpu);
1122	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1123	}
1124	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1125
1126	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1127	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1128	rc));
1129	if ( RT_SUCCESS(rc)
1130	&& pVCpu)
1131	{
1132	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1133	}
1134	}
1135
1136
1137	/**
1138	* Flushes the TLB using VPID.
1139	*
1140	* @returns VBox status code.
1141	* @param pVM Pointer to the VM.
1142	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1143	* enmFlush).
1144	* @param enmFlush Type of flush.
1145	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1146	* on @a enmFlush).
1147	*/
1148	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1149	{
1150	AssertPtr(pVM);
1151	Assert(pVM->hm.s.vmx.fVpid);
1152
1153	uint64_t descriptor[2];
1154	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1155	{
1156	descriptor[0] = 0;
1157	descriptor[1] = 0;
1158	}
1159	else
1160	{
1161	AssertPtr(pVCpu);
1162	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1163	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1164	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1165	descriptor[1] = GCPtr;
1166	}
1167
1168	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1169	AssertMsg(rc == VINF_SUCCESS,
1170	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1171	if ( RT_SUCCESS(rc)
1172	&& pVCpu)
1173	{
1174	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1175	}
1176	}
1177
1178
1179	/**
1180	* Invalidates a guest page by guest virtual address. Only relevant for
1181	* EPT/VPID, otherwise there is nothing really to invalidate.
1182	*
1183	* @returns VBox status code.
1184	* @param pVM Pointer to the VM.
1185	* @param pVCpu Pointer to the VMCPU.
1186	* @param GCVirt Guest virtual address of the page to invalidate.
1187	*/
1188	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1189	{
1190	AssertPtr(pVM);
1191	AssertPtr(pVCpu);
1192	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1193
1194	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	if (!fFlushPending)
1196	{
1197	/*
1198	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1199	* See @bugref{6043} and @bugref{6177}.
1200	*
1201	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1202	* function maybe called in a loop with individual addresses.
1203	*/
1204	if (pVM->hm.s.vmx.fVpid)
1205	{
1206	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1207	{
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1210	}
1211	else
1212	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1213	}
1214	else if (pVM->hm.s.fNestedPaging)
1215	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1216	}
1217
1218	return VINF_SUCCESS;
1219	}
1220
1221
1222	/**
1223	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1224	* otherwise there is nothing really to invalidate.
1225	*
1226	* @returns VBox status code.
1227	* @param pVM Pointer to the VM.
1228	* @param pVCpu Pointer to the VMCPU.
1229	* @param GCPhys Guest physical address of the page to invalidate.
1230	*/
1231	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1232	{
1233	LogFlowFunc(("%RGp\n", GCPhys));
1234
1235	/*
1236	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1237	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1238	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1239	*/
1240	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1242	return VINF_SUCCESS;
1243	}
1244
1245
1246	/**
1247	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1248	* case where neither EPT nor VPID is supported by the CPU.
1249	*
1250	* @param pVM Pointer to the VM.
1251	* @param pVCpu Pointer to the VMCPU.
1252	*
1253	* @remarks Called with interrupts disabled.
1254	*/
1255	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1256	{
1257	NOREF(pVM);
1258	AssertPtr(pVCpu);
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1263	AssertPtr(pCpu);
1264
1265	pVCpu->hm.s.TlbShootdown.cPages = 0;
1266	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1267	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1268	pVCpu->hm.s.fForceTLBFlush = false;
1269	return;
1270	}
1271
1272
1273	/**
1274	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1275	*
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	* @remarks All references to "ASID" in this function pertains to "VPID" in
1279	* Intel's nomenclature. The reason is, to avoid confusion in compare
1280	* statements since the host-CPU copies are named "ASID".
1281	*
1282	* @remarks Called with interrupts disabled.
1283	*/
1284	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1285	{
1286	#ifdef VBOX_WITH_STATISTICS
1287	bool fTlbFlushed = false;
1288	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1289	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1290	if (!fTlbFlushed) \
1291	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1292	} while (0)
1293	#else
1294	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1295	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1296	#endif
1297
1298	AssertPtr(pVM);
1299	AssertPtr(pVCpu);
1300	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1301	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1302	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1303
1304	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1305	AssertPtr(pCpu);
1306
1307	/*
1308	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1309	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1310	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1311	*/
1312	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1313	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1314	{
1315	++pCpu->uCurrentAsid;
1316	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1317	{
1318	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1319	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1320	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1321	}
1322
1323	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1324	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1325	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1326
1327	/*
1328	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1329	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1330	*/
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1333	HMVMX_SET_TAGGED_TLB_FLUSHED();
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1335	}
1336
1337	/* Check for explicit TLB shootdowns. */
1338	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1339	{
1340	/*
1341	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1342	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1343	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1344	* but not guest-physical mappings.
1345	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1346	*/
1347	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	HMVMX_SET_TAGGED_TLB_FLUSHED();
1350	}
1351
1352	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1353	* not be executed. See hmQueueInvlPage() where it is commented
1354	* out. Support individual entry flushing someday. */
1355	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1356	{
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1358
1359	/*
1360	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1361	* as supported by the CPU.
1362	*/
1363	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1364	{
1365	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1366	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1367	}
1368	else
1369	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370
1371	HMVMX_SET_TAGGED_TLB_FLUSHED();
1372	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1373	}
1374
1375	pVCpu->hm.s.TlbShootdown.cPages = 0;
1376	pVCpu->hm.s.fForceTLBFlush = false;
1377
1378	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1379
1380	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1381	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1382	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1383	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1384	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1385	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1386	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1387	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1388
1389	/* Update VMCS with the VPID. */
1390	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1391	AssertRC(rc);
1392
1393	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1394	}
1395
1396
1397	/**
1398	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1399	*
1400	* @returns VBox status code.
1401	* @param pVM Pointer to the VM.
1402	* @param pVCpu Pointer to the VMCPU.
1403	*
1404	* @remarks Called with interrupts disabled.
1405	*/
1406	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1407	{
1408	AssertPtr(pVM);
1409	AssertPtr(pVCpu);
1410	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1411	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1412
1413	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1414	AssertPtr(pCpu);
1415
1416	/*
1417	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1418	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1419	*/
1420	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1421	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1422	{
1423	pVCpu->hm.s.fForceTLBFlush = true;
1424	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1425	}
1426
1427	/* Check for explicit TLB shootdown flushes. */
1428	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1429	{
1430	pVCpu->hm.s.fForceTLBFlush = true;
1431	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1432	}
1433
1434	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1435	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1436
1437	if (pVCpu->hm.s.fForceTLBFlush)
1438	{
1439	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1440	pVCpu->hm.s.fForceTLBFlush = false;
1441	}
1442	else
1443	{
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1451	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1452	}
1453	else
1454	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1455	}
1456
1457	pVCpu->hm.s.TlbShootdown.cPages = 0;
1458	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1459	}
1460
1461
1462	/**
1463	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1464	*
1465	* @returns VBox status code.
1466	* @param pVM Pointer to the VM.
1467	* @param pVCpu Pointer to the VMCPU.
1468	*
1469	* @remarks Called with interrupts disabled.
1470	*/
1471	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1472	{
1473	AssertPtr(pVM);
1474	AssertPtr(pVCpu);
1475	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1476	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1477
1478	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1479
1480	/*
1481	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1482	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1483	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1484	*/
1485	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1486	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1487	{
1488	pVCpu->hm.s.fForceTLBFlush = true;
1489	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1490	}
1491
1492	/* Check for explicit TLB shootdown flushes. */
1493	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1494	{
1495	/*
1496	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1497	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1498	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1499	*/
1500	pVCpu->hm.s.fForceTLBFlush = true;
1501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1502	}
1503
1504	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1505	if (pVCpu->hm.s.fForceTLBFlush)
1506	{
1507	++pCpu->uCurrentAsid;
1508	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1509	{
1510	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1511	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1512	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1513	}
1514
1515	pVCpu->hm.s.fForceTLBFlush = false;
1516	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1517	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1518	if (pCpu->fFlushAsidBeforeUse)
1519	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1520	}
1521	else
1522	{
1523	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1524	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1525	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1526	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1527
1528	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1529	* not be executed. See hmQueueInvlPage() where it is commented
1530	* out. Support individual entry flushing someday. */
1531	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1532	{
1533	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1534	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1535	{
1536	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1537	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1538	}
1539	else
1540	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1541	}
1542	else
1543	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1544	}
1545
1546	pVCpu->hm.s.TlbShootdown.cPages = 0;
1547	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1548
1549	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1550	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1551	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1552	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1553	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1554	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1555
1556	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1557	AssertRC(rc);
1558	}
1559
1560
1561	/**
1562	* Flushes the guest TLB entry based on CPU capabilities.
1563	*
1564	* @param pVCpu Pointer to the VMCPU.
1565	*/
1566	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1567	{
1568	PVM pVM = pVCpu->CTX_SUFF(pVM);
1569	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1570	{
1571	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1572	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1573	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1574	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1575	default:
1576	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1577	break;
1578	}
1579	}
1580
1581
1582	/**
1583	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1584	* TLB entries from the host TLB before VM-entry.
1585	*
1586	* @returns VBox status code.
1587	* @param pVM Pointer to the VM.
1588	*/
1589	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1590	{
1591	/*
1592	* Determine optimal flush type for Nested Paging.
1593	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1594	* guest execution (see hmR3InitFinalizeR0()).
1595	*/
1596	if (pVM->hm.s.fNestedPaging)
1597	{
1598	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1599	{
1600	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1601	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1602	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1603	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1604	else
1605	{
1606	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1607	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1608	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1609	}
1610
1611	/* Make sure the write-back cacheable memory type for EPT is supported. */
1612	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1613	{
1614	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1615	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1617	}
1618	}
1619	else
1620	{
1621	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1622	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624	}
1625	}
1626
1627	/*
1628	* Determine optimal flush type for VPID.
1629	*/
1630	if (pVM->hm.s.vmx.fVpid)
1631	{
1632	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1633	{
1634	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1635	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1636	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1637	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1638	else
1639	{
1640	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1641	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1642	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1643	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1644	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1645	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1646	pVM->hm.s.vmx.fVpid = false;
1647	}
1648	}
1649	else
1650	{
1651	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1652	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1653	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1654	pVM->hm.s.vmx.fVpid = false;
1655	}
1656	}
1657
1658	/*
1659	* Setup the handler for flushing tagged-TLBs.
1660	*/
1661	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1662	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1663	else if (pVM->hm.s.fNestedPaging)
1664	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1665	else if (pVM->hm.s.vmx.fVpid)
1666	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1667	else
1668	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1669	return VINF_SUCCESS;
1670	}
1671
1672
1673	/**
1674	* Sets up pin-based VM-execution controls in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVM Pointer to the VM.
1678	* @param pVCpu Pointer to the VMCPU.
1679	*/
1680	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1681	{
1682	AssertPtr(pVM);
1683	AssertPtr(pVCpu);
1684
1685	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1686	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1687
1688	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1689	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1690	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1691
1692	/* Enable the VMX preemption timer. */
1693	if (pVM->hm.s.vmx.fUsePreemptTimer)
1694	{
1695	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1696	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1702	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1707	AssertRCReturn(rc, rc);
1708
1709	/* Update VCPU with the currently set pin-based VM-execution controls. */
1710	pVCpu->hm.s.vmx.u32PinCtls = val;
1711	return rc;
1712	}
1713
1714
1715	/**
1716	* Sets up processor-based VM-execution controls in the VMCS.
1717	*
1718	* @returns VBox status code.
1719	* @param pVM Pointer to the VM.
1720	* @param pVMCPU Pointer to the VMCPU.
1721	*/
1722	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1723	{
1724	AssertPtr(pVM);
1725	AssertPtr(pVCpu);
1726
1727	int rc = VERR_INTERNAL_ERROR_5;
1728	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1729	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1730
1731	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1732	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1733	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1734	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1735	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1736	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1737	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1738
1739	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1740	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1741	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1742	{
1743	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1744	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1745	}
1746
1747	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1748	if (!pVM->hm.s.fNestedPaging)
1749	{
1750	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1751	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1752	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1753	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1754	}
1755
1756	/* Use TPR shadowing if supported by the CPU. */
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1758	{
1759	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1763	AssertRCReturn(rc, rc);
1764
1765	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1766	/* CR8 writes causes a VM-exit based on TPR threshold. */
1767	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1768	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1769	}
1770	else
1771	{
1772	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1773	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1774	}
1775
1776	/* Use MSR-bitmaps if supported by the CPU. */
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1778	{
1779	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1780
1781	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1782	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1783	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1784	AssertRCReturn(rc, rc);
1785
1786	/*
1787	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1788	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1789	*/
1790	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1791	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1792	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1793	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1794	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1795	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1796	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1797	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1798	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1799	}
1800
1801	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1803	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1804
1805	if ((val & zap) != val)
1806	{
1807	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1808	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1809	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1810	}
1811
1812	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1813	AssertRCReturn(rc, rc);
1814
1815	/* Update VCPU with the currently set processor-based VM-execution controls. */
1816	pVCpu->hm.s.vmx.u32ProcCtls = val;
1817
1818	/*
1819	* Secondary processor-based VM-execution controls.
1820	*/
1821	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1822	{
1823	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1824	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1825
1826	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1827	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1828
1829	if (pVM->hm.s.fNestedPaging)
1830	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1831	else
1832	{
1833	/*
1834	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1835	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1836	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1837	*/
1838	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1839	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1840	}
1841
1842	if (pVM->hm.s.vmx.fVpid)
1843	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1844
1845	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1846	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1847
1848	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1849	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1850	* done dynamically. */
1851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1852	{
1853	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1854	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1855	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1856	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1857	AssertRCReturn(rc, rc);
1858	}
1859
1860	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1861	{
1862	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1864	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1865	}
1866
1867	if ((val & zap) != val)
1868	{
1869	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1870	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1871	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1872	}
1873
1874	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1875	AssertRCReturn(rc, rc);
1876
1877	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1878	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1879	}
1880	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1881	{
1882	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1883	"available\n"));
1884	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Sets up miscellaneous (everything other than Pin & Processor-based
1893	* VM-execution) control fields in the VMCS.
1894	*
1895	* @returns VBox status code.
1896	* @param pVM Pointer to the VM.
1897	* @param pVCpu Pointer to the VMCPU.
1898	*/
1899	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1900	{
1901	AssertPtr(pVM);
1902	AssertPtr(pVCpu);
1903
1904	int rc = VERR_GENERAL_FAILURE;
1905
1906	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1907	#if 0
1908	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1909	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1910	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1911
1912	/*
1913	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1914	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1915	* We thus use the exception bitmap to control it rather than use both.
1916	*/
1917	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1918	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1919
1920	/** @todo Explore possibility of using IO-bitmaps. */
1921	/* All IO & IOIO instructions cause VM-exits. */
1922	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1923	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1924
1925	/* Initialize the MSR-bitmap area. */
1926	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1927	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1928	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1929	#endif
1930
1931	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1932	/* Setup MSR autoloading/storing. */
1933	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1934	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1935	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1936	AssertRCReturn(rc, rc);
1937	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1938	AssertRCReturn(rc, rc);
1939
1940	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1941	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1942	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1943	AssertRCReturn(rc, rc);
1944	#endif
1945
1946	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1947	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1948	AssertRCReturn(rc, rc);
1949
1950	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1951	#if 0
1952	/* Setup debug controls */
1953	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1954	AssertRCReturn(rc, rc);
1955	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1956	AssertRCReturn(rc, rc);
1957	#endif
1958
1959	return rc;
1960	}
1961
1962
1963	/**
1964	* Sets up the initial exception bitmap in the VMCS based on static conditions
1965	* (i.e. conditions that cannot ever change at runtime).
1966	*
1967	* @returns VBox status code.
1968	* @param pVM Pointer to the VM.
1969	* @param pVCpu Pointer to the VMCPU.
1970	*/
1971	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1972	{
1973	AssertPtr(pVM);
1974	AssertPtr(pVCpu);
1975
1976	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1977
1978	uint32_t u32XcptBitmap = 0;
1979
1980	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1981	if (!pVM->hm.s.fNestedPaging)
1982	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1983
1984	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1985	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1986	AssertRCReturn(rc, rc);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Sets up the initial guest-state mask. The guest-state mask is consulted
1993	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1994	* for the nested virtualization case (as it would cause a VM-exit).
1995	*
1996	* @param pVCpu Pointer to the VMCPU.
1997	*/
1998	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1999	{
2000	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2001	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2002	return VINF_SUCCESS;
2003	}
2004
2005
2006	/**
2007	* Does per-VM VT-x initialization.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	*/
2012	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2013	{
2014	LogFlowFunc(("pVM=%p\n", pVM));
2015
2016	int rc = hmR0VmxStructsAlloc(pVM);
2017	if (RT_FAILURE(rc))
2018	{
2019	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2020	return rc;
2021	}
2022
2023	return VINF_SUCCESS;
2024	}
2025
2026
2027	/**
2028	* Does per-VM VT-x termination.
2029	*
2030	* @returns VBox status code.
2031	* @param pVM Pointer to the VM.
2032	*/
2033	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2034	{
2035	LogFlowFunc(("pVM=%p\n", pVM));
2036
2037	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2038	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2039	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2040	#endif
2041	hmR0VmxStructsFree(pVM);
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Sets up the VM for execution under VT-x.
2048	* This function is only called once per-VM during initialization.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	*/
2053	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2054	{
2055	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2057
2058	LogFlowFunc(("pVM=%p\n", pVM));
2059
2060	/*
2061	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2062	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2063	*/
2064	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2065	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2066	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2067	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2068	{
2069	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2070	return VERR_INTERNAL_ERROR;
2071	}
2072
2073	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2074	/*
2075	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2076	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2077	*/
2078	if ( pVM->hm.s.fAllow64BitGuests
2079	&& !HMVMX_IS_64BIT_HOST_MODE())
2080	{
2081	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2082	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2083	}
2084	#endif
2085
2086	/* Initialize these always, see hmR3InitFinalizeR0().*/
2087	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2088	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2089
2090	/* Setup the tagged-TLB flush handlers. */
2091	int rc = hmR0VmxSetupTaggedTlb(pVM);
2092	if (RT_FAILURE(rc))
2093	{
2094	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2095	return rc;
2096	}
2097
2098	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2099	{
2100	PVMCPU pVCpu = &pVM->aCpus[i];
2101	AssertPtr(pVCpu);
2102	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2103
2104	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2105	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2106
2107	/* Set revision dword at the beginning of the VMCS structure. */
2108	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2109
2110	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2111	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2112	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2113	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2114
2115	/* Load this VMCS as the current VMCS. */
2116	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2117	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2118	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2119
2120	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2121	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2122	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2123
2124	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2125	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2126	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2127
2128	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2129	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2130	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2131
2132	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2133	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2134	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2135
2136	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2137	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2141	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2142	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2143	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2144	#endif
2145
2146	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2147	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2148	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2152	}
2153
2154	return VINF_SUCCESS;
2155	}
2156
2157
2158	/**
2159	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2160	* the VMCS.
2161	*
2162	* @returns VBox status code.
2163	* @param pVM Pointer to the VM.
2164	* @param pVCpu Pointer to the VMCPU.
2165	*/
2166	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2167	{
2168	RTCCUINTREG uReg = ASMGetCR0();
2169	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2170	AssertRCReturn(rc, rc);
2171
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2174	if (HMVMX_IS_64BIT_HOST_MODE())
2175	{
2176	uint64_t uRegCR3 = HMR0Get64bitCR3();
2177	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2178	}
2179	else
2180	#endif
2181	{
2182	uReg = ASMGetCR3();
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2184	}
2185	AssertRCReturn(rc, rc);
2186
2187	uReg = ASMGetCR4();
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2189	AssertRCReturn(rc, rc);
2190	return rc;
2191	}
2192
2193
2194	/**
2195	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2196	* the host-state area in the VMCS.
2197	*
2198	* @returns VBox status code.
2199	* @param pVM Pointer to the VM.
2200	* @param pVCpu Pointer to the VMCPU.
2201	*/
2202	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2203	{
2204	int rc = VERR_INTERNAL_ERROR_5;
2205	RTSEL uSelDS = 0;
2206	RTSEL uSelES = 0;
2207	RTSEL uSelFS = 0;
2208	RTSEL uSelGS = 0;
2209	RTSEL uSelTR = 0;
2210
2211	/*
2212	* Host DS, ES, FS and GS segment registers.
2213	*/
2214	#if HC_ARCH_BITS == 64
2215	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2216	uSelDS = ASMGetDS();
2217	uSelES = ASMGetES();
2218	uSelFS = ASMGetFS();
2219	uSelGS = ASMGetGS();
2220	#endif
2221
2222	/*
2223	* Host CS and SS segment registers.
2224	*/
2225	RTSEL uSelCS;
2226	RTSEL uSelSS;
2227	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2228	if (HMVMX_IS_64BIT_HOST_MODE())
2229	{
2230	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2231	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2232	}
2233	else
2234	{
2235	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2236	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2237	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2238	}
2239	#else
2240	uSelCS = ASMGetCS();
2241	uSelSS = ASMGetSS();
2242	#endif
2243
2244	/*
2245	* Host TR segment register.
2246	*/
2247	uSelTR = ASMGetTR();
2248
2249	#if HC_ARCH_BITS == 64
2250	/*
2251	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2252	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2253	*/
2254	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2255	{
2256	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2257	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2258	uSelDS = 0;
2259	}
2260	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2261	{
2262	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2263	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2264	uSelES = 0;
2265	}
2266	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2267	{
2268	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2269	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2270	uSelFS = 0;
2271	}
2272	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2273	{
2274	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2275	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2276	uSelGS = 0;
2277	}
2278	#endif
2279
2280	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2281	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2282	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2283	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2284	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2285	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2286	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2287	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2288	Assert(uSelCS);
2289	Assert(uSelTR);
2290
2291	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2292	#if 0
2293	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2294	Assert(uSelSS != 0);
2295	#endif
2296
2297	/* Write these host selector fields into the host-state area in the VMCS. */
2298	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2299	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2300	#if HC_ARCH_BITS == 64
2301	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2302	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2303	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2304	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2305	#endif
2306	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2307
2308	/*
2309	* Host GDTR and IDTR.
2310	*/
2311	RTGDTR Gdtr;
2312	RT_ZERO(Gdtr);
2313	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2314	if (HMVMX_IS_64BIT_HOST_MODE())
2315	{
2316	X86XDTR64 Gdtr64;
2317	X86XDTR64 Idtr64;
2318	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2319	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2321
2322	Gdtr.cbGdt = Gdtr64.cb;
2323	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2324	}
2325	else
2326	#endif
2327	{
2328	RTIDTR Idtr;
2329	ASMGetGDTR(&Gdtr);
2330	ASMGetIDTR(&Idtr);
2331	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2332	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2337	* maximum limit (0xffff) on every VM-exit.
2338	*/
2339	if (Gdtr.cbGdt != 0xffff)
2340	{
2341	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2342	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2343	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2344	}
2345
2346	/*
2347	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2348	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2349	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2350	*/
2351	if (Idtr.cbIdt < 0x0fff)
2352	{
2353	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2354	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2355	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2356	}
2357	#endif
2358	}
2359
2360	/*
2361	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2362	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2363	*/
2364	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2365	{
2366	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2367	return VERR_VMX_INVALID_HOST_STATE;
2368	}
2369
2370	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2371	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2372	if (HMVMX_IS_64BIT_HOST_MODE())
2373	{
2374	/* We need the 64-bit TR base for hybrid darwin. */
2375	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2376	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2377	}
2378	else
2379	#endif
2380	{
2381	uintptr_t uTRBase;
2382	#if HC_ARCH_BITS == 64
2383	uTRBase = X86DESC64_BASE(pDesc);
2384
2385	/*
2386	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2387	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2388	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2389	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2390	*
2391	* [1] See Intel spec. 3.5 "System Descriptor Types".
2392	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2393	*/
2394	Assert(pDesc->System.u4Type == 11);
2395	if ( pDesc->System.u16LimitLow != 0x67
2396	\|\| pDesc->System.u4LimitHigh)
2397	{
2398	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2399	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2400
2401	/* Store the GDTR here as we need it while restoring TR. */
2402	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2403	}
2404	#else
2405	uTRBase = X86DESC_BASE(pDesc);
2406	#endif
2407	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2408	}
2409	AssertRCReturn(rc, rc);
2410
2411	/*
2412	* Host FS base and GS base.
2413	*/
2414	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2415	if (HMVMX_IS_64BIT_HOST_MODE())
2416	{
2417	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2418	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2419	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2420	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2421
2422	# if HC_ARCH_BITS == 64
2423	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2424	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2425	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2426	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2427	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2428	# endif
2429	}
2430	#endif
2431	return rc;
2432	}
2433
2434
2435	/**
2436	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2437	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2438	* the host after every successful VM exit.
2439	*
2440	* @returns VBox status code.
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	*/
2444	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2445	{
2446	AssertPtr(pVCpu);
2447	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2448
2449	int rc = VINF_SUCCESS;
2450	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2451	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2452	uint32_t cHostMsrs = 0;
2453	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2454
2455	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2456	{
2457	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2458
2459	# if HC_ARCH_BITS == 64
2460	/* Paranoia. 64-bit code requires these bits to be set always. */
2461	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2462
2463	/*
2464	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2465	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2466	* some reason (e.g. allow transparent reads) we would activate the code below.
2467	*/
2468	# if 0
2469	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2470	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2471	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2472	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2473	if (CPUMIsGuestInLongMode(pVCpu))
2474	{
2475	uint64_t u64GuestEfer;
2476	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2477	AssertRC(rc);
2478
2479	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2480	{
2481	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2482	pHostMsr->u32Reserved = 0;
2483	pHostMsr->u64Value = u64HostEfer;
2484	pHostMsr++; cHostMsrs++;
2485	}
2486	}
2487	# endif
2488	# else /* HC_ARCH_BITS != 64 */
2489	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2490	pHostMsr->u32Reserved = 0;
2491	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2492	if (CPUMIsGuestInLongMode(pVCpu))
2493	{
2494	/* Must match the EFER value in our 64 bits switcher. */
2495	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2496	}
2497	else
2498	# endif
2499	pHostMsr->u64Value = u64HostEfer;
2500	pHostMsr++; cHostMsrs++;
2501	# endif /* HC_ARCH_BITS == 64 */
2502	}
2503
2504	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2505	if (HMVMX_IS_64BIT_HOST_MODE())
2506	{
2507	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2508	pHostMsr->u32Reserved = 0;
2509	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2510	pHostMsr++; cHostMsrs++;
2511	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2512	pHostMsr->u32Reserved = 0;
2513	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2514	pHostMsr++; cHostMsrs++;
2515	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2516	pHostMsr->u32Reserved = 0;
2517	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2518	pHostMsr++; cHostMsrs++;
2519	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2520	pHostMsr->u32Reserved = 0;
2521	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2522	pHostMsr++; cHostMsrs++;
2523	}
2524	# endif
2525
2526	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2527	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2528	{
2529	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531	}
2532
2533	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2534	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2535
2536	/*
2537	* Host Sysenter MSRs.
2538	*/
2539	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2540	AssertRCReturn(rc, rc);
2541	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2542	if (HMVMX_IS_64BIT_HOST_MODE())
2543	{
2544	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2545	AssertRCReturn(rc, rc);
2546	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2547	}
2548	else
2549	{
2550	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2551	AssertRCReturn(rc, rc);
2552	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2553	}
2554	#elif HC_ARCH_BITS == 32
2555	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2556	AssertRCReturn(rc, rc);
2557	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2558	#else
2559	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2560	AssertRCReturn(rc, rc);
2561	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2562	#endif
2563	AssertRCReturn(rc, rc);
2564
2565	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2566	* hmR0VmxSetupExitCtls() !! */
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2573	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2574	* controls".
2575	*
2576	* @returns VBox status code.
2577	* @param pVCpu Pointer to the VMCPU.
2578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2579	* out-of-sync. Make sure to update the required fields
2580	* before using them.
2581	*
2582	* @remarks No-long-jump zone!!!
2583	*/
2584	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2585	{
2586	int rc = VINF_SUCCESS;
2587	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2588	{
2589	PVM pVM = pVCpu->CTX_SUFF(pVM);
2590	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2591	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2592
2593	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2594	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2595
2596	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2597	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2598	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2599	else
2600	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2601
2602	/*
2603	* The following should not be set (since we're not in SMM mode):
2604	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2605	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2606	*/
2607
2608	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2609	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2610	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2611
2612	if ((val & zap) != val)
2613	{
2614	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2615	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2616	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2617	}
2618
2619	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2620	AssertRCReturn(rc, rc);
2621
2622	/* Update VCPU with the currently set VM-exit controls. */
2623	pVCpu->hm.s.vmx.u32EntryCtls = val;
2624	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2625	}
2626	return rc;
2627	}
2628
2629
2630	/**
2631	* Sets up the VM-exit controls in the VMCS.
2632	*
2633	* @returns VBox status code.
2634	* @param pVM Pointer to the VM.
2635	* @param pVCpu Pointer to the VMCPU.
2636	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2637	* out-of-sync. Make sure to update the required fields
2638	* before using them.
2639	*
2640	* @remarks requires EFER.
2641	*/
2642	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2643	{
2644	int rc = VINF_SUCCESS;
2645	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2646	{
2647	PVM pVM = pVCpu->CTX_SUFF(pVM);
2648	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2649	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2650
2651	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2652	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2653
2654	/*
2655	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2656	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2657	*/
2658	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2659	if (HMVMX_IS_64BIT_HOST_MODE())
2660	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2661	else
2662	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2663	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2664	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2665	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2666	else
2667	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2668	#endif
2669
2670	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2671	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2672
2673	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2674	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2675	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2676	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2677	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2678
2679	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2680	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2681
2682	if ((val & zap) != val)
2683	{
2684	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2685	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2687	}
2688
2689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2690	AssertRCReturn(rc, rc);
2691
2692	/* Update VCPU with the currently set VM-exit controls. */
2693	pVCpu->hm.s.vmx.u32ExitCtls = val;
2694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2695	}
2696	return rc;
2697	}
2698
2699
2700	/**
2701	* Loads the guest APIC and related state.
2702	*
2703	* @returns VBox status code.
2704	* @param pVM Pointer to the VM.
2705	* @param pVCpu Pointer to the VMCPU.
2706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2707	* out-of-sync. Make sure to update the required fields
2708	* before using them.
2709	*/
2710	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2711	{
2712	int rc = VINF_SUCCESS;
2713	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2714	{
2715	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2716	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2717	{
2718	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2719
2720	bool fPendingIntr = false;
2721	uint8_t u8Tpr = 0;
2722	uint8_t u8PendingIntr = 0;
2723	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2724	AssertRCReturn(rc, rc);
2725
2726	/*
2727	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2728	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2729	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2730	* the interrupt when we VM-exit for other reasons.
2731	*/
2732	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2733	uint32_t u32TprThreshold = 0;
2734	if (fPendingIntr)
2735	{
2736	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2737	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2738	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2739	if (u8PendingPriority <= u8TprPriority)
2740	u32TprThreshold = u8PendingPriority;
2741	else
2742	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2743	}
2744	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2745
2746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2747	AssertRCReturn(rc, rc);
2748	}
2749
2750	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2751	}
2752	return rc;
2753	}
2754
2755
2756	/**
2757	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2758	*
2759	* @returns Guest's interruptibility-state.
2760	* @param pVCpu Pointer to the VMCPU.
2761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2762	* out-of-sync. Make sure to update the required fields
2763	* before using them.
2764	*
2765	* @remarks No-long-jump zone!!!
2766	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2767	*/
2768	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2769	{
2770	/*
2771	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2772	* inhibit interrupts or clear any existing interrupt-inhibition.
2773	*/
2774	uint32_t uIntrState = 0;
2775	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2776	{
2777	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2778	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2779	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2780	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2781	{
2782	/*
2783	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2784	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2785	*/
2786	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2787	}
2788	else if (pMixedCtx->eflags.Bits.u1IF)
2789	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2790	else
2791	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2792	}
2793	return uIntrState;
2794	}
2795
2796
2797	/**
2798	* Loads the guest's interruptibility-state into the guest-state area in the
2799	* VMCS.
2800	*
2801	* @returns VBox status code.
2802	* @param pVCpu Pointer to the VMCPU.
2803	* @param uIntrState The interruptibility-state to set.
2804	*/
2805	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2806	{
2807	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2808	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2809	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2810	AssertRCReturn(rc, rc);
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Loads the guest's RIP into the guest-state area in the VMCS.
2817	*
2818	* @returns VBox status code.
2819	* @param pVCpu Pointer to the VMCPU.
2820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2821	* out-of-sync. Make sure to update the required fields
2822	* before using them.
2823	*
2824	* @remarks No-long-jump zone!!!
2825	*/
2826	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2827	{
2828	int rc = VINF_SUCCESS;
2829	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2830	{
2831	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2832	AssertRCReturn(rc, rc);
2833	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2834	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2835	}
2836	return rc;
2837	}
2838
2839
2840	/**
2841	* Loads the guest's RSP into the guest-state area in the VMCS.
2842	*
2843	* @returns VBox status code.
2844	* @param pVCpu Pointer to the VMCPU.
2845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2846	* out-of-sync. Make sure to update the required fields
2847	* before using them.
2848	*
2849	* @remarks No-long-jump zone!!!
2850	*/
2851	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2852	{
2853	int rc = VINF_SUCCESS;
2854	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2855	{
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2857	AssertRCReturn(rc, rc);
2858	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2860	}
2861	return rc;
2862	}
2863
2864
2865	/**
2866	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2867	*
2868	* @returns VBox status code.
2869	* @param pVCpu Pointer to the VMCPU.
2870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2871	* out-of-sync. Make sure to update the required fields
2872	* before using them.
2873	*
2874	* @remarks No-long-jump zone!!!
2875	*/
2876	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2877	{
2878	int rc = VINF_SUCCESS;
2879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2880	{
2881	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2882	Let us assert it as such and use 32-bit VMWRITE. */
2883	Assert(!(pMixedCtx->rflags.u64 >> 32));
2884	X86EFLAGS Eflags = pMixedCtx->eflags;
2885	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2886	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2887
2888	/*
2889	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2890	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2891	*/
2892	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2893	{
2894	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2895	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2896	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2897	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2898	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2899	}
2900
2901	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2902	AssertRCReturn(rc, rc);
2903
2904	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2905	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2906	}
2907	return rc;
2908	}
2909
2910
2911	/**
2912	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2913	*
2914	* @returns VBox status code.
2915	* @param pVCpu Pointer to the VMCPU.
2916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2917	* out-of-sync. Make sure to update the required fields
2918	* before using them.
2919	*
2920	* @remarks No-long-jump zone!!!
2921	*/
2922	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2923	{
2924	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2925	AssertRCReturn(rc, rc);
2926	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2927	AssertRCReturn(rc, rc);
2928	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2929	AssertRCReturn(rc, rc);
2930	return rc;
2931	}
2932
2933
2934	/**
2935	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2936	* in the VMCS.
2937	*
2938	* @returns VBox status code.
2939	* @param pVM Pointer to the VM.
2940	* @param pVCpu Pointer to the VMCPU.
2941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2942	* out-of-sync. Make sure to update the required fields
2943	* before using them.
2944	*
2945	* @remarks No-long-jump zone!!!
2946	*/
2947	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2948	{
2949	int rc = VINF_SUCCESS;
2950	PVM pVM = pVCpu->CTX_SUFF(pVM);
2951
2952	/*
2953	* Guest CR0.
2954	* Guest FPU.
2955	*/
2956	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2957	{
2958	Assert(!(pCtx->cr0 >> 32));
2959	uint32_t u32GuestCR0 = pCtx->cr0;
2960
2961	/* The guest's view (read access) of its CR0 is unblemished. */
2962	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2963	AssertRCReturn(rc, rc);
2964	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2965
2966	/* Setup VT-x's view of the guest CR0. */
2967	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2968	if (pVM->hm.s.fNestedPaging)
2969	{
2970	if (CPUMIsGuestPagingEnabledEx(pCtx))
2971	{
2972	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2973	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2974	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2975	}
2976	else
2977	{
2978	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2980	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2981	}
2982
2983	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2984	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2985	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2986
2987	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2988	AssertRCReturn(rc, rc);
2989	}
2990	else
2991	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2992
2993	/*
2994	* Guest FPU bits.
2995	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2996	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2997	*/
2998	u32GuestCR0 \|= X86_CR0_NE;
2999	bool fInterceptNM = false;
3000	if (CPUMIsGuestFPUStateActive(pVCpu))
3001	{
3002	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3003	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3004	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3005	}
3006	else
3007	{
3008	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3009	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3010	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3011	}
3012
3013	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3014	bool fInterceptMF = false;
3015	if (!(pCtx->cr0 & X86_CR0_NE))
3016	fInterceptMF = true;
3017
3018	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3019	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3020	{
3021	Assert(PDMVmmDevHeapIsEnabled(pVM));
3022	Assert(pVM->hm.s.vmx.pRealModeTSS);
3023	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3024	fInterceptNM = true;
3025	fInterceptMF = true;
3026	}
3027	else
3028	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3029
3030	if (fInterceptNM)
3031	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3032	else
3033	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3034
3035	if (fInterceptMF)
3036	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3037	else
3038	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3039
3040	/* Additional intercepts for debugging, define these yourself explicitly. */
3041	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3042	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3043	\| RT_BIT(X86_XCPT_BP)
3044	\| RT_BIT(X86_XCPT_DB)
3045	\| RT_BIT(X86_XCPT_DE)
3046	\| RT_BIT(X86_XCPT_NM)
3047	\| RT_BIT(X86_XCPT_UD)
3048	\| RT_BIT(X86_XCPT_NP)
3049	\| RT_BIT(X86_XCPT_SS)
3050	\| RT_BIT(X86_XCPT_GP)
3051	\| RT_BIT(X86_XCPT_PF)
3052	\| RT_BIT(X86_XCPT_MF)
3053	;
3054	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3055	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3056	#endif
3057
3058	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3059
3060	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3061	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3062	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3063	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3064	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3065	else
3066	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3067
3068	u32GuestCR0 \|= uSetCR0;
3069	u32GuestCR0 &= uZapCR0;
3070	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3071
3072	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3073	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3074	AssertRCReturn(rc, rc);
3075	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3076	AssertRCReturn(rc, rc);
3077	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3078
3079	/*
3080	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3081	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3082	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3083	*/
3084	uint32_t u32CR0Mask = 0;
3085	u32CR0Mask = X86_CR0_PE
3086	\| X86_CR0_NE
3087	\| X86_CR0_WP
3088	\| X86_CR0_PG
3089	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3090	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3091	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3092	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3093	u32CR0Mask &= ~X86_CR0_PE;
3094	if (pVM->hm.s.fNestedPaging)
3095	u32CR0Mask &= ~X86_CR0_WP;
3096
3097	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3098	if (fInterceptNM)
3099	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3100	else
3101	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3102
3103	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3104	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3105	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3106	AssertRCReturn(rc, rc);
3107
3108	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3109	}
3110
3111	/*
3112	* Guest CR2.
3113	* It's always loaded in the assembler code. Nothing to do here.
3114	*/
3115
3116	/*
3117	* Guest CR3.
3118	*/
3119	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3120	{
3121	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3122	if (pVM->hm.s.fNestedPaging)
3123	{
3124	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3125
3126	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3127	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3128	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3129	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3130
3131	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3132	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3133	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3134
3135	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3136	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3137	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3138	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3139
3140	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3141	AssertRCReturn(rc, rc);
3142	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3143
3144	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3145	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3146	{
3147	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3148	if (CPUMIsGuestInPAEModeEx(pCtx))
3149	{
3150	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3151	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3152	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3153	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3154	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3155	}
3156
3157	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3158	have Unrestricted Execution to handle the guest when it's not using paging. */
3159	GCPhysGuestCR3 = pCtx->cr3;
3160	}
3161	else
3162	{
3163	/*
3164	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3165	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3166	* EPT takes care of translating it to host-physical addresses.
3167	*/
3168	RTGCPHYS GCPhys;
3169	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3170	Assert(PDMVmmDevHeapIsEnabled(pVM));
3171
3172	/* We obtain it here every time as the guest could have relocated this PCI region. */
3173	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3174	AssertRCReturn(rc, rc);
3175
3176	GCPhysGuestCR3 = GCPhys;
3177	}
3178
3179	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3180	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3181	}
3182	else
3183	{
3184	/* Non-nested paging case, just use the hypervisor's CR3. */
3185	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3186
3187	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3188	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3189	}
3190	AssertRCReturn(rc, rc);
3191
3192	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3193	}
3194
3195	/*
3196	* Guest CR4.
3197	*/
3198	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3199	{
3200	Assert(!(pCtx->cr4 >> 32));
3201	uint32_t u32GuestCR4 = pCtx->cr4;
3202
3203	/* The guest's view of its CR4 is unblemished. */
3204	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3205	AssertRCReturn(rc, rc);
3206	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3207
3208	/* Setup VT-x's view of the guest CR4. */
3209	/*
3210	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3211	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3212	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3213	*/
3214	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3215	{
3216	Assert(pVM->hm.s.vmx.pRealModeTSS);
3217	Assert(PDMVmmDevHeapIsEnabled(pVM));
3218	u32GuestCR4 &= ~X86_CR4_VME;
3219	}
3220
3221	if (pVM->hm.s.fNestedPaging)
3222	{
3223	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3224	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3225	{
3226	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3227	u32GuestCR4 \|= X86_CR4_PSE;
3228	/* Our identity mapping is a 32 bits page directory. */
3229	u32GuestCR4 &= ~X86_CR4_PAE;
3230	}
3231	/* else use guest CR4.*/
3232	}
3233	else
3234	{
3235	/*
3236	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3237	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3238	*/
3239	switch (pVCpu->hm.s.enmShadowMode)
3240	{
3241	case PGMMODE_REAL: /* Real-mode. */
3242	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3243	case PGMMODE_32_BIT: /* 32-bit paging. */
3244	{
3245	u32GuestCR4 &= ~X86_CR4_PAE;
3246	break;
3247	}
3248
3249	case PGMMODE_PAE: /* PAE paging. */
3250	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3251	{
3252	u32GuestCR4 \|= X86_CR4_PAE;
3253	break;
3254	}
3255
3256	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3257	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3258	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3259	break;
3260	#endif
3261	default:
3262	AssertFailed();
3263	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3264	}
3265	}
3266
3267	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3268	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3269	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3270	u32GuestCR4 \|= uSetCR4;
3271	u32GuestCR4 &= uZapCR4;
3272
3273	/* Write VT-x's view of the guest CR4 into the VMCS. */
3274	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3275	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3276	AssertRCReturn(rc, rc);
3277
3278	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3279	uint32_t u32CR4Mask = 0;
3280	u32CR4Mask = X86_CR4_VME
3281	\| X86_CR4_PAE
3282	\| X86_CR4_PGE
3283	\| X86_CR4_PSE
3284	\| X86_CR4_VMXE;
3285	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3286	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3287	AssertRCReturn(rc, rc);
3288
3289	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3290	}
3291	return rc;
3292	}
3293
3294
3295	/**
3296	* Loads the guest debug registers into the guest-state area in the VMCS.
3297	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3302	* out-of-sync. Make sure to update the required fields
3303	* before using them.
3304	*
3305	* @remarks No-long-jump zone!!!
3306	*/
3307	static int hmR0VmxLoadGuestDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3308	{
3309	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3310	return VINF_SUCCESS;
3311
3312	#ifdef VBOX_STRICT
3313	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3314	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3315	{
3316	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3317	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3318	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3319	}
3320	#endif
3321
3322	int rc;
3323	PVM pVM = pVCpu->CTX_SUFF(pVM);
3324	bool fInterceptDB = false;
3325	bool fInterceptMovDRx = false;
3326	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3327	{
3328	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3329	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3330	{
3331	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3333	AssertRCReturn(rc, rc);
3334	Assert(fInterceptDB == false);
3335	}
3336	else
3337	{
3338	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3339	pVCpu->hm.s.fClearTrapFlag = true;
3340	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3341	fInterceptDB = true;
3342	}
3343	}
3344
3345	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3346	{
3347	/*
3348	* Use the combined guest and host DRx values found in the hypervisor
3349	* register set because the debugger has breakpoints active or someone
3350	* is single stepping on the host side without a monitor trap flag.
3351	*
3352	* Note! DBGF expects a clean DR6 state before executing guest code.
3353	*/
3354	if (!CPUMIsHyperDebugStateActive(pVCpu))
3355	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3356	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3357	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3358
3359	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3360	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3361	AssertRCReturn(rc, rc);
3362
3363	fInterceptDB = true;
3364	fInterceptMovDRx = true;
3365	}
3366	else
3367	{
3368	/*
3369	* If the guest has enabled debug registers, we need to load them prior to
3370	* executing guest code so they'll trigger at the right time.
3371	*/
3372	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3373	{
3374	if (!CPUMIsGuestDebugStateActive(pVCpu))
3375	{
3376	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3377	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3378	}
3379	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3380	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3381	}
3382	/*
3383	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3384	* must intercept #DB in order to maintain a correct DR6 guest value.
3385	*/
3386	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3387	{
3388	fInterceptMovDRx = true;
3389	fInterceptDB = true;
3390	}
3391
3392	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3393	AssertRCReturn(rc, rc);
3394	}
3395
3396	/*
3397	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3398	*/
3399	if (fInterceptDB)
3400	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3401	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3402	{
3403	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3404	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3405	#endif
3406	}
3407	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3408	AssertRCReturn(rc, rc);
3409
3410	/*
3411	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3412	*/
3413	if (fInterceptMovDRx)
3414	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3415	else
3416	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3418	AssertRCReturn(rc, rc);
3419
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3421	return VINF_SUCCESS;
3422	}
3423
3424
3425	#ifdef VBOX_STRICT
3426	/**
3427	* Strict function to validate segment registers.
3428	*
3429	* @remarks Requires CR0.
3430	*/
3431	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3432	{
3433	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3434	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3435	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3436	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3437	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3438	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3439	{
3440	/* Protected mode checks */
3441	/* CS */
3442	Assert(pCtx->cs.Attr.n.u1Present);
3443	Assert(!(pCtx->cs.Attr.u & 0xf00));
3444	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3445	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3446	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3447	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3448	\|\| (pCtx->cs.Attr.n.u1Granularity));
3449	/* CS cannot be loaded with NULL in protected mode. */
3450	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3451	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3452	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3453	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3454	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3455	else
3456	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3457	/* SS */
3458	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3459	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3460	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3461	if ( !(pCtx->cr0 & X86_CR0_PE)
3462	\|\| pCtx->cs.Attr.n.u4Type == 3)
3463	{
3464	Assert(!pCtx->ss.Attr.n.u2Dpl);
3465	}
3466	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3467	{
3468	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3469	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3470	Assert(pCtx->ss.Attr.n.u1Present);
3471	Assert(!(pCtx->ss.Attr.u & 0xf00));
3472	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3473	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3474	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3475	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3476	\|\| (pCtx->ss.Attr.n.u1Granularity));
3477	}
3478	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3479	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3480	{
3481	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3482	Assert(pCtx->ds.Attr.n.u1Present);
3483	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3484	Assert(!(pCtx->ds.Attr.u & 0xf00));
3485	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3486	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3487	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3488	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3489	\|\| (pCtx->ds.Attr.n.u1Granularity));
3490	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3491	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3492	}
3493	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3494	{
3495	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3496	Assert(pCtx->es.Attr.n.u1Present);
3497	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3498	Assert(!(pCtx->es.Attr.u & 0xf00));
3499	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3500	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3501	\|\| !(pCtx->es.Attr.n.u1Granularity));
3502	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3503	\|\| (pCtx->es.Attr.n.u1Granularity));
3504	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3505	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3506	}
3507	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3508	{
3509	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3510	Assert(pCtx->fs.Attr.n.u1Present);
3511	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3512	Assert(!(pCtx->fs.Attr.u & 0xf00));
3513	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3514	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3515	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3516	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3517	\|\| (pCtx->fs.Attr.n.u1Granularity));
3518	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3519	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3520	}
3521	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3522	{
3523	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3524	Assert(pCtx->gs.Attr.n.u1Present);
3525	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3526	Assert(!(pCtx->gs.Attr.u & 0xf00));
3527	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3528	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3529	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3530	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3531	\|\| (pCtx->gs.Attr.n.u1Granularity));
3532	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3533	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3534	}
3535	/* 64-bit capable CPUs. */
3536	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3537	Assert(!(pCtx->cs.u64Base >> 32));
3538	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3539	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3540	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3541	# endif
3542	}
3543	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3544	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3545	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3546	{
3547	/* Real and v86 mode checks. */
3548	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3549	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3550	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3551	{
3552	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3553	}
3554	else
3555	{
3556	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3557	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3558	}
3559
3560	/* CS */
3561	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3562	Assert(pCtx->cs.u32Limit == 0xffff);
3563	Assert(u32CSAttr == 0xf3);
3564	/* SS */
3565	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3566	Assert(pCtx->ss.u32Limit == 0xffff);
3567	Assert(u32SSAttr == 0xf3);
3568	/* DS */
3569	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3570	Assert(pCtx->ds.u32Limit == 0xffff);
3571	Assert(u32DSAttr == 0xf3);
3572	/* ES */
3573	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3574	Assert(pCtx->es.u32Limit == 0xffff);
3575	Assert(u32ESAttr == 0xf3);
3576	/* FS */
3577	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3578	Assert(pCtx->fs.u32Limit == 0xffff);
3579	Assert(u32FSAttr == 0xf3);
3580	/* GS */
3581	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3582	Assert(pCtx->gs.u32Limit == 0xffff);
3583	Assert(u32GSAttr == 0xf3);
3584	/* 64-bit capable CPUs. */
3585	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3586	Assert(!(pCtx->cs.u64Base >> 32));
3587	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3588	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3589	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3590	# endif
3591	}
3592	}
3593	#endif /* VBOX_STRICT */
3594
3595
3596	/**
3597	* Writes a guest segment register into the guest-state area in the VMCS.
3598	*
3599	* @returns VBox status code.
3600	* @param pVCpu Pointer to the VMCPU.
3601	* @param idxSel Index of the selector in the VMCS.
3602	* @param idxLimit Index of the segment limit in the VMCS.
3603	* @param idxBase Index of the segment base in the VMCS.
3604	* @param idxAccess Index of the access rights of the segment in the VMCS.
3605	* @param pSelReg Pointer to the segment selector.
3606	* @param pCtx Pointer to the guest-CPU context.
3607	*
3608	* @remarks No-long-jump zone!!!
3609	*/
3610	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3611	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3612	{
3613	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3614	AssertRCReturn(rc, rc);
3615	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3616	AssertRCReturn(rc, rc);
3617	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3618	AssertRCReturn(rc, rc);
3619
3620	uint32_t u32Access = pSelReg->Attr.u;
3621	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3622	{
3623	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3624	u32Access = 0xf3;
3625	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3626	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3627	}
3628	else
3629	{
3630	/*
3631	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3632	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3633	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3634	* loaded in protected-mode have their attribute as 0.
3635	*/
3636	if (!u32Access)
3637	u32Access = X86DESCATTR_UNUSABLE;
3638	}
3639
3640	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3641	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3642	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3643
3644	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3645	AssertRCReturn(rc, rc);
3646	return rc;
3647	}
3648
3649
3650	/**
3651	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3652	* into the guest-state area in the VMCS.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCPU Pointer to the VMCPU.
3657	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658	* out-of-sync. Make sure to update the required fields
3659	* before using them.
3660	*
3661	* @remarks Requires CR0 (strict builds validation).
3662	* @remarks No-long-jump zone!!!
3663	*/
3664	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665	{
3666	int rc = VERR_INTERNAL_ERROR_5;
3667	PVM pVM = pVCpu->CTX_SUFF(pVM);
3668
3669	/*
3670	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3671	*/
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3673	{
3674	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3675	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3676	{
3677	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3678	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3679	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3680	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3681	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3682	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3683	}
3684
3685	#ifdef VBOX_WITH_REM
3686	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3687	{
3688	Assert(pVM->hm.s.vmx.pRealModeTSS);
3689	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3690	if ( pVCpu->hm.s.vmx.fWasInRealMode
3691	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3692	{
3693	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3694	in real-mode (e.g. OpenBSD 4.0) */
3695	REMFlushTBs(pVM);
3696	Log4(("Load: Switch to protected mode detected!\n"));
3697	pVCpu->hm.s.vmx.fWasInRealMode = false;
3698	}
3699	}
3700	#endif
3701	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3702	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3703	AssertRCReturn(rc, rc);
3704	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3705	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3706	AssertRCReturn(rc, rc);
3707	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3708	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3709	AssertRCReturn(rc, rc);
3710	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3711	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3712	AssertRCReturn(rc, rc);
3713	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3714	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3715	AssertRCReturn(rc, rc);
3716	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3717	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3718	AssertRCReturn(rc, rc);
3719
3720	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3721	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3722	#ifdef VBOX_STRICT
3723	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3724	#endif
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3726	}
3727
3728	/*
3729	* Guest TR.
3730	*/
3731	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3732	{
3733	/*
3734	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3735	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3736	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3737	*/
3738	uint16_t u16Sel = 0;
3739	uint32_t u32Limit = 0;
3740	uint64_t u64Base = 0;
3741	uint32_t u32AccessRights = 0;
3742
3743	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3744	{
3745	u16Sel = pMixedCtx->tr.Sel;
3746	u32Limit = pMixedCtx->tr.u32Limit;
3747	u64Base = pMixedCtx->tr.u64Base;
3748	u32AccessRights = pMixedCtx->tr.Attr.u;
3749	}
3750	else
3751	{
3752	Assert(pVM->hm.s.vmx.pRealModeTSS);
3753	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3754
3755	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3756	RTGCPHYS GCPhys;
3757	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3758	AssertRCReturn(rc, rc);
3759
3760	X86DESCATTR DescAttr;
3761	DescAttr.u = 0;
3762	DescAttr.n.u1Present = 1;
3763	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3764
3765	u16Sel = 0;
3766	u32Limit = HM_VTX_TSS_SIZE;
3767	u64Base = GCPhys; /* in real-mode phys = virt. */
3768	u32AccessRights = DescAttr.u;
3769	}
3770
3771	/* Validate. */
3772	Assert(!(u16Sel & RT_BIT(2)));
3773	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3774	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3775	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3776	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3777	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3778	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3779	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3780	Assert( (u32Limit & 0xfff) == 0xfff
3781	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3782	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3783	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3784
3785	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3786	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3787	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3788	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3789
3790	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3791	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3792	}
3793
3794	/*
3795	* Guest GDTR.
3796	*/
3797	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3798	{
3799	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3800	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3801
3802	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3803	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3804	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3805	}
3806
3807	/*
3808	* Guest LDTR.
3809	*/
3810	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3811	{
3812	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3813	uint32_t u32Access = 0;
3814	if (!pMixedCtx->ldtr.Attr.u)
3815	u32Access = X86DESCATTR_UNUSABLE;
3816	else
3817	u32Access = pMixedCtx->ldtr.Attr.u;
3818
3819	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3820	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3821	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3822	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3823
3824	/* Validate. */
3825	if (!(u32Access & X86DESCATTR_UNUSABLE))
3826	{
3827	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3828	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3829	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3830	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3831	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3832	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3833	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3834	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3835	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3836	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3837	}
3838
3839	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3840	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3841	}
3842
3843	/*
3844	* Guest IDTR.
3845	*/
3846	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3847	{
3848	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3849	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3850
3851	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3852	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3853	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3854	}
3855
3856	return VINF_SUCCESS;
3857	}
3858
3859
3860	/**
3861	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3862	* areas. These MSRs will automatically be loaded to the host CPU on every
3863	* successful VM entry and stored from the host CPU on every successful VM exit.
3864	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu Pointer to the VMCPU.
3868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3869	* out-of-sync. Make sure to update the required fields
3870	* before using them.
3871	*
3872	* @remarks No-long-jump zone!!!
3873	*/
3874	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3875	{
3876	AssertPtr(pVCpu);
3877	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3878
3879	/*
3880	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3881	*/
3882	int rc = VINF_SUCCESS;
3883	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3884	{
3885	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3886	PVM pVM = pVCpu->CTX_SUFF(pVM);
3887	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3888	uint32_t cGuestMsrs = 0;
3889
3890	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3891	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3892	* when the guest really is in 64-bit mode. */
3893	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3894	if (fSupportsLongMode)
3895	{
3896	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3897	pGuestMsr->u32Reserved = 0;
3898	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3899	pGuestMsr++; cGuestMsrs++;
3900	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3901	pGuestMsr->u32Reserved = 0;
3902	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3903	pGuestMsr++; cGuestMsrs++;
3904	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3905	pGuestMsr->u32Reserved = 0;
3906	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3907	pGuestMsr++; cGuestMsrs++;
3908	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3909	pGuestMsr->u32Reserved = 0;
3910	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3911	pGuestMsr++; cGuestMsrs++;
3912	}
3913
3914	/*
3915	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3916	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3917	*/
3918	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3919	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3920	{
3921	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3922	pGuestMsr->u32Reserved = 0;
3923	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3924	AssertRCReturn(rc, rc);
3925	pGuestMsr++; cGuestMsrs++;
3926	}
3927
3928	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3929	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3930	{
3931	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3932	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3933	}
3934
3935	/* Update the VCPU's copy of the guest MSR count. */
3936	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3937	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3938	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3939	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3940
3941	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3942	}
3943
3944	/*
3945	* Guest Sysenter MSRs.
3946	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3947	* VM-exits on WRMSRs for these MSRs.
3948	*/
3949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3950	{
3951	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3952	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3953	}
3954	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3955	{
3956	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3957	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3958	}
3959	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3960	{
3961	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3962	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3963	}
3964
3965	return rc;
3966	}
3967
3968
3969	/**
3970	* Loads the guest activity state into the guest-state area in the VMCS.
3971	*
3972	* @returns VBox status code.
3973	* @param pVCpu Pointer to the VMCPU.
3974	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3975	* out-of-sync. Make sure to update the required fields
3976	* before using them.
3977	*
3978	* @remarks No-long-jump zone!!!
3979	*/
3980	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3981	{
3982	/** @todo See if we can make use of other states, e.g.
3983	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3984	int rc = VINF_SUCCESS;
3985	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3986	{
3987	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3988	AssertRCReturn(rc, rc);
3989	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3990	}
3991	return rc;
3992	}
3993
3994
3995	/**
3996	* Sets up the appropriate function to run guest code.
3997	*
3998	* @returns VBox status code.
3999	* @param pVCpu Pointer to the VMCPU.
4000	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4001	* out-of-sync. Make sure to update the required fields
4002	* before using them.
4003	*
4004	* @remarks No-long-jump zone!!!
4005	*/
4006	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4007	{
4008	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4009	{
4010	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4011	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4012	#endif
4013	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4014	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4015	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4016	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4017	#else
4018	/* 64-bit host or hybrid host. */
4019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4020	#endif
4021	}
4022	else
4023	{
4024	/* Guest is not in long mode, use the 32-bit handler. */
4025	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4026	}
4027	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4028	return VINF_SUCCESS;
4029	}
4030
4031
4032	/**
4033	* Wrapper for running the guest code in VT-x.
4034	*
4035	* @returns VBox strict status code.
4036	* @param pVM Pointer to the VM.
4037	* @param pVCpu Pointer to the VMCPU.
4038	* @param pCtx Pointer to the guest-CPU context.
4039	*
4040	* @remarks No-long-jump zone!!!
4041	*/
4042	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4043	{
4044	/*
4045	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4046	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4047	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4048	*/
4049	#ifdef VBOX_WITH_KERNEL_USING_XMM
4050	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4051	#else
4052	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4053	#endif
4054	}
4055
4056
4057	/**
4058	* Reports world-switch error and dumps some useful debug info.
4059	*
4060	* @param pVM Pointer to the VM.
4061	* @param pVCpu Pointer to the VMCPU.
4062	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4063	* @param pCtx Pointer to the guest-CPU context.
4064	* @param pVmxTransient Pointer to the VMX transient structure (only
4065	* exitReason updated).
4066	*/
4067	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4068	{
4069	Assert(pVM);
4070	Assert(pVCpu);
4071	Assert(pCtx);
4072	Assert(pVmxTransient);
4073	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4074
4075	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4076	switch (rcVMRun)
4077	{
4078	case VERR_VMX_INVALID_VMXON_PTR:
4079	AssertFailed();
4080	break;
4081	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4082	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4083	{
4084	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4085	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4086	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4087	AssertRC(rc);
4088
4089	#ifdef VBOX_STRICT
4090	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4091	pVmxTransient->uExitReason));
4092	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4093	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4094	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4095	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4096	else
4097	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4098
4099	/* VMX control bits. */
4100	uint32_t u32Val;
4101	uint64_t u64Val;
4102	HMVMXHCUINTREG uHCReg;
4103	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4104	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4105	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4106	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4107	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4108	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4110	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4112	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4114	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4116	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4118	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4119	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4120	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4121	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4122	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4123	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4124	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4125	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4126	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4127	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4128	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4129	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4130	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4131	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4132	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4133	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4134	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4135	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4136	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4137	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4138	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4139	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4140	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4141	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4142	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4143	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4144	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4145
4146	/* Guest bits. */
4147	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4148	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4149	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4150	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4151	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4152	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4153	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4154	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4155
4156	/* Host bits. */
4157	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4158	Log4(("Host CR0 %#RHr\n", uHCReg));
4159	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4160	Log4(("Host CR3 %#RHr\n", uHCReg));
4161	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4162	Log4(("Host CR4 %#RHr\n", uHCReg));
4163
4164	RTGDTR HostGdtr;
4165	PCX86DESCHC pDesc;
4166	ASMGetGDTR(&HostGdtr);
4167	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4168	Log4(("Host CS %#08x\n", u32Val));
4169	if (u32Val < HostGdtr.cbGdt)
4170	{
4171	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4172	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4173	}
4174
4175	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4176	Log4(("Host DS %#08x\n", u32Val));
4177	if (u32Val < HostGdtr.cbGdt)
4178	{
4179	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4180	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4181	}
4182
4183	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4184	Log4(("Host ES %#08x\n", u32Val));
4185	if (u32Val < HostGdtr.cbGdt)
4186	{
4187	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4188	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4189	}
4190
4191	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4192	Log4(("Host FS %#08x\n", u32Val));
4193	if (u32Val < HostGdtr.cbGdt)
4194	{
4195	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4196	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4197	}
4198
4199	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4200	Log4(("Host GS %#08x\n", u32Val));
4201	if (u32Val < HostGdtr.cbGdt)
4202	{
4203	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4204	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4205	}
4206
4207	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4208	Log4(("Host SS %#08x\n", u32Val));
4209	if (u32Val < HostGdtr.cbGdt)
4210	{
4211	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4212	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4213	}
4214
4215	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4216	Log4(("Host TR %#08x\n", u32Val));
4217	if (u32Val < HostGdtr.cbGdt)
4218	{
4219	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4220	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4221	}
4222
4223	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4224	Log4(("Host TR Base %#RHv\n", uHCReg));
4225	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4226	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4227	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4228	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4229	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4230	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4231	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4232	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4233	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4234	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4235	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4236	Log4(("Host RSP %#RHv\n", uHCReg));
4237	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4238	Log4(("Host RIP %#RHv\n", uHCReg));
4239	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4240	if (HMVMX_IS_64BIT_HOST_MODE())
4241	{
4242	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4243	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4244	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4245	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4246	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4247	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4248	}
4249	# endif
4250	#endif /* VBOX_STRICT */
4251	break;
4252	}
4253
4254	default:
4255	/* Impossible */
4256	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4257	break;
4258	}
4259	NOREF(pVM);
4260	}
4261
4262
4263	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4264	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4265	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4266	#endif
4267	#ifdef VBOX_STRICT
4268	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4269	{
4270	switch (idxField)
4271	{
4272	case VMX_VMCS_GUEST_RIP:
4273	case VMX_VMCS_GUEST_RSP:
4274	case VMX_VMCS_GUEST_SYSENTER_EIP:
4275	case VMX_VMCS_GUEST_SYSENTER_ESP:
4276	case VMX_VMCS_GUEST_GDTR_BASE:
4277	case VMX_VMCS_GUEST_IDTR_BASE:
4278	case VMX_VMCS_GUEST_CS_BASE:
4279	case VMX_VMCS_GUEST_DS_BASE:
4280	case VMX_VMCS_GUEST_ES_BASE:
4281	case VMX_VMCS_GUEST_FS_BASE:
4282	case VMX_VMCS_GUEST_GS_BASE:
4283	case VMX_VMCS_GUEST_SS_BASE:
4284	case VMX_VMCS_GUEST_LDTR_BASE:
4285	case VMX_VMCS_GUEST_TR_BASE:
4286	case VMX_VMCS_GUEST_CR3:
4287	return true;
4288	}
4289	return false;
4290	}
4291
4292	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4293	{
4294	switch (idxField)
4295	{
4296	/* Read-only fields. */
4297	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4298	return true;
4299	}
4300	/* Remaining readable fields should also be writable. */
4301	return hmR0VmxIsValidWriteField(idxField);
4302	}
4303	#endif /* VBOX_STRICT */
4304
4305
4306	/**
4307	* Executes the specified handler in 64-bit mode.
4308	*
4309	* @returns VBox status code.
4310	* @param pVM Pointer to the VM.
4311	* @param pVCpu Pointer to the VMCPU.
4312	* @param pCtx Pointer to the guest CPU context.
4313	* @param enmOp The operation to perform.
4314	* @param cbParam Number of parameters.
4315	* @param paParam Array of 32-bit parameters.
4316	*/
4317	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4318	uint32_t *paParam)
4319	{
4320	int rc, rc2;
4321	PHMGLOBALCPUINFO pCpu;
4322	RTHCPHYS HCPhysCpuPage;
4323	RTCCUINTREG uOldEflags;
4324
4325	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4326	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4327	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4328	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4329
4330	#ifdef VBOX_STRICT
4331	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4332	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4333
4334	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4335	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4336	#endif
4337
4338	/* Disable interrupts. */
4339	uOldEflags = ASMIntDisableFlags();
4340
4341	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4342	RTCPUID idHostCpu = RTMpCpuId();
4343	CPUMR0SetLApic(pVM, idHostCpu);
4344	#endif
4345
4346	pCpu = HMR0GetCurrentCpu();
4347	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4348
4349	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4350	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4351
4352	/* Leave VMX Root Mode. */
4353	VMXDisable();
4354
4355	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4356
4357	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4358	CPUMSetHyperEIP(pVCpu, enmOp);
4359	for (int i = (int)cbParam - 1; i >= 0; i--)
4360	CPUMPushHyper(pVCpu, paParam[i]);
4361
4362	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4363
4364	/* Call the switcher. */
4365	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4366	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4367
4368	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4369	/* Make sure the VMX instructions don't cause #UD faults. */
4370	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4371
4372	/* Re-enter VMX Root Mode */
4373	rc2 = VMXEnable(HCPhysCpuPage);
4374	if (RT_FAILURE(rc2))
4375	{
4376	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4377	ASMSetFlags(uOldEflags);
4378	return rc2;
4379	}
4380
4381	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4382	AssertRC(rc2);
4383	Assert(!(ASMGetFlags() & X86_EFL_IF));
4384	ASMSetFlags(uOldEflags);
4385	return rc;
4386	}
4387
4388
4389	/**
4390	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4391	* supporting 64-bit guests.
4392	*
4393	* @returns VBox status code.
4394	* @param fResume Whether to VMLAUNCH or VMRESUME.
4395	* @param pCtx Pointer to the guest-CPU context.
4396	* @param pCache Pointer to the VMCS cache.
4397	* @param pVM Pointer to the VM.
4398	* @param pVCpu Pointer to the VMCPU.
4399	*/
4400	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4401	{
4402	uint32_t aParam[6];
4403	PHMGLOBALCPUINFO pCpu = NULL;
4404	RTHCPHYS HCPhysCpuPage = 0;
4405	int rc = VERR_INTERNAL_ERROR_5;
4406
4407	pCpu = HMR0GetCurrentCpu();
4408	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4409
4410	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4411	pCache->uPos = 1;
4412	pCache->interPD = PGMGetInterPaeCR3(pVM);
4413	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4414	#endif
4415
4416	#ifdef VBOX_STRICT
4417	pCache->TestIn.HCPhysCpuPage = 0;
4418	pCache->TestIn.HCPhysVmcs = 0;
4419	pCache->TestIn.pCache = 0;
4420	pCache->TestOut.HCPhysVmcs = 0;
4421	pCache->TestOut.pCache = 0;
4422	pCache->TestOut.pCtx = 0;
4423	pCache->TestOut.eflags = 0;
4424	#endif
4425
4426	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4427	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4428	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4429	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4430	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4431	aParam[5] = 0;
4432
4433	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4434	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4435	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4436	#endif
4437	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4438
4439	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4440	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4441	Assert(pCtx->dr[4] == 10);
4442	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4443	#endif
4444
4445	#ifdef VBOX_STRICT
4446	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4447	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4448	pVCpu->hm.s.vmx.HCPhysVmcs));
4449	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4450	pCache->TestOut.HCPhysVmcs));
4451	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4452	pCache->TestOut.pCache));
4453	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4454	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4455	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4456	pCache->TestOut.pCtx));
4457	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4458	#endif
4459	return rc;
4460	}
4461
4462
4463	/**
4464	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4465	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4466	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4467	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4468	*
4469	* @returns VBox status code.
4470	* @param pVM Pointer to the VM.
4471	* @param pVCpu Pointer to the VMCPU.
4472	*/
4473	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4474	{
4475	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4476	{ \
4477	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4478	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4479	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4480	++cReadFields; \
4481	}
4482
4483	AssertPtr(pVM);
4484	AssertPtr(pVCpu);
4485	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4486	uint32_t cReadFields = 0;
4487
4488	/*
4489	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4490	* and serve to indicate exceptions to the rules.
4491	*/
4492
4493	/* Guest-natural selector base fields. */
4494	#if 0
4495	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4496	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4497	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4498	#endif
4499	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4500	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4501	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4502	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4503	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4504	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4505	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4506	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4507	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4508	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4509	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4510	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4511	#if 0
4512	/* Unused natural width guest-state fields. */
4513	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4515	#endif
4516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4518
4519	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4520	#if 0
4521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4530	#endif
4531
4532	/* Natural width guest-state fields. */
4533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4534	#if 0
4535	/* Currently unused field. */
4536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4537	#endif
4538
4539	if (pVM->hm.s.fNestedPaging)
4540	{
4541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4542	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4543	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4544	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4545	}
4546	else
4547	{
4548	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4549	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4550	}
4551
4552	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4553	return VINF_SUCCESS;
4554	}
4555
4556
4557	/**
4558	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4559	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4560	* darwin, running 64-bit guests).
4561	*
4562	* @returns VBox status code.
4563	* @param pVCpu Pointer to the VMCPU.
4564	* @param idxField The VMCS field encoding.
4565	* @param u64Val 16, 32 or 64 bits value.
4566	*/
4567	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4568	{
4569	int rc;
4570	switch (idxField)
4571	{
4572	/*
4573	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4574	*/
4575	/* 64-bit Control fields. */
4576	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4577	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4578	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4579	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4580	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4581	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4582	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4583	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4584	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4585	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4586	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4587	case VMX_VMCS64_CTRL_EPTP_FULL:
4588	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4589	/* 64-bit Guest-state fields. */
4590	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4591	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4592	case VMX_VMCS64_GUEST_PAT_FULL:
4593	case VMX_VMCS64_GUEST_EFER_FULL:
4594	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4595	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4596	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4597	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4598	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4599	/* 64-bit Host-state fields. */
4600	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4601	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4602	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4603	{
4604	rc = VMXWriteVmcs32(idxField, u64Val);
4605	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4606	break;
4607	}
4608
4609	/*
4610	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4611	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4612	*/
4613	/* Natural-width Guest-state fields. */
4614	case VMX_VMCS_GUEST_CR3:
4615	case VMX_VMCS_GUEST_ES_BASE:
4616	case VMX_VMCS_GUEST_CS_BASE:
4617	case VMX_VMCS_GUEST_SS_BASE:
4618	case VMX_VMCS_GUEST_DS_BASE:
4619	case VMX_VMCS_GUEST_FS_BASE:
4620	case VMX_VMCS_GUEST_GS_BASE:
4621	case VMX_VMCS_GUEST_LDTR_BASE:
4622	case VMX_VMCS_GUEST_TR_BASE:
4623	case VMX_VMCS_GUEST_GDTR_BASE:
4624	case VMX_VMCS_GUEST_IDTR_BASE:
4625	case VMX_VMCS_GUEST_RSP:
4626	case VMX_VMCS_GUEST_RIP:
4627	case VMX_VMCS_GUEST_SYSENTER_ESP:
4628	case VMX_VMCS_GUEST_SYSENTER_EIP:
4629	{
4630	if (!(u64Val >> 32))
4631	{
4632	/* If this field is 64-bit, VT-x will zero out the top bits. */
4633	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4634	}
4635	else
4636	{
4637	/* Assert that only the 32->64 switcher case should ever come here. */
4638	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4639	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4640	}
4641	break;
4642	}
4643
4644	default:
4645	{
4646	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4647	rc = VERR_INVALID_PARAMETER;
4648	break;
4649	}
4650	}
4651	AssertRCReturn(rc, rc);
4652	return rc;
4653	}
4654
4655
4656	/**
4657	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4658	* hosts (except darwin) for 64-bit guests.
4659	*
4660	* @param pVCpu Pointer to the VMCPU.
4661	* @param idxField The VMCS field encoding.
4662	* @param u64Val 16, 32 or 64 bits value.
4663	*/
4664	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4665	{
4666	AssertPtr(pVCpu);
4667	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4668
4669	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4670	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4671
4672	/* Make sure there are no duplicates. */
4673	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4674	{
4675	if (pCache->Write.aField[i] == idxField)
4676	{
4677	pCache->Write.aFieldVal[i] = u64Val;
4678	return VINF_SUCCESS;
4679	}
4680	}
4681
4682	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4683	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4684	pCache->Write.cValidEntries++;
4685	return VINF_SUCCESS;
4686	}
4687
4688	/* Enable later when the assembly code uses these as callbacks. */
4689	#if 0
4690	/*
4691	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4692	*
4693	* @param pVCpu Pointer to the VMCPU.
4694	* @param pCache Pointer to the VMCS cache.
4695	*
4696	* @remarks No-long-jump zone!!!
4697	*/
4698	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4699	{
4700	AssertPtr(pCache);
4701	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4702	{
4703	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4704	AssertRC(rc);
4705	}
4706	pCache->Write.cValidEntries = 0;
4707	}
4708
4709
4710	/**
4711	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4712	*
4713	* @param pVCpu Pointer to the VMCPU.
4714	* @param pCache Pointer to the VMCS cache.
4715	*
4716	* @remarks No-long-jump zone!!!
4717	*/
4718	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4719	{
4720	AssertPtr(pCache);
4721	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4722	{
4723	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4724	AssertRC(rc);
4725	}
4726	}
4727	#endif
4728	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4729
4730
4731	/**
4732	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4733	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4734	* timer.
4735	*
4736	* @returns VBox status code.
4737	* @param pVCpu Pointer to the VMCPU.
4738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4739	* out-of-sync. Make sure to update the required fields
4740	* before using them.
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744	{
4745	int rc = VERR_INTERNAL_ERROR_5;
4746	bool fOffsettedTsc = false;
4747	PVM pVM = pVCpu->CTX_SUFF(pVM);
4748	if (pVM->hm.s.vmx.fUsePreemptTimer)
4749	{
4750	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4751
4752	/* Make sure the returned values have sane upper and lower boundaries. */
4753	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4754	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4755	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4756	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4757
4758	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4759	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4760	}
4761	else
4762	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4763
4764	if (fOffsettedTsc)
4765	{
4766	uint64_t u64CurTSC = ASMReadTSC();
4767	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4768	{
4769	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4770	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4771
4772	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4773	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4774	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4775	}
4776	else
4777	{
4778	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4779	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4781	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4782	}
4783	}
4784	else
4785	{
4786	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4787	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4788	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4789	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4790	}
4791	}
4792
4793
4794	/**
4795	* Determines if an exception is a contributory exception. Contributory
4796	* exceptions are ones which can cause double-faults. Page-fault is
4797	* intentionally not included here as it's a conditional contributory exception.
4798	*
4799	* @returns true if the exception is contributory, false otherwise.
4800	* @param uVector The exception vector.
4801	*/
4802	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4803	{
4804	switch (uVector)
4805	{
4806	case X86_XCPT_GP:
4807	case X86_XCPT_SS:
4808	case X86_XCPT_NP:
4809	case X86_XCPT_TS:
4810	case X86_XCPT_DE:
4811	return true;
4812	default:
4813	break;
4814	}
4815	return false;
4816	}
4817
4818
4819	/**
4820	* Sets an event as a pending event to be injected into the guest.
4821	*
4822	* @param pVCpu Pointer to the VMCPU.
4823	* @param u32IntrInfo The VM-entry interruption-information field.
4824	* @param cbInstr The VM-entry instruction length in bytes (for software
4825	* interrupts, exceptions and privileged software
4826	* exceptions).
4827	* @param u32ErrCode The VM-entry exception error code.
4828	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4829	* page-fault.
4830	*
4831	* @remarks Statistics counter assumes this is a guest event being injected or
4832	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4833	* always incremented.
4834	*/
4835	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4836	RTGCUINTPTR GCPtrFaultAddress)
4837	{
4838	Assert(!pVCpu->hm.s.Event.fPending);
4839	pVCpu->hm.s.Event.fPending = true;
4840	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4841	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4842	pVCpu->hm.s.Event.cbInstr = cbInstr;
4843	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4844
4845	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4846	}
4847
4848
4849	/**
4850	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4851	*
4852	* @param pVCpu Pointer to the VMCPU.
4853	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4854	* out-of-sync. Make sure to update the required fields
4855	* before using them.
4856	*/
4857	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4858	{
4859	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4860	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4861	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4862	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4863	}
4864
4865
4866	/**
4867	* Handle a condition that occurred while delivering an event through the guest
4868	* IDT.
4869	*
4870	* @returns VBox status code (informational error codes included).
4871	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4872	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4873	* continue execution of the guest which will delivery the #DF.
4874	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4875	*
4876	* @param pVCpu Pointer to the VMCPU.
4877	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4878	* out-of-sync. Make sure to update the required fields
4879	* before using them.
4880	* @param pVmxTransient Pointer to the VMX transient structure.
4881	*
4882	* @remarks No-long-jump zone!!!
4883	*/
4884	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4885	{
4886	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4887	AssertRC(rc);
4888	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4889	{
4890	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4891	AssertRCReturn(rc, rc);
4892
4893	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4894	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4895	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4896
4897	typedef enum
4898	{
4899	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4900	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4901	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4902	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4903	} VMXREFLECTXCPT;
4904
4905	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4906	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4907	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4908	{
4909	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4910	{
4911	enmReflect = VMXREFLECTXCPT_XCPT;
4912	#ifdef VBOX_STRICT
4913	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4914	&& uExitVector == X86_XCPT_PF)
4915	{
4916	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4917	}
4918	#endif
4919	if ( uExitVector == X86_XCPT_PF
4920	&& uIdtVector == X86_XCPT_PF)
4921	{
4922	pVmxTransient->fVectoringPF = true;
4923	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4924	}
4925	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4926	&& hmR0VmxIsContributoryXcpt(uExitVector)
4927	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4928	\|\| uIdtVector == X86_XCPT_PF))
4929	{
4930	enmReflect = VMXREFLECTXCPT_DF;
4931	}
4932	else if (uIdtVector == X86_XCPT_DF)
4933	enmReflect = VMXREFLECTXCPT_TF;
4934	}
4935	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4936	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4937	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4938	{
4939	/*
4940	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4941	* (whatever they are) as they reoccur when restarting the instruction.
4942	*/
4943	enmReflect = VMXREFLECTXCPT_XCPT;
4944	}
4945	}
4946	else
4947	{
4948	/*
4949	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4950	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4951	* original exception to the guest after handling the VM-exit.
4952	*/
4953	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4954	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4955	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4956	{
4957	enmReflect = VMXREFLECTXCPT_XCPT;
4958	}
4959	}
4960
4961	switch (enmReflect)
4962	{
4963	case VMXREFLECTXCPT_XCPT:
4964	{
4965	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4966	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4967	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4968
4969	uint32_t u32ErrCode = 0;
4970	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4971	{
4972	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4973	AssertRCReturn(rc, rc);
4974	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4975	}
4976
4977	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4978	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4979	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4980	rc = VINF_SUCCESS;
4981	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4982	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4983
4984	break;
4985	}
4986
4987	case VMXREFLECTXCPT_DF:
4988	{
4989	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4990	rc = VINF_HM_DOUBLE_FAULT;
4991	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4992	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4993
4994	break;
4995	}
4996
4997	case VMXREFLECTXCPT_TF:
4998	{
4999	rc = VINF_EM_RESET;
5000	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5001	uExitVector));
5002	break;
5003	}
5004
5005	default:
5006	Assert(rc == VINF_SUCCESS);
5007	break;
5008	}
5009	}
5010	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5011	return rc;
5012	}
5013
5014
5015	/**
5016	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5029	{
5030	uint32_t uVal = 0;
5031	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5032	AssertRCReturn(rc, rc);
5033	uint32_t uShadow = 0;
5034	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5035	AssertRCReturn(rc, rc);
5036
5037	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5038	CPUMSetGuestCR0(pVCpu, uVal);
5039	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5040	}
5041	return VINF_SUCCESS;
5042	}
5043
5044
5045	/**
5046	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	int rc = VINF_SUCCESS;
5059	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5060	{
5061	uint32_t uVal = 0;
5062	uint32_t uShadow = 0;
5063	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5064	AssertRCReturn(rc, rc);
5065	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5066	AssertRCReturn(rc, rc);
5067
5068	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5069	CPUMSetGuestCR4(pVCpu, uVal);
5070	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5071	}
5072	return rc;
5073	}
5074
5075
5076	/**
5077	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	int rc = VINF_SUCCESS;
5090	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5091	{
5092	uint64_t u64Val = 0;
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5094	AssertRCReturn(rc, rc);
5095
5096	pMixedCtx->rip = u64Val;
5097	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5098	}
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5109	* out-of-sync. Make sure to update the required fields
5110	* before using them.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5115	{
5116	int rc = VINF_SUCCESS;
5117	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5118	{
5119	uint64_t u64Val = 0;
5120	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5121	AssertRCReturn(rc, rc);
5122
5123	pMixedCtx->rsp = u64Val;
5124	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5125	}
5126	return rc;
5127	}
5128
5129
5130	/**
5131	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5132	*
5133	* @returns VBox status code.
5134	* @param pVCpu Pointer to the VMCPU.
5135	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5136	* out-of-sync. Make sure to update the required fields
5137	* before using them.
5138	*
5139	* @remarks No-long-jump zone!!!
5140	*/
5141	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5142	{
5143	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5144	{
5145	uint32_t uVal = 0;
5146	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5147	AssertRCReturn(rc, rc);
5148
5149	pMixedCtx->eflags.u32 = uVal;
5150	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5151	{
5152	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5153	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5154
5155	pMixedCtx->eflags.Bits.u1VM = 0;
5156	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5157	}
5158
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5160	}
5161	return VINF_SUCCESS;
5162	}
5163
5164
5165	/**
5166	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5167	* guest-CPU context.
5168	*/
5169	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5170	{
5171	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5172	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5173	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5174	return rc;
5175	}
5176
5177
5178	/**
5179	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5180	* from the guest-state area in the VMCS.
5181	*
5182	* @param pVCpu Pointer to the VMCPU.
5183	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184	* out-of-sync. Make sure to update the required fields
5185	* before using them.
5186	*
5187	* @remarks No-long-jump zone!!!
5188	*/
5189	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190	{
5191	uint32_t uIntrState = 0;
5192	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5193	AssertRC(rc);
5194
5195	if (!uIntrState)
5196	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5197	else
5198	{
5199	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5200	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5201	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5202	AssertRC(rc);
5203	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5204	AssertRC(rc);
5205
5206	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5207	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5208	}
5209	}
5210
5211
5212	/**
5213	* Saves the guest's activity state.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5218	* out-of-sync. Make sure to update the required fields
5219	* before using them.
5220	*
5221	* @remarks No-long-jump zone!!!
5222	*/
5223	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5224	{
5225	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5226	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5227	return VINF_SUCCESS;
5228	}
5229
5230
5231	/**
5232	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5233	* the current VMCS into the guest-CPU context.
5234	*
5235	* @returns VBox status code.
5236	* @param pVCpu Pointer to the VMCPU.
5237	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5238	* out-of-sync. Make sure to update the required fields
5239	* before using them.
5240	*
5241	* @remarks No-long-jump zone!!!
5242	*/
5243	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5244	{
5245	int rc = VINF_SUCCESS;
5246	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5247	{
5248	uint32_t u32Val = 0;
5249	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5250	pMixedCtx->SysEnter.cs = u32Val;
5251	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5252	}
5253
5254	uint64_t u64Val = 0;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5256	{
5257	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5258	pMixedCtx->SysEnter.eip = u64Val;
5259	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5260	}
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5262	{
5263	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5264	pMixedCtx->SysEnter.esp = u64Val;
5265	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5266	}
5267	return rc;
5268	}
5269
5270
5271	/**
5272	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5273	* context.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu Pointer to the VMCPU.
5277	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5278	* out-of-sync. Make sure to update the required fields
5279	* before using them.
5280	*
5281	* @remarks No-long-jump zone!!!
5282	*/
5283	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5284	{
5285	int rc = VINF_SUCCESS;
5286	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5287	{
5288	uint64_t u64Val = 0;
5289	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5290	pMixedCtx->fs.u64Base = u64Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5292	}
5293	return rc;
5294	}
5295
5296
5297	/**
5298	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5299	* context.
5300	*
5301	* @returns VBox status code.
5302	* @param pVCpu Pointer to the VMCPU.
5303	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304	* out-of-sync. Make sure to update the required fields
5305	* before using them.
5306	*
5307	* @remarks No-long-jump zone!!!
5308	*/
5309	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310	{
5311	int rc = VINF_SUCCESS;
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5313	{
5314	uint64_t u64Val = 0;
5315	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5316	pMixedCtx->gs.u64Base = u64Val;
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5318	}
5319	return rc;
5320	}
5321
5322
5323	/**
5324	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5325	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5326	* and TSC_AUX.
5327	*
5328	* @returns VBox status code.
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5339	return VINF_SUCCESS;
5340
5341	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5343	{
5344	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5345	pMsr += i;
5346	switch (pMsr->u32IndexMSR)
5347	{
5348	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5349	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5350	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5351	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5352	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5353	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5354	default:
5355	{
5356	AssertFailed();
5357	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5358	}
5359	}
5360	}
5361	#endif
5362
5363	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5364	return VINF_SUCCESS;
5365	}
5366
5367
5368	/**
5369	* Saves the guest control registers from the current VMCS into the guest-CPU
5370	* context.
5371	*
5372	* @returns VBox status code.
5373	* @param pVCpu Pointer to the VMCPU.
5374	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5375	* out-of-sync. Make sure to update the required fields
5376	* before using them.
5377	*
5378	* @remarks No-long-jump zone!!!
5379	*/
5380	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5381	{
5382	/* Guest CR0. Guest FPU. */
5383	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5384	AssertRCReturn(rc, rc);
5385
5386	/* Guest CR4. */
5387	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5388	AssertRCReturn(rc, rc);
5389
5390	/* Guest CR2 - updated always during the world-switch or in #PF. */
5391	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5392	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5393	{
5394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5395	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5396
5397	PVM pVM = pVCpu->CTX_SUFF(pVM);
5398	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5399	\|\| ( pVM->hm.s.fNestedPaging
5400	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5401	{
5402	uint64_t u64Val = 0;
5403	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5404	if (pMixedCtx->cr3 != u64Val)
5405	{
5406	CPUMSetGuestCR3(pVCpu, u64Val);
5407	if (VMMRZCallRing3IsEnabled(pVCpu))
5408	{
5409	PGMUpdateCR3(pVCpu, u64Val);
5410	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5411	}
5412	else
5413	{
5414	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5415	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5416	}
5417	}
5418
5419	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5420	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5421	{
5422	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5423	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5424	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5425	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5426
5427	if (VMMRZCallRing3IsEnabled(pVCpu))
5428	{
5429	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5430	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5431	}
5432	else
5433	{
5434	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5435	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5436	}
5437	}
5438	}
5439
5440	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5441	}
5442
5443	/*
5444	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5445	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5446	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5447	*
5448	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5449	*/
5450	if (VMMRZCallRing3IsEnabled(pVCpu))
5451	{
5452	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5453	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5454
5455	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5456	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5457
5458	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5459	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5460	}
5461
5462	return rc;
5463	}
5464
5465
5466	/**
5467	* Reads a guest segment register from the current VMCS into the guest-CPU
5468	* context.
5469	*
5470	* @returns VBox status code.
5471	* @param pVCpu Pointer to the VMCPU.
5472	* @param idxSel Index of the selector in the VMCS.
5473	* @param idxLimit Index of the segment limit in the VMCS.
5474	* @param idxBase Index of the segment base in the VMCS.
5475	* @param idxAccess Index of the access rights of the segment in the VMCS.
5476	* @param pSelReg Pointer to the segment selector.
5477	*
5478	* @remarks No-long-jump zone!!!
5479	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5480	* macro as that takes care of whether to read from the VMCS cache or
5481	* not.
5482	*/
5483	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5484	PCPUMSELREG pSelReg)
5485	{
5486	uint32_t u32Val = 0;
5487	int rc = VMXReadVmcs32(idxSel, &u32Val);
5488	AssertRCReturn(rc, rc);
5489	pSelReg->Sel = (uint16_t)u32Val;
5490	pSelReg->ValidSel = (uint16_t)u32Val;
5491	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5492
5493	rc = VMXReadVmcs32(idxLimit, &u32Val);
5494	AssertRCReturn(rc, rc);
5495	pSelReg->u32Limit = u32Val;
5496
5497	uint64_t u64Val = 0;
5498	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5499	AssertRCReturn(rc, rc);
5500	pSelReg->u64Base = u64Val;
5501
5502	rc = VMXReadVmcs32(idxAccess, &u32Val);
5503	AssertRCReturn(rc, rc);
5504	pSelReg->Attr.u = u32Val;
5505
5506	/*
5507	* If VT-x marks the segment as unusable, most other bits remain undefined:
5508	* - For CS the L, D and G bits have meaning.
5509	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5510	* - For the remaining data segments no bits are defined.
5511	*
5512	* The present bit and the unusable bit has been observed to be set at the
5513	* same time (the selector was supposed to invalid as we started executing
5514	* a V8086 interrupt in ring-0).
5515	*
5516	* What should be important for the rest of the VBox code that the P bit is
5517	* cleared. Some of the other VBox code recognizes the unusable bit, but
5518	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5519	* safe side here, we'll strip off P and other bits we don't care about. If
5520	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5521	*
5522	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5523	*/
5524	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5525	{
5526	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5527
5528	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5529	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5530	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5531
5532	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5533	#ifdef DEBUG_bird
5534	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5535	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5536	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5537	#endif
5538	}
5539	return VINF_SUCCESS;
5540	}
5541
5542
5543	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5544	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5545	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5546	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5547	#else
5548	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5549	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5550	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5551	#endif
5552
5553
5554	/**
5555	* Saves the guest segment registers from the current VMCS into the guest-CPU
5556	* context.
5557	*
5558	* @returns VBox status code.
5559	* @param pVCpu Pointer to the VMCPU.
5560	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5561	* out-of-sync. Make sure to update the required fields
5562	* before using them.
5563	*
5564	* @remarks No-long-jump zone!!!
5565	*/
5566	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5567	{
5568	/* Guest segment registers. */
5569	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5570	{
5571	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5572	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5573	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5574	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5575	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5576	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5577	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5578
5579	/* Restore segment attributes for real-on-v86 mode hack. */
5580	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5581	{
5582	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5583	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5584	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5585	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5586	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5587	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5588	}
5589	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5590	}
5591
5592	return VINF_SUCCESS;
5593	}
5594
5595
5596	/**
5597	* Saves the guest descriptor table registers and task register from the current
5598	* VMCS into the guest-CPU context.
5599	*
5600	* @returns VBox status code.
5601	* @param pVCpu Pointer to the VMCPU.
5602	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5603	* out-of-sync. Make sure to update the required fields
5604	* before using them.
5605	*
5606	* @remarks No-long-jump zone!!!
5607	*/
5608	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5609	{
5610	int rc = VINF_SUCCESS;
5611
5612	/* Guest LDTR. */
5613	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5614	{
5615	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5616	AssertRCReturn(rc, rc);
5617	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5618	}
5619
5620	/* Guest GDTR. */
5621	uint64_t u64Val = 0;
5622	uint32_t u32Val = 0;
5623	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5624	{
5625	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5626	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5627	pMixedCtx->gdtr.pGdt = u64Val;
5628	pMixedCtx->gdtr.cbGdt = u32Val;
5629	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5630	}
5631
5632	/* Guest IDTR. */
5633	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5634	{
5635	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5636	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5637	pMixedCtx->idtr.pIdt = u64Val;
5638	pMixedCtx->idtr.cbIdt = u32Val;
5639	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5640	}
5641
5642	/* Guest TR. */
5643	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5644	{
5645	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5646	AssertRCReturn(rc, rc);
5647
5648	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5649	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5650	{
5651	rc = VMXLOCAL_READ_SEG(TR, tr);
5652	AssertRCReturn(rc, rc);
5653	}
5654	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5655	}
5656	return rc;
5657	}
5658
5659	#undef VMXLOCAL_READ_SEG
5660
5661
5662	/**
5663	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5664	* context.
5665	*
5666	* @returns VBox status code.
5667	* @param pVCpu Pointer to the VMCPU.
5668	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669	* out-of-sync. Make sure to update the required fields
5670	* before using them.
5671	*
5672	* @remarks No-long-jump zone!!!
5673	*/
5674	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675	{
5676	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5677	{
5678	if (!CPUMIsHyperDebugStateActive(pVCpu))
5679	{
5680	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5681	uint32_t u32Val;
5682	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5683	pMixedCtx->dr[7] = u32Val;
5684	}
5685
5686	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5687	}
5688	return VINF_SUCCESS;
5689	}
5690
5691
5692	/**
5693	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
5694	*
5695	* @returns VBox status code.
5696	* @param pVCpu Pointer to the VMCPU.
5697	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5698	* out-of-sync. Make sure to update the required fields
5699	* before using them.
5700	*
5701	* @remarks No-long-jump zone!!!
5702	*/
5703	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5704	{
5705	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5706	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5707	return VINF_SUCCESS;
5708	}
5709
5710
5711	/**
5712	* Saves the entire guest state from the currently active VMCS into the
5713	* guest-CPU context. This essentially VMREADs all guest-data.
5714	*
5715	* @returns VBox status code.
5716	* @param pVCpu Pointer to the VMCPU.
5717	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5718	* out-of-sync. Make sure to update the required fields
5719	* before using them.
5720	*/
5721	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5722	{
5723	Assert(pVCpu);
5724	Assert(pMixedCtx);
5725
5726	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5727	return VINF_SUCCESS;
5728
5729	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5730	there is no real need to. */
5731	if (VMMRZCallRing3IsEnabled(pVCpu))
5732	VMMR0LogFlushDisable(pVCpu);
5733	else
5734	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5735	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5736
5737	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5738	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5739
5740	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5741	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5742
5743	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5744	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5745
5746	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5747	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5748
5749	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5750	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5751
5752	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5754
5755	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5757
5758	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5760
5761	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5763
5764	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5765	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5766
5767	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5768	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5769
5770	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5771	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5772
5773	if (VMMRZCallRing3IsEnabled(pVCpu))
5774	VMMR0LogFlushEnable(pVCpu);
5775
5776	return rc;
5777	}
5778
5779
5780	/**
5781	* Check per-VM and per-VCPU force flag actions that require us to go back to
5782	* ring-3 for one reason or another.
5783	*
5784	* @returns VBox status code (information status code included).
5785	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5786	* ring-3.
5787	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5788	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5789	* interrupts)
5790	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5791	* all EMTs to be in ring-3.
5792	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5793	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5794	* to the EM loop.
5795	*
5796	* @param pVM Pointer to the VM.
5797	* @param pVCpu Pointer to the VMCPU.
5798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5799	* out-of-sync. Make sure to update the required fields
5800	* before using them.
5801	*/
5802	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5803	{
5804	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5805
5806	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5807	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5808	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5809	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5810	{
5811	/* We need the control registers now, make sure the guest-CPU context is updated. */
5812	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5813	AssertRCReturn(rc3, rc3);
5814
5815	/* Pending HM CR3 sync. */
5816	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5817	{
5818	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5819	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
5820	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5821	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5822	}
5823
5824	/* Pending HM PAE PDPEs. */
5825	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5826	{
5827	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5828	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5829	}
5830
5831	/* Pending PGM C3 sync. */
5832	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5833	{
5834	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5835	if (rc2 != VINF_SUCCESS)
5836	{
5837	AssertRC(rc2);
5838	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5839	return rc2;
5840	}
5841	}
5842
5843	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5844	/* -XXX- what was that about single stepping? */
5845	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5846	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5847	{
5848	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5849	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5850	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5851	return rc2;
5852	}
5853
5854	/* Pending VM request packets, such as hardware interrupts. */
5855	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5856	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5857	{
5858	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5859	return VINF_EM_PENDING_REQUEST;
5860	}
5861
5862	/* Pending PGM pool flushes. */
5863	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5864	{
5865	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5866	return VINF_PGM_POOL_FLUSH_PENDING;
5867	}
5868
5869	/* Pending DMA requests. */
5870	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5871	{
5872	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5873	return VINF_EM_RAW_TO_R3;
5874	}
5875	}
5876
5877	/* Paranoia. */
5878	return VINF_SUCCESS;
5879	}
5880
5881
5882	/**
5883	* Converts any TRPM trap into a pending HM event. This is typically used when
5884	* entering from ring-3 (not longjmp returns).
5885	*
5886	* @param pVCpu Pointer to the VMCPU.
5887	*/
5888	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5889	{
5890	Assert(TRPMHasTrap(pVCpu));
5891	Assert(!pVCpu->hm.s.Event.fPending);
5892
5893	uint8_t uVector;
5894	TRPMEVENT enmTrpmEvent;
5895	RTGCUINT uErrCode;
5896	RTGCUINTPTR GCPtrFaultAddress;
5897	uint8_t cbInstr;
5898
5899	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5900	AssertRC(rc);
5901
5902	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5903	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5904	if (enmTrpmEvent == TRPM_TRAP)
5905	{
5906	switch (uVector)
5907	{
5908	case X86_XCPT_BP:
5909	case X86_XCPT_OF:
5910	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5911	break;
5912
5913	case X86_XCPT_PF:
5914	case X86_XCPT_DF:
5915	case X86_XCPT_TS:
5916	case X86_XCPT_NP:
5917	case X86_XCPT_SS:
5918	case X86_XCPT_GP:
5919	case X86_XCPT_AC:
5920	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5921	/* no break! */
5922	default:
5923	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5924	break;
5925	}
5926	}
5927	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5928	{
5929	if (uVector == X86_XCPT_NMI)
5930	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5931	else
5932	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5933	}
5934	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5935	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5936	else
5937	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5938
5939	rc = TRPMResetTrap(pVCpu);
5940	AssertRC(rc);
5941	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5942	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5943
5944	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5945	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5946	}
5947
5948
5949	/**
5950	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5951	* VT-x to execute any instruction.
5952	*
5953	* @param pvCpu Pointer to the VMCPU.
5954	*/
5955	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5956	{
5957	Assert(pVCpu->hm.s.Event.fPending);
5958
5959	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5960	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5961	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5962	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5963
5964	/* If a trap was already pending, we did something wrong! */
5965	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5966
5967	TRPMEVENT enmTrapType;
5968	switch (uVectorType)
5969	{
5970	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5971	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5972	enmTrapType = TRPM_HARDWARE_INT;
5973	break;
5974
5975	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5976	enmTrapType = TRPM_SOFTWARE_INT;
5977	break;
5978
5979	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5980	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5981	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5982	enmTrapType = TRPM_TRAP;
5983	break;
5984
5985	default:
5986	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5987	enmTrapType = TRPM_32BIT_HACK;
5988	break;
5989	}
5990
5991	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5992
5993	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5994	AssertRC(rc);
5995
5996	if (fErrorCodeValid)
5997	TRPMSetErrorCode(pVCpu, uErrorCode);
5998
5999	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6000	&& uVector == X86_XCPT_PF)
6001	{
6002	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6003	}
6004	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6005	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6006	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6007	{
6008	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6009	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6010	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6011	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6012	}
6013	pVCpu->hm.s.Event.fPending = false;
6014	}
6015
6016
6017	/**
6018	* Does the necessary state syncing before returning to ring-3 for any reason
6019	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6020	*
6021	* @param pVM Pointer to the VM.
6022	* @param pVCpu Pointer to the VMCPU.
6023	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6024	* out-of-sync. Make sure to update the required fields
6025	* before using them.
6026	*
6027	* @remarks No-long-jmp zone!!!
6028	*/
6029	static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6030	{
6031	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6032	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6033
6034	/* Avoid repeating this work when thread-context hooks are used and we had been preempted before
6035	which would've done this work from the VMXR0ThreadCtxCallback(). */
6036	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6037	bool fPreemptDisabled = false;
6038	if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
6039	{
6040	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6041	RTThreadPreemptDisable(&PreemptState);
6042	fPreemptDisabled = true;
6043	}
6044
6045	if (!pVCpu->hm.s.fLeaveDone)
6046	{
6047	/* Save the guest state if necessary. */
6048	if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6049	{
6050	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6051	AssertRC(rc);
6052	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6053	}
6054
6055	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6056	if (CPUMIsGuestFPUStateActive(pVCpu))
6057	{
6058	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6059	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6060	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
6061	}
6062
6063	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6064	#ifdef VBOX_STRICT
6065	if (CPUMIsHyperDebugStateActive(pVCpu))
6066	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6067	#endif
6068	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6069	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
6070	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6071	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6072
6073	/* Restore host-state bits that VT-x only restores partially. */
6074	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6075	{
6076	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6077	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6078	}
6079
6080	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6081	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6082	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6083	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6084	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6085	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6086	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6087	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6088
6089	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6090
6091	pVCpu->hm.s.fLeaveDone = true;
6092	}
6093
6094	/* Restore preemption if we previous disabled it ourselves. */
6095	if (fPreemptDisabled)
6096	RTThreadPreemptRestore(&PreemptState);
6097	}
6098
6099
6100	/**
6101	* Does the necessary state syncing before doing a longjmp to ring-3.
6102	*
6103	* @param pVM Pointer to the VM.
6104	* @param pVCpu Pointer to the VMCPU.
6105	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6106	* out-of-sync. Make sure to update the required fields
6107	* before using them.
6108	*
6109	* @remarks No-long-jmp zone!!!
6110	*/
6111	DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6112	{
6113	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6114	}
6115
6116
6117	/**
6118	* Take necessary actions before going back to ring-3.
6119	*
6120	* An action requires us to go back to ring-3. This function does the necessary
6121	* steps before we can safely return to ring-3. This is not the same as longjmps
6122	* to ring-3, this is voluntary and prepares the guest so it may continue
6123	* executing outside HM (recompiler/IEM).
6124	*
6125	* @param pVM Pointer to the VM.
6126	* @param pVCpu Pointer to the VMCPU.
6127	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6128	* out-of-sync. Make sure to update the required fields
6129	* before using them.
6130	* @param rcExit The reason for exiting to ring-3. Can be
6131	* VINF_VMM_UNKNOWN_RING3_CALL.
6132	*/
6133	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6134	{
6135	Assert(pVM);
6136	Assert(pVCpu);
6137	Assert(pMixedCtx);
6138	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6139
6140	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6141	{
6142	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6143	return;
6144	}
6145	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6146	{
6147	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6148	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6149	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6150	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6151	return;
6152	}
6153
6154	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6155	VMMRZCallRing3Disable(pVCpu);
6156	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6157
6158	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6159	if (pVCpu->hm.s.Event.fPending)
6160	{
6161	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6162	Assert(!pVCpu->hm.s.Event.fPending);
6163	}
6164
6165	/* Save guest state and restore host state bits. */
6166	hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6167	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6168
6169	/* Sync recompiler state. */
6170	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6171	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6172	\| CPUM_CHANGED_LDTR
6173	\| CPUM_CHANGED_GDTR
6174	\| CPUM_CHANGED_IDTR
6175	\| CPUM_CHANGED_TR
6176	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6177	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6178	if ( pVM->hm.s.fNestedPaging
6179	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6180	{
6181	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6182	}
6183
6184	/*
6185	* Clear the X86_EFL_TF if necessary .
6186	*/
6187	if (pVCpu->hm.s.fClearTrapFlag)
6188	{
6189	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6190	pMixedCtx->eflags.Bits.u1TF = 0;
6191	pVCpu->hm.s.fClearTrapFlag = false;
6192	}
6193	/** @todo there seems to be issues with the resume flag when the monitor trap
6194	* flag is pending without being used. Seen early in bios init when
6195	* accessing APIC page in prot mode. */
6196
6197	/* On our way back from ring-3 the following needs to be done. */
6198	/** @todo This can change with preemption hooks. */
6199	if (rcExit == VINF_EM_RAW_INTERRUPT)
6200	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6201	else
6202	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6203
6204	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6205	VMMRZCallRing3Enable(pVCpu);
6206	}
6207
6208
6209	/**
6210	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6211	* longjump to ring-3 and possibly get preempted.
6212	*
6213	* @param pVCpu Pointer to the VMCPU.
6214	* @param enmOperation The operation causing the ring-3 longjump.
6215	* @param pvUser The user argument (pointer to the possibly
6216	* out-of-date guest-CPU context).
6217	*
6218	* @remarks Must never be called with @a enmOperation ==
6219	* VMMCALLRING3_VM_R0_ASSERTION.
6220	*/
6221	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6222	{
6223	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6224	Assert(pVCpu);
6225	Assert(pvUser);
6226	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6227	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6228
6229	VMMRZCallRing3Disable(pVCpu);
6230	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6231
6232	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6233	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6234
6235	VMMRZCallRing3Enable(pVCpu);
6236	}
6237
6238
6239	/**
6240	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6241	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6242	*
6243	* @param pVCpu Pointer to the VMCPU.
6244	*/
6245	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6246	{
6247	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6248	{
6249	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6250	{
6251	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6252	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6253	AssertRC(rc);
6254	}
6255	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6256	}
6257
6258
6259	/**
6260	* Injects any pending events into the guest if the guest is in a state to
6261	* receive them.
6262	*
6263	* @returns VBox status code (informational status codes included).
6264	* @param pVCpu Pointer to the VMCPU.
6265	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6266	* out-of-sync. Make sure to update the required fields
6267	* before using them.
6268	*/
6269	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6270	{
6271	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6272	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6273	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6274	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6275
6276	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6277	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6278	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6279	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6280	Assert(!TRPMHasTrap(pVCpu));
6281
6282	int rc = VINF_SUCCESS;
6283	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6284	{
6285	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6286	bool fInject = true;
6287	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6288	{
6289	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6290	AssertRCReturn(rc, rc);
6291	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6292	if ( fBlockInt
6293	\|\| fBlockSti
6294	\|\| fBlockMovSS)
6295	{
6296	fInject = false;
6297	}
6298	}
6299	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6300	&& ( fBlockMovSS
6301	\|\| fBlockSti))
6302	{
6303	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6304	fInject = false;
6305	}
6306
6307	if (fInject)
6308	{
6309	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6310	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6311	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6312	AssertRCReturn(rc, rc);
6313	pVCpu->hm.s.Event.fPending = false;
6314
6315	#ifdef VBOX_WITH_STATISTICS
6316	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6317	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6318	else
6319	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6320	#endif
6321	}
6322	else
6323	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6324	} /** @todo SMI. SMIs take priority over NMIs. */
6325	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6326	{
6327	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6328	if ( !fBlockMovSS
6329	&& !fBlockSti)
6330	{
6331	Log4(("Injecting NMI\n"));
6332	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6333	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6334	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6335	0 /* GCPtrFaultAddress */, &uIntrState);
6336	AssertRCReturn(rc, rc);
6337	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6338
6339	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6340	}
6341	else
6342	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6343	}
6344	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6345	&& !pVCpu->hm.s.fSingleInstruction)
6346	{
6347	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6348	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6349	AssertRCReturn(rc, rc);
6350	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6351	if ( !fBlockInt
6352	&& !fBlockSti
6353	&& !fBlockMovSS)
6354	{
6355	uint8_t u8Interrupt;
6356	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6357	if (RT_SUCCESS(rc))
6358	{
6359	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6360	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6361	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6362	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6363	0 /* GCPtrFaultAddress */, &uIntrState);
6364
6365	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6366	}
6367	else
6368	{
6369	/** @todo Does this actually happen? If not turn it into an assertion. */
6370	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6371	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6372	rc = VINF_SUCCESS;
6373	}
6374	}
6375	else
6376	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6377	}
6378
6379	/*
6380	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6381	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6382	*/
6383	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6384	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6385	int rc2 = VINF_SUCCESS;
6386	if ( fBlockSti
6387	\|\| fBlockMovSS)
6388	{
6389	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6390	{
6391	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6392	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6393	{
6394	/*
6395	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6396	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6397	* See Intel spec. 27.3.4 "Saving Non-Register State".
6398	*/
6399	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6400	AssertRCReturn(rc, rc);
6401	}
6402	}
6403	else
6404	{
6405	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6406	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6407	uIntrState = 0;
6408	}
6409	}
6410
6411	/*
6412	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6413	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6414	*/
6415	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6416	AssertRC(rc2);
6417
6418	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6419	return rc;
6420	}
6421
6422
6423	/**
6424	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6425	*
6426	* @param pVCpu Pointer to the VMCPU.
6427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6428	* out-of-sync. Make sure to update the required fields
6429	* before using them.
6430	*/
6431	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6432	{
6433	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6434	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6435	}
6436
6437
6438	/**
6439	* Injects a double-fault (#DF) exception into the VM.
6440	*
6441	* @returns VBox status code (informational status code included).
6442	* @param pVCpu Pointer to the VMCPU.
6443	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6444	* out-of-sync. Make sure to update the required fields
6445	* before using them.
6446	*/
6447	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6448	{
6449	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6450	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6451	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6452	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6453	puIntrState);
6454	}
6455
6456
6457	/**
6458	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6459	*
6460	* @param pVCpu Pointer to the VMCPU.
6461	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6462	* out-of-sync. Make sure to update the required fields
6463	* before using them.
6464	*/
6465	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6466	{
6467	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6468	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6469	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6470	}
6471
6472
6473	/**
6474	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6475	*
6476	* @param pVCpu Pointer to the VMCPU.
6477	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6478	* out-of-sync. Make sure to update the required fields
6479	* before using them.
6480	* @param cbInstr The value of RIP that is to be pushed on the guest
6481	* stack.
6482	*/
6483	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6484	{
6485	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6486	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6488	}
6489
6490
6491	/**
6492	* Injects a general-protection (#GP) fault into the VM.
6493	*
6494	* @returns VBox status code (informational status code included).
6495	* @param pVCpu Pointer to the VMCPU.
6496	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6497	* out-of-sync. Make sure to update the required fields
6498	* before using them.
6499	* @param u32ErrorCode The error code associated with the #GP.
6500	*/
6501	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6502	uint32_t *puIntrState)
6503	{
6504	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6505	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6506	if (fErrorCodeValid)
6507	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6508	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6509	puIntrState);
6510	}
6511
6512
6513	/**
6514	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6515	*
6516	* @param pVCpu Pointer to the VMCPU.
6517	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6518	* out-of-sync. Make sure to update the required fields
6519	* before using them.
6520	* @param uVector The software interrupt vector number.
6521	* @param cbInstr The value of RIP that is to be pushed on the guest
6522	* stack.
6523	*/
6524	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6525	{
6526	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6527	if ( uVector == X86_XCPT_BP
6528	\|\| uVector == X86_XCPT_OF)
6529	{
6530	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6531	}
6532	else
6533	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6534	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6535	}
6536
6537
6538	/**
6539	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6540	* stack.
6541	*
6542	* @returns VBox status code (information status code included).
6543	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6544	* @param pVM Pointer to the VM.
6545	* @param pMixedCtx Pointer to the guest-CPU context.
6546	* @param uValue The value to push to the guest stack.
6547	*/
6548	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6549	{
6550	/*
6551	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6552	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6553	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6554	*/
6555	if (pMixedCtx->sp == 1)
6556	return VINF_EM_RESET;
6557	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6558	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6559	AssertRCReturn(rc, rc);
6560	return rc;
6561	}
6562
6563
6564	/**
6565	* Injects an event into the guest upon VM-entry by updating the relevant fields
6566	* in the VM-entry area in the VMCS.
6567	*
6568	* @returns VBox status code (informational error codes included).
6569	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6570	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6571	*
6572	* @param pVCpu Pointer to the VMCPU.
6573	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6574	* be out-of-sync. Make sure to update the required
6575	* fields before using them.
6576	* @param u64IntrInfo The VM-entry interruption-information field.
6577	* @param cbInstr The VM-entry instruction length in bytes (for
6578	* software interrupts, exceptions and privileged
6579	* software exceptions).
6580	* @param u32ErrCode The VM-entry exception error code.
6581	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6582	* @param puIntrState Pointer to the current guest interruptibility-state.
6583	* This interruptibility-state will be updated if
6584	* necessary. This cannot not be NULL.
6585	*
6586	* @remarks No-long-jump zone!!!
6587	* @remarks Requires CR0!
6588	*/
6589	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6590	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6591	{
6592	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6593	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6594	Assert(puIntrState);
6595	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6596
6597	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6598	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6599
6600	#ifdef VBOX_STRICT
6601	/* Validate the error-code-valid bit for hardware exceptions. */
6602	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6603	{
6604	switch (uVector)
6605	{
6606	case X86_XCPT_PF:
6607	case X86_XCPT_DF:
6608	case X86_XCPT_TS:
6609	case X86_XCPT_NP:
6610	case X86_XCPT_SS:
6611	case X86_XCPT_GP:
6612	case X86_XCPT_AC:
6613	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6614	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6615	/* fallthru */
6616	default:
6617	break;
6618	}
6619	}
6620	#endif
6621
6622	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6623	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6624	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6625
6626	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6627
6628	/* We require CR0 to check if the guest is in real-mode. */
6629	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6630	AssertRCReturn(rc, rc);
6631
6632	/*
6633	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6634	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6635	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6636	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6637	*/
6638	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6639	{
6640	PVM pVM = pVCpu->CTX_SUFF(pVM);
6641	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6642	{
6643	Assert(PDMVmmDevHeapIsEnabled(pVM));
6644	Assert(pVM->hm.s.vmx.pRealModeTSS);
6645
6646	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6647	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6648	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6649	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6650	AssertRCReturn(rc, rc);
6651	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6652
6653	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6654	const size_t cbIdtEntry = 4;
6655	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6656	{
6657	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6658	if (uVector == X86_XCPT_DF)
6659	return VINF_EM_RESET;
6660	else if (uVector == X86_XCPT_GP)
6661	{
6662	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6663	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6664	}
6665
6666	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6667	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6668	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6669	}
6670
6671	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6672	uint16_t uGuestIp = pMixedCtx->ip;
6673	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6674	{
6675	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6676	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6677	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6678	}
6679	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6680	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6681
6682	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6683	uint16_t offIdtEntry = 0;
6684	RTSEL selIdtEntry = 0;
6685	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6686	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6687	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6688	AssertRCReturn(rc, rc);
6689
6690	/* Construct the stack frame for the interrupt/exception handler. */
6691	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6692	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6693	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6694	AssertRCReturn(rc, rc);
6695
6696	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6697	if (rc == VINF_SUCCESS)
6698	{
6699	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6700	pMixedCtx->rip = offIdtEntry;
6701	pMixedCtx->cs.Sel = selIdtEntry;
6702	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6703	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6704	&& uVector == X86_XCPT_PF)
6705	{
6706	pMixedCtx->cr2 = GCPtrFaultAddress;
6707	}
6708	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6709	\| HM_CHANGED_GUEST_RIP
6710	\| HM_CHANGED_GUEST_RFLAGS
6711	\| HM_CHANGED_GUEST_RSP;
6712
6713	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6714	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6715	{
6716	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6717	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6718	Log4(("Clearing inhibition due to STI.\n"));
6719	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6720	}
6721	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6722	}
6723	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6724	return rc;
6725	}
6726	else
6727	{
6728	/*
6729	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6730	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6731	*/
6732	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6733	}
6734	}
6735
6736	/* Validate. */
6737	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6738	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6739	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6740
6741	/* Inject. */
6742	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6743	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6744	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6745	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6746
6747	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6748	&& uVector == X86_XCPT_PF)
6749	{
6750	pMixedCtx->cr2 = GCPtrFaultAddress;
6751	}
6752
6753	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6754	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6755
6756	AssertRCReturn(rc, rc);
6757	return rc;
6758	}
6759
6760
6761	/**
6762	* Enters the VT-x session.
6763	*
6764	* @returns VBox status code.
6765	* @param pVM Pointer to the VM.
6766	* @param pVCpu Pointer to the VMCPU.
6767	* @param pCpu Pointer to the CPU info struct.
6768	*/
6769	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6770	{
6771	AssertPtr(pVM);
6772	AssertPtr(pVCpu);
6773	Assert(pVM->hm.s.vmx.fSupported);
6774	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6775	NOREF(pCpu);
6776
6777	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6778
6779	#ifdef VBOX_STRICT
6780	/* Make sure we're in VMX root mode. */
6781	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6782	if (!(u32HostCR4 & X86_CR4_VMXE))
6783	{
6784	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6785	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6786	}
6787	#endif
6788
6789	/* Load the active VMCS as the current one. */
6790	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6791	if (RT_FAILURE(rc))
6792	return rc;
6793
6794	/** @todo this will change with preemption hooks where can VMRESUME as long
6795	* as we're no preempted. */
6796	pVCpu->hm.s.fResumeVM = false;
6797	pVCpu->hm.s.fLeaveDone = false;
6798	return VINF_SUCCESS;
6799	}
6800
6801
6802	/**
6803	* The thread-context callback (only on platforms which support it).
6804	*
6805	* @param enmEvent The thread-context event.
6806	* @param pVCpu Pointer to the VMCPU.
6807	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6808	* @thread EMT.
6809	*/
6810	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6811	{
6812	switch (enmEvent)
6813	{
6814	case RTTHREADCTXEVENT_PREEMPTING:
6815	{
6816	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6817	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu)); /* Paranoia. */
6818
6819	PVM pVM = pVCpu->CTX_SUFF(pVM);
6820	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6821	VMMRZCallRing3Disable(pVCpu); /* No longjmps (log-flush, locks) in this fragile context. */
6822	hmR0VmxLeave(pVM, pVCpu, pMixedCtx); /* Save the guest-state, restore host-state (FPU, debug etc.). */
6823
6824	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs); /* Flush VMCS CPU state to VMCS region in memory. */
6825	AssertRC(rc); NOREF(rc);
6826
6827	rc = HMR0LeaveEx(pVCpu); /* Leave HM context, takes care of local init (term). */
6828	AssertRC(rc); NOREF(rc);
6829
6830	VMMRZCallRing3Enable(pVCpu); /* Restore longjmp state. */
6831	break;
6832	}
6833
6834	case RTTHREADCTXEVENT_RESUMED:
6835	{
6836	/* Disable preemption, we don't want to be migrated to another CPU while re-initializing VT-x state. */
6837	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6838	RTThreadPreemptDisable(&PreemptState);
6839
6840	/* Initialize the bare minimum state required for HM. This takes care of
6841	initializing VT-x if necessary (onlined CPUs, local init etc.) */
6842	HMR0EnterEx(pVCpu);
6843	Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_GUEST_CR0));
6844
6845	/* Load the active VMCS as the current one. */
6846	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6847	AssertRC(rc);
6848
6849	pVCpu->hm.s.fResumeVM = false;
6850	pVCpu->hm.s.fLeaveDone = false;
6851
6852	/* Restore preemption, migrating to another CPU should be fine now. */
6853	RTThreadPreemptRestore(&PreemptState);
6854	break;
6855	}
6856
6857	default:
6858	break;
6859	}
6860	}
6861
6862
6863	/**
6864	* Leaves the VT-x session.
6865	*
6866	* @returns VBox status code.
6867	* @param pVM Pointer to the VM.
6868	* @param pVCpu Pointer to the VMCPU.
6869	* @param pCtx Pointer to the guest-CPU context.
6870	*/
6871	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6872	{
6873	AssertPtr(pVCpu);
6874	NOREF(pVM);
6875	NOREF(pCtx);
6876
6877	if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6878	{
6879	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6880
6881	/*
6882	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6883	* and mark the VMCS launch-state as "clear".
6884	*/
6885	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6886	return rc;
6887	}
6888
6889	/* With thread-context hooks, nothing to do here. It's taken care of in VMXR0ThreadCtxCallback(). */
6890	return VINF_SUCCESS;
6891	}
6892
6893
6894	/**
6895	* Saves the host state in the VMCS host-state.
6896	* Sets up the VM-exit MSR-load area.
6897	*
6898	* The CPU state will be loaded from these fields on every successful VM-exit.
6899	*
6900	* @returns VBox status code.
6901	* @param pVM Pointer to the VM.
6902	* @param pVCpu Pointer to the VMCPU.
6903	*
6904	* @remarks No-long-jump zone!!!
6905	*/
6906	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6907	{
6908	AssertPtr(pVM);
6909	AssertPtr(pVCpu);
6910	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6911
6912	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6913
6914	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6915	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6916	return VINF_SUCCESS;
6917
6918	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6919	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6920
6921	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6922	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6923
6924	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6925	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6926
6927	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6928	return rc;
6929	}
6930
6931
6932	/**
6933	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6934	* loaded from these fields on every successful VM-entry.
6935	*
6936	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6937	* Sets up the VM-entry controls.
6938	* Sets up the appropriate VMX non-root function to execute guest code based on
6939	* the guest CPU mode.
6940	*
6941	* @returns VBox status code.
6942	* @param pVM Pointer to the VM.
6943	* @param pVCpu Pointer to the VMCPU.
6944	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6945	* out-of-sync. Make sure to update the required fields
6946	* before using them.
6947	*
6948	* @remarks No-long-jump zone!!!
6949	*/
6950	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6951	{
6952	AssertPtr(pVM);
6953	AssertPtr(pVCpu);
6954	AssertPtr(pMixedCtx);
6955	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6956
6957	#ifdef LOG_ENABLED
6958	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6959	* probably not initialized yet? Anyway this will do for now. */
6960	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6961	VMMR0LogFlushDisable(pVCpu);
6962	#endif
6963
6964	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6965
6966	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6967
6968	/* Determine real-on-v86 mode. */
6969	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6970	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6971	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6972	{
6973	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6974	}
6975
6976	/*
6977	* Load the guest-state into the VMCS.
6978	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6979	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6980	*/
6981	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6982	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6983
6984	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6985	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6986
6987	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6988	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6989
6990	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6991	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6992
6993	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6994	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6995	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6996
6997	rc = hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
6998	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugState: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6999
7000	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7001	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7002
7003	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7004	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7005
7006	/* Must be done after hmR0VmxLoadGuestDebugState() as it may have updated eflags.TF for debugging purposes. */
7007	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7008	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7009
7010	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7011	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7012
7013	/* Clear any unused and reserved bits. */
7014	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7015
7016	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
7017	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
7018	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
7019
7020	#ifdef LOG_ENABLED
7021	/* Only reenable log-flushing if the caller has it enabled. */
7022	if (!fCallerDisabledLogFlush)
7023	VMMR0LogFlushEnable(pVCpu);
7024	#endif
7025
7026	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7027	return rc;
7028	}
7029
7030
7031	/**
7032	* Loads the guest state into the VMCS guest-state area.
7033	*
7034	* @returns VBox status code.
7035	* @param pVM Pointer to the VM.
7036	* @param pVCpu Pointer to the VMCPU.
7037	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7038	* out-of-sync. Make sure to update the required fields
7039	* before using them.
7040	*
7041	* @remarks No-long-jump zone!!!
7042	*/
7043	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7044	{
7045	/*
7046	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
7047	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
7048	* finally executing guest code.
7049	*/
7050	return VINF_SUCCESS;
7051	}
7052
7053
7054	/**
7055	* Does the preparations before executing guest code in VT-x.
7056	*
7057	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7058	* recompiler. We must be cautious what we do here regarding committing
7059	* guest-state information into the VMCS assuming we assuredly execute the
7060	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
7061	* clearing the common-state (TRPM/forceflags), we must undo those changes so
7062	* that the recompiler can (and should) use them when it resumes guest
7063	* execution. Otherwise such operations must be done when we can no longer
7064	* exit to ring-3.
7065	*
7066	* @returns VBox status code (informational status codes included).
7067	* @retval VINF_SUCCESS if we can proceed with running the guest.
7068	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
7069	* into the guest.
7070	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7071	*
7072	* @param pVM Pointer to the VM.
7073	* @param pVCpu Pointer to the VMCPU.
7074	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7075	* out-of-sync. Make sure to update the required fields
7076	* before using them.
7077	* @param pVmxTransient Pointer to the VMX transient structure.
7078	*
7079	* @remarks Called with preemption disabled.
7080	*/
7081	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7082	{
7083	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7084
7085	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7086	PGMRZDynMapFlushAutoSet(pVCpu);
7087	#endif
7088
7089	/* Check force flag actions that might require us to go back to ring-3. */
7090	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7091	if (rc != VINF_SUCCESS)
7092	return rc;
7093
7094	#ifndef IEM_VERIFICATION_MODE_FULL
7095	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7096	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7097	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7098	{
7099	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7100	RTGCPHYS GCPhysApicBase;
7101	GCPhysApicBase = pMixedCtx->msrApicBase;
7102	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7103
7104	/* Unalias any existing mapping. */
7105	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7106	AssertRCReturn(rc, rc);
7107
7108	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7109	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7110	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7111	AssertRCReturn(rc, rc);
7112
7113	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7114	}
7115	#endif /* !IEM_VERIFICATION_MODE_FULL */
7116
7117	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7118	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
7119	pVmxTransient->uEflags = ASMIntDisableFlags();
7120	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7121	{
7122	ASMSetFlags(pVmxTransient->uEflags);
7123	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7124	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
7125	return VINF_EM_RAW_INTERRUPT;
7126	}
7127	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7128	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7129	#endif
7130
7131	/*
7132	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
7133	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
7134	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
7135	*/
7136	/** @todo Rework event evaluation and injection to be completely separate.
7137	* Update: Tried it, problem with handling halts. Control never returns to VT-x
7138	* if we exit VT-x with external interrupt pending in a TRPM event. */
7139	if (TRPMHasTrap(pVCpu))
7140	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7141
7142	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7143	AssertRCReturn(rc, rc);
7144	return rc;
7145	}
7146
7147
7148	/**
7149	* Prepares to run guest code in VT-x and we've committed to doing so. This
7150	* means there is no backing out to ring-3 or anywhere else at this
7151	* point.
7152	*
7153	* @param pVM Pointer to the VM.
7154	* @param pVCpu Pointer to the VMCPU.
7155	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7156	* out-of-sync. Make sure to update the required fields
7157	* before using them.
7158	* @param pVmxTransient Pointer to the VMX transient structure.
7159	*
7160	* @remarks Called with preemption disabled.
7161	* @remarks No-long-jump zone!!!
7162	*/
7163	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7164	{
7165	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7166	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7167
7168	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7169	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
7170	pVmxTransient->uEflags = ASMIntDisableFlags();
7171	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7172	#endif
7173
7174	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
7175	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7176	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7177	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7178	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
7179	#endif
7180	int rc = VINF_SUCCESS;
7181	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7182	{
7183	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7184	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7185	}
7186	else if (pVCpu->hm.s.fContextUseFlags)
7187	{
7188	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7189	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7190	}
7191	AssertRC(rc);
7192	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
7193
7194	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7195	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7196	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7197	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7198	#endif
7199
7200	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
7201	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7202	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7203
7204	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7205	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7206	{
7207	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7208	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7209	}
7210
7211	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7212	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7213	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
7214
7215	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7216
7217	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7218	to start executing. */
7219
7220	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7221	/*
7222	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7223	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7224	*/
7225	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7226	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7227	{
7228	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7229	uint64_t u64HostTscAux = 0;
7230	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7231	AssertRC(rc2);
7232	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7233	}
7234	#endif
7235	}
7236
7237
7238	/**
7239	* Performs some essential restoration of state after running guest code in
7240	* VT-x.
7241	*
7242	* @param pVM Pointer to the VM.
7243	* @param pVCpu Pointer to the VMCPU.
7244	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7245	* out-of-sync. Make sure to update the required fields
7246	* before using them.
7247	* @param pVmxTransient Pointer to the VMX transient structure.
7248	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7249	*
7250	* @remarks Called with interrupts disabled.
7251	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7252	* unconditionally when it is safe to do so.
7253	*/
7254	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7255	{
7256	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7257
7258	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7259	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7260	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7261	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7262	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7263
7264	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7265	{
7266	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7267	/* Restore host's TSC_AUX. */
7268	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7269	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7270	#endif
7271	/** @todo Find a way to fix hardcoding a guestimate. */
7272	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7273	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7274	}
7275
7276	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7277	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7278	Assert(!(ASMGetFlags() & X86_EFL_IF));
7279	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7280
7281	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7282	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7283
7284	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7285	uint32_t uExitReason;
7286	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7287	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7288	AssertRC(rc);
7289	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7290	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7291
7292	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7293	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7294
7295	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7296	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7297	{
7298	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7299	pVmxTransient->fVMEntryFailed));
7300	return;
7301	}
7302
7303	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7304	{
7305	/* Update the guest interruptibility-state from the VMCS. */
7306	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7307	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7308	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7309	AssertRC(rc);
7310	#endif
7311	/*
7312	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7313	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7314	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7315	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7316	*/
7317	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7318	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7319	{
7320	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7321	AssertRC(rc);
7322	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7323	}
7324	}
7325	}
7326
7327
7328
7329	/**
7330	* Runs the guest code using VT-x the normal way.
7331	*
7332	* @returns VBox status code.
7333	* @param pVM Pointer to the VM.
7334	* @param pVCpu Pointer to the VMCPU.
7335	* @param pCtx Pointer to the guest-CPU context.
7336	*
7337	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
7338	* @remarks Called with preemption disabled.
7339	*/
7340	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7341	{
7342	VMXTRANSIENT VmxTransient;
7343	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7344	int rc = VERR_INTERNAL_ERROR_5;
7345	uint32_t cLoops = 0;
7346
7347	for (;; cLoops++)
7348	{
7349	Assert(!HMR0SuspendPending());
7350	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7351	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7352	(unsigned)RTMpCpuId(), cLoops));
7353
7354	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7355	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7356	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7357	if (rc != VINF_SUCCESS)
7358	break;
7359
7360	/*
7361	* No longjmps to ring-3 from this point on!!!
7362	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7363	* This also disables flushing of the R0-logger instance (if any).
7364	*/
7365	VMMRZCallRing3Disable(pVCpu);
7366	VMMRZCallRing3RemoveNotification(pVCpu);
7367	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7368
7369	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7370	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7371
7372	/*
7373	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7374	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7375	*/
7376	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7377	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7378	{
7379	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7380	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7381	return rc;
7382	}
7383
7384	/* Handle the VM-exit. */
7385	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7386	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7387	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7388	HMVMX_START_EXIT_DISPATCH_PROF();
7389	#ifdef HMVMX_USE_FUNCTION_TABLE
7390	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7391	#else
7392	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7393	#endif
7394	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7395	if (rc != VINF_SUCCESS)
7396	break;
7397	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7398	{
7399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7400	rc = VINF_EM_RAW_INTERRUPT;
7401	break;
7402	}
7403	}
7404
7405	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7406	return rc;
7407	}
7408
7409
7410	/**
7411	* Single steps guest code using VT-x.
7412	*
7413	* @returns VBox status code.
7414	* @param pVM Pointer to the VM.
7415	* @param pVCpu Pointer to the VMCPU.
7416	* @param pCtx Pointer to the guest-CPU context.
7417	*
7418	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7419	* @remarks Called with preemption disabled.
7420	*/
7421	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7422	{
7423	VMXTRANSIENT VmxTransient;
7424	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7425	int rc = VERR_INTERNAL_ERROR_5;
7426	uint32_t cLoops = 0;
7427	uint16_t uCsStart = pCtx->cs.Sel;
7428	uint64_t uRipStart = pCtx->rip;
7429
7430	for (;; cLoops++)
7431	{
7432	Assert(!HMR0SuspendPending());
7433	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7434	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7435	(unsigned)RTMpCpuId(), cLoops));
7436
7437	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7438	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7439	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7440	if (rc != VINF_SUCCESS)
7441	break;
7442
7443	/*
7444	* No longjmps to ring-3 from this point on!!!
7445	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7446	* This also disables flushing of the R0-logger instance (if any).
7447	*/
7448	VMMRZCallRing3Disable(pVCpu);
7449	VMMRZCallRing3RemoveNotification(pVCpu);
7450	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7451
7452	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7453	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7454
7455	/*
7456	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7457	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7458	*/
7459	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7460	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7461	{
7462	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7463	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7464	return rc;
7465	}
7466
7467	/* Handle the VM-exit. */
7468	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7469	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7470	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7471	HMVMX_START_EXIT_DISPATCH_PROF();
7472	#ifdef HMVMX_USE_FUNCTION_TABLE
7473	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7474	#else
7475	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7476	#endif
7477	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7478	if (rc != VINF_SUCCESS)
7479	break;
7480	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7481	{
7482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7483	rc = VINF_EM_RAW_INTERRUPT;
7484	break;
7485	}
7486
7487	/*
7488	* Did the RIP change, if so, consider it a single step.
7489	* Otherwise, make sure one of the TFs gets set.
7490	*/
7491	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7492	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7493	AssertRCReturn(rc2, rc2);
7494	if ( pCtx->rip != uRipStart
7495	\|\| pCtx->cs.Sel != uCsStart)
7496	{
7497	rc = VINF_EM_DBG_STEPPED;
7498	break;
7499	}
7500	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7501	}
7502
7503	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7504	return rc;
7505	}
7506
7507
7508	/**
7509	* Runs the guest code using VT-x.
7510	*
7511	* @returns VBox status code.
7512	* @param pVM Pointer to the VM.
7513	* @param pVCpu Pointer to the VMCPU.
7514	* @param pCtx Pointer to the guest-CPU context.
7515	*
7516	* @remarks Called with preemption disabled.
7517	*/
7518	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7519	{
7520	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7521	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7522
7523	int rc;
7524	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7525	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7526	else
7527	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7528
7529	if (rc == VERR_EM_INTERPRETER)
7530	rc = VINF_EM_RAW_EMULATE_INSTR;
7531	else if (rc == VINF_EM_RESET)
7532	rc = VINF_EM_TRIPLE_FAULT;
7533	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7534	return rc;
7535	}
7536
7537
7538	#ifndef HMVMX_USE_FUNCTION_TABLE
7539	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7540	{
7541	int rc;
7542	switch (rcReason)
7543	{
7544	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7545	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7546	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7547	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7548	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7549	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7550	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7551	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7552	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7553	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7554	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7555	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7556	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7557	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7558	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7559	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7560	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7561	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7562	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7563	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7564	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7565	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7566	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7567	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7568	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7569	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7570	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7571	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7572	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7573	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7574	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7575	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7576	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7577
7578	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7579	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7580	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7581	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7582	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7583	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7584	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7585	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7586	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7587
7588	case VMX_EXIT_VMCALL:
7589	case VMX_EXIT_VMCLEAR:
7590	case VMX_EXIT_VMLAUNCH:
7591	case VMX_EXIT_VMPTRLD:
7592	case VMX_EXIT_VMPTRST:
7593	case VMX_EXIT_VMREAD:
7594	case VMX_EXIT_VMRESUME:
7595	case VMX_EXIT_VMWRITE:
7596	case VMX_EXIT_VMXOFF:
7597	case VMX_EXIT_VMXON:
7598	case VMX_EXIT_INVEPT:
7599	case VMX_EXIT_INVVPID:
7600	case VMX_EXIT_VMFUNC:
7601	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7602	break;
7603	default:
7604	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7605	break;
7606	}
7607	return rc;
7608	}
7609	#endif
7610
7611	#ifdef DEBUG
7612	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7613	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7614	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7615
7616	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7617	do \
7618	{ \
7619	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7620	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7621	} while (0)
7622
7623	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7624	do { \
7625	AssertPtr(pVCpu); \
7626	AssertPtr(pMixedCtx); \
7627	AssertPtr(pVmxTransient); \
7628	Assert(pVmxTransient->fVMEntryFailed == false); \
7629	Assert(ASMIntAreEnabled()); \
7630	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7631	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7632	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7633	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7634	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7635	HMVMX_ASSERT_PREEMPT_CPUID(); \
7636	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7637	} while (0)
7638
7639	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7640	do { \
7641	Log4Func(("\n")); \
7642	} while(0)
7643	#else /* Release builds */
7644	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7645	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7646	#endif
7647
7648
7649	/**
7650	* Advances the guest RIP after reading it from the VMCS.
7651	*
7652	* @returns VBox status code.
7653	* @param pVCpu Pointer to the VMCPU.
7654	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7655	* out-of-sync. Make sure to update the required fields
7656	* before using them.
7657	* @param pVmxTransient Pointer to the VMX transient structure.
7658	*
7659	* @remarks No-long-jump zone!!!
7660	*/
7661	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7662	{
7663	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7664	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7665	AssertRCReturn(rc, rc);
7666
7667	pMixedCtx->rip += pVmxTransient->cbInstr;
7668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7669	return rc;
7670	}
7671
7672
7673	/**
7674	* Tries to determine what part of the guest-state VT-x has deemed as invalid
7675	* and update error record fields accordingly.
7676	*
7677	* @return VMX_IGS_* return codes.
7678	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7679	* wrong with the guest state.
7680	*
7681	* @param pVM Pointer to the VM.
7682	* @param pVCpu Pointer to the VMCPU.
7683	* @param pCtx Pointer to the guest-CPU state.
7684	*/
7685	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7686	{
7687	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7688	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7689	uError = (err); \
7690	break; \
7691	} else do {} while (0)
7692	/* Duplicate of IEM_IS_CANONICAL(). */
7693	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7694
7695	int rc;
7696	uint64_t u64Val;
7697	uint32_t u32Val;
7698	uint32_t uError = VMX_IGS_ERROR;
7699	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7700
7701	do
7702	{
7703	/*
7704	* CR0.
7705	*/
7706	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7707	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7708	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7709	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7710	if (fUnrestrictedGuest)
7711	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
7712
7713	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7714	AssertRCBreak(rc);
7715	HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7716	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7717	if ( !fUnrestrictedGuest
7718	&& (u32Val & X86_CR0_PG)
7719	&& !(u32Val & X86_CR0_PE))
7720	{
7721	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7722	}
7723
7724	/*
7725	* CR4.
7726	*/
7727	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7728	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7729	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7730	AssertRCBreak(rc);
7731	HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7732	HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7733
7734	/*
7735	* IA32_DEBUGCTL MSR.
7736	*/
7737	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7738	AssertRCBreak(rc);
7739	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7740	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7741	{
7742	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7743	}
7744	uint64_t u64DebugCtlMsr = u64Val;
7745
7746	#ifdef VBOX_STRICT
7747	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7748	AssertRCBreak(rc);
7749	Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7750	#endif
7751	bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7752
7753	/*
7754	* RIP and RFLAGS.
7755	*/
7756	uint32_t u32Eflags;
7757	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7758	if (HMVMX_IS_64BIT_HOST_MODE())
7759	{
7760	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7761	AssertRCBreak(rc);
7762	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7763	if ( !fLongModeGuest
7764	\|\| !pCtx->cs.Attr.n.u1Long)
7765	{
7766	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7767	}
7768	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7769	* must be identical if the "IA32e mode guest" VM-entry control is 1
7770	* and CS.L is 1. No check applies if the CPU supports 64
7771	* linear-address bits. */
7772
7773	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7774	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7775	AssertRCBreak(rc);
7776	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7777	VMX_IGS_RFLAGS_RESERVED);
7778	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7779	u32Eflags = u64Val;
7780	}
7781	else
7782	#endif
7783	{
7784	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
7785	AssertRCBreak(rc);
7786	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
7787	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7788	}
7789
7790	if ( fLongModeGuest
7791	\|\| !(pCtx->cr0 & X86_CR0_PE))
7792	{
7793	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7794	}
7795
7796	uint32_t u32EntryInfo;
7797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7798	AssertRCBreak(rc);
7799	if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7800	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7801	{
7802	HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7803	}
7804
7805	/*
7806	* 64-bit checks.
7807	*/
7808	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7809	if (HMVMX_IS_64BIT_HOST_MODE())
7810	{
7811	if ( fLongModeGuest
7812	&& !fUnrestrictedGuest)
7813	{
7814	HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
7815	HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
7816	}
7817
7818	if ( !fLongModeGuest
7819	&& (pCtx->cr4 & X86_CR4_PCIDE))
7820	{
7821	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7822	}
7823
7824	/** @todo CR3 field must be such that bits 63:52 and bits in the range
7825	* 51:32 beyond the processor's physical-address width are 0. */
7826
7827	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7828	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7829	{
7830	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7831	}
7832
7833	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
7834	AssertRCBreak(rc);
7835	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
7836
7837	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
7838	AssertRCBreak(rc);
7839	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
7840	}
7841	#endif
7842
7843	/*
7844	* PERF_GLOBAL MSR.
7845	*/
7846	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
7847	{
7848	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
7849	AssertRCBreak(rc);
7850	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
7851	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
7852	}
7853
7854	/*
7855	* PAT MSR.
7856	*/
7857	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
7858	{
7859	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
7860	AssertRCBreak(rc);
7861	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
7862	for (unsigned i = 0; i < 8; i++)
7863	{
7864	uint8_t u8Val = (u64Val & 0x7);
7865	if ( u8Val != 0 /* UC */
7866	\|\| u8Val != 1 /* WC */
7867	\|\| u8Val != 4 /* WT */
7868	\|\| u8Val != 5 /* WP */
7869	\|\| u8Val != 6 /* WB */
7870	\|\| u8Val != 7 /* UC- */)
7871	{
7872	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
7873	}
7874	u64Val >>= 3;
7875	}
7876	}
7877
7878	/*
7879	* EFER MSR.
7880	*/
7881	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
7882	{
7883	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
7884	AssertRCBreak(rc);
7885	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
7886	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
7887	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
7888	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
7889	HMVMX_CHECK_BREAK( fUnrestrictedGuest
7890	\|\| (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
7891	}
7892
7893	/*
7894	* Segment registers.
7895	*/
7896	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
7897	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
7898	if (!(u32Eflags & X86_EFL_VM))
7899	{
7900	/* CS */
7901	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
7902	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
7903	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
7904	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
7905	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7906	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
7907	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
7908	/* CS cannot be loaded with NULL in protected mode. */
7909	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
7910	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
7911	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
7912	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
7913	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
7914	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
7915	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
7916	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
7917	else
7918	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
7919
7920	/* SS */
7921	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7922	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
7923	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
7924	if ( !(pCtx->cr0 & X86_CR0_PE)
7925	\|\| pCtx->cs.Attr.n.u4Type == 3)
7926	{
7927	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
7928	}
7929	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
7930	{
7931	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
7932	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
7933	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
7934	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
7935	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
7936	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7937	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
7938	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
7939	}
7940
7941	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
7942	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
7943	{
7944	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
7945	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
7946	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7947	\|\| pCtx->ds.Attr.n.u4Type > 11
7948	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7949	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
7950	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
7951	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
7952	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7953	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
7954	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
7955	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7956	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
7957	}
7958	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
7959	{
7960	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
7961	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
7962	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7963	\|\| pCtx->es.Attr.n.u4Type > 11
7964	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
7965	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
7966	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
7967	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
7968	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7969	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
7970	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
7971	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7972	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
7973	}
7974	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
7975	{
7976	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
7977	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
7978	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7979	\|\| pCtx->fs.Attr.n.u4Type > 11
7980	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
7981	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
7982	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
7983	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
7984	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7985	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
7986	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
7987	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
7988	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
7989	}
7990	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
7991	{
7992	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
7993	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
7994	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
7995	\|\| pCtx->gs.Attr.n.u4Type > 11
7996	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
7997	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
7998	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
7999	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8000	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8001	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8002	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8003	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8004	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8005	}
8006	/* 64-bit capable CPUs. */
8007	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8008	if (HMVMX_IS_64BIT_HOST_MODE())
8009	{
8010	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8011	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8012	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8013	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8014	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8015	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8016	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8017	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8018	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8019	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8020	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8021	}
8022	#endif
8023	}
8024	else
8025	{
8026	/* V86 mode checks. */
8027	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8028	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8029	{
8030	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8031	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8032	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8033	}
8034	else
8035	{
8036	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8037	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8038	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8039	}
8040
8041	/* CS */
8042	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8043	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8044	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8045	/* SS */
8046	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8047	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8048	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8049	/* DS */
8050	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8051	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8052	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8053	/* ES */
8054	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8055	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8056	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8057	/* FS */
8058	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8059	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8060	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8061	/* GS */
8062	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8063	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8064	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8065	/* 64-bit capable CPUs. */
8066	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8067	if (HMVMX_IS_64BIT_HOST_MODE())
8068	{
8069	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8070	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8071	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8072	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8073	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8074	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8075	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8076	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8077	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8078	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8079	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8080	}
8081	#endif
8082	}
8083
8084	/*
8085	* TR.
8086	*/
8087	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8088	/* 64-bit capable CPUs. */
8089	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8090	if (HMVMX_IS_64BIT_HOST_MODE())
8091	{
8092	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8093	}
8094	#endif
8095	if (fLongModeGuest)
8096	{
8097	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8098	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8099	}
8100	else
8101	{
8102	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8103	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8104	VMX_IGS_TR_ATTR_TYPE_INVALID);
8105	}
8106	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8107	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8108	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8109	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8110	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8111	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8112	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8113	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8114
8115	/*
8116	* GDTR and IDTR.
8117	*/
8118	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8119	if (HMVMX_IS_64BIT_HOST_MODE())
8120	{
8121	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8122	AssertRCBreak(rc);
8123	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8124
8125	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8126	AssertRCBreak(rc);
8127	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8128	}
8129	#endif
8130
8131	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8132	AssertRCBreak(rc);
8133	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8134
8135	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8136	AssertRCBreak(rc);
8137	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8138
8139	/*
8140	* Guest Non-Register State.
8141	*/
8142	/* Activity State. */
8143	uint32_t u32ActivityState;
8144	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8145	AssertRCBreak(rc);
8146	HMVMX_CHECK_BREAK( !u32ActivityState
8147	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.msr.vmx_misc)),
8148	VMX_IGS_ACTIVITY_STATE_INVALID);
8149	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8150	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8151	uint32_t u32IntrState;
8152	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8153	AssertRCBreak(rc);
8154	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8155	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8156	{
8157	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8158	}
8159
8160	/** @todo Activity state and injecting interrupts. Left as a todo since we
8161	* currently don't use activity states but ACTIVE. */
8162
8163	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8164	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8165
8166	/* Guest interruptibility-state. */
8167	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8168	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8169	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8170	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8171	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8172	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8173	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8174	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8175	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8176	if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8177	{
8178	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8179	{
8180	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8181	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8182	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8183	}
8184	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8185	{
8186	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8187	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8188	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8189	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8190	}
8191	}
8192	/** @todo Assumes the processor is not in SMM. */
8193	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8194	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8195	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8196	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8197	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8198	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8199	&& VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8200	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8201	{
8202	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8203	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8204	}
8205
8206	/* Pending debug exceptions. */
8207	if (HMVMX_IS_64BIT_HOST_MODE())
8208	{
8209	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8210	AssertRCBreak(rc);
8211	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8212	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8213	u32Val = u64Val; /* For pending debug exceptions checks below. */
8214	}
8215	else
8216	{
8217	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8218	AssertRCBreak(rc);
8219	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8220	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8221	}
8222
8223	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8224	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8225	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8226	{
8227	if ( (u32Eflags & X86_EFL_TF)
8228	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8229	{
8230	/* Bit 14 is PendingDebug.BS. */
8231	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8232	}
8233	if ( !(u32Eflags & X86_EFL_TF)
8234	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8235	{
8236	/* Bit 14 is PendingDebug.BS. */
8237	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8238	}
8239	}
8240
8241	/* VMCS link pointer. */
8242	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8243	AssertRCBreak(rc);
8244	if (u64Val != UINT64_C(0xffffffffffffffff))
8245	{
8246	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8247	/** @todo Bits beyond the processor's physical-address width MBZ. */
8248	/** @todo 32-bit located in memory referenced by value of this field (as a
8249	* physical address) must contain the processor's VMCS revision ID. */
8250	/** @todo SMM checks. */
8251	}
8252
8253	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8254
8255	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8256	if (uError == VMX_IGS_ERROR)
8257	uError = VMX_IGS_REASON_NOT_FOUND;
8258	} while (0);
8259
8260	pVCpu->hm.s.u32HMError = uError;
8261	return uError;
8262
8263	#undef HMVMX_ERROR_BREAK
8264	#undef HMVMX_CHECK_BREAK
8265	#undef HMVMX_IS_CANONICAL
8266	}
8267
8268	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8269	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8270	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8271
8272	/** @name VM-exit handlers.
8273	* @{
8274	*/
8275
8276	/**
8277	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8278	*/
8279	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8280	{
8281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8283	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8284	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
8285	Assert(ASMIntAreEnabled());
8286	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8287	return VINF_SUCCESS;
8288	#endif
8289	return VINF_EM_RAW_INTERRUPT;
8290	}
8291
8292
8293	/**
8294	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8295	*/
8296	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8297	{
8298	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8299	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8300
8301	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8302	AssertRCReturn(rc, rc);
8303
8304	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8305	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8306	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8307	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8308
8309	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8310	{
8311	/*
8312	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8313	* anything we inject is not going to cause a VM-exit directly for the event being injected.
8314	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8315	*
8316	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8317	*/
8318	VMXDispatchHostNmi();
8319	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8320	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8321	return VINF_SUCCESS;
8322	}
8323
8324	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8325	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8326	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8327	{
8328	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8329	return VINF_SUCCESS;
8330	}
8331	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8332	{
8333	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8334	return rc;
8335	}
8336
8337	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8338	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8339	switch (uIntrType)
8340	{
8341	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8342	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8343	/* no break */
8344	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8345	{
8346	switch (uVector)
8347	{
8348	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8349	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8350	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8351	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8352	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8353	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8354	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8355	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8356	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8357	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8358	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8359	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8360	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8361	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8362	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8363	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8364	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8365	#endif
8366	default:
8367	{
8368	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8369	AssertRCReturn(rc, rc);
8370
8371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8372	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8373	{
8374	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8375	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8376	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8377	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8378	AssertRCReturn(rc, rc);
8379	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8380	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8381	0 /* GCPtrFaultAddress */);
8382	AssertRCReturn(rc, rc);
8383	}
8384	else
8385	{
8386	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8387	pVCpu->hm.s.u32HMError = uVector;
8388	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8389	}
8390	break;
8391	}
8392	}
8393	break;
8394	}
8395
8396	default:
8397	{
8398	pVCpu->hm.s.u32HMError = uExitIntrInfo;
8399	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8400	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8401	break;
8402	}
8403	}
8404	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8405	return rc;
8406	}
8407
8408
8409	/**
8410	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8411	*/
8412	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8413	{
8414	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8415
8416	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8417	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8418	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8419	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8420	AssertRCReturn(rc, rc);
8421
8422	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8424	return VINF_SUCCESS;
8425	}
8426
8427
8428	/**
8429	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8430	*/
8431	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8432	{
8433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8434	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8435	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8436	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8437	}
8438
8439
8440	/**
8441	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8442	*/
8443	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8444	{
8445	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8447	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8448	}
8449
8450
8451	/**
8452	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8453	*/
8454	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8455	{
8456	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8457	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8458	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8459	}
8460
8461
8462	/**
8463	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8464	*/
8465	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8466	{
8467	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8468	PVM pVM = pVCpu->CTX_SUFF(pVM);
8469	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8470	if (RT_LIKELY(rc == VINF_SUCCESS))
8471	{
8472	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8473	Assert(pVmxTransient->cbInstr == 2);
8474	}
8475	else
8476	{
8477	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8478	rc = VERR_EM_INTERPRETER;
8479	}
8480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8481	return rc;
8482	}
8483
8484
8485	/**
8486	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8487	*/
8488	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8489	{
8490	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8491	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8492	AssertRCReturn(rc, rc);
8493
8494	if (pMixedCtx->cr4 & X86_CR4_SMXE)
8495	return VINF_EM_RAW_EMULATE_INSTR;
8496
8497	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8498	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8499	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8500	}
8501
8502
8503	/**
8504	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8505	*/
8506	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8507	{
8508	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8509	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8510	AssertRCReturn(rc, rc);
8511
8512	PVM pVM = pVCpu->CTX_SUFF(pVM);
8513	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8514	if (RT_LIKELY(rc == VINF_SUCCESS))
8515	{
8516	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8517	Assert(pVmxTransient->cbInstr == 2);
8518	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8519	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8520	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8521	}
8522	else
8523	{
8524	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8525	rc = VERR_EM_INTERPRETER;
8526	}
8527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8528	return rc;
8529	}
8530
8531
8532	/**
8533	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8534	*/
8535	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8536	{
8537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8538	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8539	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8540	AssertRCReturn(rc, rc);
8541
8542	PVM pVM = pVCpu->CTX_SUFF(pVM);
8543	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8544	if (RT_LIKELY(rc == VINF_SUCCESS))
8545	{
8546	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8547	Assert(pVmxTransient->cbInstr == 3);
8548	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8549	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8550	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8551	}
8552	else
8553	{
8554	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8555	rc = VERR_EM_INTERPRETER;
8556	}
8557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8558	return rc;
8559	}
8560
8561
8562	/**
8563	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8564	*/
8565	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8566	{
8567	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8568	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8569	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8570	AssertRCReturn(rc, rc);
8571
8572	PVM pVM = pVCpu->CTX_SUFF(pVM);
8573	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8574	if (RT_LIKELY(rc == VINF_SUCCESS))
8575	{
8576	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8577	Assert(pVmxTransient->cbInstr == 2);
8578	}
8579	else
8580	{
8581	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8582	rc = VERR_EM_INTERPRETER;
8583	}
8584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8585	return rc;
8586	}
8587
8588
8589	/**
8590	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8591	*/
8592	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8593	{
8594	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8595	PVM pVM = pVCpu->CTX_SUFF(pVM);
8596	Assert(!pVM->hm.s.fNestedPaging);
8597
8598	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8599	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8600	AssertRCReturn(rc, rc);
8601
8602	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8603	rc = VBOXSTRICTRC_VAL(rc2);
8604	if (RT_LIKELY(rc == VINF_SUCCESS))
8605	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8606	else
8607	{
8608	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8609	pVmxTransient->uExitQualification, rc));
8610	}
8611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8612	return rc;
8613	}
8614
8615
8616	/**
8617	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8618	*/
8619	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8620	{
8621	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8622	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8623	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8624	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8625	AssertRCReturn(rc, rc);
8626
8627	PVM pVM = pVCpu->CTX_SUFF(pVM);
8628	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8629	if (RT_LIKELY(rc == VINF_SUCCESS))
8630	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8631	else
8632	{
8633	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8634	rc = VERR_EM_INTERPRETER;
8635	}
8636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8637	return rc;
8638	}
8639
8640
8641	/**
8642	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8643	*/
8644	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8645	{
8646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8647	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8648	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8649	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8650	AssertRCReturn(rc, rc);
8651
8652	PVM pVM = pVCpu->CTX_SUFF(pVM);
8653	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8654	rc = VBOXSTRICTRC_VAL(rc2);
8655	if (RT_LIKELY( rc == VINF_SUCCESS
8656	\|\| rc == VINF_EM_HALT))
8657	{
8658	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8659	AssertRCReturn(rc3, rc3);
8660
8661	if ( rc == VINF_EM_HALT
8662	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8663	{
8664	rc = VINF_SUCCESS;
8665	}
8666	}
8667	else
8668	{
8669	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8670	rc = VERR_EM_INTERPRETER;
8671	}
8672	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
8673	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8675	return rc;
8676	}
8677
8678
8679	/**
8680	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8681	*/
8682	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8683	{
8684	/*
8685	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8686	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8687	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
8688	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8689	*/
8690	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8691	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8692	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8693	}
8694
8695
8696	/**
8697	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8698	*/
8699	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8700	{
8701	/*
8702	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8703	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8704	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
8705	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8706	*/
8707	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8708	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8709	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8710	}
8711
8712
8713	/**
8714	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8715	*/
8716	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8717	{
8718	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8719	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8720	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8721	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8722	}
8723
8724
8725	/**
8726	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8727	*/
8728	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8729	{
8730	/*
8731	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8732	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8733	* See Intel spec. 25.3 "Other Causes of VM-exits".
8734	*/
8735	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8736	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8737	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8738	}
8739
8740
8741	/**
8742	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8743	* VM-exit.
8744	*/
8745	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8746	{
8747	/*
8748	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8749	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8750	*
8751	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8752	* See Intel spec. "23.8 Restrictions on VMX operation".
8753	*/
8754	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8755	return VINF_SUCCESS;
8756	}
8757
8758
8759	/**
8760	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8761	* VM-exit.
8762	*/
8763	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8764	{
8765	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8766	return VINF_EM_RESET;
8767	}
8768
8769
8770	/**
8771	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8772	*/
8773	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8774	{
8775	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8776	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8777	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8778	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8779	AssertRCReturn(rc, rc);
8780
8781	pMixedCtx->rip++;
8782	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8783	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
8784	rc = VINF_SUCCESS;
8785	else
8786	rc = VINF_EM_HALT;
8787
8788	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8789	return rc;
8790	}
8791
8792
8793	/**
8794	* VM-exit handler for instructions that result in a #UD exception delivered to
8795	* the guest.
8796	*/
8797	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8798	{
8799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8800	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
8801	return VINF_SUCCESS;
8802	}
8803
8804
8805	/**
8806	* VM-exit handler for expiry of the VMX preemption timer.
8807	*/
8808	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8809	{
8810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8811
8812	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
8813	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8814
8815	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8816	PVM pVM = pVCpu->CTX_SUFF(pVM);
8817	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8818	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
8819	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8820	}
8821
8822
8823	/**
8824	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8825	*/
8826	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8827	{
8828	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8829
8830	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
8831	/** @todo check if XSETBV is supported by the recompiler. */
8832	return VERR_EM_INTERPRETER;
8833	}
8834
8835
8836	/**
8837	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
8838	*/
8839	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8840	{
8841	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8842
8843	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
8844	/** @todo implement EMInterpretInvpcid() */
8845	return VERR_EM_INTERPRETER;
8846	}
8847
8848
8849	/**
8850	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
8851	* Error VM-exit.
8852	*/
8853	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8854	{
8855	uint32_t uIntrState;
8856	HMVMXHCUINTREG uHCReg;
8857	uint64_t u64Val;
8858	uint32_t u32Val;
8859
8860	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
8861	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
8862	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
8863	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
8864	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8865	AssertRCReturn(rc, rc);
8866
8867	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
8868	NOREF(uInvalidReason);
8869
8870	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
8871	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
8872	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
8873	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
8874
8875	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
8876	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
8877	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
8878	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
8879	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
8880	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8881	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
8882	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
8883	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
8884	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
8885	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
8886	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
8887
8888	PVM pVM = pVCpu->CTX_SUFF(pVM);
8889	HMDumpRegs(pVM, pVCpu, pMixedCtx);
8890
8891	return VERR_VMX_INVALID_GUEST_STATE;
8892	}
8893
8894
8895	/**
8896	* VM-exit handler for VM-entry failure due to an MSR-load
8897	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
8898	*/
8899	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8900	{
8901	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8902	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8903	}
8904
8905
8906	/**
8907	* VM-exit handler for VM-entry failure due to a machine-check event
8908	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
8909	*/
8910	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8911	{
8912	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8913	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8914	}
8915
8916
8917	/**
8918	* VM-exit handler for all undefined reasons. Should never ever happen.. in
8919	* theory.
8920	*/
8921	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8922	{
8923	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
8924	return VERR_VMX_UNDEFINED_EXIT_CODE;
8925	}
8926
8927
8928	/**
8929	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
8930	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
8931	* Conditional VM-exit.
8932	*/
8933	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8934	{
8935	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8936
8937	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
8938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
8939	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
8940	return VERR_EM_INTERPRETER;
8941	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8942	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8943	}
8944
8945
8946	/**
8947	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
8948	*/
8949	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8950	{
8951	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8952
8953	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
8954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
8955	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
8956	return VERR_EM_INTERPRETER;
8957	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8958	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8959	}
8960
8961
8962	/**
8963	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8964	*/
8965	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8966	{
8967	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8968
8969	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8970	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8971	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8972	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8973	AssertRCReturn(rc, rc);
8974
8975	PVM pVM = pVCpu->CTX_SUFF(pVM);
8976	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8977	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8978	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8980
8981	if (RT_LIKELY(rc == VINF_SUCCESS))
8982	{
8983	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8984	Assert(pVmxTransient->cbInstr == 2);
8985	}
8986	return rc;
8987	}
8988
8989
8990	/**
8991	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8992	*/
8993	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8994	{
8995	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8996	PVM pVM = pVCpu->CTX_SUFF(pVM);
8997	int rc = VINF_SUCCESS;
8998
8999	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9000	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9001	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9002	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9003	AssertRCReturn(rc, rc);
9004	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9005
9006	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9007	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9008	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9009
9010	if (RT_LIKELY(rc == VINF_SUCCESS))
9011	{
9012	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9013
9014	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9015	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9016	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9017	{
9018	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9019	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9020	EMInterpretWrmsr() changes it. */
9021	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9022	}
9023	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9024	{
9025	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9026	AssertRCReturn(rc, rc);
9027	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9028	}
9029	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9030	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9031
9032	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9033	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9034	{
9035	switch (pMixedCtx->ecx)
9036	{
9037	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9038	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9039	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9040	case MSR_K8_FS_BASE: /* no break */
9041	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9042	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9043	}
9044	}
9045	#ifdef VBOX_STRICT
9046	else
9047	{
9048	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9049	switch (pMixedCtx->ecx)
9050	{
9051	case MSR_IA32_SYSENTER_CS:
9052	case MSR_IA32_SYSENTER_EIP:
9053	case MSR_IA32_SYSENTER_ESP:
9054	case MSR_K8_FS_BASE:
9055	case MSR_K8_GS_BASE:
9056	{
9057	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9058	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9059	}
9060
9061	case MSR_K8_LSTAR:
9062	case MSR_K6_STAR:
9063	case MSR_K8_SF_MASK:
9064	case MSR_K8_TSC_AUX:
9065	case MSR_K8_KERNEL_GS_BASE:
9066	{
9067	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9068	pMixedCtx->ecx));
9069	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9070	}
9071	}
9072	}
9073	#endif /* VBOX_STRICT */
9074	}
9075	return rc;
9076	}
9077
9078
9079	/**
9080	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9081	*/
9082	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9083	{
9084	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9085
9086	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9088	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9089	return VERR_EM_INTERPRETER;
9090	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9091	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9092	}
9093
9094
9095	/**
9096	* VM-exit handler for when the TPR value is lowered below the specified
9097	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9098	*/
9099	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9100	{
9101	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9102	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9103
9104	/*
9105	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9106	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9107	* resume guest execution.
9108	*/
9109	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9111	return VINF_SUCCESS;
9112	}
9113
9114
9115	/**
9116	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9117	* VM-exit.
9118	*
9119	* @retval VINF_SUCCESS when guest execution can continue.
9120	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9121	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9122	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9123	* recompiler.
9124	*/
9125	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9126	{
9127	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9128	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9129	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9130	AssertRCReturn(rc, rc);
9131
9132	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9133	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9134	PVM pVM = pVCpu->CTX_SUFF(pVM);
9135	switch (uAccessType)
9136	{
9137	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9138	{
9139	#if 0
9140	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9141	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9142	#else
9143	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9144	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9145	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9146	#endif
9147	AssertRCReturn(rc, rc);
9148
9149	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9150	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9151	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9152	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9153
9154	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9155	{
9156	case 0: /* CR0 */
9157	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9158	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9159	break;
9160	case 2: /* C2 **/
9161	/* Nothing to do here, CR2 it's not part of the VMCS. */
9162	break;
9163	case 3: /* CR3 */
9164	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9165	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9166	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
9167	break;
9168	case 4: /* CR4 */
9169	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9170	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
9171	break;
9172	case 8: /* CR8 */
9173	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9174	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9175	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
9176	break;
9177	default:
9178	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9179	break;
9180	}
9181
9182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9183	break;
9184	}
9185
9186	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9187	{
9188	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9189	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9190	AssertRCReturn(rc, rc);
9191	Assert( !pVM->hm.s.fNestedPaging
9192	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9193	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9194
9195	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9196	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9197	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9198
9199	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9200	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9201	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9202	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9204	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9205	break;
9206	}
9207
9208	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9209	{
9210	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9211	AssertRCReturn(rc, rc);
9212	rc = EMInterpretCLTS(pVM, pVCpu);
9213	AssertRCReturn(rc, rc);
9214	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9216	Log4(("CRX CLTS write rc=%d\n", rc));
9217	break;
9218	}
9219
9220	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9221	{
9222	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9223	AssertRCReturn(rc, rc);
9224	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9225	if (RT_LIKELY(rc == VINF_SUCCESS))
9226	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9228	Log4(("CRX LMSW write rc=%d\n", rc));
9229	break;
9230	}
9231
9232	default:
9233	{
9234	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9235	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9236	}
9237	}
9238
9239	/* Validate possible error codes. */
9240	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9241	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9242	if (RT_SUCCESS(rc))
9243	{
9244	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9245	AssertRCReturn(rc2, rc2);
9246	}
9247
9248	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9249	return rc;
9250	}
9251
9252
9253	/**
9254	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9255	* VM-exit.
9256	*/
9257	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9258	{
9259	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9260	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9261
9262	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9263	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9264	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9265	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9266	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9267	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9268	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9269	AssertRCReturn(rc2, rc2);
9270
9271	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9272	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9273	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9274	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9275	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9276	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9277	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9278
9279	/* I/O operation lookup arrays. */
9280	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9281	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9282
9283	VBOXSTRICTRC rcStrict;
9284	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9285	const uint32_t cbInstr = pVmxTransient->cbInstr;
9286	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9287	PVM pVM = pVCpu->CTX_SUFF(pVM);
9288	if (fIOString)
9289	{
9290	/*
9291	* INS/OUTS - I/O String instruction.
9292	*
9293	* Use instruction-information if available, otherwise fall back on
9294	* interpreting the instruction.
9295	*/
9296	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9297	#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9298	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9299	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
9300	{
9301	rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9302	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9303	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9304	AssertRCReturn(rc2, rc2);
9305	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9306	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9307	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9308	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9309	if (fIOWrite)
9310	{
9311	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9312	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9313	//if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9314	// hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9315	// pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9316	}
9317	else
9318	{
9319	AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9320	("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9321	VERR_HMVMX_IPE_4);
9322	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9323	//if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9324	// hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9325	}
9326	}
9327	else
9328	{
9329	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9330	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9331	AssertRCReturn(rc2, rc2);
9332	rcStrict = IEMExecOne(pVCpu);
9333	}
9334	/** @todo IEM needs to be setting these flags somehow. */
9335	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9336	fUpdateRipAlready = true;
9337
9338	#else
9339	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9340	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9341	if (RT_SUCCESS(rcStrict))
9342	{
9343	if (fIOWrite)
9344	{
9345	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9346	(DISCPUMODE)pDis->uAddrMode, cbValue);
9347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9348	}
9349	else
9350	{
9351	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9352	(DISCPUMODE)pDis->uAddrMode, cbValue);
9353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9354	}
9355	}
9356	else
9357	{
9358	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9359	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9360	}
9361	#endif
9362	}
9363	else
9364	{
9365	/*
9366	* IN/OUT - I/O instruction.
9367	*/
9368	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9369	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9370	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9371	if (fIOWrite)
9372	{
9373	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9374	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9375	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9377	}
9378	else
9379	{
9380	uint32_t u32Result = 0;
9381	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9382	if (IOM_SUCCESS(rcStrict))
9383	{
9384	/* Save result of I/O IN instr. in AL/AX/EAX. */
9385	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
9386	}
9387	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9388	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9390	}
9391	}
9392
9393	if (IOM_SUCCESS(rcStrict))
9394	{
9395	if (!fUpdateRipAlready)
9396	{
9397	pMixedCtx->rip += cbInstr;
9398	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9399	}
9400
9401	/*
9402	* If any I/O breakpoints are armed, we need to check if one triggered
9403	* and take appropriate action.
9404	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9405	*/
9406	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9407	AssertRCReturn(rc2, rc2);
9408
9409	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9410	* execution engines about whether hyper BPs and such are pending. */
9411	uint32_t const uDr7 = pMixedCtx->dr[7];
9412	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9413	&& X86_DR7_ANY_RW_IO(uDr7)
9414	&& (pMixedCtx->cr4 & X86_CR4_DE))
9415	\|\| DBGFBpIsHwIoArmed(pVM)))
9416	{
9417	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9418	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
9419
9420	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9421	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9422	{
9423	/* Raise #DB. */
9424	if (fIsGuestDbgActive)
9425	ASMSetDR6(pMixedCtx->dr[6]);
9426	if (pMixedCtx->dr[7] != uDr7)
9427	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9428
9429	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9430	}
9431	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9432	else if ( rcStrict2 != VINF_SUCCESS
9433	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
9434	rcStrict = rcStrict2;
9435	}
9436	}
9437
9438	#ifdef DEBUG
9439	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9440	Assert(!fIOWrite);
9441	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9442	Assert(fIOWrite);
9443	else
9444	{
9445	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9446	* statuses, that the VMM device and some others may return. See
9447	* IOM_SUCCESS() for guidance. */
9448	AssertMsg( RT_FAILURE(rcStrict)
9449	\|\| rcStrict == VINF_SUCCESS
9450	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
9451	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
9452	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
9453	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9454	}
9455	#endif
9456
9457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9458	return VBOXSTRICTRC_TODO(rcStrict);
9459	}
9460
9461
9462	/**
9463	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9464	* VM-exit.
9465	*/
9466	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9467	{
9468	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9469
9470	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
9471	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9472	AssertRCReturn(rc, rc);
9473	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9474	{
9475	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9476	AssertRCReturn(rc, rc);
9477	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9478	{
9479	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9480
9481	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9482	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9483	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9484	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9485	{
9486	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9487	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9488
9489	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9490	Assert(!pVCpu->hm.s.Event.fPending);
9491	pVCpu->hm.s.Event.fPending = true;
9492	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9493	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9494	AssertRCReturn(rc, rc);
9495	if (fErrorCodeValid)
9496	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9497	else
9498	pVCpu->hm.s.Event.u32ErrCode = 0;
9499	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9500	&& uVector == X86_XCPT_PF)
9501	{
9502	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9503	}
9504
9505	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9506	}
9507	}
9508	}
9509
9510	/** @todo Emulate task switch someday, currently just going back to ring-3 for
9511	* emulation. */
9512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9513	return VERR_EM_INTERPRETER;
9514	}
9515
9516
9517	/**
9518	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9519	*/
9520	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9521	{
9522	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9523	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9524	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9525	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9526	AssertRCReturn(rc, rc);
9527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9528	return VINF_EM_DBG_STEPPED;
9529	}
9530
9531
9532	/**
9533	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9534	*/
9535	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9536	{
9537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9538
9539	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9540	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9541	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9542	return VINF_SUCCESS;
9543	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9544	return rc;
9545
9546	#if 0
9547	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9548	* just sync the whole thing. */
9549	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9550	#else
9551	/* Aggressive state sync. for now. */
9552	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9553	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9554	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9555	#endif
9556	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9557	AssertRCReturn(rc, rc);
9558
9559	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9560	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9561	switch (uAccessType)
9562	{
9563	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9564	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9565	{
9566	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9567	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9568	{
9569	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9570	}
9571
9572	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9573	GCPhys &= PAGE_BASE_GC_MASK;
9574	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9575	PVM pVM = pVCpu->CTX_SUFF(pVM);
9576	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9577	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9578
9579	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9580	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9581	CPUMCTX2CORE(pMixedCtx), GCPhys);
9582	rc = VBOXSTRICTRC_VAL(rc2);
9583	Log4(("ApicAccess rc=%d\n", rc));
9584	if ( rc == VINF_SUCCESS
9585	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9586	\|\| rc == VERR_PAGE_NOT_PRESENT)
9587	{
9588	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9589	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9590	rc = VINF_SUCCESS;
9591	}
9592	break;
9593	}
9594
9595	default:
9596	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9597	rc = VINF_EM_RAW_EMULATE_INSTR;
9598	break;
9599	}
9600
9601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9602	return rc;
9603	}
9604
9605
9606	/**
9607	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9608	* VM-exit.
9609	*/
9610	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9611	{
9612	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9613
9614	/* We should -not- get this VM-exit if the guest is debugging. */
9615	if (CPUMIsGuestDebugStateActive(pVCpu))
9616	{
9617	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9618	return VERR_VMX_UNEXPECTED_EXIT_CODE;
9619	}
9620
9621	int rc = VERR_INTERNAL_ERROR_5;
9622	if ( !DBGFIsStepping(pVCpu)
9623	&& !pVCpu->hm.s.fSingleInstruction
9624	&& !CPUMIsHyperDebugStateActive(pVCpu))
9625	{
9626	/* Don't intercept MOV DRx and #DB any more. */
9627	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9628	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9629	AssertRCReturn(rc, rc);
9630
9631	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9632	{
9633	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9634	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9635	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9636	AssertRCReturn(rc, rc);
9637	#endif
9638	}
9639
9640	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9641	PVM pVM = pVCpu->CTX_SUFF(pVM);
9642	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9643	Assert(CPUMIsGuestDebugStateActive(pVCpu));
9644
9645	#ifdef VBOX_WITH_STATISTICS
9646	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9647	AssertRCReturn(rc, rc);
9648	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9650	else
9651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9652	#endif
9653	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9654	return VINF_SUCCESS;
9655	}
9656
9657	/*
9658	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9659	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9660	*/
9661	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9662	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9663	AssertRCReturn(rc, rc);
9664
9665	PVM pVM = pVCpu->CTX_SUFF(pVM);
9666	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9667	{
9668	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9669	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9670	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9671	if (RT_SUCCESS(rc))
9672	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
9673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9674	}
9675	else
9676	{
9677	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9678	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9679	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9681	}
9682
9683	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9684	if (RT_SUCCESS(rc))
9685	{
9686	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9687	AssertRCReturn(rc2, rc2);
9688	}
9689	return rc;
9690	}
9691
9692
9693	/**
9694	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9695	* Conditional VM-exit.
9696	*/
9697	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9698	{
9699	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9700	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9701
9702	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9703	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9704	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9705	return VINF_SUCCESS;
9706	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9707	return rc;
9708
9709	RTGCPHYS GCPhys = 0;
9710	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9711
9712	#if 0
9713	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9714	#else
9715	/* Aggressive state sync. for now. */
9716	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9717	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9718	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9719	#endif
9720	AssertRCReturn(rc, rc);
9721
9722	/*
9723	* If we succeed, resume guest execution.
9724	* If we fail in interpreting the instruction because we couldn't get the guest physical address
9725	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9726	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9727	* weird case. See @bugref{6043}.
9728	*/
9729	PVM pVM = pVCpu->CTX_SUFF(pVM);
9730	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9731	rc = VBOXSTRICTRC_VAL(rc2);
9732	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9733	if ( rc == VINF_SUCCESS
9734	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9735	\|\| rc == VERR_PAGE_NOT_PRESENT)
9736	{
9737	/* Successfully handled MMIO operation. */
9738	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9739	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9740	rc = VINF_SUCCESS;
9741	}
9742	return rc;
9743	}
9744
9745
9746	/**
9747	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9748	* VM-exit.
9749	*/
9750	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9751	{
9752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9753	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9754
9755	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9756	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9757	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9758	return VINF_SUCCESS;
9759	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9760	return rc;
9761
9762	RTGCPHYS GCPhys = 0;
9763	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9764	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9765	#if 0
9766	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9767	#else
9768	/* Aggressive state sync. for now. */
9769	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9770	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9771	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9772	#endif
9773	AssertRCReturn(rc, rc);
9774
9775	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
9776	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
9777
9778	RTGCUINT uErrorCode = 0;
9779	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
9780	uErrorCode \|= X86_TRAP_PF_ID;
9781	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
9782	uErrorCode \|= X86_TRAP_PF_RW;
9783	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
9784	uErrorCode \|= X86_TRAP_PF_P;
9785
9786	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9787
9788	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
9789	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9790
9791	/* Handle the pagefault trap for the nested shadow table. */
9792	PVM pVM = pVCpu->CTX_SUFF(pVM);
9793	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
9794	TRPMResetTrap(pVCpu);
9795
9796	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9797	if ( rc == VINF_SUCCESS
9798	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
9799	\|\| rc == VERR_PAGE_NOT_PRESENT)
9800	{
9801	/* Successfully synced our nested page tables. */
9802	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
9803	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9804	return VINF_SUCCESS;
9805	}
9806
9807	Log4(("EPT return to ring-3 rc=%d\n"));
9808	return rc;
9809	}
9810
9811	/** @} */
9812
9813	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9814	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
9815	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9816
9817	/** @name VM-exit exception handlers.
9818	* @{
9819	*/
9820
9821	/**
9822	* VM-exit exception handler for #MF (Math Fault: floating point exception).
9823	*/
9824	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9825	{
9826	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
9828
9829	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9830	AssertRCReturn(rc, rc);
9831
9832	if (!(pMixedCtx->cr0 & X86_CR0_NE))
9833	{
9834	/* Old-style FPU error reporting needs some extra work. */
9835	/** @todo don't fall back to the recompiler, but do it manually. */
9836	return VERR_EM_INTERPRETER;
9837	}
9838
9839	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9840	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9841	return rc;
9842	}
9843
9844
9845	/**
9846	* VM-exit exception handler for #BP (Breakpoint exception).
9847	*/
9848	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9849	{
9850	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
9852
9853	/** @todo Try optimize this by not saving the entire guest state unless
9854	* really needed. */
9855	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9856	AssertRCReturn(rc, rc);
9857
9858	PVM pVM = pVCpu->CTX_SUFF(pVM);
9859	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9860	if (rc == VINF_EM_RAW_GUEST_TRAP)
9861	{
9862	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9863	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9864	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9865	AssertRCReturn(rc, rc);
9866
9867	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9868	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9869	}
9870
9871	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
9872	return rc;
9873	}
9874
9875
9876	/**
9877	* VM-exit exception handler for #DB (Debug exception).
9878	*/
9879	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9880	{
9881	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
9883	Log6(("XcptDB\n"));
9884
9885	/*
9886	* Get the DR6-like values from the exit qualification and pass it to DBGF
9887	* for processing.
9888	*/
9889	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9890	AssertRCReturn(rc, rc);
9891
9892	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
9893	uint64_t uDR6 = X86_DR6_INIT_VAL;
9894	uDR6 \|= ( pVmxTransient->uExitQualification
9895	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
9896
9897	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
9898	if (rc == VINF_EM_RAW_GUEST_TRAP)
9899	{
9900	/*
9901	* The exception was for the guest. Update DR6, DR7.GD and
9902	* IA32_DEBUGCTL.LBR before forwarding it.
9903	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
9904	*/
9905	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
9906	pMixedCtx->dr[6] \|= uDR6;
9907	if (CPUMIsGuestDebugStateActive(pVCpu))
9908	ASMSetDR6(pMixedCtx->dr[6]);
9909
9910	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9911	AssertRCReturn(rc, rc);
9912
9913	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
9914	pMixedCtx->dr[7] &= ~X86_DR7_GD;
9915
9916	/* Paranoia. */
9917	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
9918	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
9919
9920	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
9921	AssertRCReturn(rc, rc);
9922
9923	/*
9924	* Raise #DB in the guest.
9925	*/
9926	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9927	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9928	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9929	AssertRCReturn(rc, rc);
9930	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9931	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9932	return VINF_SUCCESS;
9933	}
9934
9935	/*
9936	* Not a guest trap, must be a hypervisor related debug event then.
9937	* Update DR6 in case someone is interested in it.
9938	*/
9939	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
9940	AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
9941	CPUMSetHyperDR6(pVCpu, uDR6);
9942
9943	return rc;
9944	}
9945
9946
9947	/**
9948	* VM-exit exception handler for #NM (Device-not-available exception: floating
9949	* point exception).
9950	*/
9951	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9952	{
9953	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9954
9955	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9956	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
9957	#endif
9958
9959	/* We require CR0 and EFER. EFER is always up-to-date. */
9960	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9961	AssertRCReturn(rc, rc);
9962
9963	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
9964	PVM pVM = pVCpu->CTX_SUFF(pVM);
9965	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
9966	if (rc == VINF_SUCCESS)
9967	{
9968	Assert(CPUMIsGuestFPUStateActive(pVCpu));
9969	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
9970	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
9971	return VINF_SUCCESS;
9972	}
9973
9974	/* Forward #NM to the guest. */
9975	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
9976	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9977	AssertRCReturn(rc, rc);
9978	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9979	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
9980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
9981	return rc;
9982	}
9983
9984
9985	/**
9986	* VM-exit exception handler for #GP (General-protection exception).
9987	*
9988	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
9989	*/
9990	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9991	{
9992	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
9994
9995	int rc = VERR_INTERNAL_ERROR_5;
9996	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9997	{
9998	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9999	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10000	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10001	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10002	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10003	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10004	AssertRCReturn(rc, rc);
10005	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10006	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10007	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10008	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10009	return rc;
10010	#else
10011	/* We don't intercept #GP. */
10012	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10013	return VERR_VMX_UNEXPECTED_EXCEPTION;
10014	#endif
10015	}
10016
10017	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10018	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10019
10020	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10021	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10022	AssertRCReturn(rc, rc);
10023
10024	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10025	uint32_t cbOp = 0;
10026	PVM pVM = pVCpu->CTX_SUFF(pVM);
10027	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10028	if (RT_SUCCESS(rc))
10029	{
10030	rc = VINF_SUCCESS;
10031	Assert(cbOp == pDis->cbInstr);
10032	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10033	switch (pDis->pCurInstr->uOpcode)
10034	{
10035	case OP_CLI:
10036	{
10037	pMixedCtx->eflags.Bits.u1IF = 0;
10038	pMixedCtx->rip += pDis->cbInstr;
10039	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10041	break;
10042	}
10043
10044	case OP_STI:
10045	{
10046	pMixedCtx->eflags.Bits.u1IF = 1;
10047	pMixedCtx->rip += pDis->cbInstr;
10048	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10049	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10050	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
10051	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10052	break;
10053	}
10054
10055	case OP_HLT:
10056	{
10057	rc = VINF_EM_HALT;
10058	pMixedCtx->rip += pDis->cbInstr;
10059	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
10060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10061	break;
10062	}
10063
10064	case OP_POPF:
10065	{
10066	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10067	uint32_t cbParm = 0;
10068	uint32_t uMask = 0;
10069	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10070	{
10071	cbParm = 4;
10072	uMask = 0xffffffff;
10073	}
10074	else
10075	{
10076	cbParm = 2;
10077	uMask = 0xffff;
10078	}
10079
10080	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10081	RTGCPTR GCPtrStack = 0;
10082	X86EFLAGS Eflags;
10083	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10084	&GCPtrStack);
10085	if (RT_SUCCESS(rc))
10086	{
10087	Assert(sizeof(Eflags.u32) >= cbParm);
10088	Eflags.u32 = 0;
10089	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10090	}
10091	if (RT_FAILURE(rc))
10092	{
10093	rc = VERR_EM_INTERPRETER;
10094	break;
10095	}
10096	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10097	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10098	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10099	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10100	pMixedCtx->eflags.Bits.u1RF = 0;
10101	pMixedCtx->esp += cbParm;
10102	pMixedCtx->esp &= uMask;
10103	pMixedCtx->rip += pDis->cbInstr;
10104	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
10105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10106	break;
10107	}
10108
10109	case OP_PUSHF:
10110	{
10111	uint32_t cbParm = 0;
10112	uint32_t uMask = 0;
10113	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10114	{
10115	cbParm = 4;
10116	uMask = 0xffffffff;
10117	}
10118	else
10119	{
10120	cbParm = 2;
10121	uMask = 0xffff;
10122	}
10123
10124	/* Get the stack pointer & push the contents of eflags onto the stack. */
10125	RTGCPTR GCPtrStack = 0;
10126	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10127	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10128	if (RT_FAILURE(rc))
10129	{
10130	rc = VERR_EM_INTERPRETER;
10131	break;
10132	}
10133	X86EFLAGS Eflags = pMixedCtx->eflags;
10134	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10135	Eflags.Bits.u1RF = 0;
10136	Eflags.Bits.u1VM = 0;
10137
10138	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10139	if (RT_FAILURE(rc))
10140	{
10141	rc = VERR_EM_INTERPRETER;
10142	break;
10143	}
10144	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10145	pMixedCtx->esp -= cbParm;
10146	pMixedCtx->esp &= uMask;
10147	pMixedCtx->rip += pDis->cbInstr;
10148	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
10149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10150	break;
10151	}
10152
10153	case OP_IRET:
10154	{
10155	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10156	* instruction reference. */
10157	RTGCPTR GCPtrStack = 0;
10158	uint32_t uMask = 0xffff;
10159	uint16_t aIretFrame[3];
10160	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10161	{
10162	rc = VERR_EM_INTERPRETER;
10163	break;
10164	}
10165	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10166	&GCPtrStack);
10167	if (RT_SUCCESS(rc))
10168	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10169	if (RT_FAILURE(rc))
10170	{
10171	rc = VERR_EM_INTERPRETER;
10172	break;
10173	}
10174	pMixedCtx->eip = 0;
10175	pMixedCtx->ip = aIretFrame[0];
10176	pMixedCtx->cs.Sel = aIretFrame[1];
10177	pMixedCtx->cs.ValidSel = aIretFrame[1];
10178	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10179	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10180	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10181	pMixedCtx->sp += sizeof(aIretFrame);
10182	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
10183	\| HM_CHANGED_GUEST_RFLAGS;
10184	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10186	break;
10187	}
10188
10189	case OP_INT:
10190	{
10191	uint16_t uVector = pDis->Param1.uValue & 0xff;
10192	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10193	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10194	break;
10195	}
10196
10197	case OP_INTO:
10198	{
10199	if (pMixedCtx->eflags.Bits.u1OF)
10200	{
10201	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10203	}
10204	break;
10205	}
10206
10207	default:
10208	{
10209	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10210	EMCODETYPE_SUPERVISOR);
10211	rc = VBOXSTRICTRC_VAL(rc2);
10212	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
10213	Log4(("#GP rc=%Rrc\n", rc));
10214	break;
10215	}
10216	}
10217	}
10218	else
10219	rc = VERR_EM_INTERPRETER;
10220
10221	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10222	("#GP Unexpected rc=%Rrc\n", rc));
10223	return rc;
10224	}
10225
10226
10227	/**
10228	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10229	* the exception reported in the VMX transient structure back into the VM.
10230	*
10231	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
10232	* up-to-date.
10233	*/
10234	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10235	{
10236	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10237
10238	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10239	hmR0VmxCheckExitDueToEventDelivery(). */
10240	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10241	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10242	AssertRCReturn(rc, rc);
10243	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10244
10245	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10246	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10247	return VINF_SUCCESS;
10248	}
10249
10250
10251	/**
10252	* VM-exit exception handler for #PF (Page-fault exception).
10253	*/
10254	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10255	{
10256	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10257	PVM pVM = pVCpu->CTX_SUFF(pVM);
10258	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10259	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10260	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10261	AssertRCReturn(rc, rc);
10262
10263	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
10264	if (pVM->hm.s.fNestedPaging)
10265	{
10266	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10267	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10268	{
10269	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10270	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10271	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10272	}
10273	else
10274	{
10275	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10276	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10277	Log4(("Pending #DF due to vectoring #PF. NP\n"));
10278	}
10279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10280	return rc;
10281	}
10282	#else
10283	Assert(!pVM->hm.s.fNestedPaging);
10284	#endif
10285
10286	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10287	AssertRCReturn(rc, rc);
10288
10289	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10290	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10291
10292	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10293	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10294	(RTGCPTR)pVmxTransient->uExitQualification);
10295
10296	Log4(("#PF: rc=%Rrc\n", rc));
10297	if (rc == VINF_SUCCESS)
10298	{
10299	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10300	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
10301	* memory? We don't update the whole state here... */
10302	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
10303	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
10304	TRPMResetTrap(pVCpu);
10305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10306	return rc;
10307	}
10308	else if (rc == VINF_EM_RAW_GUEST_TRAP)
10309	{
10310	if (!pVmxTransient->fVectoringPF)
10311	{
10312	/* It's a guest page fault and needs to be reflected to the guest. */
10313	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10314	TRPMResetTrap(pVCpu);
10315	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10316	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10317	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10318	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10319	}
10320	else
10321	{
10322	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10323	TRPMResetTrap(pVCpu);
10324	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10325	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10326	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10327	}
10328
10329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10330	return VINF_SUCCESS;
10331	}
10332
10333	TRPMResetTrap(pVCpu);
10334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10335	return rc;
10336	}
10337
10338	/** @} */
10339

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47826

以其他格式下載: