HMVMXR0.cpp@ 50655

最後變更在這個檔案從50655是 50655,由 vboxsync 提交於 11 年前
VMM/HMVMXR0: First entry into VT-x may already have saved the host-context, skip that case for the assertion checking for guest mode changes..
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 477.1 KB

行
1	/* $Id: HMVMXR0.cpp 50655 2014-02-28 16:53:46Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns true if the MSR was added -and- its value was updated, false
1239	* otherwise.
1240	* @param pVCpu Pointer to the VMCPU.
1241	* @param uMsr The MSR.
1242	* @param uGuestMsr Value of the guest MSR.
1243	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1244	* necessary.
1245	*/
1246	static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1247	{
1248	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1249	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1250	uint32_t i;
1251	for (i = 0; i < cMsrs; i++)
1252	{
1253	if (pGuestMsr->u32Msr == uMsr)
1254	break;
1255	pGuestMsr++;
1256	}
1257
1258	bool fAdded = false;
1259	if (i == cMsrs)
1260	{
1261	++cMsrs;
1262	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1263	AssertRC(rc);
1264
1265	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1266	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1267	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1268
1269	fAdded = true;
1270	}
1271
1272	/* Update the MSR values in the auto-load/store MSR area. */
1273	pGuestMsr->u32Msr = uMsr;
1274	pGuestMsr->u64Value = uGuestMsrValue;
1275
1276	/* Create/update the MSR slot in the host MSR area. */
1277	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1278	pHostMsr += i;
1279	pHostMsr->u32Msr = uMsr;
1280
1281	/*
1282	* Update the host MSR only when requested by the caller AND when we're
1283	* adding it to the auto-load/store area. Otherwise, it would have been
1284	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1285	*/
1286	bool fUpdatedMsrValue = false;
1287	if ( fAdded
1288	&& fUpdateHostMsr)
1289	{
1290	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1292	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1293	fUpdatedMsrValue = true;
1294	}
1295
1296	return fUpdatedMsrValue;
1297	}
1298
1299
1300	/**
1301	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1302	* auto-load/store MSR area in the VMCS.
1303	*
1304	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1305	* area.
1306	*
1307	* @returns VBox status code.
1308	* @param pVCpu Pointer to the VMCPU.
1309	* @param uMsr The MSR.
1310	*/
1311	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1312	{
1313	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315	for (uint32_t i = 0; i < cMsrs; i++)
1316	{
1317	/* Find the MSR. */
1318	if (pGuestMsr->u32Msr == uMsr)
1319	{
1320	/* If it's the last MSR, simply reduce the count. */
1321	if (i == cMsrs - 1)
1322	{
1323	--cMsrs;
1324	break;
1325	}
1326
1327	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1328	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1329	pLastGuestMsr += cMsrs;
1330	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1331	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1332
1333	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335	pLastHostMsr += cMsrs;
1336	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1337	pHostMsr->u64Value = pLastHostMsr->u64Value;
1338	--cMsrs;
1339	break;
1340	}
1341	pGuestMsr++;
1342	}
1343
1344	/* Update the VMCS if the count changed (meaning the MSR was found). */
1345	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1346	{
1347	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1348	AssertRCReturn(rc, rc);
1349
1350	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1351	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1352	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1353	}
1354
1355	return VINF_SUCCESS;
1356	}
1357
1358
1359	/**
1360	* Checks if the specified guest MSR is part of the auto-load/store area in
1361	* the VMCS.
1362	*
1363	* @returns true if found, false otherwise.
1364	* @param pVCpu Pointer to the VMCPU.
1365	* @param uMsr The MSR to find.
1366	*/
1367	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1368	{
1369	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1370	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1371
1372	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1373	{
1374	if (pGuestMsr->u32Msr == uMsr)
1375	return true;
1376	}
1377	return false;
1378	}
1379
1380
1381	/**
1382	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1383	*
1384	* @param pVCpu Pointer to the VMCPU.
1385	*
1386	* @remarks No-long-jump zone!!!
1387	*/
1388	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1389	{
1390	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1391	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1392	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1393	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1394
1395	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1396	{
1397	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1398	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1399	}
1400
1401	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1402	}
1403
1404
1405	#if HC_ARCH_BITS == 64
1406	/**
1407	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1408	* perform lazy restoration of the host MSRs while leaving VT-x.
1409	*
1410	* @param pVCpu Pointer to the VMCPU.
1411	*
1412	* @remarks No-long-jump zone!!!
1413	*/
1414	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1415	{
1416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1417
1418	/*
1419	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1420	*/
1421	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1422	{
1423	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1424	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1425	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1426	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1427	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1428	}
1429	}
1430
1431
1432	/**
1433	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1434	* lazily while leaving VT-x.
1435	*
1436	* @returns true if it does, false otherwise.
1437	* @param pVCpu Pointer to the VMCPU.
1438	* @param uMsr The MSR to check.
1439	*/
1440	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1441	{
1442	NOREF(pVCpu);
1443	switch (uMsr)
1444	{
1445	case MSR_K8_LSTAR:
1446	case MSR_K6_STAR:
1447	case MSR_K8_SF_MASK:
1448	case MSR_K8_KERNEL_GS_BASE:
1449	return true;
1450	}
1451	return false;
1452	}
1453
1454
1455	/**
1456	* Saves a set of guests MSRs back into the guest-CPU context.
1457	*
1458	* @param pVCpu Pointer to the VMCPU.
1459	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1460	* out-of-sync. Make sure to update the required fields
1461	* before using them.
1462	*
1463	* @remarks No-long-jump zone!!!
1464	*/
1465	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1466	{
1467	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1468	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1469
1470	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1471	{
1472	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1473	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1474	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1475	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1476	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1477	}
1478	}
1479
1480
1481	/**
1482	* Loads a set of guests MSRs to allow read/passthru to the guest.
1483	*
1484	* The name of this function is slightly confusing. This function does NOT
1485	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1486	* common prefix for functions dealing with "lazy restoration" of the shared
1487	* MSRs.
1488	*
1489	* @param pVCpu Pointer to the VMCPU.
1490	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1491	* out-of-sync. Make sure to update the required fields
1492	* before using them.
1493	*
1494	* @remarks No-long-jump zone!!!
1495	*/
1496	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1497	{
1498	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1499	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1500
1501	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1502	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1503	{
1504	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1505	do { \
1506	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1507	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1508	else \
1509	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1510	} while (0)
1511
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1513	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1514	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1515	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1516	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1517	}
1518	else
1519	{
1520	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1521	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1522	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1523	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1524	}
1525	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1526	}
1527
1528
1529	/**
1530	* Performs lazy restoration of the set of host MSRs if they were previously
1531	* loaded with guest MSR values.
1532	*
1533	* @param pVCpu Pointer to the VMCPU.
1534	*
1535	* @remarks No-long-jump zone!!!
1536	* @remarks The guest MSRs should have been saved back into the guest-CPU
1537	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1538	*/
1539	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1540	{
1541	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1542	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1543
1544	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1545	{
1546	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1547	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1548	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1549	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1550	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1551	}
1552	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1553	}
1554	#endif /* HC_ARCH_BITS == 64 */
1555
1556
1557	#ifdef VBOX_STRICT
1558	/**
1559	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1560	* VMCS are correct.
1561	*
1562	* @param pVCpu Pointer to the VMCPU.
1563	*/
1564	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1565	{
1566	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567
1568	/* Verify MSR counts in the VMCS are what we think it should be. */
1569	uint32_t cMsrs;
1570	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1577	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1578
1579	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1580	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1581	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1582	{
1583	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1584	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1585	pGuestMsr->u32Msr));
1586
1587	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1588	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1589	pHostMsr->u64Value, u64Msr));
1590
1591	/* Verify that the permissions are as expected in the MSR bitmap. */
1592	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1593	{
1594	VMXMSREXITREAD enmRead;
1595	VMXMSREXITWRITE enmWrite;
1596	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1597	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1598	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1599	pGuestMsr->u32Msr));
1600	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1601	pGuestMsr->u32Msr));
1602	}
1603	}
1604	}
1605	# endif /* VBOX_STRICT */
1606
1607
1608	/**
1609	* Flushes the TLB using EPT.
1610	*
1611	* @returns VBox status code.
1612	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1613	* enmFlush).
1614	* @param enmFlush Type of flush.
1615	*
1616	* @remarks Caller is responsible for making sure this function is called only
1617	* when NestedPaging is supported and providing @a enmFlush that is
1618	* supported by the CPU.
1619	* @remarks Can be called with interrupts disabled.
1620	*/
1621	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1622	{
1623	uint64_t au64Descriptor[2];
1624	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1625	au64Descriptor[0] = 0;
1626	else
1627	{
1628	Assert(pVCpu);
1629	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1630	}
1631	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1632
1633	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1634	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1635	rc));
1636	if ( RT_SUCCESS(rc)
1637	&& pVCpu)
1638	{
1639	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1640	}
1641	}
1642
1643
1644	/**
1645	* Flushes the TLB using VPID.
1646	*
1647	* @returns VBox status code.
1648	* @param pVM Pointer to the VM.
1649	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1650	* enmFlush).
1651	* @param enmFlush Type of flush.
1652	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1653	* on @a enmFlush).
1654	*
1655	* @remarks Can be called with interrupts disabled.
1656	*/
1657	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1658	{
1659	NOREF(pVM);
1660	AssertPtr(pVM);
1661	Assert(pVM->hm.s.vmx.fVpid);
1662
1663	uint64_t au64Descriptor[2];
1664	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1665	{
1666	au64Descriptor[0] = 0;
1667	au64Descriptor[1] = 0;
1668	}
1669	else
1670	{
1671	AssertPtr(pVCpu);
1672	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1673	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1674	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1675	au64Descriptor[1] = GCPtr;
1676	}
1677
1678	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1679	AssertMsg(rc == VINF_SUCCESS,
1680	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1681	if ( RT_SUCCESS(rc)
1682	&& pVCpu)
1683	{
1684	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1685	}
1686	}
1687
1688
1689	/**
1690	* Invalidates a guest page by guest virtual address. Only relevant for
1691	* EPT/VPID, otherwise there is nothing really to invalidate.
1692	*
1693	* @returns VBox status code.
1694	* @param pVM Pointer to the VM.
1695	* @param pVCpu Pointer to the VMCPU.
1696	* @param GCVirt Guest virtual address of the page to invalidate.
1697	*/
1698	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1699	{
1700	AssertPtr(pVM);
1701	AssertPtr(pVCpu);
1702	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1703
1704	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1705	if (!fFlushPending)
1706	{
1707	/*
1708	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1709	* See @bugref{6043} and @bugref{6177}.
1710	*
1711	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1712	* function maybe called in a loop with individual addresses.
1713	*/
1714	if (pVM->hm.s.vmx.fVpid)
1715	{
1716	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1717	{
1718	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1719	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1720	}
1721	else
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724	else if (pVM->hm.s.fNestedPaging)
1725	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1726	}
1727
1728	return VINF_SUCCESS;
1729	}
1730
1731
1732	/**
1733	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1734	* otherwise there is nothing really to invalidate.
1735	*
1736	* @returns VBox status code.
1737	* @param pVM Pointer to the VM.
1738	* @param pVCpu Pointer to the VMCPU.
1739	* @param GCPhys Guest physical address of the page to invalidate.
1740	*/
1741	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1742	{
1743	NOREF(pVM); NOREF(GCPhys);
1744	LogFlowFunc(("%RGp\n", GCPhys));
1745
1746	/*
1747	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1748	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1749	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1750	*/
1751	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1752	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1753	return VINF_SUCCESS;
1754	}
1755
1756
1757	/**
1758	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1759	* case where neither EPT nor VPID is supported by the CPU.
1760	*
1761	* @param pVM Pointer to the VM.
1762	* @param pVCpu Pointer to the VMCPU.
1763	* @param pCpu Pointer to the global HM struct.
1764	*
1765	* @remarks Called with interrupts disabled.
1766	*/
1767	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1768	{
1769	AssertPtr(pVCpu);
1770	AssertPtr(pCpu);
1771	NOREF(pVM);
1772
1773	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1774
1775	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1776	#if 0
1777	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1778	pVCpu->hm.s.TlbShootdown.cPages = 0;
1779	#endif
1780
1781	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1782	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1783	pVCpu->hm.s.fForceTLBFlush = false;
1784	return;
1785	}
1786
1787
1788	/**
1789	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1790	*
1791	* @param pVM Pointer to the VM.
1792	* @param pVCpu Pointer to the VMCPU.
1793	* @param pCpu Pointer to the global HM CPU struct.
1794	* @remarks All references to "ASID" in this function pertains to "VPID" in
1795	* Intel's nomenclature. The reason is, to avoid confusion in compare
1796	* statements since the host-CPU copies are named "ASID".
1797	*
1798	* @remarks Called with interrupts disabled.
1799	*/
1800	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1801	{
1802	#ifdef VBOX_WITH_STATISTICS
1803	bool fTlbFlushed = false;
1804	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1805	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1806	if (!fTlbFlushed) \
1807	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1808	} while (0)
1809	#else
1810	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1811	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1812	#endif
1813
1814	AssertPtr(pVM);
1815	AssertPtr(pCpu);
1816	AssertPtr(pVCpu);
1817	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1818	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1819	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1820
1821
1822	/*
1823	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1824	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1825	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1826	*/
1827	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1828	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1829	{
1830	++pCpu->uCurrentAsid;
1831	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1832	{
1833	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1834	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1835	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1836	}
1837
1838	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1839	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1840	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1841
1842	/*
1843	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1844	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1845	*/
1846	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1847	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1848	HMVMX_SET_TAGGED_TLB_FLUSHED();
1849	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1850	}
1851
1852	/* Check for explicit TLB shootdowns. */
1853	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1854	{
1855	/*
1856	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1857	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1858	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1859	* but not guest-physical mappings.
1860	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1861	*/
1862	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1863	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1864	HMVMX_SET_TAGGED_TLB_FLUSHED();
1865	}
1866
1867	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1868	* where it is commented out. Support individual entry flushing
1869	* someday. */
1870	#if 0
1871	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1872	{
1873	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1874
1875	/*
1876	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1877	* as supported by the CPU.
1878	*/
1879	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1880	{
1881	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1882	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1883	}
1884	else
1885	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1886
1887	HMVMX_SET_TAGGED_TLB_FLUSHED();
1888	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1889	pVCpu->hm.s.TlbShootdown.cPages = 0;
1890	}
1891	#endif
1892
1893	pVCpu->hm.s.fForceTLBFlush = false;
1894
1895	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1896
1897	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1898	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1899	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1900	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1901	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1902	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1903	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1904	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1905
1906	/* Update VMCS with the VPID. */
1907	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1908	AssertRC(rc);
1909
1910	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1911	}
1912
1913
1914	/**
1915	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1916	*
1917	* @returns VBox status code.
1918	* @param pVM Pointer to the VM.
1919	* @param pVCpu Pointer to the VMCPU.
1920	* @param pCpu Pointer to the global HM CPU struct.
1921	*
1922	* @remarks Called with interrupts disabled.
1923	*/
1924	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1925	{
1926	AssertPtr(pVM);
1927	AssertPtr(pVCpu);
1928	AssertPtr(pCpu);
1929	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1930	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1931
1932	/*
1933	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1934	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1935	*/
1936	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1937	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1938	{
1939	pVCpu->hm.s.fForceTLBFlush = true;
1940	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1941	}
1942
1943	/* Check for explicit TLB shootdown flushes. */
1944	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1945	{
1946	pVCpu->hm.s.fForceTLBFlush = true;
1947	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1948	}
1949
1950	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1951	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1952
1953	if (pVCpu->hm.s.fForceTLBFlush)
1954	{
1955	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1956	pVCpu->hm.s.fForceTLBFlush = false;
1957	}
1958	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1959	* where it is commented out. Support individual entry flushing
1960	* someday. */
1961	#if 0
1962	else
1963	{
1964	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1965	{
1966	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1967	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1968	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969	}
1970	else
1971	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1972
1973	pVCpu->hm.s.TlbShootdown.cPages = 0;
1974	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1975	}
1976	#endif
1977	}
1978
1979
1980	/**
1981	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1982	*
1983	* @returns VBox status code.
1984	* @param pVM Pointer to the VM.
1985	* @param pVCpu Pointer to the VMCPU.
1986	* @param pCpu Pointer to the global HM CPU struct.
1987	*
1988	* @remarks Called with interrupts disabled.
1989	*/
1990	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1991	{
1992	AssertPtr(pVM);
1993	AssertPtr(pVCpu);
1994	AssertPtr(pCpu);
1995	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1996	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1997
1998	/*
1999	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2000	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2001	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2002	*/
2003	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2004	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2005	{
2006	pVCpu->hm.s.fForceTLBFlush = true;
2007	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2008	}
2009
2010	/* Check for explicit TLB shootdown flushes. */
2011	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2012	{
2013	/*
2014	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2015	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2016	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2017	*/
2018	pVCpu->hm.s.fForceTLBFlush = true;
2019	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2020	}
2021
2022	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2023	if (pVCpu->hm.s.fForceTLBFlush)
2024	{
2025	++pCpu->uCurrentAsid;
2026	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2027	{
2028	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2029	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2030	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2031	}
2032
2033	pVCpu->hm.s.fForceTLBFlush = false;
2034	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2035	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2036	if (pCpu->fFlushAsidBeforeUse)
2037	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2038	}
2039	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2040	* where it is commented out. Support individual entry flushing
2041	* someday. */
2042	#if 0
2043	else
2044	{
2045	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2046	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2047	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2048	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2049
2050	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2051	{
2052	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2053	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2054	{
2055	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2056	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2057	}
2058	else
2059	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2060
2061	pVCpu->hm.s.TlbShootdown.cPages = 0;
2062	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2063	}
2064	else
2065	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2066	}
2067	#endif
2068
2069	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2070	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2071	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2072	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2073	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2074	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2075
2076	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2077	AssertRC(rc);
2078	}
2079
2080
2081	/**
2082	* Flushes the guest TLB entry based on CPU capabilities.
2083	*
2084	* @param pVCpu Pointer to the VMCPU.
2085	* @param pCpu Pointer to the global HM CPU struct.
2086	*/
2087	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2088	{
2089	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2090	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2091	#endif
2092	PVM pVM = pVCpu->CTX_SUFF(pVM);
2093	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2094	{
2095	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2096	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2097	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2098	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2099	default:
2100	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2101	break;
2102	}
2103
2104	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2105	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2106
2107	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2108	}
2109
2110
2111	/**
2112	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2113	* TLB entries from the host TLB before VM-entry.
2114	*
2115	* @returns VBox status code.
2116	* @param pVM Pointer to the VM.
2117	*/
2118	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2119	{
2120	/*
2121	* Determine optimal flush type for Nested Paging.
2122	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2123	* guest execution (see hmR3InitFinalizeR0()).
2124	*/
2125	if (pVM->hm.s.fNestedPaging)
2126	{
2127	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2128	{
2129	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2130	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2131	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2132	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2133	else
2134	{
2135	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2136	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2137	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2138	}
2139
2140	/* Make sure the write-back cacheable memory type for EPT is supported. */
2141	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2142	{
2143	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2144	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2145	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146	}
2147	}
2148	else
2149	{
2150	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2151	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2152	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2153	}
2154	}
2155
2156	/*
2157	* Determine optimal flush type for VPID.
2158	*/
2159	if (pVM->hm.s.vmx.fVpid)
2160	{
2161	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2162	{
2163	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2164	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2165	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2166	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2167	else
2168	{
2169	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2170	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2171	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2172	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2173	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2174	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2175	pVM->hm.s.vmx.fVpid = false;
2176	}
2177	}
2178	else
2179	{
2180	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2181	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2182	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2183	pVM->hm.s.vmx.fVpid = false;
2184	}
2185	}
2186
2187	/*
2188	* Setup the handler for flushing tagged-TLBs.
2189	*/
2190	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2191	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2192	else if (pVM->hm.s.fNestedPaging)
2193	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2194	else if (pVM->hm.s.vmx.fVpid)
2195	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2196	else
2197	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2198	return VINF_SUCCESS;
2199	}
2200
2201
2202	/**
2203	* Sets up pin-based VM-execution controls in the VMCS.
2204	*
2205	* @returns VBox status code.
2206	* @param pVM Pointer to the VM.
2207	* @param pVCpu Pointer to the VMCPU.
2208	*/
2209	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2210	{
2211	AssertPtr(pVM);
2212	AssertPtr(pVCpu);
2213
2214	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2215	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2216
2217	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2218	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2219	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2220
2221	/* Enable the VMX preemption timer. */
2222	if (pVM->hm.s.vmx.fUsePreemptTimer)
2223	{
2224	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2225	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2226	}
2227
2228	if ((val & zap) != val)
2229	{
2230	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2231	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2232	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2233	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234	}
2235
2236	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2237	AssertRCReturn(rc, rc);
2238
2239	/* Update VCPU with the currently set pin-based VM-execution controls. */
2240	pVCpu->hm.s.vmx.u32PinCtls = val;
2241	return rc;
2242	}
2243
2244
2245	/**
2246	* Sets up processor-based VM-execution controls in the VMCS.
2247	*
2248	* @returns VBox status code.
2249	* @param pVM Pointer to the VM.
2250	* @param pVMCPU Pointer to the VMCPU.
2251	*/
2252	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2253	{
2254	AssertPtr(pVM);
2255	AssertPtr(pVCpu);
2256
2257	int rc = VERR_INTERNAL_ERROR_5;
2258	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2259	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2260
2261	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2265	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2266	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2267	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2268
2269	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2270	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2271	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2272	{
2273	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2274	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2275	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2276	}
2277
2278	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2279	if (!pVM->hm.s.fNestedPaging)
2280	{
2281	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2282	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2283	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2284	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2285	}
2286
2287	/* Use TPR shadowing if supported by the CPU. */
2288	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2289	{
2290	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2292	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2293	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2294	AssertRCReturn(rc, rc);
2295
2296	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2297	/* CR8 writes causes a VM-exit based on TPR threshold. */
2298	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2299	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2300	}
2301	else
2302	{
2303	/*
2304	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2305	* Set this control only for 64-bit guests.
2306	*/
2307	if (pVM->hm.s.fAllow64BitGuests)
2308	{
2309	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2310	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2311	}
2312	}
2313
2314	/* Use MSR-bitmaps if supported by the CPU. */
2315	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2316	{
2317	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2318
2319	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2320	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2321	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2322	AssertRCReturn(rc, rc);
2323
2324	/*
2325	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2326	* automatically as dedicated fields in the VMCS.
2327	*/
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2331	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2332	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2333
2334	#if HC_ARCH_BITS == 64
2335	/*
2336	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2337	*/
2338	if (pVM->hm.s.fAllow64BitGuests)
2339	{
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2342	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2343	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2344	}
2345	#endif
2346	}
2347
2348	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2349	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2350	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2351
2352	if ((val & zap) != val)
2353	{
2354	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2355	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2356	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2357	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2358	}
2359
2360	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2361	AssertRCReturn(rc, rc);
2362
2363	/* Update VCPU with the currently set processor-based VM-execution controls. */
2364	pVCpu->hm.s.vmx.u32ProcCtls = val;
2365
2366	/*
2367	* Secondary processor-based VM-execution controls.
2368	*/
2369	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2370	{
2371	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2372	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2373
2374	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2376
2377	if (pVM->hm.s.fNestedPaging)
2378	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2379	else
2380	{
2381	/*
2382	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2383	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2384	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2385	*/
2386	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2387	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2388	}
2389
2390	if (pVM->hm.s.vmx.fVpid)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2392
2393	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2394	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2395
2396	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2397	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2398	* done dynamically. */
2399	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2400	{
2401	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2402	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2403	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2404	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2405	AssertRCReturn(rc, rc);
2406	}
2407
2408	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2409	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2410
2411	if ((val & zap) != val)
2412	{
2413	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2414	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2415	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2416	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2417	}
2418
2419	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2420	AssertRCReturn(rc, rc);
2421
2422	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2423	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2424	}
2425	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2426	{
2427	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2428	"available\n"));
2429	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2430	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2431	}
2432
2433	return VINF_SUCCESS;
2434	}
2435
2436
2437	/**
2438	* Sets up miscellaneous (everything other than Pin & Processor-based
2439	* VM-execution) control fields in the VMCS.
2440	*
2441	* @returns VBox status code.
2442	* @param pVM Pointer to the VM.
2443	* @param pVCpu Pointer to the VMCPU.
2444	*/
2445	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2446	{
2447	NOREF(pVM);
2448	AssertPtr(pVM);
2449	AssertPtr(pVCpu);
2450
2451	int rc = VERR_GENERAL_FAILURE;
2452
2453	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2454	#if 0
2455	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2457	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2458
2459	/*
2460	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2461	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2462	* We thus use the exception bitmap to control it rather than use both.
2463	*/
2464	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2465	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2466
2467	/** @todo Explore possibility of using IO-bitmaps. */
2468	/* All IO & IOIO instructions cause VM-exits. */
2469	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2470	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2471
2472	/* Initialize the MSR-bitmap area. */
2473	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2474	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2476	#endif
2477
2478	/* Setup MSR auto-load/store area. */
2479	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2480	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2481	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2482	AssertRCReturn(rc, rc);
2483	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2484	AssertRCReturn(rc, rc);
2485
2486	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2487	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2488	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2489	AssertRCReturn(rc, rc);
2490
2491	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2492	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2493	AssertRCReturn(rc, rc);
2494
2495	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2496	#if 0
2497	/* Setup debug controls */
2498	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2499	AssertRCReturn(rc, rc);
2500	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2501	AssertRCReturn(rc, rc);
2502	#endif
2503
2504	return rc;
2505	}
2506
2507
2508	/**
2509	* Sets up the initial exception bitmap in the VMCS based on static conditions
2510	* (i.e. conditions that cannot ever change after starting the VM).
2511	*
2512	* @returns VBox status code.
2513	* @param pVM Pointer to the VM.
2514	* @param pVCpu Pointer to the VMCPU.
2515	*/
2516	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2517	{
2518	AssertPtr(pVM);
2519	AssertPtr(pVCpu);
2520
2521	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2522
2523	uint32_t u32XcptBitmap = 0;
2524
2525	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2526	if (!pVM->hm.s.fNestedPaging)
2527	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2528
2529	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2530	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2531	AssertRCReturn(rc, rc);
2532	return rc;
2533	}
2534
2535
2536	/**
2537	* Sets up the initial guest-state mask. The guest-state mask is consulted
2538	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2539	* for the nested virtualization case (as it would cause a VM-exit).
2540	*
2541	* @param pVCpu Pointer to the VMCPU.
2542	*/
2543	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2544	{
2545	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2546	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2547	return VINF_SUCCESS;
2548	}
2549
2550
2551	/**
2552	* Does per-VM VT-x initialization.
2553	*
2554	* @returns VBox status code.
2555	* @param pVM Pointer to the VM.
2556	*/
2557	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2558	{
2559	LogFlowFunc(("pVM=%p\n", pVM));
2560
2561	int rc = hmR0VmxStructsAlloc(pVM);
2562	if (RT_FAILURE(rc))
2563	{
2564	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2565	return rc;
2566	}
2567
2568	return VINF_SUCCESS;
2569	}
2570
2571
2572	/**
2573	* Does per-VM VT-x termination.
2574	*
2575	* @returns VBox status code.
2576	* @param pVM Pointer to the VM.
2577	*/
2578	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2579	{
2580	LogFlowFunc(("pVM=%p\n", pVM));
2581
2582	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2583	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2584	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2585	#endif
2586	hmR0VmxStructsFree(pVM);
2587	return VINF_SUCCESS;
2588	}
2589
2590
2591	/**
2592	* Sets up the VM for execution under VT-x.
2593	* This function is only called once per-VM during initialization.
2594	*
2595	* @returns VBox status code.
2596	* @param pVM Pointer to the VM.
2597	*/
2598	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2599	{
2600	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2601	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2602
2603	LogFlowFunc(("pVM=%p\n", pVM));
2604
2605	/*
2606	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2607	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2608	*/
2609	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2610	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2611	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2612	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2613	{
2614	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2615	return VERR_INTERNAL_ERROR;
2616	}
2617
2618	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2619	/*
2620	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2621	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2622	*/
2623	if ( pVM->hm.s.fAllow64BitGuests
2624	&& !HMVMX_IS_64BIT_HOST_MODE())
2625	{
2626	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2627	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2628	}
2629	#endif
2630
2631	/* Initialize these always, see hmR3InitFinalizeR0().*/
2632	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2633	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2634
2635	/* Setup the tagged-TLB flush handlers. */
2636	int rc = hmR0VmxSetupTaggedTlb(pVM);
2637	if (RT_FAILURE(rc))
2638	{
2639	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2640	return rc;
2641	}
2642
2643	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2644	{
2645	PVMCPU pVCpu = &pVM->aCpus[i];
2646	AssertPtr(pVCpu);
2647	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2648
2649	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2650	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2651
2652	/* Set revision dword at the beginning of the VMCS structure. */
2653	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2654
2655	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2656	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2657	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2658	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2659
2660	/* Load this VMCS as the current VMCS. */
2661	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2662	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2663	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2664
2665	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2666	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2667	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2668
2669	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2670	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2671	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2672
2673	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2674	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2675	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2676
2677	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2678	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2679	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2680
2681	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2682	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2683	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2684
2685	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2686	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2687	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2688	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2689	#endif
2690
2691	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2692	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2693	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2694	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2695
2696	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2697
2698	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2699	}
2700
2701	return VINF_SUCCESS;
2702	}
2703
2704
2705	/**
2706	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2707	* the VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM Pointer to the VM.
2711	* @param pVCpu Pointer to the VMCPU.
2712	*/
2713	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2714	{
2715	NOREF(pVM); NOREF(pVCpu);
2716
2717	RTCCUINTREG uReg = ASMGetCR0();
2718	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2719	AssertRCReturn(rc, rc);
2720
2721	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2722	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2723	if (HMVMX_IS_64BIT_HOST_MODE())
2724	{
2725	uint64_t uRegCR3 = HMR0Get64bitCR3();
2726	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2727	}
2728	else
2729	#endif
2730	{
2731	uReg = ASMGetCR3();
2732	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2733	}
2734	AssertRCReturn(rc, rc);
2735
2736	uReg = ASMGetCR4();
2737	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2738	AssertRCReturn(rc, rc);
2739	return rc;
2740	}
2741
2742
2743	#if HC_ARCH_BITS == 64
2744	/**
2745	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2746	* requirements. See hmR0VmxSaveHostSegmentRegs().
2747	*/
2748	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2749	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2750	{ \
2751	bool fValidSelector = true; \
2752	if ((selValue) & X86_SEL_LDT) \
2753	{ \
2754	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2755	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2756	} \
2757	if (fValidSelector) \
2758	{ \
2759	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2760	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2761	} \
2762	(selValue) = 0; \
2763	}
2764	#endif
2765
2766
2767	/**
2768	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2769	* the host-state area in the VMCS.
2770	*
2771	* @returns VBox status code.
2772	* @param pVM Pointer to the VM.
2773	* @param pVCpu Pointer to the VMCPU.
2774	*/
2775	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2776	{
2777	NOREF(pVM);
2778	int rc = VERR_INTERNAL_ERROR_5;
2779
2780	#if HC_ARCH_BITS == 64
2781	/*
2782	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2783	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2784	*/
2785	AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2786	("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2787	#endif
2788
2789	/*
2790	* Host DS, ES, FS and GS segment registers.
2791	*/
2792	#if HC_ARCH_BITS == 64
2793	RTSEL uSelDS = ASMGetDS();
2794	RTSEL uSelES = ASMGetES();
2795	RTSEL uSelFS = ASMGetFS();
2796	RTSEL uSelGS = ASMGetGS();
2797	#else
2798	RTSEL uSelDS = 0;
2799	RTSEL uSelES = 0;
2800	RTSEL uSelFS = 0;
2801	RTSEL uSelGS = 0;
2802	#endif
2803
2804	/* Recalculate which host-state bits need to be manually restored. */
2805	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2806
2807	/*
2808	* Host CS and SS segment registers.
2809	*/
2810	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2811	RTSEL uSelCS;
2812	RTSEL uSelSS;
2813	if (HMVMX_IS_64BIT_HOST_MODE())
2814	{
2815	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2816	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2817	}
2818	else
2819	{
2820	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2821	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2822	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2823	}
2824	#else
2825	RTSEL uSelCS = ASMGetCS();
2826	RTSEL uSelSS = ASMGetSS();
2827	#endif
2828
2829	/*
2830	* Host TR segment register.
2831	*/
2832	RTSEL uSelTR = ASMGetTR();
2833
2834	#if HC_ARCH_BITS == 64
2835	/*
2836	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2837	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2838	*/
2839	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2840	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2841	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2842	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2843	# undef VMXLOCAL_ADJUST_HOST_SEG
2844	#endif
2845
2846	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2847	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2848	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2849	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2850	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2851	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2852	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2853	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2854	Assert(uSelCS);
2855	Assert(uSelTR);
2856
2857	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2858	#if 0
2859	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2860	Assert(uSelSS != 0);
2861	#endif
2862
2863	/* Write these host selector fields into the host-state area in the VMCS. */
2864	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2865	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2866	#if HC_ARCH_BITS == 64
2867	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2868	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2869	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2870	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2871	#endif
2872	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2873
2874	/*
2875	* Host GDTR and IDTR.
2876	*/
2877	RTGDTR Gdtr;
2878	RT_ZERO(Gdtr);
2879	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2880	if (HMVMX_IS_64BIT_HOST_MODE())
2881	{
2882	X86XDTR64 Gdtr64;
2883	X86XDTR64 Idtr64;
2884	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2885	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2887
2888	Gdtr.cbGdt = Gdtr64.cb;
2889	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2890	}
2891	else
2892	#endif
2893	{
2894	RTIDTR Idtr;
2895	ASMGetGDTR(&Gdtr);
2896	ASMGetIDTR(&Idtr);
2897	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2898	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2899
2900	#if HC_ARCH_BITS == 64
2901	/*
2902	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2903	* maximum limit (0xffff) on every VM-exit.
2904	*/
2905	if (Gdtr.cbGdt != 0xffff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2908	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2910	}
2911
2912	/*
2913	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2914	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2915	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2916	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2917	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2918	* hosts where we are pretty sure it won't cause trouble.
2919	*/
2920	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
2921	if (Idtr.cbIdt < 0x0fff)
2922	# else
2923	if (Idtr.cbIdt != 0xffff)
2924	# endif
2925	{
2926	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2927	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2928	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2929	}
2930	#endif
2931	}
2932
2933	/*
2934	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2935	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2936	*/
2937	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2938	{
2939	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2940	return VERR_VMX_INVALID_HOST_STATE;
2941	}
2942
2943	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2944	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2945	if (HMVMX_IS_64BIT_HOST_MODE())
2946	{
2947	/* We need the 64-bit TR base for hybrid darwin. */
2948	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2949	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2950	}
2951	else
2952	#endif
2953	{
2954	uintptr_t uTRBase;
2955	#if HC_ARCH_BITS == 64
2956	uTRBase = X86DESC64_BASE(pDesc);
2957
2958	/*
2959	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2960	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2961	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2962	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2963	*
2964	* [1] See Intel spec. 3.5 "System Descriptor Types".
2965	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2966	*/
2967	Assert(pDesc->System.u4Type == 11);
2968	if ( pDesc->System.u16LimitLow != 0x67
2969	\|\| pDesc->System.u4LimitHigh)
2970	{
2971	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2972	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2973
2974	/* Store the GDTR here as we need it while restoring TR. */
2975	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2976	}
2977	#else
2978	uTRBase = X86DESC_BASE(pDesc);
2979	#endif
2980	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2981	}
2982	AssertRCReturn(rc, rc);
2983
2984	/*
2985	* Host FS base and GS base.
2986	*/
2987	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2988	if (HMVMX_IS_64BIT_HOST_MODE())
2989	{
2990	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2991	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2992	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2993	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2994
2995	# if HC_ARCH_BITS == 64
2996	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2997	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2998	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2999	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3000	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3001	# endif
3002	}
3003	#endif
3004	return rc;
3005	}
3006
3007
3008	/**
3009	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3010	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3011	* the host after every successful VM-exit.
3012	*
3013	* @returns VBox status code.
3014	* @param pVM Pointer to the VM.
3015	* @param pVCpu Pointer to the VMCPU.
3016	*
3017	* @remarks No-long-jump zone!!!
3018	*/
3019	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3020	{
3021	NOREF(pVM);
3022
3023	AssertPtr(pVCpu);
3024	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3025
3026	int rc = VINF_SUCCESS;
3027	#if HC_ARCH_BITS == 64
3028	if (pVM->hm.s.fAllow64BitGuests)
3029	hmR0VmxLazySaveHostMsrs(pVCpu);
3030	#endif
3031
3032	/*
3033	* Host Sysenter MSRs.
3034	*/
3035	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3036	AssertRCReturn(rc, rc);
3037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3038	if (HMVMX_IS_64BIT_HOST_MODE())
3039	{
3040	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3041	AssertRCReturn(rc, rc);
3042	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3043	}
3044	else
3045	{
3046	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3047	AssertRCReturn(rc, rc);
3048	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3049	}
3050	#elif HC_ARCH_BITS == 32
3051	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3052	AssertRCReturn(rc, rc);
3053	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3054	#else
3055	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3056	AssertRCReturn(rc, rc);
3057	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3058	#endif
3059	AssertRCReturn(rc, rc);
3060
3061	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3062	* hmR0VmxSetupExitCtls() !! */
3063	return rc;
3064	}
3065
3066
3067	/**
3068	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3069	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3070	* controls".
3071	*
3072	* @returns VBox status code.
3073	* @param pVCpu Pointer to the VMCPU.
3074	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3075	* out-of-sync. Make sure to update the required fields
3076	* before using them.
3077	*
3078	* @remarks No-long-jump zone!!!
3079	*/
3080	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3081	{
3082	int rc = VINF_SUCCESS;
3083	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3084	{
3085	PVM pVM = pVCpu->CTX_SUFF(pVM);
3086	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3087	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3088
3089	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3090	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3091
3092	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3093	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3094	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3095	else
3096	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3097
3098	/*
3099	* The following should -not- be set (since we're not in SMM mode):
3100	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3101	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3102	*/
3103
3104	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3105	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3106	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3107
3108	if ((val & zap) != val)
3109	{
3110	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3111	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3112	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3113	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3114	}
3115
3116	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3117	AssertRCReturn(rc, rc);
3118
3119	/* Update VCPU with the currently set VM-exit controls. */
3120	pVCpu->hm.s.vmx.u32EntryCtls = val;
3121	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3122	}
3123	return rc;
3124	}
3125
3126
3127	/**
3128	* Sets up the VM-exit controls in the VMCS.
3129	*
3130	* @returns VBox status code.
3131	* @param pVM Pointer to the VM.
3132	* @param pVCpu Pointer to the VMCPU.
3133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3134	* out-of-sync. Make sure to update the required fields
3135	* before using them.
3136	*
3137	* @remarks requires EFER.
3138	*/
3139	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3140	{
3141	NOREF(pMixedCtx);
3142
3143	int rc = VINF_SUCCESS;
3144	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3145	{
3146	PVM pVM = pVCpu->CTX_SUFF(pVM);
3147	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3148	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3149
3150	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3151	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3152
3153	/*
3154	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3155	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3156	*/
3157	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3158	if (HMVMX_IS_64BIT_HOST_MODE())
3159	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3160	else
3161	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3162	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3163	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3164	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3165	else
3166	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3167	#endif
3168
3169	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3170	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3171
3172	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3173	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3174	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3175	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3176	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3177
3178	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3179	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3180
3181	if ((val & zap) != val)
3182	{
3183	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3184	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3185	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3186	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3187	}
3188
3189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3190	AssertRCReturn(rc, rc);
3191
3192	/* Update VCPU with the currently set VM-exit controls. */
3193	pVCpu->hm.s.vmx.u32ExitCtls = val;
3194	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3195	}
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Loads the guest APIC and related state.
3202	*
3203	* @returns VBox status code.
3204	* @param pVM Pointer to the VM.
3205	* @param pVCpu Pointer to the VMCPU.
3206	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3207	* out-of-sync. Make sure to update the required fields
3208	* before using them.
3209	*/
3210	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3211	{
3212	NOREF(pMixedCtx);
3213
3214	int rc = VINF_SUCCESS;
3215	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3216	{
3217	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3218	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3219	{
3220	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3221
3222	bool fPendingIntr = false;
3223	uint8_t u8Tpr = 0;
3224	uint8_t u8PendingIntr = 0;
3225	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3226	AssertRCReturn(rc, rc);
3227
3228	/*
3229	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3230	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3231	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3232	* the interrupt when we VM-exit for other reasons.
3233	*/
3234	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3235	uint32_t u32TprThreshold = 0;
3236	if (fPendingIntr)
3237	{
3238	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3239	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3240	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3241	if (u8PendingPriority <= u8TprPriority)
3242	u32TprThreshold = u8PendingPriority;
3243	else
3244	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3245	}
3246	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3247
3248	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3249	AssertRCReturn(rc, rc);
3250	}
3251
3252	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3253	}
3254	return rc;
3255	}
3256
3257
3258	/**
3259	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3260	*
3261	* @returns Guest's interruptibility-state.
3262	* @param pVCpu Pointer to the VMCPU.
3263	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3264	* out-of-sync. Make sure to update the required fields
3265	* before using them.
3266	*
3267	* @remarks No-long-jump zone!!!
3268	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3269	*/
3270	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3271	{
3272	/*
3273	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3274	* inhibit interrupts or clear any existing interrupt-inhibition.
3275	*/
3276	uint32_t uIntrState = 0;
3277	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3278	{
3279	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3280	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3281	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3282	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3283	{
3284	/*
3285	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3286	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3287	*/
3288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3289	}
3290	else if (pMixedCtx->eflags.Bits.u1IF)
3291	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3292	else
3293	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3294	}
3295	return uIntrState;
3296	}
3297
3298
3299	/**
3300	* Loads the guest's interruptibility-state into the guest-state area in the
3301	* VMCS.
3302	*
3303	* @returns VBox status code.
3304	* @param pVCpu Pointer to the VMCPU.
3305	* @param uIntrState The interruptibility-state to set.
3306	*/
3307	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3308	{
3309	NOREF(pVCpu);
3310	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3311	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3312	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3313	AssertRCReturn(rc, rc);
3314	return rc;
3315	}
3316
3317
3318	/**
3319	* Loads the guest's RIP into the guest-state area in the VMCS.
3320	*
3321	* @returns VBox status code.
3322	* @param pVCpu Pointer to the VMCPU.
3323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3324	* out-of-sync. Make sure to update the required fields
3325	* before using them.
3326	*
3327	* @remarks No-long-jump zone!!!
3328	*/
3329	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3330	{
3331	int rc = VINF_SUCCESS;
3332	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3333	{
3334	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3335	AssertRCReturn(rc, rc);
3336
3337	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3338	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3339	}
3340	return rc;
3341	}
3342
3343
3344	/**
3345	* Loads the guest's RSP into the guest-state area in the VMCS.
3346	*
3347	* @returns VBox status code.
3348	* @param pVCpu Pointer to the VMCPU.
3349	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350	* out-of-sync. Make sure to update the required fields
3351	* before using them.
3352	*
3353	* @remarks No-long-jump zone!!!
3354	*/
3355	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356	{
3357	int rc = VINF_SUCCESS;
3358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3359	{
3360	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3361	AssertRCReturn(rc, rc);
3362
3363	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3364	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3365	}
3366	return rc;
3367	}
3368
3369
3370	/**
3371	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3372	*
3373	* @returns VBox status code.
3374	* @param pVCpu Pointer to the VMCPU.
3375	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3376	* out-of-sync. Make sure to update the required fields
3377	* before using them.
3378	*
3379	* @remarks No-long-jump zone!!!
3380	*/
3381	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3382	{
3383	int rc = VINF_SUCCESS;
3384	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3385	{
3386	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3387	Let us assert it as such and use 32-bit VMWRITE. */
3388	Assert(!(pMixedCtx->rflags.u64 >> 32));
3389	X86EFLAGS Eflags = pMixedCtx->eflags;
3390	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3391	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3392
3393	/*
3394	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3395	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3396	*/
3397	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3398	{
3399	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3400	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3401	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3402	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3403	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3404	}
3405
3406	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3407	AssertRCReturn(rc, rc);
3408
3409	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3410	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3411	}
3412	return rc;
3413	}
3414
3415
3416	/**
3417	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3418	*
3419	* @returns VBox status code.
3420	* @param pVCpu Pointer to the VMCPU.
3421	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3422	* out-of-sync. Make sure to update the required fields
3423	* before using them.
3424	*
3425	* @remarks No-long-jump zone!!!
3426	*/
3427	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3428	{
3429	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3430	AssertRCReturn(rc, rc);
3431	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3432	AssertRCReturn(rc, rc);
3433	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3434	AssertRCReturn(rc, rc);
3435	return rc;
3436	}
3437
3438
3439	/**
3440	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3441	* CR0 is partially shared with the host and we have to consider the FPU bits.
3442	*
3443	* @returns VBox status code.
3444	* @param pVM Pointer to the VM.
3445	* @param pVCpu Pointer to the VMCPU.
3446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3447	* out-of-sync. Make sure to update the required fields
3448	* before using them.
3449	*
3450	* @remarks No-long-jump zone!!!
3451	*/
3452	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3453	{
3454	/*
3455	* Guest CR0.
3456	* Guest FPU.
3457	*/
3458	int rc = VINF_SUCCESS;
3459	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3460	{
3461	Assert(!(pMixedCtx->cr0 >> 32));
3462	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3463	PVM pVM = pVCpu->CTX_SUFF(pVM);
3464
3465	/* The guest's view (read access) of its CR0 is unblemished. */
3466	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3467	AssertRCReturn(rc, rc);
3468	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3469
3470	/* Setup VT-x's view of the guest CR0. */
3471	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3472	if (pVM->hm.s.fNestedPaging)
3473	{
3474	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3475	{
3476	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3477	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3478	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3479	}
3480	else
3481	{
3482	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3483	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3484	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3485	}
3486
3487	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3488	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3489	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3490
3491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3492	AssertRCReturn(rc, rc);
3493	}
3494	else
3495	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3496
3497	/*
3498	* Guest FPU bits.
3499	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3500	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3501	*/
3502	u32GuestCR0 \|= X86_CR0_NE;
3503	bool fInterceptNM = false;
3504	if (CPUMIsGuestFPUStateActive(pVCpu))
3505	{
3506	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3507	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3508	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3509	}
3510	else
3511	{
3512	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3513	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3514	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3515	}
3516
3517	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3518	bool fInterceptMF = false;
3519	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3520	fInterceptMF = true;
3521
3522	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3523	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3524	{
3525	Assert(PDMVmmDevHeapIsEnabled(pVM));
3526	Assert(pVM->hm.s.vmx.pRealModeTSS);
3527	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3528	fInterceptNM = true;
3529	fInterceptMF = true;
3530	}
3531	else
3532	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3533
3534	if (fInterceptNM)
3535	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3536	else
3537	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3538
3539	if (fInterceptMF)
3540	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3541	else
3542	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3543
3544	/* Additional intercepts for debugging, define these yourself explicitly. */
3545	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3546	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3547	\| RT_BIT(X86_XCPT_BP)
3548	\| RT_BIT(X86_XCPT_DB)
3549	\| RT_BIT(X86_XCPT_DE)
3550	\| RT_BIT(X86_XCPT_NM)
3551	\| RT_BIT(X86_XCPT_UD)
3552	\| RT_BIT(X86_XCPT_NP)
3553	\| RT_BIT(X86_XCPT_SS)
3554	\| RT_BIT(X86_XCPT_GP)
3555	\| RT_BIT(X86_XCPT_PF)
3556	\| RT_BIT(X86_XCPT_MF)
3557	;
3558	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3559	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3560	#endif
3561
3562	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3563
3564	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3565	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3566	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3567	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3568	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3569	else
3570	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3571
3572	u32GuestCR0 \|= uSetCR0;
3573	u32GuestCR0 &= uZapCR0;
3574	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3575
3576	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3577	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3578	AssertRCReturn(rc, rc);
3579	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3580	AssertRCReturn(rc, rc);
3581	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3582
3583	/*
3584	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3585	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3586	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3587	*/
3588	uint32_t u32CR0Mask = 0;
3589	u32CR0Mask = X86_CR0_PE
3590	\| X86_CR0_NE
3591	\| X86_CR0_WP
3592	\| X86_CR0_PG
3593	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3594	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3595	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3596
3597	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3598	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3599	* and @bugref{6944}. */
3600	#if 0
3601	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3602	u32CR0Mask &= ~X86_CR0_PE;
3603	#endif
3604	if (pVM->hm.s.fNestedPaging)
3605	u32CR0Mask &= ~X86_CR0_WP;
3606
3607	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3608	if (fInterceptNM)
3609	{
3610	u32CR0Mask \|= X86_CR0_TS
3611	\| X86_CR0_MP;
3612	}
3613
3614	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3615	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3616	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3617	AssertRCReturn(rc, rc);
3618	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3619
3620	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3621	}
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest control registers (CR3, CR4) into the guest-state area
3628	* in the VMCS.
3629	*
3630	* @returns VBox status code.
3631	* @param pVM Pointer to the VM.
3632	* @param pVCpu Pointer to the VMCPU.
3633	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3634	* out-of-sync. Make sure to update the required fields
3635	* before using them.
3636	*
3637	* @remarks No-long-jump zone!!!
3638	*/
3639	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3640	{
3641	int rc = VINF_SUCCESS;
3642	PVM pVM = pVCpu->CTX_SUFF(pVM);
3643
3644	/*
3645	* Guest CR2.
3646	* It's always loaded in the assembler code. Nothing to do here.
3647	*/
3648
3649	/*
3650	* Guest CR3.
3651	*/
3652	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3653	{
3654	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3655	if (pVM->hm.s.fNestedPaging)
3656	{
3657	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3658
3659	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3660	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3661	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3662	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3663
3664	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3665	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3666	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3667
3668	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3669	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3670	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3671	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3672
3673	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3674	AssertRCReturn(rc, rc);
3675	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3676
3677	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3678	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3679	{
3680	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3681	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3682	{
3683	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3684	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3685	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3686	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3687	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3688	}
3689
3690	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3691	have Unrestricted Execution to handle the guest when it's not using paging. */
3692	GCPhysGuestCR3 = pMixedCtx->cr3;
3693	}
3694	else
3695	{
3696	/*
3697	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3698	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3699	* EPT takes care of translating it to host-physical addresses.
3700	*/
3701	RTGCPHYS GCPhys;
3702	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3703	Assert(PDMVmmDevHeapIsEnabled(pVM));
3704
3705	/* We obtain it here every time as the guest could have relocated this PCI region. */
3706	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3707	AssertRCReturn(rc, rc);
3708
3709	GCPhysGuestCR3 = GCPhys;
3710	}
3711
3712	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3713	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3714	}
3715	else
3716	{
3717	/* Non-nested paging case, just use the hypervisor's CR3. */
3718	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3719
3720	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3721	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3722	}
3723	AssertRCReturn(rc, rc);
3724
3725	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3726	}
3727
3728	/*
3729	* Guest CR4.
3730	*/
3731	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3732	{
3733	Assert(!(pMixedCtx->cr4 >> 32));
3734	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3735
3736	/* The guest's view of its CR4 is unblemished. */
3737	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3738	AssertRCReturn(rc, rc);
3739	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3740
3741	/* Setup VT-x's view of the guest CR4. */
3742	/*
3743	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3744	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3745	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3746	*/
3747	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3748	{
3749	Assert(pVM->hm.s.vmx.pRealModeTSS);
3750	Assert(PDMVmmDevHeapIsEnabled(pVM));
3751	u32GuestCR4 &= ~X86_CR4_VME;
3752	}
3753
3754	if (pVM->hm.s.fNestedPaging)
3755	{
3756	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3757	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3758	{
3759	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3760	u32GuestCR4 \|= X86_CR4_PSE;
3761	/* Our identity mapping is a 32-bit page directory. */
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	}
3764	/* else use guest CR4.*/
3765	}
3766	else
3767	{
3768	/*
3769	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3770	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3771	*/
3772	switch (pVCpu->hm.s.enmShadowMode)
3773	{
3774	case PGMMODE_REAL: /* Real-mode. */
3775	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3776	case PGMMODE_32_BIT: /* 32-bit paging. */
3777	{
3778	u32GuestCR4 &= ~X86_CR4_PAE;
3779	break;
3780	}
3781
3782	case PGMMODE_PAE: /* PAE paging. */
3783	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3784	{
3785	u32GuestCR4 \|= X86_CR4_PAE;
3786	break;
3787	}
3788
3789	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3790	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3791	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3792	break;
3793	#endif
3794	default:
3795	AssertFailed();
3796	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3797	}
3798	}
3799
3800	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3801	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3802	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3803	u32GuestCR4 \|= uSetCR4;
3804	u32GuestCR4 &= uZapCR4;
3805
3806	/* Write VT-x's view of the guest CR4 into the VMCS. */
3807	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3808	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3809	AssertRCReturn(rc, rc);
3810
3811	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3812	uint32_t u32CR4Mask = 0;
3813	u32CR4Mask = X86_CR4_VME
3814	\| X86_CR4_PAE
3815	\| X86_CR4_PGE
3816	\| X86_CR4_PSE
3817	\| X86_CR4_VMXE;
3818	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3819	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3820	AssertRCReturn(rc, rc);
3821
3822	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3823	}
3824	return rc;
3825	}
3826
3827
3828	/**
3829	* Loads the guest debug registers into the guest-state area in the VMCS.
3830	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3831	*
3832	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3833	*
3834	* @returns VBox status code.
3835	* @param pVCpu Pointer to the VMCPU.
3836	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3837	* out-of-sync. Make sure to update the required fields
3838	* before using them.
3839	*
3840	* @remarks No-long-jump zone!!!
3841	*/
3842	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3843	{
3844	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3845	return VINF_SUCCESS;
3846
3847	#ifdef VBOX_STRICT
3848	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3849	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3850	{
3851	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3852	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3853	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3854	}
3855	#endif
3856
3857	int rc;
3858	PVM pVM = pVCpu->CTX_SUFF(pVM);
3859	bool fInterceptDB = false;
3860	bool fInterceptMovDRx = false;
3861	if ( pVCpu->hm.s.fSingleInstruction
3862	\|\| DBGFIsStepping(pVCpu))
3863	{
3864	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3865	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3866	{
3867	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3868	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3869	AssertRCReturn(rc, rc);
3870	Assert(fInterceptDB == false);
3871	}
3872	else
3873	{
3874	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3875	pVCpu->hm.s.fClearTrapFlag = true;
3876	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3877	fInterceptDB = true;
3878	}
3879	}
3880
3881	if ( fInterceptDB
3882	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3883	{
3884	/*
3885	* Use the combined guest and host DRx values found in the hypervisor
3886	* register set because the debugger has breakpoints active or someone
3887	* is single stepping on the host side without a monitor trap flag.
3888	*
3889	* Note! DBGF expects a clean DR6 state before executing guest code.
3890	*/
3891	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3892	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3893	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3894	{
3895	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3896	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3897	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3898	}
3899	else
3900	#endif
3901	if (!CPUMIsHyperDebugStateActive(pVCpu))
3902	{
3903	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3904	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3905	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3906	}
3907
3908	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3909	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3910	AssertRCReturn(rc, rc);
3911
3912	pVCpu->hm.s.fUsingHyperDR7 = true;
3913	fInterceptDB = true;
3914	fInterceptMovDRx = true;
3915	}
3916	else
3917	{
3918	/*
3919	* If the guest has enabled debug registers, we need to load them prior to
3920	* executing guest code so they'll trigger at the right time.
3921	*/
3922	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3923	{
3924	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3925	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3926	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3927	{
3928	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3929	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3930	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3931	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3932	}
3933	else
3934	#endif
3935	if (!CPUMIsGuestDebugStateActive(pVCpu))
3936	{
3937	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3938	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3939	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3940	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3941	}
3942	Assert(!fInterceptDB);
3943	Assert(!fInterceptMovDRx);
3944	}
3945	/*
3946	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3947	* must intercept #DB in order to maintain a correct DR6 guest value.
3948	*/
3949	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3950	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3951	&& !CPUMIsGuestDebugStateActive(pVCpu))
3952	#else
3953	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3954	#endif
3955	{
3956	fInterceptMovDRx = true;
3957	fInterceptDB = true;
3958	}
3959
3960	/* Update guest DR7. */
3961	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3962	AssertRCReturn(rc, rc);
3963
3964	pVCpu->hm.s.fUsingHyperDR7 = false;
3965	}
3966
3967	/*
3968	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3969	*/
3970	if (fInterceptDB)
3971	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3972	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3973	{
3974	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3975	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3976	#endif
3977	}
3978	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3983	*/
3984	if (fInterceptMovDRx)
3985	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3986	else
3987	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3988	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3989	AssertRCReturn(rc, rc);
3990
3991	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3992	return VINF_SUCCESS;
3993	}
3994
3995
3996	#ifdef VBOX_STRICT
3997	/**
3998	* Strict function to validate segment registers.
3999	*
4000	* @remarks ASSUMES CR0 is up to date.
4001	*/
4002	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4003	{
4004	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4005	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4006	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4007	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4008	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4009	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4010	{
4011	/* Protected mode checks */
4012	/* CS */
4013	Assert(pCtx->cs.Attr.n.u1Present);
4014	Assert(!(pCtx->cs.Attr.u & 0xf00));
4015	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4016	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4017	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4018	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4019	\|\| (pCtx->cs.Attr.n.u1Granularity));
4020	/* CS cannot be loaded with NULL in protected mode. */
4021	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4022	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4023	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4024	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4025	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4026	else
4027	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4028	/* SS */
4029	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4030	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4031	if ( !(pCtx->cr0 & X86_CR0_PE)
4032	\|\| pCtx->cs.Attr.n.u4Type == 3)
4033	{
4034	Assert(!pCtx->ss.Attr.n.u2Dpl);
4035	}
4036	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4037	{
4038	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4039	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4040	Assert(pCtx->ss.Attr.n.u1Present);
4041	Assert(!(pCtx->ss.Attr.u & 0xf00));
4042	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4043	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4044	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4045	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4046	\|\| (pCtx->ss.Attr.n.u1Granularity));
4047	}
4048	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4049	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4050	{
4051	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4052	Assert(pCtx->ds.Attr.n.u1Present);
4053	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4054	Assert(!(pCtx->ds.Attr.u & 0xf00));
4055	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4056	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4057	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4058	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4059	\|\| (pCtx->ds.Attr.n.u1Granularity));
4060	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4061	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4062	}
4063	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4064	{
4065	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4066	Assert(pCtx->es.Attr.n.u1Present);
4067	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4068	Assert(!(pCtx->es.Attr.u & 0xf00));
4069	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4070	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4071	\|\| !(pCtx->es.Attr.n.u1Granularity));
4072	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4073	\|\| (pCtx->es.Attr.n.u1Granularity));
4074	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4075	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4076	}
4077	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4078	{
4079	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4080	Assert(pCtx->fs.Attr.n.u1Present);
4081	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4082	Assert(!(pCtx->fs.Attr.u & 0xf00));
4083	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4084	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4085	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4087	\|\| (pCtx->fs.Attr.n.u1Granularity));
4088	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4089	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4090	}
4091	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4092	{
4093	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4094	Assert(pCtx->gs.Attr.n.u1Present);
4095	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4096	Assert(!(pCtx->gs.Attr.u & 0xf00));
4097	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4098	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4099	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4100	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4101	\|\| (pCtx->gs.Attr.n.u1Granularity));
4102	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4103	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4104	}
4105	/* 64-bit capable CPUs. */
4106	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4107	if (HMVMX_IS_64BIT_HOST_MODE())
4108	{
4109	Assert(!(pCtx->cs.u64Base >> 32));
4110	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4111	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4112	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4113	}
4114	# endif
4115	}
4116	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4117	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4118	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4119	{
4120	/* Real and v86 mode checks. */
4121	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4122	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4123	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4124	{
4125	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4126	}
4127	else
4128	{
4129	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4130	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4131	}
4132
4133	/* CS */
4134	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4135	Assert(pCtx->cs.u32Limit == 0xffff);
4136	Assert(u32CSAttr == 0xf3);
4137	/* SS */
4138	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4139	Assert(pCtx->ss.u32Limit == 0xffff);
4140	Assert(u32SSAttr == 0xf3);
4141	/* DS */
4142	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4143	Assert(pCtx->ds.u32Limit == 0xffff);
4144	Assert(u32DSAttr == 0xf3);
4145	/* ES */
4146	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4147	Assert(pCtx->es.u32Limit == 0xffff);
4148	Assert(u32ESAttr == 0xf3);
4149	/* FS */
4150	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4151	Assert(pCtx->fs.u32Limit == 0xffff);
4152	Assert(u32FSAttr == 0xf3);
4153	/* GS */
4154	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4155	Assert(pCtx->gs.u32Limit == 0xffff);
4156	Assert(u32GSAttr == 0xf3);
4157	/* 64-bit capable CPUs. */
4158	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4159	if (HMVMX_IS_64BIT_HOST_MODE())
4160	{
4161	Assert(!(pCtx->cs.u64Base >> 32));
4162	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4163	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4164	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4165	}
4166	# endif
4167	}
4168	}
4169	#endif /* VBOX_STRICT */
4170
4171
4172	/**
4173	* Writes a guest segment register into the guest-state area in the VMCS.
4174	*
4175	* @returns VBox status code.
4176	* @param pVCpu Pointer to the VMCPU.
4177	* @param idxSel Index of the selector in the VMCS.
4178	* @param idxLimit Index of the segment limit in the VMCS.
4179	* @param idxBase Index of the segment base in the VMCS.
4180	* @param idxAccess Index of the access rights of the segment in the VMCS.
4181	* @param pSelReg Pointer to the segment selector.
4182	*
4183	* @remarks No-long-jump zone!!!
4184	*/
4185	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4186	uint32_t idxAccess, PCPUMSELREG pSelReg)
4187	{
4188	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4189	AssertRCReturn(rc, rc);
4190	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4191	AssertRCReturn(rc, rc);
4192	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4193	AssertRCReturn(rc, rc);
4194
4195	uint32_t u32Access = pSelReg->Attr.u;
4196	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4197	{
4198	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4199	u32Access = 0xf3;
4200	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4201	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4202	}
4203	else
4204	{
4205	/*
4206	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4207	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4208	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4209	* loaded in protected-mode have their attribute as 0.
4210	*/
4211	if (!u32Access)
4212	u32Access = X86DESCATTR_UNUSABLE;
4213	}
4214
4215	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4216	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4217	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4218
4219	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4220	AssertRCReturn(rc, rc);
4221	return rc;
4222	}
4223
4224
4225	/**
4226	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4227	* into the guest-state area in the VMCS.
4228	*
4229	* @returns VBox status code.
4230	* @param pVM Pointer to the VM.
4231	* @param pVCPU Pointer to the VMCPU.
4232	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4233	* out-of-sync. Make sure to update the required fields
4234	* before using them.
4235	*
4236	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4237	* @remarks No-long-jump zone!!!
4238	*/
4239	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4240	{
4241	int rc = VERR_INTERNAL_ERROR_5;
4242	PVM pVM = pVCpu->CTX_SUFF(pVM);
4243
4244	/*
4245	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4246	*/
4247	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4248	{
4249	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4250	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4251	{
4252	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4253	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4254	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4255	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4256	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4257	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4258	}
4259
4260	#ifdef VBOX_WITH_REM
4261	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4262	{
4263	Assert(pVM->hm.s.vmx.pRealModeTSS);
4264	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4265	if ( pVCpu->hm.s.vmx.fWasInRealMode
4266	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4267	{
4268	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4269	in real-mode (e.g. OpenBSD 4.0) */
4270	REMFlushTBs(pVM);
4271	Log4(("Load: Switch to protected mode detected!\n"));
4272	pVCpu->hm.s.vmx.fWasInRealMode = false;
4273	}
4274	}
4275	#endif
4276	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4277	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4278	AssertRCReturn(rc, rc);
4279	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4280	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4281	AssertRCReturn(rc, rc);
4282	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4283	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4284	AssertRCReturn(rc, rc);
4285	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4286	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4287	AssertRCReturn(rc, rc);
4288	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4289	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4290	AssertRCReturn(rc, rc);
4291	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4292	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4293	AssertRCReturn(rc, rc);
4294
4295	#ifdef VBOX_STRICT
4296	/* Validate. */
4297	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4298	#endif
4299
4300	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4301	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4302	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4303	}
4304
4305	/*
4306	* Guest TR.
4307	*/
4308	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4309	{
4310	/*
4311	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4312	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4313	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4314	*/
4315	uint16_t u16Sel = 0;
4316	uint32_t u32Limit = 0;
4317	uint64_t u64Base = 0;
4318	uint32_t u32AccessRights = 0;
4319
4320	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4321	{
4322	u16Sel = pMixedCtx->tr.Sel;
4323	u32Limit = pMixedCtx->tr.u32Limit;
4324	u64Base = pMixedCtx->tr.u64Base;
4325	u32AccessRights = pMixedCtx->tr.Attr.u;
4326	}
4327	else
4328	{
4329	Assert(pVM->hm.s.vmx.pRealModeTSS);
4330	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4331
4332	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4333	RTGCPHYS GCPhys;
4334	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4335	AssertRCReturn(rc, rc);
4336
4337	X86DESCATTR DescAttr;
4338	DescAttr.u = 0;
4339	DescAttr.n.u1Present = 1;
4340	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4341
4342	u16Sel = 0;
4343	u32Limit = HM_VTX_TSS_SIZE;
4344	u64Base = GCPhys; /* in real-mode phys = virt. */
4345	u32AccessRights = DescAttr.u;
4346	}
4347
4348	/* Validate. */
4349	Assert(!(u16Sel & RT_BIT(2)));
4350	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4351	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4352	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4353	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4354	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4355	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4356	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4357	Assert( (u32Limit & 0xfff) == 0xfff
4358	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4359	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4360	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4361
4362	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4363	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4364	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4365	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4366
4367	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4368	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4369	}
4370
4371	/*
4372	* Guest GDTR.
4373	*/
4374	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4375	{
4376	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4377	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4378
4379	/* Validate. */
4380	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4381
4382	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4383	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4384	}
4385
4386	/*
4387	* Guest LDTR.
4388	*/
4389	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4390	{
4391	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4392	uint32_t u32Access = 0;
4393	if (!pMixedCtx->ldtr.Attr.u)
4394	u32Access = X86DESCATTR_UNUSABLE;
4395	else
4396	u32Access = pMixedCtx->ldtr.Attr.u;
4397
4398	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4399	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4400	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4401	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4402
4403	/* Validate. */
4404	if (!(u32Access & X86DESCATTR_UNUSABLE))
4405	{
4406	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4407	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4408	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4409	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4410	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4411	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4412	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4413	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4414	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4415	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4416	}
4417
4418	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4419	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4420	}
4421
4422	/*
4423	* Guest IDTR.
4424	*/
4425	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4426	{
4427	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4428	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4429
4430	/* Validate. */
4431	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4432
4433	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4434	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4435	}
4436
4437	return VINF_SUCCESS;
4438	}
4439
4440
4441	/**
4442	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4443	* areas. These MSRs will automatically be loaded to the host CPU on every
4444	* successful VM entry and stored from the host CPU on every successful VM-exit.
4445	*
4446	* This also creates/updates MSR slots for the host MSRs. The actual host
4447	* MSR values are -not- updated here for performance reasons. See
4448	* hmR0VmxSaveHostMsrs().
4449	*
4450	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4451	*
4452	* @returns VBox status code.
4453	* @param pVCpu Pointer to the VMCPU.
4454	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4455	* out-of-sync. Make sure to update the required fields
4456	* before using them.
4457	*
4458	* @remarks No-long-jump zone!!!
4459	*/
4460	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4461	{
4462	AssertPtr(pVCpu);
4463	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4464
4465	/*
4466	* MSRs that we use the auto-load/store MSR area in the VMCS.
4467	*/
4468	PVM pVM = pVCpu->CTX_SUFF(pVM);
4469	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4470	{
4471	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4472	if (pVM->hm.s.fAllow64BitGuests)
4473	{
4474	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4475	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4476	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4477	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4478	# ifdef DEBUG
4479	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4480	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4481	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4482	# endif
4483	}
4484	#endif
4485	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4486	}
4487
4488	/*
4489	* Guest Sysenter MSRs.
4490	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4491	* VM-exits on WRMSRs for these MSRs.
4492	*/
4493	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4494	{
4495	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4496	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4497	}
4498
4499	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4500	{
4501	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4502	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4503	}
4504
4505	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4506	{
4507	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4508	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4509	}
4510
4511	return VINF_SUCCESS;
4512	}
4513
4514
4515	/**
4516	* Loads the guest activity state into the guest-state area in the VMCS.
4517	*
4518	* @returns VBox status code.
4519	* @param pVCpu Pointer to the VMCPU.
4520	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4521	* out-of-sync. Make sure to update the required fields
4522	* before using them.
4523	*
4524	* @remarks No-long-jump zone!!!
4525	*/
4526	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4527	{
4528	NOREF(pCtx);
4529	/** @todo See if we can make use of other states, e.g.
4530	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4531	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4532	{
4533	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4534	AssertRCReturn(rc, rc);
4535
4536	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4537	}
4538	return VINF_SUCCESS;
4539	}
4540
4541
4542	/**
4543	* Sets up the appropriate function to run guest code.
4544	*
4545	* @returns VBox status code.
4546	* @param pVCpu Pointer to the VMCPU.
4547	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4548	* out-of-sync. Make sure to update the required fields
4549	* before using them.
4550	*
4551	* @remarks No-long-jump zone!!!
4552	*/
4553	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4554	{
4555	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4556	{
4557	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4558	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4559	#endif
4560	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4561	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4562	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4563	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4564	{
4565	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4566	{
4567	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4568	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS),
4569	("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4570	}
4571	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4572	}
4573	#else
4574	/* 64-bit host or hybrid host. */
4575	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4576	#endif
4577	}
4578	else
4579	{
4580	/* Guest is not in long mode, use the 32-bit handler. */
4581	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4582	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4583	{
4584	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4585	{
4586	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4587	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS),
4588	("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4589	}
4590	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4591	}
4592	#else
4593	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4594	#endif
4595	}
4596	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4597	return VINF_SUCCESS;
4598	}
4599
4600
4601	/**
4602	* Wrapper for running the guest code in VT-x.
4603	*
4604	* @returns VBox strict status code.
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param pCtx Pointer to the guest-CPU context.
4608	*
4609	* @remarks No-long-jump zone!!!
4610	*/
4611	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4612	{
4613	/*
4614	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4615	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4616	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4617	*/
4618	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4619	/** @todo Add stats for resume vs launch. */
4620	#ifdef VBOX_WITH_KERNEL_USING_XMM
4621	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4622	#else
4623	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4624	#endif
4625	}
4626
4627
4628	/**
4629	* Reports world-switch error and dumps some useful debug info.
4630	*
4631	* @param pVM Pointer to the VM.
4632	* @param pVCpu Pointer to the VMCPU.
4633	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4634	* @param pCtx Pointer to the guest-CPU context.
4635	* @param pVmxTransient Pointer to the VMX transient structure (only
4636	* exitReason updated).
4637	*/
4638	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4639	{
4640	Assert(pVM);
4641	Assert(pVCpu);
4642	Assert(pCtx);
4643	Assert(pVmxTransient);
4644	HMVMX_ASSERT_PREEMPT_SAFE();
4645
4646	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4647	switch (rcVMRun)
4648	{
4649	case VERR_VMX_INVALID_VMXON_PTR:
4650	AssertFailed();
4651	break;
4652	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4653	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4654	{
4655	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4656	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4657	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4658	AssertRC(rc);
4659
4660	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4661	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4662	Cannot do it here as we may have been long preempted. */
4663
4664	#ifdef VBOX_STRICT
4665	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4666	pVmxTransient->uExitReason));
4667	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4668	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4669	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4670	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4671	else
4672	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4673	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4674	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4675
4676	/* VMX control bits. */
4677	uint32_t u32Val;
4678	uint64_t u64Val;
4679	HMVMXHCUINTREG uHCReg;
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4686	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4687	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4688	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4689	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4690	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4691	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4692	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4693	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4695	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4696	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4697	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4698	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4699	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4700	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4701	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4702	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4703	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4707	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4708	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4709	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4710	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4711	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4713	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4714	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4715	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4716	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4717	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4718	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4719	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4720	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4721	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4722
4723	/* Guest bits. */
4724	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4725	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4726	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4727	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4728	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4729	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4730	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4731	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4732
4733	/* Host bits. */
4734	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4735	Log4(("Host CR0 %#RHr\n", uHCReg));
4736	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4737	Log4(("Host CR3 %#RHr\n", uHCReg));
4738	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4739	Log4(("Host CR4 %#RHr\n", uHCReg));
4740
4741	RTGDTR HostGdtr;
4742	PCX86DESCHC pDesc;
4743	ASMGetGDTR(&HostGdtr);
4744	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4745	Log4(("Host CS %#08x\n", u32Val));
4746	if (u32Val < HostGdtr.cbGdt)
4747	{
4748	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4749	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4750	}
4751
4752	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4753	Log4(("Host DS %#08x\n", u32Val));
4754	if (u32Val < HostGdtr.cbGdt)
4755	{
4756	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4757	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4758	}
4759
4760	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4761	Log4(("Host ES %#08x\n", u32Val));
4762	if (u32Val < HostGdtr.cbGdt)
4763	{
4764	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4765	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4766	}
4767
4768	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4769	Log4(("Host FS %#08x\n", u32Val));
4770	if (u32Val < HostGdtr.cbGdt)
4771	{
4772	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4773	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4774	}
4775
4776	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4777	Log4(("Host GS %#08x\n", u32Val));
4778	if (u32Val < HostGdtr.cbGdt)
4779	{
4780	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4781	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4782	}
4783
4784	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4785	Log4(("Host SS %#08x\n", u32Val));
4786	if (u32Val < HostGdtr.cbGdt)
4787	{
4788	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4789	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4790	}
4791
4792	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4793	Log4(("Host TR %#08x\n", u32Val));
4794	if (u32Val < HostGdtr.cbGdt)
4795	{
4796	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4797	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4798	}
4799
4800	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4801	Log4(("Host TR Base %#RHv\n", uHCReg));
4802	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4803	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4804	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4805	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4806	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4807	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4808	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4809	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4810	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4811	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4812	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4813	Log4(("Host RSP %#RHv\n", uHCReg));
4814	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4815	Log4(("Host RIP %#RHv\n", uHCReg));
4816	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4817	if (HMVMX_IS_64BIT_HOST_MODE())
4818	{
4819	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4820	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4821	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4822	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4823	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4824	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4825	}
4826	# endif
4827	#endif /* VBOX_STRICT */
4828	break;
4829	}
4830
4831	default:
4832	/* Impossible */
4833	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4834	break;
4835	}
4836	NOREF(pVM); NOREF(pCtx);
4837	}
4838
4839
4840	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4841	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4842	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4843	#endif
4844	#ifdef VBOX_STRICT
4845	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4846	{
4847	switch (idxField)
4848	{
4849	case VMX_VMCS_GUEST_RIP:
4850	case VMX_VMCS_GUEST_RSP:
4851	case VMX_VMCS_GUEST_SYSENTER_EIP:
4852	case VMX_VMCS_GUEST_SYSENTER_ESP:
4853	case VMX_VMCS_GUEST_GDTR_BASE:
4854	case VMX_VMCS_GUEST_IDTR_BASE:
4855	case VMX_VMCS_GUEST_CS_BASE:
4856	case VMX_VMCS_GUEST_DS_BASE:
4857	case VMX_VMCS_GUEST_ES_BASE:
4858	case VMX_VMCS_GUEST_FS_BASE:
4859	case VMX_VMCS_GUEST_GS_BASE:
4860	case VMX_VMCS_GUEST_SS_BASE:
4861	case VMX_VMCS_GUEST_LDTR_BASE:
4862	case VMX_VMCS_GUEST_TR_BASE:
4863	case VMX_VMCS_GUEST_CR3:
4864	return true;
4865	}
4866	return false;
4867	}
4868
4869	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4870	{
4871	switch (idxField)
4872	{
4873	/* Read-only fields. */
4874	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4875	return true;
4876	}
4877	/* Remaining readable fields should also be writable. */
4878	return hmR0VmxIsValidWriteField(idxField);
4879	}
4880	#endif /* VBOX_STRICT */
4881
4882
4883	/**
4884	* Executes the specified handler in 64-bit mode.
4885	*
4886	* @returns VBox status code.
4887	* @param pVM Pointer to the VM.
4888	* @param pVCpu Pointer to the VMCPU.
4889	* @param pCtx Pointer to the guest CPU context.
4890	* @param enmOp The operation to perform.
4891	* @param cbParam Number of parameters.
4892	* @param paParam Array of 32-bit parameters.
4893	*/
4894	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4895	uint32_t *paParam)
4896	{
4897	int rc, rc2;
4898	PHMGLOBALCPUINFO pCpu;
4899	RTHCPHYS HCPhysCpuPage;
4900	RTCCUINTREG uOldEflags;
4901
4902	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4903	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4904	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4905	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4906
4907	#ifdef VBOX_STRICT
4908	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4909	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4910
4911	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4912	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4913	#endif
4914
4915	/* Disable interrupts. */
4916	uOldEflags = ASMIntDisableFlags();
4917
4918	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4919	RTCPUID idHostCpu = RTMpCpuId();
4920	CPUMR0SetLApic(pVCpu, idHostCpu);
4921	#endif
4922
4923	pCpu = HMR0GetCurrentCpu();
4924	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4925
4926	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4927	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4928
4929	/* Leave VMX Root Mode. */
4930	VMXDisable();
4931
4932	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4933
4934	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4935	CPUMSetHyperEIP(pVCpu, enmOp);
4936	for (int i = (int)cbParam - 1; i >= 0; i--)
4937	CPUMPushHyper(pVCpu, paParam[i]);
4938
4939	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4940
4941	/* Call the switcher. */
4942	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4943	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4944
4945	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4946	/* Make sure the VMX instructions don't cause #UD faults. */
4947	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4948
4949	/* Re-enter VMX Root Mode */
4950	rc2 = VMXEnable(HCPhysCpuPage);
4951	if (RT_FAILURE(rc2))
4952	{
4953	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4954	ASMSetFlags(uOldEflags);
4955	return rc2;
4956	}
4957
4958	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4959	AssertRC(rc2);
4960	Assert(!(ASMGetFlags() & X86_EFL_IF));
4961	ASMSetFlags(uOldEflags);
4962	return rc;
4963	}
4964
4965
4966	/**
4967	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4968	* supporting 64-bit guests.
4969	*
4970	* @returns VBox status code.
4971	* @param fResume Whether to VMLAUNCH or VMRESUME.
4972	* @param pCtx Pointer to the guest-CPU context.
4973	* @param pCache Pointer to the VMCS cache.
4974	* @param pVM Pointer to the VM.
4975	* @param pVCpu Pointer to the VMCPU.
4976	*/
4977	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4978	{
4979	uint32_t aParam[6];
4980	PHMGLOBALCPUINFO pCpu = NULL;
4981	RTHCPHYS HCPhysCpuPage = 0;
4982	int rc = VERR_INTERNAL_ERROR_5;
4983
4984	pCpu = HMR0GetCurrentCpu();
4985	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4986
4987	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4988	pCache->uPos = 1;
4989	pCache->interPD = PGMGetInterPaeCR3(pVM);
4990	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4991	#endif
4992
4993	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4994	pCache->TestIn.HCPhysCpuPage = 0;
4995	pCache->TestIn.HCPhysVmcs = 0;
4996	pCache->TestIn.pCache = 0;
4997	pCache->TestOut.HCPhysVmcs = 0;
4998	pCache->TestOut.pCache = 0;
4999	pCache->TestOut.pCtx = 0;
5000	pCache->TestOut.eflags = 0;
5001	#endif
5002
5003	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5004	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5005	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5006	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5007	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5008	aParam[5] = 0;
5009
5010	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5011	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5012	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5013	#endif
5014	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5015
5016	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5017	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5018	Assert(pCtx->dr[4] == 10);
5019	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5020	#endif
5021
5022	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5023	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5024	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5025	pVCpu->hm.s.vmx.HCPhysVmcs));
5026	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5027	pCache->TestOut.HCPhysVmcs));
5028	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5029	pCache->TestOut.pCache));
5030	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5031	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5032	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5033	pCache->TestOut.pCtx));
5034	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5035	#endif
5036	return rc;
5037	}
5038
5039
5040	/**
5041	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5042	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5043	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5044	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5045	*
5046	* @returns VBox status code.
5047	* @param pVM Pointer to the VM.
5048	* @param pVCpu Pointer to the VMCPU.
5049	*/
5050	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5051	{
5052	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5053	{ \
5054	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5055	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5056	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5057	++cReadFields; \
5058	}
5059
5060	AssertPtr(pVM);
5061	AssertPtr(pVCpu);
5062	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5063	uint32_t cReadFields = 0;
5064
5065	/*
5066	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5067	* and serve to indicate exceptions to the rules.
5068	*/
5069
5070	/* Guest-natural selector base fields. */
5071	#if 0
5072	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5075	#endif
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5081	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5082	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5083	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5085	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5086	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5088	#if 0
5089	/* Unused natural width guest-state fields. */
5090	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5091	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5092	#endif
5093	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5094	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5095
5096	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5097	#if 0
5098	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5099	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5100	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5101	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5102	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5103	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5104	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5105	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5106	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5107	#endif
5108
5109	/* Natural width guest-state fields. */
5110	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5111	#if 0
5112	/* Currently unused field. */
5113	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5114	#endif
5115
5116	if (pVM->hm.s.fNestedPaging)
5117	{
5118	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5119	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5120	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5121	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5122	}
5123	else
5124	{
5125	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5126	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5127	}
5128
5129	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5130	return VINF_SUCCESS;
5131	}
5132
5133
5134	/**
5135	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5136	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5137	* darwin, running 64-bit guests).
5138	*
5139	* @returns VBox status code.
5140	* @param pVCpu Pointer to the VMCPU.
5141	* @param idxField The VMCS field encoding.
5142	* @param u64Val 16, 32 or 64-bit value.
5143	*/
5144	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5145	{
5146	int rc;
5147	switch (idxField)
5148	{
5149	/*
5150	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5151	*/
5152	/* 64-bit Control fields. */
5153	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5154	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5155	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5156	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5157	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5158	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5159	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5160	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5161	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5162	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5163	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5164	case VMX_VMCS64_CTRL_EPTP_FULL:
5165	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5166	/* 64-bit Guest-state fields. */
5167	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5168	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5169	case VMX_VMCS64_GUEST_PAT_FULL:
5170	case VMX_VMCS64_GUEST_EFER_FULL:
5171	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5172	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5173	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5174	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5175	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5176	/* 64-bit Host-state fields. */
5177	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5178	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5179	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5180	{
5181	rc = VMXWriteVmcs32(idxField, u64Val);
5182	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5183	break;
5184	}
5185
5186	/*
5187	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5188	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5189	*/
5190	/* Natural-width Guest-state fields. */
5191	case VMX_VMCS_GUEST_CR3:
5192	case VMX_VMCS_GUEST_ES_BASE:
5193	case VMX_VMCS_GUEST_CS_BASE:
5194	case VMX_VMCS_GUEST_SS_BASE:
5195	case VMX_VMCS_GUEST_DS_BASE:
5196	case VMX_VMCS_GUEST_FS_BASE:
5197	case VMX_VMCS_GUEST_GS_BASE:
5198	case VMX_VMCS_GUEST_LDTR_BASE:
5199	case VMX_VMCS_GUEST_TR_BASE:
5200	case VMX_VMCS_GUEST_GDTR_BASE:
5201	case VMX_VMCS_GUEST_IDTR_BASE:
5202	case VMX_VMCS_GUEST_RSP:
5203	case VMX_VMCS_GUEST_RIP:
5204	case VMX_VMCS_GUEST_SYSENTER_ESP:
5205	case VMX_VMCS_GUEST_SYSENTER_EIP:
5206	{
5207	if (!(u64Val >> 32))
5208	{
5209	/* If this field is 64-bit, VT-x will zero out the top bits. */
5210	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5211	}
5212	else
5213	{
5214	/* Assert that only the 32->64 switcher case should ever come here. */
5215	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5216	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5217	}
5218	break;
5219	}
5220
5221	default:
5222	{
5223	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5224	rc = VERR_INVALID_PARAMETER;
5225	break;
5226	}
5227	}
5228	AssertRCReturn(rc, rc);
5229	return rc;
5230	}
5231
5232
5233	/**
5234	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5235	* hosts (except darwin) for 64-bit guests.
5236	*
5237	* @param pVCpu Pointer to the VMCPU.
5238	* @param idxField The VMCS field encoding.
5239	* @param u64Val 16, 32 or 64-bit value.
5240	*/
5241	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5242	{
5243	AssertPtr(pVCpu);
5244	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5245
5246	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5247	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5248
5249	/* Make sure there are no duplicates. */
5250	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5251	{
5252	if (pCache->Write.aField[i] == idxField)
5253	{
5254	pCache->Write.aFieldVal[i] = u64Val;
5255	return VINF_SUCCESS;
5256	}
5257	}
5258
5259	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5260	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5261	pCache->Write.cValidEntries++;
5262	return VINF_SUCCESS;
5263	}
5264
5265	/* Enable later when the assembly code uses these as callbacks. */
5266	#if 0
5267	/*
5268	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5269	*
5270	* @param pVCpu Pointer to the VMCPU.
5271	* @param pCache Pointer to the VMCS cache.
5272	*
5273	* @remarks No-long-jump zone!!!
5274	*/
5275	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5276	{
5277	AssertPtr(pCache);
5278	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5279	{
5280	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5281	AssertRC(rc);
5282	}
5283	pCache->Write.cValidEntries = 0;
5284	}
5285
5286
5287	/**
5288	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5289	*
5290	* @param pVCpu Pointer to the VMCPU.
5291	* @param pCache Pointer to the VMCS cache.
5292	*
5293	* @remarks No-long-jump zone!!!
5294	*/
5295	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5296	{
5297	AssertPtr(pCache);
5298	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5299	{
5300	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5301	AssertRC(rc);
5302	}
5303	}
5304	#endif
5305	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5306
5307
5308	/**
5309	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5310	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5311	* timer.
5312	*
5313	* @returns VBox status code.
5314	* @param pVCpu Pointer to the VMCPU.
5315	*
5316	* @remarks No-long-jump zone!!!
5317	*/
5318	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5319	{
5320	int rc = VERR_INTERNAL_ERROR_5;
5321	bool fOffsettedTsc = false;
5322	PVM pVM = pVCpu->CTX_SUFF(pVM);
5323	if (pVM->hm.s.vmx.fUsePreemptTimer)
5324	{
5325	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5326
5327	/* Make sure the returned values have sane upper and lower boundaries. */
5328	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5329	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5330	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5331	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5332
5333	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5334	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5335	}
5336	else
5337	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5338
5339	if (fOffsettedTsc)
5340	{
5341	uint64_t u64CurTSC = ASMReadTSC();
5342	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5343	{
5344	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5345	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5346
5347	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5348	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5349	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5350	}
5351	else
5352	{
5353	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5354	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5355	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5357	}
5358	}
5359	else
5360	{
5361	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5362	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5363	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5364	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5365	}
5366	}
5367
5368
5369	/**
5370	* Determines if an exception is a contributory exception. Contributory
5371	* exceptions are ones which can cause double-faults. Page-fault is
5372	* intentionally not included here as it's a conditional contributory exception.
5373	*
5374	* @returns true if the exception is contributory, false otherwise.
5375	* @param uVector The exception vector.
5376	*/
5377	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5378	{
5379	switch (uVector)
5380	{
5381	case X86_XCPT_GP:
5382	case X86_XCPT_SS:
5383	case X86_XCPT_NP:
5384	case X86_XCPT_TS:
5385	case X86_XCPT_DE:
5386	return true;
5387	default:
5388	break;
5389	}
5390	return false;
5391	}
5392
5393
5394	/**
5395	* Sets an event as a pending event to be injected into the guest.
5396	*
5397	* @param pVCpu Pointer to the VMCPU.
5398	* @param u32IntInfo The VM-entry interruption-information field.
5399	* @param cbInstr The VM-entry instruction length in bytes (for software
5400	* interrupts, exceptions and privileged software
5401	* exceptions).
5402	* @param u32ErrCode The VM-entry exception error code.
5403	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5404	* page-fault.
5405	*
5406	* @remarks Statistics counter assumes this is a guest event being injected or
5407	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5408	* always incremented.
5409	*/
5410	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5411	RTGCUINTPTR GCPtrFaultAddress)
5412	{
5413	Assert(!pVCpu->hm.s.Event.fPending);
5414	pVCpu->hm.s.Event.fPending = true;
5415	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5416	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5417	pVCpu->hm.s.Event.cbInstr = cbInstr;
5418	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5419
5420	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5421	}
5422
5423
5424	/**
5425	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5426	*
5427	* @param pVCpu Pointer to the VMCPU.
5428	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5429	* out-of-sync. Make sure to update the required fields
5430	* before using them.
5431	*/
5432	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5433	{
5434	NOREF(pMixedCtx);
5435	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5436	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5437	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5438	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5439	}
5440
5441
5442	/**
5443	* Handle a condition that occurred while delivering an event through the guest
5444	* IDT.
5445	*
5446	* @returns VBox status code (informational error codes included).
5447	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5448	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5449	* continue execution of the guest which will delivery the #DF.
5450	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5451	*
5452	* @param pVCpu Pointer to the VMCPU.
5453	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5454	* out-of-sync. Make sure to update the required fields
5455	* before using them.
5456	* @param pVmxTransient Pointer to the VMX transient structure.
5457	*
5458	* @remarks No-long-jump zone!!!
5459	*/
5460	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5461	{
5462	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5463	AssertRCReturn(rc, rc);
5464	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5465	{
5466	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5467	AssertRCReturn(rc, rc);
5468
5469	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5470	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5471	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5472
5473	typedef enum
5474	{
5475	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5476	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5477	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5478	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5479	} VMXREFLECTXCPT;
5480
5481	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5482	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5483	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5484	{
5485	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5486	{
5487	enmReflect = VMXREFLECTXCPT_XCPT;
5488	#ifdef VBOX_STRICT
5489	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5490	&& uExitVector == X86_XCPT_PF)
5491	{
5492	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5493	}
5494	#endif
5495	if ( uExitVector == X86_XCPT_PF
5496	&& uIdtVector == X86_XCPT_PF)
5497	{
5498	pVmxTransient->fVectoringPF = true;
5499	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5500	}
5501	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5502	&& hmR0VmxIsContributoryXcpt(uExitVector)
5503	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5504	\|\| uIdtVector == X86_XCPT_PF))
5505	{
5506	enmReflect = VMXREFLECTXCPT_DF;
5507	}
5508	else if (uIdtVector == X86_XCPT_DF)
5509	enmReflect = VMXREFLECTXCPT_TF;
5510	}
5511	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5512	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5513	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5514	{
5515	/*
5516	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5517	* (whatever they are) as they reoccur when restarting the instruction.
5518	*/
5519	enmReflect = VMXREFLECTXCPT_XCPT;
5520	}
5521	}
5522	else
5523	{
5524	/*
5525	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5526	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5527	* original exception to the guest after handling the VM-exit.
5528	*/
5529	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5530	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5531	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5532	{
5533	enmReflect = VMXREFLECTXCPT_XCPT;
5534	}
5535	}
5536
5537	switch (enmReflect)
5538	{
5539	case VMXREFLECTXCPT_XCPT:
5540	{
5541	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5542	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5543	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5544
5545	uint32_t u32ErrCode = 0;
5546	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5547	{
5548	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5549	AssertRCReturn(rc, rc);
5550	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5551	}
5552
5553	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5554	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5555	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5556	rc = VINF_SUCCESS;
5557	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5558	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5559
5560	break;
5561	}
5562
5563	case VMXREFLECTXCPT_DF:
5564	{
5565	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5566	rc = VINF_HM_DOUBLE_FAULT;
5567	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5568	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5569
5570	break;
5571	}
5572
5573	case VMXREFLECTXCPT_TF:
5574	{
5575	rc = VINF_EM_RESET;
5576	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5577	uExitVector));
5578	break;
5579	}
5580
5581	default:
5582	Assert(rc == VINF_SUCCESS);
5583	break;
5584	}
5585	}
5586	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5587	return rc;
5588	}
5589
5590
5591	/**
5592	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5593	*
5594	* @returns VBox status code.
5595	* @param pVCpu Pointer to the VMCPU.
5596	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5597	* out-of-sync. Make sure to update the required fields
5598	* before using them.
5599	*
5600	* @remarks No-long-jump zone!!!
5601	*/
5602	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5603	{
5604	NOREF(pMixedCtx);
5605
5606	/*
5607	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5608	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5609	*/
5610	VMMRZCallRing3Disable(pVCpu);
5611	HM_DISABLE_PREEMPT_IF_NEEDED();
5612
5613	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5614	{
5615	uint32_t uVal = 0;
5616	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5617	AssertRCReturn(rc, rc);
5618
5619	uint32_t uShadow = 0;
5620	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5621	AssertRCReturn(rc, rc);
5622
5623	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5624	CPUMSetGuestCR0(pVCpu, uVal);
5625	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5626	}
5627
5628	HM_RESTORE_PREEMPT_IF_NEEDED();
5629	VMMRZCallRing3Enable(pVCpu);
5630	return VINF_SUCCESS;
5631	}
5632
5633
5634	/**
5635	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5636	*
5637	* @returns VBox status code.
5638	* @param pVCpu Pointer to the VMCPU.
5639	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5640	* out-of-sync. Make sure to update the required fields
5641	* before using them.
5642	*
5643	* @remarks No-long-jump zone!!!
5644	*/
5645	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5646	{
5647	NOREF(pMixedCtx);
5648
5649	int rc = VINF_SUCCESS;
5650	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5651	{
5652	uint32_t uVal = 0;
5653	uint32_t uShadow = 0;
5654	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5655	AssertRCReturn(rc, rc);
5656	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5657	AssertRCReturn(rc, rc);
5658
5659	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5660	CPUMSetGuestCR4(pVCpu, uVal);
5661	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5662	}
5663	return rc;
5664	}
5665
5666
5667	/**
5668	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5669	*
5670	* @returns VBox status code.
5671	* @param pVCpu Pointer to the VMCPU.
5672	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5673	* out-of-sync. Make sure to update the required fields
5674	* before using them.
5675	*
5676	* @remarks No-long-jump zone!!!
5677	*/
5678	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5679	{
5680	int rc = VINF_SUCCESS;
5681	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5682	{
5683	uint64_t u64Val = 0;
5684	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5685	AssertRCReturn(rc, rc);
5686
5687	pMixedCtx->rip = u64Val;
5688	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5689	}
5690	return rc;
5691	}
5692
5693
5694	/**
5695	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5696	*
5697	* @returns VBox status code.
5698	* @param pVCpu Pointer to the VMCPU.
5699	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5700	* out-of-sync. Make sure to update the required fields
5701	* before using them.
5702	*
5703	* @remarks No-long-jump zone!!!
5704	*/
5705	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5706	{
5707	int rc = VINF_SUCCESS;
5708	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5709	{
5710	uint64_t u64Val = 0;
5711	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5712	AssertRCReturn(rc, rc);
5713
5714	pMixedCtx->rsp = u64Val;
5715	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5716	}
5717	return rc;
5718	}
5719
5720
5721	/**
5722	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5723	*
5724	* @returns VBox status code.
5725	* @param pVCpu Pointer to the VMCPU.
5726	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5727	* out-of-sync. Make sure to update the required fields
5728	* before using them.
5729	*
5730	* @remarks No-long-jump zone!!!
5731	*/
5732	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5735	{
5736	uint32_t uVal = 0;
5737	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5738	AssertRCReturn(rc, rc);
5739
5740	pMixedCtx->eflags.u32 = uVal;
5741	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5742	{
5743	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5744	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5745
5746	pMixedCtx->eflags.Bits.u1VM = 0;
5747	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5748	}
5749
5750	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5751	}
5752	return VINF_SUCCESS;
5753	}
5754
5755
5756	/**
5757	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5758	* guest-CPU context.
5759	*/
5760	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5761	{
5762	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5763	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5764	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5765	return rc;
5766	}
5767
5768
5769	/**
5770	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5771	* from the guest-state area in the VMCS.
5772	*
5773	* @param pVCpu Pointer to the VMCPU.
5774	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5775	* out-of-sync. Make sure to update the required fields
5776	* before using them.
5777	*
5778	* @remarks No-long-jump zone!!!
5779	*/
5780	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5781	{
5782	uint32_t uIntrState = 0;
5783	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5784	AssertRC(rc);
5785
5786	if (!uIntrState)
5787	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5788	else
5789	{
5790	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5791	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5792	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5793	AssertRC(rc);
5794	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5795	AssertRC(rc);
5796
5797	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5798	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5799	}
5800	}
5801
5802
5803	/**
5804	* Saves the guest's activity state.
5805	*
5806	* @returns VBox status code.
5807	* @param pVCpu Pointer to the VMCPU.
5808	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5809	* out-of-sync. Make sure to update the required fields
5810	* before using them.
5811	*
5812	* @remarks No-long-jump zone!!!
5813	*/
5814	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5815	{
5816	NOREF(pMixedCtx);
5817	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5818	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5819	return VINF_SUCCESS;
5820	}
5821
5822
5823	/**
5824	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5825	* the current VMCS into the guest-CPU context.
5826	*
5827	* @returns VBox status code.
5828	* @param pVCpu Pointer to the VMCPU.
5829	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5830	* out-of-sync. Make sure to update the required fields
5831	* before using them.
5832	*
5833	* @remarks No-long-jump zone!!!
5834	*/
5835	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5836	{
5837	int rc = VINF_SUCCESS;
5838	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5839	{
5840	uint32_t u32Val = 0;
5841	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5842	pMixedCtx->SysEnter.cs = u32Val;
5843	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5844	}
5845
5846	uint64_t u64Val = 0;
5847	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5848	{
5849	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5850	pMixedCtx->SysEnter.eip = u64Val;
5851	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5852	}
5853	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5854	{
5855	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5856	pMixedCtx->SysEnter.esp = u64Val;
5857	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5858	}
5859	return rc;
5860	}
5861
5862
5863	/**
5864	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5865	* the CPU back into the guest-CPU context.
5866	*
5867	* @returns VBox status code.
5868	* @param pVCpu Pointer to the VMCPU.
5869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5870	* out-of-sync. Make sure to update the required fields
5871	* before using them.
5872	*
5873	* @remarks No-long-jump zone!!!
5874	*/
5875	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5876	{
5877	#if HC_ARCH_BITS == 64
5878	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5879	{
5880	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5881	VMMRZCallRing3Disable(pVCpu);
5882	HM_DISABLE_PREEMPT_IF_NEEDED();
5883
5884	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5885	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5886	{
5887	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5888	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5889	}
5890
5891	HM_RESTORE_PREEMPT_IF_NEEDED();
5892	VMMRZCallRing3Enable(pVCpu);
5893	}
5894	else
5895	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5896	#else
5897	NOREF(pMixedCtx);
5898	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5899	#endif
5900
5901	return VINF_SUCCESS;
5902	}
5903
5904
5905	/**
5906	* Saves the auto load/store'd guest MSRs from the current VMCS into
5907	* the guest-CPU context.
5908	*
5909	* @returns VBox status code.
5910	* @param pVCpu Pointer to the VMCPU.
5911	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5912	* out-of-sync. Make sure to update the required fields
5913	* before using them.
5914	*
5915	* @remarks No-long-jump zone!!!
5916	*/
5917	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5918	{
5919	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5920	return VINF_SUCCESS;
5921
5922	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5923	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5924	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5925	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5926	{
5927	switch (pMsr->u32Msr)
5928	{
5929	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5930	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5931	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5932	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5933	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5934	default:
5935	{
5936	AssertFailed();
5937	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5938	}
5939	}
5940	}
5941
5942	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5943	return VINF_SUCCESS;
5944	}
5945
5946
5947	/**
5948	* Saves the guest control registers from the current VMCS into the guest-CPU
5949	* context.
5950	*
5951	* @returns VBox status code.
5952	* @param pVCpu Pointer to the VMCPU.
5953	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5954	* out-of-sync. Make sure to update the required fields
5955	* before using them.
5956	*
5957	* @remarks No-long-jump zone!!!
5958	*/
5959	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5960	{
5961	/* Guest CR0. Guest FPU. */
5962	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5963	AssertRCReturn(rc, rc);
5964
5965	/* Guest CR4. */
5966	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5967	AssertRCReturn(rc, rc);
5968
5969	/* Guest CR2 - updated always during the world-switch or in #PF. */
5970	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5971	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5972	{
5973	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5974	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5975
5976	PVM pVM = pVCpu->CTX_SUFF(pVM);
5977	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5978	\|\| ( pVM->hm.s.fNestedPaging
5979	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5980	{
5981	uint64_t u64Val = 0;
5982	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5983	if (pMixedCtx->cr3 != u64Val)
5984	{
5985	CPUMSetGuestCR3(pVCpu, u64Val);
5986	if (VMMRZCallRing3IsEnabled(pVCpu))
5987	{
5988	PGMUpdateCR3(pVCpu, u64Val);
5989	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5990	}
5991	else
5992	{
5993	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5994	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5995	}
5996	}
5997
5998	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5999	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6000	{
6001	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6002	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6003	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6004	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6005
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6009	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6010	}
6011	else
6012	{
6013	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6014	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6015	}
6016	}
6017	}
6018
6019	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6020	}
6021
6022	/*
6023	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6024	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6025	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6026	*
6027	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6028	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6029	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6030	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6031	*
6032	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6033	*/
6034	if (VMMRZCallRing3IsEnabled(pVCpu))
6035	{
6036	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6037	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6038
6039	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6040	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6041
6042	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6043	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6044	}
6045
6046	return rc;
6047	}
6048
6049
6050	/**
6051	* Reads a guest segment register from the current VMCS into the guest-CPU
6052	* context.
6053	*
6054	* @returns VBox status code.
6055	* @param pVCpu Pointer to the VMCPU.
6056	* @param idxSel Index of the selector in the VMCS.
6057	* @param idxLimit Index of the segment limit in the VMCS.
6058	* @param idxBase Index of the segment base in the VMCS.
6059	* @param idxAccess Index of the access rights of the segment in the VMCS.
6060	* @param pSelReg Pointer to the segment selector.
6061	*
6062	* @remarks No-long-jump zone!!!
6063	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6064	* macro as that takes care of whether to read from the VMCS cache or
6065	* not.
6066	*/
6067	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6068	PCPUMSELREG pSelReg)
6069	{
6070	NOREF(pVCpu);
6071
6072	uint32_t u32Val = 0;
6073	int rc = VMXReadVmcs32(idxSel, &u32Val);
6074	AssertRCReturn(rc, rc);
6075	pSelReg->Sel = (uint16_t)u32Val;
6076	pSelReg->ValidSel = (uint16_t)u32Val;
6077	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6078
6079	rc = VMXReadVmcs32(idxLimit, &u32Val);
6080	AssertRCReturn(rc, rc);
6081	pSelReg->u32Limit = u32Val;
6082
6083	uint64_t u64Val = 0;
6084	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6085	AssertRCReturn(rc, rc);
6086	pSelReg->u64Base = u64Val;
6087
6088	rc = VMXReadVmcs32(idxAccess, &u32Val);
6089	AssertRCReturn(rc, rc);
6090	pSelReg->Attr.u = u32Val;
6091
6092	/*
6093	* If VT-x marks the segment as unusable, most other bits remain undefined:
6094	* - For CS the L, D and G bits have meaning.
6095	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6096	* - For the remaining data segments no bits are defined.
6097	*
6098	* The present bit and the unusable bit has been observed to be set at the
6099	* same time (the selector was supposed to invalid as we started executing
6100	* a V8086 interrupt in ring-0).
6101	*
6102	* What should be important for the rest of the VBox code, is that the P bit is
6103	* cleared. Some of the other VBox code recognizes the unusable bit, but
6104	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6105	* safe side here, we'll strip off P and other bits we don't care about. If
6106	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6107	*
6108	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6109	*/
6110	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6111	{
6112	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6113
6114	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6115	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6116	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6117
6118	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6119	#ifdef DEBUG_bird
6120	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6121	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6122	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6123	#endif
6124	}
6125	return VINF_SUCCESS;
6126	}
6127
6128
6129	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6130	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6131	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6132	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6133	#else
6134	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6135	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6136	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6137	#endif
6138
6139
6140	/**
6141	* Saves the guest segment registers from the current VMCS into the guest-CPU
6142	* context.
6143	*
6144	* @returns VBox status code.
6145	* @param pVCpu Pointer to the VMCPU.
6146	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6147	* out-of-sync. Make sure to update the required fields
6148	* before using them.
6149	*
6150	* @remarks No-long-jump zone!!!
6151	*/
6152	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6153	{
6154	/* Guest segment registers. */
6155	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6156	{
6157	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6158	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6159	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6160	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6161	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6162	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6163	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6164
6165	/* Restore segment attributes for real-on-v86 mode hack. */
6166	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6167	{
6168	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6169	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6170	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6171	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6172	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6173	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6174	}
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6176	}
6177
6178	return VINF_SUCCESS;
6179	}
6180
6181
6182	/**
6183	* Saves the guest descriptor table registers and task register from the current
6184	* VMCS into the guest-CPU context.
6185	*
6186	* @returns VBox status code.
6187	* @param pVCpu Pointer to the VMCPU.
6188	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6189	* out-of-sync. Make sure to update the required fields
6190	* before using them.
6191	*
6192	* @remarks No-long-jump zone!!!
6193	*/
6194	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6195	{
6196	int rc = VINF_SUCCESS;
6197
6198	/* Guest LDTR. */
6199	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6200	{
6201	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6202	AssertRCReturn(rc, rc);
6203	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6204	}
6205
6206	/* Guest GDTR. */
6207	uint64_t u64Val = 0;
6208	uint32_t u32Val = 0;
6209	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6210	{
6211	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6212	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6213	pMixedCtx->gdtr.pGdt = u64Val;
6214	pMixedCtx->gdtr.cbGdt = u32Val;
6215	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6216	}
6217
6218	/* Guest IDTR. */
6219	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6220	{
6221	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6222	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6223	pMixedCtx->idtr.pIdt = u64Val;
6224	pMixedCtx->idtr.cbIdt = u32Val;
6225	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6226	}
6227
6228	/* Guest TR. */
6229	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6230	{
6231	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6232	AssertRCReturn(rc, rc);
6233
6234	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6235	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6236	{
6237	rc = VMXLOCAL_READ_SEG(TR, tr);
6238	AssertRCReturn(rc, rc);
6239	}
6240	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6241	}
6242	return rc;
6243	}
6244
6245	#undef VMXLOCAL_READ_SEG
6246
6247
6248	/**
6249	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6250	* context.
6251	*
6252	* @returns VBox status code.
6253	* @param pVCpu Pointer to the VMCPU.
6254	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6255	* out-of-sync. Make sure to update the required fields
6256	* before using them.
6257	*
6258	* @remarks No-long-jump zone!!!
6259	*/
6260	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6261	{
6262	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6263	{
6264	if (!pVCpu->hm.s.fUsingHyperDR7)
6265	{
6266	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6267	uint32_t u32Val;
6268	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6269	pMixedCtx->dr[7] = u32Val;
6270	}
6271
6272	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6273	}
6274	return VINF_SUCCESS;
6275	}
6276
6277
6278	/**
6279	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6280	*
6281	* @returns VBox status code.
6282	* @param pVCpu Pointer to the VMCPU.
6283	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6284	* out-of-sync. Make sure to update the required fields
6285	* before using them.
6286	*
6287	* @remarks No-long-jump zone!!!
6288	*/
6289	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6290	{
6291	NOREF(pMixedCtx);
6292
6293	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6294	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6295	return VINF_SUCCESS;
6296	}
6297
6298
6299	/**
6300	* Saves the entire guest state from the currently active VMCS into the
6301	* guest-CPU context. This essentially VMREADs all guest-data.
6302	*
6303	* @returns VBox status code.
6304	* @param pVCpu Pointer to the VMCPU.
6305	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6306	* out-of-sync. Make sure to update the required fields
6307	* before using them.
6308	*/
6309	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6310	{
6311	Assert(pVCpu);
6312	Assert(pMixedCtx);
6313
6314	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6315	return VINF_SUCCESS;
6316
6317	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6318	again on the ring-3 callback path, there is no real need to. */
6319	if (VMMRZCallRing3IsEnabled(pVCpu))
6320	VMMR0LogFlushDisable(pVCpu);
6321	else
6322	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6323	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6324
6325	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6327
6328	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6330
6331	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6332	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6333
6334	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6335	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6336
6337	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6338	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6339
6340	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6341	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6342
6343	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6344	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6345
6346	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6347	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6348
6349	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6350	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6351
6352	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6353	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6354
6355	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6356	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6357
6358	if (VMMRZCallRing3IsEnabled(pVCpu))
6359	VMMR0LogFlushEnable(pVCpu);
6360
6361	return rc;
6362	}
6363
6364
6365	/**
6366	* Check per-VM and per-VCPU force flag actions that require us to go back to
6367	* ring-3 for one reason or another.
6368	*
6369	* @returns VBox status code (information status code included).
6370	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6371	* ring-3.
6372	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6373	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6374	* interrupts)
6375	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6376	* all EMTs to be in ring-3.
6377	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6378	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6379	* to the EM loop.
6380	*
6381	* @param pVM Pointer to the VM.
6382	* @param pVCpu Pointer to the VMCPU.
6383	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6384	* out-of-sync. Make sure to update the required fields
6385	* before using them.
6386	*/
6387	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6388	{
6389	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6390
6391	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6392	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6393	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6394	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6395	{
6396	/* We need the control registers now, make sure the guest-CPU context is updated. */
6397	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6398	AssertRCReturn(rc3, rc3);
6399
6400	/* Pending HM CR3 sync. */
6401	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6402	{
6403	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6404	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6405	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6406	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6407	}
6408
6409	/* Pending HM PAE PDPEs. */
6410	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6411	{
6412	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6413	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6414	}
6415
6416	/* Pending PGM C3 sync. */
6417	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6418	{
6419	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6420	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6421	if (rc2 != VINF_SUCCESS)
6422	{
6423	AssertRC(rc2);
6424	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6425	return rc2;
6426	}
6427	}
6428
6429	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6430	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6431	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6432	{
6433	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6434	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6435	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6436	return rc2;
6437	}
6438
6439	/* Pending VM request packets, such as hardware interrupts. */
6440	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6441	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6442	{
6443	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6444	return VINF_EM_PENDING_REQUEST;
6445	}
6446
6447	/* Pending PGM pool flushes. */
6448	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6449	{
6450	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6451	return VINF_PGM_POOL_FLUSH_PENDING;
6452	}
6453
6454	/* Pending DMA requests. */
6455	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6456	{
6457	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6458	return VINF_EM_RAW_TO_R3;
6459	}
6460	}
6461
6462	return VINF_SUCCESS;
6463	}
6464
6465
6466	/**
6467	* Converts any TRPM trap into a pending HM event. This is typically used when
6468	* entering from ring-3 (not longjmp returns).
6469	*
6470	* @param pVCpu Pointer to the VMCPU.
6471	*/
6472	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6473	{
6474	Assert(TRPMHasTrap(pVCpu));
6475	Assert(!pVCpu->hm.s.Event.fPending);
6476
6477	uint8_t uVector;
6478	TRPMEVENT enmTrpmEvent;
6479	RTGCUINT uErrCode;
6480	RTGCUINTPTR GCPtrFaultAddress;
6481	uint8_t cbInstr;
6482
6483	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6484	AssertRC(rc);
6485
6486	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6487	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6488	if (enmTrpmEvent == TRPM_TRAP)
6489	{
6490	switch (uVector)
6491	{
6492	case X86_XCPT_BP:
6493	case X86_XCPT_OF:
6494	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6495	break;
6496
6497	case X86_XCPT_PF:
6498	case X86_XCPT_DF:
6499	case X86_XCPT_TS:
6500	case X86_XCPT_NP:
6501	case X86_XCPT_SS:
6502	case X86_XCPT_GP:
6503	case X86_XCPT_AC:
6504	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6505	/* no break! */
6506	default:
6507	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6508	break;
6509	}
6510	}
6511	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6512	{
6513	if (uVector == X86_XCPT_NMI)
6514	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6515	else
6516	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6517	}
6518	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6519	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6520	else
6521	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6522
6523	rc = TRPMResetTrap(pVCpu);
6524	AssertRC(rc);
6525	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6526	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6527
6528	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6529	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6530	}
6531
6532
6533	/**
6534	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6535	* VT-x to execute any instruction.
6536	*
6537	* @param pvCpu Pointer to the VMCPU.
6538	*/
6539	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6540	{
6541	Assert(pVCpu->hm.s.Event.fPending);
6542
6543	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6544	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6545	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6546	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6547
6548	/* If a trap was already pending, we did something wrong! */
6549	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6550
6551	TRPMEVENT enmTrapType;
6552	switch (uVectorType)
6553	{
6554	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6555	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6556	enmTrapType = TRPM_HARDWARE_INT;
6557	break;
6558
6559	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6560	enmTrapType = TRPM_SOFTWARE_INT;
6561	break;
6562
6563	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6564	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6565	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6566	enmTrapType = TRPM_TRAP;
6567	break;
6568
6569	default:
6570	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6571	enmTrapType = TRPM_32BIT_HACK;
6572	break;
6573	}
6574
6575	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6576
6577	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6578	AssertRC(rc);
6579
6580	if (fErrorCodeValid)
6581	TRPMSetErrorCode(pVCpu, uErrorCode);
6582
6583	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6584	&& uVector == X86_XCPT_PF)
6585	{
6586	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6587	}
6588	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6589	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6590	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6591	{
6592	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6593	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6594	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6595	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6596	}
6597	pVCpu->hm.s.Event.fPending = false;
6598	}
6599
6600
6601	/**
6602	* Does the necessary state syncing before returning to ring-3 for any reason
6603	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6604	*
6605	* @returns VBox status code.
6606	* @param pVM Pointer to the VM.
6607	* @param pVCpu Pointer to the VMCPU.
6608	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6609	* be out-of-sync. Make sure to update the required
6610	* fields before using them.
6611	* @param fSaveGuestState Whether to save the guest state or not.
6612	*
6613	* @remarks No-long-jmp zone!!!
6614	*/
6615	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6616	{
6617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6618	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6619
6620	RTCPUID idCpu = RTMpCpuId();
6621	Log4Func(("HostCpuId=%u\n", idCpu));
6622
6623	/*
6624	* !!! IMPORTANT !!!
6625	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6626	*/
6627
6628	/* Save the guest state if necessary. */
6629	if ( fSaveGuestState
6630	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6631	{
6632	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6633	AssertRCReturn(rc, rc);
6634	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6635	}
6636
6637	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6638	if (CPUMIsGuestFPUStateActive(pVCpu))
6639	{
6640	/* We shouldn't reload CR0 without saving it first. */
6641	if (!fSaveGuestState)
6642	{
6643	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6644	AssertRCReturn(rc, rc);
6645	}
6646	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6647	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6648	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6649	}
6650
6651	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6652	#ifdef VBOX_STRICT
6653	if (CPUMIsHyperDebugStateActive(pVCpu))
6654	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6655	#endif
6656	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6657	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6658	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6659	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6660
6661	#if HC_ARCH_BITS == 64
6662	/* Restore host-state bits that VT-x only restores partially. */
6663	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6664	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6665	{
6666	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6667	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6668	}
6669	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6670	#endif
6671
6672	#if HC_ARCH_BITS == 64
6673	/* Restore the host MSRs as we're leaving VT-x context. */
6674	if ( pVM->hm.s.fAllow64BitGuests
6675	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6676	{
6677	/* We shouldn't reload the guest MSRs without saving it first. */
6678	if (!fSaveGuestState)
6679	{
6680	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6681	AssertRCReturn(rc, rc);
6682	}
6683	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6684	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6685	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6686	}
6687	#endif
6688
6689	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6690	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6691
6692	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6693	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6694	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6695	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6696	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6697	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6698	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6699	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6700
6701	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6702
6703	/** @todo This partially defeats the purpose of having preemption hooks.
6704	* The problem is, deregistering the hooks should be moved to a place that
6705	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6706	* context.
6707	*/
6708	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6709	{
6710	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6711	AssertRCReturn(rc, rc);
6712
6713	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6714	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6715	}
6716	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6717	NOREF(idCpu);
6718
6719	return VINF_SUCCESS;
6720	}
6721
6722
6723	/**
6724	* Leaves the VT-x session.
6725	*
6726	* @returns VBox status code.
6727	* @param pVM Pointer to the VM.
6728	* @param pVCpu Pointer to the VMCPU.
6729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6730	* out-of-sync. Make sure to update the required fields
6731	* before using them.
6732	*
6733	* @remarks No-long-jmp zone!!!
6734	*/
6735	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6736	{
6737	HM_DISABLE_PREEMPT_IF_NEEDED();
6738	HMVMX_ASSERT_CPU_SAFE();
6739	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6740	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6741
6742	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6743	and done this from the VMXR0ThreadCtxCallback(). */
6744	if (!pVCpu->hm.s.fLeaveDone)
6745	{
6746	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6747	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6748	pVCpu->hm.s.fLeaveDone = true;
6749	}
6750
6751	/*
6752	* !!! IMPORTANT !!!
6753	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6754	*/
6755
6756	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6757	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6758	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6759	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6760	VMMR0ThreadCtxHooksDeregister(pVCpu);
6761
6762	/* Leave HM context. This takes care of local init (term). */
6763	int rc = HMR0LeaveCpu(pVCpu);
6764
6765	HM_RESTORE_PREEMPT_IF_NEEDED();
6766
6767	return rc;
6768	}
6769
6770
6771	/**
6772	* Does the necessary state syncing before doing a longjmp to ring-3.
6773	*
6774	* @returns VBox status code.
6775	* @param pVM Pointer to the VM.
6776	* @param pVCpu Pointer to the VMCPU.
6777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6778	* out-of-sync. Make sure to update the required fields
6779	* before using them.
6780	*
6781	* @remarks No-long-jmp zone!!!
6782	*/
6783	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6784	{
6785	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6786	}
6787
6788
6789	/**
6790	* Take necessary actions before going back to ring-3.
6791	*
6792	* An action requires us to go back to ring-3. This function does the necessary
6793	* steps before we can safely return to ring-3. This is not the same as longjmps
6794	* to ring-3, this is voluntary and prepares the guest so it may continue
6795	* executing outside HM (recompiler/IEM).
6796	*
6797	* @returns VBox status code.
6798	* @param pVM Pointer to the VM.
6799	* @param pVCpu Pointer to the VMCPU.
6800	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6801	* out-of-sync. Make sure to update the required fields
6802	* before using them.
6803	* @param rcExit The reason for exiting to ring-3. Can be
6804	* VINF_VMM_UNKNOWN_RING3_CALL.
6805	*/
6806	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6807	{
6808	Assert(pVM);
6809	Assert(pVCpu);
6810	Assert(pMixedCtx);
6811	HMVMX_ASSERT_PREEMPT_SAFE();
6812
6813	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6814	{
6815	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6816	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6817	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6818	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6819	}
6820
6821	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6822	VMMRZCallRing3Disable(pVCpu);
6823	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6824
6825	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6826	if (pVCpu->hm.s.Event.fPending)
6827	{
6828	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6829	Assert(!pVCpu->hm.s.Event.fPending);
6830	}
6831
6832	/* Save guest state and restore host state bits. */
6833	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6834	AssertRCReturn(rc, rc);
6835	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6836
6837	/* Sync recompiler state. */
6838	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6839	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6840	\| CPUM_CHANGED_LDTR
6841	\| CPUM_CHANGED_GDTR
6842	\| CPUM_CHANGED_IDTR
6843	\| CPUM_CHANGED_TR
6844	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6845	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6846	if ( pVM->hm.s.fNestedPaging
6847	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6848	{
6849	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6850	}
6851
6852	Assert(!pVCpu->hm.s.fClearTrapFlag);
6853
6854	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6855	if (rcExit != VINF_EM_RAW_INTERRUPT)
6856	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6857
6858	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6859
6860	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6861	VMMRZCallRing3RemoveNotification(pVCpu);
6862	VMMRZCallRing3Enable(pVCpu);
6863
6864	return rc;
6865	}
6866
6867
6868	/**
6869	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6870	* longjump to ring-3 and possibly get preempted.
6871	*
6872	* @returns VBox status code.
6873	* @param pVCpu Pointer to the VMCPU.
6874	* @param enmOperation The operation causing the ring-3 longjump.
6875	* @param pvUser Opaque pointer to the guest-CPU context. The data
6876	* may be out-of-sync. Make sure to update the required
6877	* fields before using them.
6878	*/
6879	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6880	{
6881	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6882	{
6883	/*
6884	* !!! IMPORTANT !!!
6885	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6886	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6887	*/
6888	VMMRZCallRing3RemoveNotification(pVCpu);
6889	VMMRZCallRing3Disable(pVCpu);
6890	HM_DISABLE_PREEMPT_IF_NEEDED();
6891
6892	PVM pVM = pVCpu->CTX_SUFF(pVM);
6893	if (CPUMIsGuestFPUStateActive(pVCpu))
6894	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6895
6896	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6897
6898	#if HC_ARCH_BITS == 64
6899	/* Restore host-state bits that VT-x only restores partially. */
6900	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6901	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6902	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6903	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6904
6905	/* Restore the host MSRs as we're leaving VT-x context. */
6906	if ( pVM->hm.s.fAllow64BitGuests
6907	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6908	{
6909	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6910	}
6911	#endif
6912	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6913	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6914	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6915	{
6916	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6917	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6918	}
6919
6920	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6921	VMMR0ThreadCtxHooksDeregister(pVCpu);
6922
6923	HMR0LeaveCpu(pVCpu);
6924	HM_RESTORE_PREEMPT_IF_NEEDED();
6925	return VINF_SUCCESS;
6926	}
6927
6928	Assert(pVCpu);
6929	Assert(pvUser);
6930	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6931	HMVMX_ASSERT_PREEMPT_SAFE();
6932
6933	VMMRZCallRing3Disable(pVCpu);
6934	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6935
6936	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6937	enmOperation));
6938
6939	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6940	AssertRCReturn(rc, rc);
6941
6942	VMMRZCallRing3Enable(pVCpu);
6943	return VINF_SUCCESS;
6944	}
6945
6946
6947	/**
6948	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6949	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6950	*
6951	* @param pVCpu Pointer to the VMCPU.
6952	*/
6953	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6954	{
6955	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6956	{
6957	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6958	{
6959	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6960	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6961	AssertRC(rc);
6962	Log4(("Setup interrupt-window exiting\n"));
6963	}
6964	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6965	}
6966
6967
6968	/**
6969	* Clears the interrupt-window exiting control in the VMCS.
6970	*
6971	* @param pVCpu Pointer to the VMCPU.
6972	*/
6973	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
6974	{
6975	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
6976	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6977	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6978	AssertRC(rc);
6979	Log4(("Cleared interrupt-window exiting\n"));
6980	}
6981
6982
6983	/**
6984	* Evaluates the event to be delivered to the guest and sets it as the pending
6985	* event.
6986	*
6987	* @param pVCpu Pointer to the VMCPU.
6988	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6989	* out-of-sync. Make sure to update the required fields
6990	* before using them.
6991	*/
6992	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6993	{
6994	Assert(!pVCpu->hm.s.Event.fPending);
6995
6996	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6997	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6998	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6999	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7000
7001	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7002	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7003	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7004	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7005	Assert(!TRPMHasTrap(pVCpu));
7006
7007	/** @todo SMI. SMIs take priority over NMIs. */
7008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7009	{
7010	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7011	if ( !fBlockMovSS
7012	&& !fBlockSti)
7013	{
7014	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7015	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7016	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7017	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7018
7019	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
7020	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7021	}
7022	else
7023	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7024	}
7025	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7026	&& !pVCpu->hm.s.fSingleInstruction)
7027	{
7028	/*
7029	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7030	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
7031	* evaluated here and not set as pending, solely based on the force-flags.
7032	*/
7033	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7034	AssertRC(rc);
7035	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7036	if ( !fBlockInt
7037	&& !fBlockSti
7038	&& !fBlockMovSS)
7039	{
7040	uint8_t u8Interrupt;
7041	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7042	if (RT_SUCCESS(rc))
7043	{
7044	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7045	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7046	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7047
7048	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7049	}
7050	else
7051	{
7052	/** @todo Does this actually happen? If not turn it into an assertion. */
7053	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7054	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7055	}
7056	}
7057	else
7058	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7059	}
7060	}
7061
7062
7063	/**
7064	* Sets a pending-debug exception to be delivered to the guest if the guest is
7065	* single-stepping.
7066	*
7067	* @param pVCpu Pointer to the VMCPU.
7068	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7069	* out-of-sync. Make sure to update the required fields
7070	* before using them.
7071	*/
7072	DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7073	{
7074	HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
7075	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7076	{
7077	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7078	AssertRC(rc);
7079	}
7080	}
7081
7082
7083	/**
7084	* Injects any pending events into the guest if the guest is in a state to
7085	* receive them.
7086	*
7087	* @returns VBox status code (informational status codes included).
7088	* @param pVCpu Pointer to the VMCPU.
7089	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7090	* out-of-sync. Make sure to update the required fields
7091	* before using them.
7092	*/
7093	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7094	{
7095	HMVMX_ASSERT_PREEMPT_SAFE();
7096	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7097
7098	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7099	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7100	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7101	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7102
7103	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7104	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7105	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7106	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7107	Assert(!TRPMHasTrap(pVCpu));
7108
7109	int rc = VINF_SUCCESS;
7110	if (pVCpu->hm.s.Event.fPending)
7111	{
7112	/*
7113	* Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7114	* VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7115	* ended up enabling interrupts outside VT-x.
7116	*/
7117	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7118	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7119	&& ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7120	\|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7121	{
7122	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7123	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7124	}
7125	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7126	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7127	{
7128	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7129	AssertRCReturn(rc, rc);
7130	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7131	Assert(!fBlockInt);
7132	Assert(!fBlockSti);
7133	Assert(!fBlockMovSS);
7134	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7135	}
7136	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7137	{
7138	Assert(!fBlockSti);
7139	Assert(!fBlockMovSS);
7140	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7141	}
7142	#endif
7143	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7144	(uint8_t)uIntType));
7145	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7146	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7147	AssertRCReturn(rc, rc);
7148
7149	/* Update the interruptibility-state as it could have been changed by
7150	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7151	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7152	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7153
7154	#ifdef VBOX_WITH_STATISTICS
7155	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7156	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7157	else
7158	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7159	#endif
7160	}
7161
7162	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7163	if ( fBlockSti
7164	\|\| fBlockMovSS)
7165	{
7166	if ( !pVCpu->hm.s.fSingleInstruction
7167	&& !DBGFIsStepping(pVCpu))
7168	{
7169	/*
7170	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7171	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7172	* See Intel spec. 27.3.4 "Saving Non-Register State".
7173	*/
7174	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7175	AssertRCReturn(rc2, rc2);
7176	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7177	}
7178	else if (pMixedCtx->eflags.Bits.u1TF)
7179	{
7180	/*
7181	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7182	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7183	*/
7184	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7185	uIntrState = 0;
7186	}
7187	}
7188
7189	/*
7190	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7191	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7192	*/
7193	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7194	AssertRC(rc2);
7195
7196	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7197	NOREF(fBlockMovSS); NOREF(fBlockSti);
7198	return rc;
7199	}
7200
7201
7202	/**
7203	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7204	*
7205	* @param pVCpu Pointer to the VMCPU.
7206	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7207	* out-of-sync. Make sure to update the required fields
7208	* before using them.
7209	*/
7210	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7211	{
7212	NOREF(pMixedCtx);
7213	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7214	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7215	}
7216
7217
7218	/**
7219	* Injects a double-fault (#DF) exception into the VM.
7220	*
7221	* @returns VBox status code (informational status code included).
7222	* @param pVCpu Pointer to the VMCPU.
7223	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7224	* out-of-sync. Make sure to update the required fields
7225	* before using them.
7226	*/
7227	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7228	{
7229	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7230	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7231	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7232	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7233	puIntrState);
7234	}
7235
7236
7237	/**
7238	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7239	*
7240	* @param pVCpu Pointer to the VMCPU.
7241	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7242	* out-of-sync. Make sure to update the required fields
7243	* before using them.
7244	*/
7245	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7246	{
7247	NOREF(pMixedCtx);
7248	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7249	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7250	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7251	}
7252
7253
7254	/**
7255	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7256	*
7257	* @param pVCpu Pointer to the VMCPU.
7258	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7259	* out-of-sync. Make sure to update the required fields
7260	* before using them.
7261	* @param cbInstr The value of RIP that is to be pushed on the guest
7262	* stack.
7263	*/
7264	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7265	{
7266	NOREF(pMixedCtx);
7267	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7268	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7269	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7270	}
7271
7272
7273	/**
7274	* Injects a general-protection (#GP) fault into the VM.
7275	*
7276	* @returns VBox status code (informational status code included).
7277	* @param pVCpu Pointer to the VMCPU.
7278	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7279	* out-of-sync. Make sure to update the required fields
7280	* before using them.
7281	* @param u32ErrorCode The error code associated with the #GP.
7282	*/
7283	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7284	uint32_t *puIntrState)
7285	{
7286	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7287	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7288	if (fErrorCodeValid)
7289	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7290	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7291	puIntrState);
7292	}
7293
7294
7295	/**
7296	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7297	*
7298	* @param pVCpu Pointer to the VMCPU.
7299	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7300	* out-of-sync. Make sure to update the required fields
7301	* before using them.
7302	* @param uVector The software interrupt vector number.
7303	* @param cbInstr The value of RIP that is to be pushed on the guest
7304	* stack.
7305	*/
7306	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7307	{
7308	NOREF(pMixedCtx);
7309	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7310	if ( uVector == X86_XCPT_BP
7311	\|\| uVector == X86_XCPT_OF)
7312	{
7313	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7314	}
7315	else
7316	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7317	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7318	}
7319
7320
7321	/**
7322	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7323	* stack.
7324	*
7325	* @returns VBox status code (information status code included).
7326	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7327	* @param pVM Pointer to the VM.
7328	* @param pMixedCtx Pointer to the guest-CPU context.
7329	* @param uValue The value to push to the guest stack.
7330	*/
7331	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7332	{
7333	/*
7334	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7335	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7336	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7337	*/
7338	if (pMixedCtx->sp == 1)
7339	return VINF_EM_RESET;
7340	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7341	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7342	AssertRCReturn(rc, rc);
7343	return rc;
7344	}
7345
7346
7347	/**
7348	* Injects an event into the guest upon VM-entry by updating the relevant fields
7349	* in the VM-entry area in the VMCS.
7350	*
7351	* @returns VBox status code (informational error codes included).
7352	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7353	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7354	*
7355	* @param pVCpu Pointer to the VMCPU.
7356	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7357	* be out-of-sync. Make sure to update the required
7358	* fields before using them.
7359	* @param u64IntInfo The VM-entry interruption-information field.
7360	* @param cbInstr The VM-entry instruction length in bytes (for
7361	* software interrupts, exceptions and privileged
7362	* software exceptions).
7363	* @param u32ErrCode The VM-entry exception error code.
7364	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7365	* @param puIntrState Pointer to the current guest interruptibility-state.
7366	* This interruptibility-state will be updated if
7367	* necessary. This cannot not be NULL.
7368	*
7369	* @remarks Requires CR0!
7370	* @remarks No-long-jump zone!!!
7371	*/
7372	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7373	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7374	{
7375	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7376	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7377	Assert(puIntrState);
7378	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7379
7380	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7381	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7382
7383	#ifdef VBOX_STRICT
7384	/* Validate the error-code-valid bit for hardware exceptions. */
7385	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7386	{
7387	switch (uVector)
7388	{
7389	case X86_XCPT_PF:
7390	case X86_XCPT_DF:
7391	case X86_XCPT_TS:
7392	case X86_XCPT_NP:
7393	case X86_XCPT_SS:
7394	case X86_XCPT_GP:
7395	case X86_XCPT_AC:
7396	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7397	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7398	/* fallthru */
7399	default:
7400	break;
7401	}
7402	}
7403	#endif
7404
7405	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7406	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7407	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7408
7409	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7410
7411	/* We require CR0 to check if the guest is in real-mode. */
7412	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7413	AssertRCReturn(rc, rc);
7414
7415	/*
7416	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7417	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7418	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7419	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7420	*/
7421	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7422	{
7423	PVM pVM = pVCpu->CTX_SUFF(pVM);
7424	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7425	{
7426	Assert(PDMVmmDevHeapIsEnabled(pVM));
7427	Assert(pVM->hm.s.vmx.pRealModeTSS);
7428
7429	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7430	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7431	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7432	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7433	AssertRCReturn(rc, rc);
7434	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7435
7436	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7437	const size_t cbIdtEntry = sizeof(X86IDTR16);
7438	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7439	{
7440	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7441	if (uVector == X86_XCPT_DF)
7442	return VINF_EM_RESET;
7443	else if (uVector == X86_XCPT_GP)
7444	{
7445	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7446	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7447	}
7448
7449	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7450	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7451	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7452	}
7453
7454	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7455	uint16_t uGuestIp = pMixedCtx->ip;
7456	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7457	{
7458	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7459	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7460	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7461	}
7462	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7463	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7464
7465	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7466	X86IDTR16 IdtEntry;
7467	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7468	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7469	AssertRCReturn(rc, rc);
7470
7471	/* Construct the stack frame for the interrupt/exception handler. */
7472	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7473	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7474	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7475	AssertRCReturn(rc, rc);
7476
7477	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7478	if (rc == VINF_SUCCESS)
7479	{
7480	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7481	pMixedCtx->rip = IdtEntry.offSel;
7482	pMixedCtx->cs.Sel = IdtEntry.uSel;
7483	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7484	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7485	&& uVector == X86_XCPT_PF)
7486	{
7487	pMixedCtx->cr2 = GCPtrFaultAddress;
7488	}
7489
7490	/* If any other guest-state bits are changed here, make sure to update
7491	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7492	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7493	\| HM_CHANGED_GUEST_RIP
7494	\| HM_CHANGED_GUEST_RFLAGS
7495	\| HM_CHANGED_GUEST_RSP);
7496
7497	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7498	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7499	{
7500	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7501	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7502	Log4(("Clearing inhibition due to STI.\n"));
7503	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7504	}
7505	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7506
7507	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7508	it, if we are returning to ring-3 before executing guest code. */
7509	pVCpu->hm.s.Event.fPending = false;
7510	}
7511	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7512	return rc;
7513	}
7514	else
7515	{
7516	/*
7517	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7518	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7519	*/
7520	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7521	}
7522	}
7523
7524	/* Validate. */
7525	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7526	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7527	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7528
7529	/* Inject. */
7530	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7531	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7532	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7533	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7534
7535	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7536	&& uVector == X86_XCPT_PF)
7537	{
7538	pMixedCtx->cr2 = GCPtrFaultAddress;
7539	}
7540
7541	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7542	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7543
7544	AssertRCReturn(rc, rc);
7545	return rc;
7546	}
7547
7548
7549	/**
7550	* Clears the interrupt-window exiting control in the VMCS and if necessary
7551	* clears the current event in the VMCS as well.
7552	*
7553	* @returns VBox status code.
7554	* @param pVCpu Pointer to the VMCPU.
7555	*
7556	* @remarks Use this function only to clear events that have not yet been
7557	* delivered to the guest but are injected in the VMCS!
7558	* @remarks No-long-jump zone!!!
7559	*/
7560	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7561	{
7562	int rc;
7563	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7564
7565	/* Clear interrupt-window exiting control. */
7566	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7567	hmR0VmxClearIntWindowExitVmcs(pVCpu);
7568
7569	if (!pVCpu->hm.s.Event.fPending)
7570	return;
7571
7572	#ifdef VBOX_STRICT
7573	uint32_t u32EntryInfo;
7574	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7575	AssertRC(rc);
7576	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7577	#endif
7578
7579	/* Clear the entry-interruption field (including the valid bit). */
7580	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7581	AssertRC(rc);
7582
7583	/* Clear the pending debug exception field. */
7584	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7585	AssertRC(rc);
7586	}
7587
7588
7589	/**
7590	* Enters the VT-x session.
7591	*
7592	* @returns VBox status code.
7593	* @param pVM Pointer to the VM.
7594	* @param pVCpu Pointer to the VMCPU.
7595	* @param pCpu Pointer to the CPU info struct.
7596	*/
7597	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7598	{
7599	AssertPtr(pVM);
7600	AssertPtr(pVCpu);
7601	Assert(pVM->hm.s.vmx.fSupported);
7602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7603	NOREF(pCpu); NOREF(pVM);
7604
7605	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7606	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7607
7608	#ifdef VBOX_STRICT
7609	/* Make sure we're in VMX root mode. */
7610	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7611	if (!(u32HostCR4 & X86_CR4_VMXE))
7612	{
7613	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7614	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7615	}
7616	#endif
7617
7618	/*
7619	* Load the VCPU's VMCS as the current (and active) one.
7620	*/
7621	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7622	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7623	if (RT_FAILURE(rc))
7624	return rc;
7625
7626	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7627	pVCpu->hm.s.fLeaveDone = false;
7628	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7629
7630	return VINF_SUCCESS;
7631	}
7632
7633
7634	/**
7635	* The thread-context callback (only on platforms which support it).
7636	*
7637	* @param enmEvent The thread-context event.
7638	* @param pVCpu Pointer to the VMCPU.
7639	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7640	* @thread EMT(pVCpu)
7641	*/
7642	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7643	{
7644	NOREF(fGlobalInit);
7645
7646	switch (enmEvent)
7647	{
7648	case RTTHREADCTXEVENT_PREEMPTING:
7649	{
7650	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7651	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7652	VMCPU_ASSERT_EMT(pVCpu);
7653
7654	PVM pVM = pVCpu->CTX_SUFF(pVM);
7655	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7656
7657	/* No longjmps (logger flushes, locks) in this fragile context. */
7658	VMMRZCallRing3Disable(pVCpu);
7659	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7660
7661	/*
7662	* Restore host-state (FPU, debug etc.)
7663	*/
7664	if (!pVCpu->hm.s.fLeaveDone)
7665	{
7666	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7667	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7668	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7669	pVCpu->hm.s.fLeaveDone = true;
7670	}
7671
7672	/* Leave HM context, takes care of local init (term). */
7673	int rc = HMR0LeaveCpu(pVCpu);
7674	AssertRC(rc); NOREF(rc);
7675
7676	/* Restore longjmp state. */
7677	VMMRZCallRing3Enable(pVCpu);
7678	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7679	break;
7680	}
7681
7682	case RTTHREADCTXEVENT_RESUMED:
7683	{
7684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7685	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7686	VMCPU_ASSERT_EMT(pVCpu);
7687
7688	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7689	VMMRZCallRing3Disable(pVCpu);
7690	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7691
7692	/* Initialize the bare minimum state required for HM. This takes care of
7693	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7694	int rc = HMR0EnterCpu(pVCpu);
7695	AssertRC(rc);
7696	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7697
7698	/* Load the active VMCS as the current one. */
7699	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7700	{
7701	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7702	AssertRC(rc); NOREF(rc);
7703	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7704	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7705	}
7706	pVCpu->hm.s.fLeaveDone = false;
7707
7708	/* Restore longjmp state. */
7709	VMMRZCallRing3Enable(pVCpu);
7710	break;
7711	}
7712
7713	default:
7714	break;
7715	}
7716	}
7717
7718
7719	/**
7720	* Saves the host state in the VMCS host-state.
7721	* Sets up the VM-exit MSR-load area.
7722	*
7723	* The CPU state will be loaded from these fields on every successful VM-exit.
7724	*
7725	* @returns VBox status code.
7726	* @param pVM Pointer to the VM.
7727	* @param pVCpu Pointer to the VMCPU.
7728	*
7729	* @remarks No-long-jump zone!!!
7730	*/
7731	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7732	{
7733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7734
7735	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7736	return VINF_SUCCESS;
7737
7738	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7739	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7740
7741	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7742	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7743
7744	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7745	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7746
7747	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7748	return rc;
7749	}
7750
7751
7752	/**
7753	* Saves the host state in the VMCS host-state.
7754	*
7755	* @returns VBox status code.
7756	* @param pVM Pointer to the VM.
7757	* @param pVCpu Pointer to the VMCPU.
7758	*
7759	* @remarks No-long-jump zone!!!
7760	*/
7761	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7762	{
7763	AssertPtr(pVM);
7764	AssertPtr(pVCpu);
7765
7766	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7767
7768	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7769	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7771	return hmR0VmxSaveHostState(pVM, pVCpu);
7772	}
7773
7774
7775	/**
7776	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7777	* loaded from these fields on every successful VM-entry.
7778	*
7779	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7780	* Sets up the VM-entry controls.
7781	* Sets up the appropriate VMX non-root function to execute guest code based on
7782	* the guest CPU mode.
7783	*
7784	* @returns VBox status code.
7785	* @param pVM Pointer to the VM.
7786	* @param pVCpu Pointer to the VMCPU.
7787	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7788	* out-of-sync. Make sure to update the required fields
7789	* before using them.
7790	*
7791	* @remarks No-long-jump zone!!!
7792	*/
7793	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7794	{
7795	AssertPtr(pVM);
7796	AssertPtr(pVCpu);
7797	AssertPtr(pMixedCtx);
7798	HMVMX_ASSERT_PREEMPT_SAFE();
7799
7800	#ifdef LOG_ENABLED
7801	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7802	* probably not initialized yet? Anyway this will do for now.
7803	*
7804	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7805	* interface and disable ring-3 calls when thread-context hooks are not
7806	* available. */
7807	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7808	VMMR0LogFlushDisable(pVCpu);
7809	#endif
7810
7811	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7812
7813	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7814
7815	/* Determine real-on-v86 mode. */
7816	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7817	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7818	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7819	{
7820	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7821	}
7822
7823	/*
7824	* Load the guest-state into the VMCS.
7825	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7826	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7827	*/
7828	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7829	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7830
7831	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7832	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7833	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7834
7835	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7836	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7837	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7838
7839	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7840	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7841
7842	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7843	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7844
7845	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7846	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7847	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7848
7849	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7850	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7851
7852	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7853	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7854
7855	/*
7856	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7857	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7858	*/
7859	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7860	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7861
7862	/* Clear any unused and reserved bits. */
7863	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7864
7865	#ifdef LOG_ENABLED
7866	/* Only reenable log-flushing if the caller has it enabled. */
7867	if (!fCallerDisabledLogFlush)
7868	VMMR0LogFlushEnable(pVCpu);
7869	#endif
7870
7871	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7872	return rc;
7873	}
7874
7875
7876	/**
7877	* Loads the state shared between the host and guest into the VMCS.
7878	*
7879	* @param pVM Pointer to the VM.
7880	* @param pVCpu Pointer to the VMCPU.
7881	* @param pCtx Pointer to the guest-CPU context.
7882	*
7883	* @remarks No-long-jump zone!!!
7884	*/
7885	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7886	{
7887	NOREF(pVM);
7888
7889	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7890	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7891
7892	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7893	{
7894	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7895	AssertRC(rc);
7896	}
7897
7898	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7899	{
7900	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7901	AssertRC(rc);
7902
7903	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7904	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7905	{
7906	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7907	AssertRC(rc);
7908	}
7909	}
7910
7911	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7912	{
7913	#if HC_ARCH_BITS == 64
7914	if (pVM->hm.s.fAllow64BitGuests)
7915	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7916	#endif
7917	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7918	}
7919
7920	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7921	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7922	}
7923
7924
7925	/**
7926	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7927	*
7928	* @param pVM Pointer to the VM.
7929	* @param pVCpu Pointer to the VMCPU.
7930	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7931	* out-of-sync. Make sure to update the required fields
7932	* before using them.
7933	*/
7934	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7935	{
7936	HMVMX_ASSERT_PREEMPT_SAFE();
7937
7938	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7939	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7940	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7941	#endif
7942
7943	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7944	{
7945	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7946	AssertRC(rc);
7947	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7948	}
7949	else if (HMCPU_CF_VALUE(pVCpu))
7950	{
7951	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7952	AssertRC(rc);
7953	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7954	}
7955
7956	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7957	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7958	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7959	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7960
7961	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7962	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7963	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7964	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7965	#endif
7966	}
7967
7968
7969	/**
7970	* Does the preparations before executing guest code in VT-x.
7971	*
7972	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7973	* recompiler. We must be cautious what we do here regarding committing
7974	* guest-state information into the VMCS assuming we assuredly execute the
7975	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7976	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7977	* so that the recompiler can (and should) use them when it resumes guest
7978	* execution. Otherwise such operations must be done when we can no longer
7979	* exit to ring-3.
7980	*
7981	* @returns Strict VBox status code.
7982	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7983	* have been disabled.
7984	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7985	* double-fault into the guest.
7986	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7987	*
7988	* @param pVM Pointer to the VM.
7989	* @param pVCpu Pointer to the VMCPU.
7990	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7991	* out-of-sync. Make sure to update the required fields
7992	* before using them.
7993	* @param pVmxTransient Pointer to the VMX transient structure.
7994	*/
7995	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7996	{
7997	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7998
7999	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8000	PGMRZDynMapFlushAutoSet(pVCpu);
8001	#endif
8002
8003	/* Check force flag actions that might require us to go back to ring-3. */
8004	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8005	if (rc != VINF_SUCCESS)
8006	return rc;
8007
8008	#ifndef IEM_VERIFICATION_MODE_FULL
8009	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8010	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8011	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8012	{
8013	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8014	RTGCPHYS GCPhysApicBase;
8015	GCPhysApicBase = pMixedCtx->msrApicBase;
8016	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8017
8018	/* Unalias any existing mapping. */
8019	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8020	AssertRCReturn(rc, rc);
8021
8022	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8023	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8024	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8025	AssertRCReturn(rc, rc);
8026
8027	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8028	}
8029	#endif /* !IEM_VERIFICATION_MODE_FULL */
8030
8031	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
8032	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8033
8034	/*
8035	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8036	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8037	*/
8038	if (TRPMHasTrap(pVCpu))
8039	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8040	else if (!pVCpu->hm.s.Event.fPending)
8041	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8042
8043	/*
8044	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8045	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8046	*/
8047	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8048	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8049	{
8050	Assert(rc == VINF_EM_RESET);
8051	return rc;
8052	}
8053
8054	/*
8055	* No longjmps to ring-3 from this point on!!!
8056	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8057	* This also disables flushing of the R0-logger instance (if any).
8058	*/
8059	VMMRZCallRing3Disable(pVCpu);
8060
8061	/*
8062	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8063	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8064	*
8065	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8066	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8067	*
8068	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8069	* executing guest code.
8070	*/
8071	pVmxTransient->uEflags = ASMIntDisableFlags();
8072	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
8073	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8074	{
8075	hmR0VmxClearEventVmcs(pVCpu);
8076	ASMSetFlags(pVmxTransient->uEflags);
8077	VMMRZCallRing3Enable(pVCpu);
8078	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8079	return VINF_EM_RAW_TO_R3;
8080	}
8081	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8082	{
8083	hmR0VmxClearEventVmcs(pVCpu);
8084	ASMSetFlags(pVmxTransient->uEflags);
8085	VMMRZCallRing3Enable(pVCpu);
8086	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8087	return VINF_EM_RAW_INTERRUPT;
8088	}
8089
8090	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8091	pVCpu->hm.s.Event.fPending = false;
8092
8093	return VINF_SUCCESS;
8094	}
8095
8096
8097	/**
8098	* Prepares to run guest code in VT-x and we've committed to doing so. This
8099	* means there is no backing out to ring-3 or anywhere else at this
8100	* point.
8101	*
8102	* @param pVM Pointer to the VM.
8103	* @param pVCpu Pointer to the VMCPU.
8104	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8105	* out-of-sync. Make sure to update the required fields
8106	* before using them.
8107	* @param pVmxTransient Pointer to the VMX transient structure.
8108	*
8109	* @remarks Called with preemption disabled.
8110	* @remarks No-long-jump zone!!!
8111	*/
8112	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8113	{
8114	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8115	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8116	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8117
8118	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8119	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8120
8121	/*
8122	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8123	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8124	* Reload only the necessary state, the assertion will catch if other parts of the code
8125	* change.
8126	*/
8127	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8128	{
8129	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8130	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8131	}
8132
8133	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8134	if (!CPUMIsGuestFPUStateActive(pVCpu))
8135	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8136	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8137	#endif
8138
8139	if ( pVCpu->hm.s.fUseGuestFpu
8140	&& !CPUMIsGuestFPUStateActive(pVCpu))
8141	{
8142	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8143	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8144	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8145	}
8146
8147	/*
8148	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
8149	*/
8150	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8151	&& pVCpu->hm.s.vmx.cMsrs > 0)
8152	{
8153	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8154	}
8155
8156	/*
8157	* Load the host state bits as we may've been preempted (only happens when
8158	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8159	*/
8160	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8161	{
8162	/* This ASSUMES that pfnStartVM has been set up already. */
8163	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8164	AssertRC(rc);
8165	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8166	}
8167	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8168
8169	/*
8170	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8171	*/
8172	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8173	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8174	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8175
8176	/* Store status of the shared guest-host state at the time of VM-entry. */
8177	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8178	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8179	{
8180	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8181	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8182	}
8183	else
8184	#endif
8185	{
8186	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8187	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8188	}
8189	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8190
8191	/*
8192	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8193	*/
8194	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8195	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8196
8197	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8198	RTCPUID idCurrentCpu = pCpu->idCpu;
8199	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8200	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8201	{
8202	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8203	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8204	}
8205
8206	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8207	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8208	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8209	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8210
8211	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8212
8213	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8214	to start executing. */
8215
8216	/*
8217	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8218	*/
8219	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8220	{
8221	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8222	{
8223	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8224	AssertRC(rc2);
8225	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8226	bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8227	true /* fUpdateHostMsr */);
8228	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8229	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8230	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8231	}
8232	else
8233	{
8234	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8235	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8236	}
8237	}
8238	#ifdef VBOX_STRICT
8239	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8240	#endif
8241	}
8242
8243
8244	/**
8245	* Performs some essential restoration of state after running guest code in
8246	* VT-x.
8247	*
8248	* @param pVM Pointer to the VM.
8249	* @param pVCpu Pointer to the VMCPU.
8250	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8251	* out-of-sync. Make sure to update the required fields
8252	* before using them.
8253	* @param pVmxTransient Pointer to the VMX transient structure.
8254	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8255	*
8256	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8257	*
8258	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8259	* unconditionally when it is safe to do so.
8260	*/
8261	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8262	{
8263	NOREF(pVM);
8264
8265	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8266
8267	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8268	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8269	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8270	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8271	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8272
8273	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8274	{
8275	/** @todo Find a way to fix hardcoding a guestimate. */
8276	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8277	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8278	}
8279
8280	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8281	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8282	Assert(!(ASMGetFlags() & X86_EFL_IF));
8283	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8284
8285	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8286	if (CPUMIsGuestFPUStateActive(pVCpu))
8287	{
8288	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8289	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8290	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8291	}
8292	#endif
8293
8294	#if HC_ARCH_BITS == 64
8295	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8296	#endif
8297	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8298	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8299	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8300
8301	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8302	uint32_t uExitReason;
8303	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8304	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8305	AssertRC(rc);
8306	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8307	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8308
8309	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8310	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8311	{
8312	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8313	pVmxTransient->fVMEntryFailed));
8314	return;
8315	}
8316
8317	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8318	{
8319	/* Update the guest interruptibility-state from the VMCS. */
8320	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8321	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8322	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8323	AssertRC(rc);
8324	#endif
8325	/*
8326	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8327	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8328	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8329	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8330	*/
8331	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8332	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8333	{
8334	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8335	AssertRC(rc);
8336	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8337	}
8338	}
8339	}
8340
8341
8342
8343	/**
8344	* Runs the guest code using VT-x the normal way.
8345	*
8346	* @returns VBox status code.
8347	* @param pVM Pointer to the VM.
8348	* @param pVCpu Pointer to the VMCPU.
8349	* @param pCtx Pointer to the guest-CPU context.
8350	*
8351	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8352	*/
8353	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8354	{
8355	VMXTRANSIENT VmxTransient;
8356	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8357	int rc = VERR_INTERNAL_ERROR_5;
8358	uint32_t cLoops = 0;
8359
8360	for (;; cLoops++)
8361	{
8362	Assert(!HMR0SuspendPending());
8363	HMVMX_ASSERT_CPU_SAFE();
8364
8365	/* Preparatory work for running guest code, this may force us to return
8366	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8367	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8368	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8369	if (rc != VINF_SUCCESS)
8370	break;
8371
8372	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8373	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8374	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8375
8376	/* Restore any residual host-state and save any bits shared between host
8377	and guest into the guest-CPU state. Re-enables interrupts! */
8378	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8379
8380	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8381	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8382	{
8383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8384	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8385	return rc;
8386	}
8387
8388	/* Handle the VM-exit. */
8389	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8391	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8392	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8393	HMVMX_START_EXIT_DISPATCH_PROF();
8394	#ifdef HMVMX_USE_FUNCTION_TABLE
8395	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8396	#else
8397	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8398	#endif
8399	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8400	if (rc != VINF_SUCCESS)
8401	break;
8402	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8403	{
8404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8405	rc = VINF_EM_RAW_INTERRUPT;
8406	break;
8407	}
8408	}
8409
8410	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8411	return rc;
8412	}
8413
8414
8415	/**
8416	* Single steps guest code using VT-x.
8417	*
8418	* @returns VBox status code.
8419	* @param pVM Pointer to the VM.
8420	* @param pVCpu Pointer to the VMCPU.
8421	* @param pCtx Pointer to the guest-CPU context.
8422	*
8423	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8424	*/
8425	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8426	{
8427	VMXTRANSIENT VmxTransient;
8428	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8429	int rc = VERR_INTERNAL_ERROR_5;
8430	uint32_t cLoops = 0;
8431	uint16_t uCsStart = pCtx->cs.Sel;
8432	uint64_t uRipStart = pCtx->rip;
8433
8434	for (;; cLoops++)
8435	{
8436	Assert(!HMR0SuspendPending());
8437	HMVMX_ASSERT_CPU_SAFE();
8438
8439	/* Preparatory work for running guest code, this may force us to return
8440	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8441	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8442	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8443	if (rc != VINF_SUCCESS)
8444	break;
8445
8446	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8447	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8448	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8449
8450	/* Restore any residual host-state and save any bits shared between host
8451	and guest into the guest-CPU state. Re-enables interrupts! */
8452	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8453
8454	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8455	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8456	{
8457	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8458	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8459	return rc;
8460	}
8461
8462	/* Handle the VM-exit. */
8463	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8465	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8466	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8467	HMVMX_START_EXIT_DISPATCH_PROF();
8468	#ifdef HMVMX_USE_FUNCTION_TABLE
8469	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8470	#else
8471	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8472	#endif
8473	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8474	if (rc != VINF_SUCCESS)
8475	break;
8476	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8477	{
8478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8479	rc = VINF_EM_RAW_INTERRUPT;
8480	break;
8481	}
8482
8483	/*
8484	* Did the RIP change, if so, consider it a single step.
8485	* Otherwise, make sure one of the TFs gets set.
8486	*/
8487	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8488	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8489	AssertRCReturn(rc2, rc2);
8490	if ( pCtx->rip != uRipStart
8491	\|\| pCtx->cs.Sel != uCsStart)
8492	{
8493	rc = VINF_EM_DBG_STEPPED;
8494	break;
8495	}
8496	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8497	}
8498
8499	/*
8500	* Clear the X86_EFL_TF if necessary.
8501	*/
8502	if (pVCpu->hm.s.fClearTrapFlag)
8503	{
8504	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8505	AssertRCReturn(rc2, rc2);
8506	pVCpu->hm.s.fClearTrapFlag = false;
8507	pCtx->eflags.Bits.u1TF = 0;
8508	}
8509	/** @todo there seems to be issues with the resume flag when the monitor trap
8510	* flag is pending without being used. Seen early in bios init when
8511	* accessing APIC page in prot mode. */
8512
8513	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8514	return rc;
8515	}
8516
8517
8518	/**
8519	* Runs the guest code using VT-x.
8520	*
8521	* @returns VBox status code.
8522	* @param pVM Pointer to the VM.
8523	* @param pVCpu Pointer to the VMCPU.
8524	* @param pCtx Pointer to the guest-CPU context.
8525	*/
8526	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8527	{
8528	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8529	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8530	HMVMX_ASSERT_PREEMPT_SAFE();
8531
8532	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8533
8534	int rc;
8535	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8536	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8537	else
8538	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8539
8540	if (rc == VERR_EM_INTERPRETER)
8541	rc = VINF_EM_RAW_EMULATE_INSTR;
8542	else if (rc == VINF_EM_RESET)
8543	rc = VINF_EM_TRIPLE_FAULT;
8544
8545	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8546	if (RT_FAILURE(rc2))
8547	{
8548	pVCpu->hm.s.u32HMError = rc;
8549	rc = rc2;
8550	}
8551	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8552	return rc;
8553	}
8554
8555
8556	#ifndef HMVMX_USE_FUNCTION_TABLE
8557	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8558	{
8559	#ifdef DEBUG_ramshankar
8560	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8561	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8562	#endif
8563	int rc;
8564	switch (rcReason)
8565	{
8566	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8567	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8568	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8569	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8570	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8571	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8572	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8573	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8574	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8575	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8576	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8577	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8578	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8579	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8580	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8581	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8582	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8583	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8584	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8585	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8586	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8587	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8588	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8589	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8590	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8591	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8592	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8593	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8594	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8595	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8596	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8597	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8598	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8599
8600	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8601	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8602	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8603	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8604	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8605	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8606	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8607	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8608	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8609
8610	case VMX_EXIT_VMCALL:
8611	case VMX_EXIT_VMCLEAR:
8612	case VMX_EXIT_VMLAUNCH:
8613	case VMX_EXIT_VMPTRLD:
8614	case VMX_EXIT_VMPTRST:
8615	case VMX_EXIT_VMREAD:
8616	case VMX_EXIT_VMRESUME:
8617	case VMX_EXIT_VMWRITE:
8618	case VMX_EXIT_VMXOFF:
8619	case VMX_EXIT_VMXON:
8620	case VMX_EXIT_INVEPT:
8621	case VMX_EXIT_INVVPID:
8622	case VMX_EXIT_VMFUNC:
8623	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8624	break;
8625	default:
8626	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8627	break;
8628	}
8629	return rc;
8630	}
8631	#endif
8632
8633	#ifdef DEBUG
8634	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8635	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8636	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8637
8638	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8639	do \
8640	{ \
8641	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8642	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8643	} while (0)
8644
8645	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8646	do { \
8647	AssertPtr(pVCpu); \
8648	AssertPtr(pMixedCtx); \
8649	AssertPtr(pVmxTransient); \
8650	Assert(pVmxTransient->fVMEntryFailed == false); \
8651	Assert(ASMIntAreEnabled()); \
8652	HMVMX_ASSERT_PREEMPT_SAFE(); \
8653	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8654	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8655	HMVMX_ASSERT_PREEMPT_SAFE(); \
8656	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8657	HMVMX_ASSERT_PREEMPT_CPUID(); \
8658	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8659	} while (0)
8660
8661	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8662	do { \
8663	Log4Func(("\n")); \
8664	} while (0)
8665	#else /* Release builds */
8666	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8667	do { \
8668	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8669	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8670	} while (0)
8671	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8672	#endif
8673
8674
8675	/**
8676	* Advances the guest RIP after reading it from the VMCS.
8677	*
8678	* @returns VBox status code.
8679	* @param pVCpu Pointer to the VMCPU.
8680	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8681	* out-of-sync. Make sure to update the required fields
8682	* before using them.
8683	* @param pVmxTransient Pointer to the VMX transient structure.
8684	*
8685	* @remarks No-long-jump zone!!!
8686	*/
8687	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8688	{
8689	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8690	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8691	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8692	AssertRCReturn(rc, rc);
8693
8694	pMixedCtx->rip += pVmxTransient->cbInstr;
8695	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8696
8697	/*
8698	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8699	* pending debug exception field as it takes care of priority of events.
8700	*
8701	* See Intel spec. 32.2.1 "Debug Exceptions".
8702	*/
8703	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8704
8705	return rc;
8706	}
8707
8708
8709	/**
8710	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8711	* and update error record fields accordingly.
8712	*
8713	* @return VMX_IGS_* return codes.
8714	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8715	* wrong with the guest state.
8716	*
8717	* @param pVM Pointer to the VM.
8718	* @param pVCpu Pointer to the VMCPU.
8719	* @param pCtx Pointer to the guest-CPU state.
8720	*/
8721	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8722	{
8723	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8724	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8725	uError = (err); \
8726	break; \
8727	} else do { } while (0)
8728
8729	int rc;
8730	uint32_t uError = VMX_IGS_ERROR;
8731	uint32_t u32Val;
8732	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8733
8734	do
8735	{
8736	/*
8737	* CR0.
8738	*/
8739	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8740	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8741	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8742	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8743	if (fUnrestrictedGuest)
8744	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8745
8746	uint32_t u32GuestCR0;
8747	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8748	AssertRCBreak(rc);
8749	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8750	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8751	if ( !fUnrestrictedGuest
8752	&& (u32GuestCR0 & X86_CR0_PG)
8753	&& !(u32GuestCR0 & X86_CR0_PE))
8754	{
8755	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8756	}
8757
8758	/*
8759	* CR4.
8760	*/
8761	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8762	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8763
8764	uint32_t u32GuestCR4;
8765	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8766	AssertRCBreak(rc);
8767	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8768	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8769
8770	/*
8771	* IA32_DEBUGCTL MSR.
8772	*/
8773	uint64_t u64Val;
8774	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8775	AssertRCBreak(rc);
8776	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8777	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8778	{
8779	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8780	}
8781	uint64_t u64DebugCtlMsr = u64Val;
8782
8783	#ifdef VBOX_STRICT
8784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8785	AssertRCBreak(rc);
8786	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8787	#endif
8788	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8789
8790	/*
8791	* RIP and RFLAGS.
8792	*/
8793	uint32_t u32Eflags;
8794	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8795	if (HMVMX_IS_64BIT_HOST_MODE())
8796	{
8797	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8798	AssertRCBreak(rc);
8799	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8800	if ( !fLongModeGuest
8801	\|\| !pCtx->cs.Attr.n.u1Long)
8802	{
8803	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8804	}
8805	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8806	* must be identical if the "IA32e mode guest" VM-entry control is 1
8807	* and CS.L is 1. No check applies if the CPU supports 64
8808	* linear-address bits. */
8809
8810	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8811	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8812	AssertRCBreak(rc);
8813	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8814	VMX_IGS_RFLAGS_RESERVED);
8815	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8816	u32Eflags = u64Val;
8817	}
8818	else
8819	#endif
8820	{
8821	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8822	AssertRCBreak(rc);
8823	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8824	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8825	}
8826
8827	if ( fLongModeGuest
8828	\|\| ( fUnrestrictedGuest
8829	&& !(u32GuestCR0 & X86_CR0_PE)))
8830	{
8831	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8832	}
8833
8834	uint32_t u32EntryInfo;
8835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8836	AssertRCBreak(rc);
8837	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8838	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8839	{
8840	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8841	}
8842
8843	/*
8844	* 64-bit checks.
8845	*/
8846	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8847	if (HMVMX_IS_64BIT_HOST_MODE())
8848	{
8849	if ( fLongModeGuest
8850	&& !fUnrestrictedGuest)
8851	{
8852	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8853	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8854	}
8855
8856	if ( !fLongModeGuest
8857	&& (u32GuestCR4 & X86_CR4_PCIDE))
8858	{
8859	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8860	}
8861
8862	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8863	* 51:32 beyond the processor's physical-address width are 0. */
8864
8865	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8866	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8867	{
8868	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8869	}
8870
8871	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8872	AssertRCBreak(rc);
8873	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8874
8875	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8876	AssertRCBreak(rc);
8877	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8878	}
8879	#endif
8880
8881	/*
8882	* PERF_GLOBAL MSR.
8883	*/
8884	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8885	{
8886	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8887	AssertRCBreak(rc);
8888	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8889	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8890	}
8891
8892	/*
8893	* PAT MSR.
8894	*/
8895	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8896	{
8897	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8898	AssertRCBreak(rc);
8899	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8900	for (unsigned i = 0; i < 8; i++)
8901	{
8902	uint8_t u8Val = (u64Val & 0x7);
8903	if ( u8Val != 0 /* UC */
8904	\|\| u8Val != 1 /* WC */
8905	\|\| u8Val != 4 /* WT */
8906	\|\| u8Val != 5 /* WP */
8907	\|\| u8Val != 6 /* WB */
8908	\|\| u8Val != 7 /* UC- */)
8909	{
8910	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8911	}
8912	u64Val >>= 3;
8913	}
8914	}
8915
8916	/*
8917	* EFER MSR.
8918	*/
8919	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8920	{
8921	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8922	AssertRCBreak(rc);
8923	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8924	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8925	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8926	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8927	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8928	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8929	}
8930
8931	/*
8932	* Segment registers.
8933	*/
8934	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8935	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8936	if (!(u32Eflags & X86_EFL_VM))
8937	{
8938	/* CS */
8939	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8940	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8941	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8942	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8943	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8944	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8945	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8946	/* CS cannot be loaded with NULL in protected mode. */
8947	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8948	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8949	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8950	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8951	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8952	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8953	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8954	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8955	else
8956	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8957
8958	/* SS */
8959	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8960	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8961	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8962	if ( !(pCtx->cr0 & X86_CR0_PE)
8963	\|\| pCtx->cs.Attr.n.u4Type == 3)
8964	{
8965	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8966	}
8967	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8968	{
8969	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8970	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8971	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8972	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8973	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8974	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8975	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8976	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8977	}
8978
8979	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8980	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8981	{
8982	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8983	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8984	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8985	\|\| pCtx->ds.Attr.n.u4Type > 11
8986	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8987	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8988	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8989	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8990	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8991	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8992	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8993	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8994	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8995	}
8996	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8997	{
8998	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8999	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9000	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9001	\|\| pCtx->es.Attr.n.u4Type > 11
9002	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9003	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9004	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9005	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9006	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9007	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9008	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9009	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9010	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9011	}
9012	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9013	{
9014	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9015	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9016	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9017	\|\| pCtx->fs.Attr.n.u4Type > 11
9018	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9019	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9020	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9021	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9022	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9023	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9024	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9025	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9026	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9027	}
9028	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9029	{
9030	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9031	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9032	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9033	\|\| pCtx->gs.Attr.n.u4Type > 11
9034	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9035	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9036	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9037	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9038	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9039	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9040	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9041	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9042	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9043	}
9044	/* 64-bit capable CPUs. */
9045	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9046	if (HMVMX_IS_64BIT_HOST_MODE())
9047	{
9048	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9049	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9050	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9051	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9052	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9053	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9054	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9055	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9056	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9057	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9058	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9059	}
9060	#endif
9061	}
9062	else
9063	{
9064	/* V86 mode checks. */
9065	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9066	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9067	{
9068	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9069	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9070	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9071	}
9072	else
9073	{
9074	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9075	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9076	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9077	}
9078
9079	/* CS */
9080	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9081	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9082	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9083	/* SS */
9084	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9085	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9086	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9087	/* DS */
9088	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9089	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9090	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9091	/* ES */
9092	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9093	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9094	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9095	/* FS */
9096	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9097	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9098	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9099	/* GS */
9100	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9101	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9102	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9103	/* 64-bit capable CPUs. */
9104	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9105	if (HMVMX_IS_64BIT_HOST_MODE())
9106	{
9107	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9108	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9109	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9110	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9111	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9112	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9113	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9114	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9115	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9116	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9117	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9118	}
9119	#endif
9120	}
9121
9122	/*
9123	* TR.
9124	*/
9125	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9126	/* 64-bit capable CPUs. */
9127	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9128	if (HMVMX_IS_64BIT_HOST_MODE())
9129	{
9130	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9131	}
9132	#endif
9133	if (fLongModeGuest)
9134	{
9135	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9136	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9137	}
9138	else
9139	{
9140	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9141	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9142	VMX_IGS_TR_ATTR_TYPE_INVALID);
9143	}
9144	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9145	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9146	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9147	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9148	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9149	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9150	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9151	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9152
9153	/*
9154	* GDTR and IDTR.
9155	*/
9156	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9157	if (HMVMX_IS_64BIT_HOST_MODE())
9158	{
9159	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9160	AssertRCBreak(rc);
9161	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9162
9163	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9164	AssertRCBreak(rc);
9165	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9166	}
9167	#endif
9168
9169	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9170	AssertRCBreak(rc);
9171	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9172
9173	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9174	AssertRCBreak(rc);
9175	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9176
9177	/*
9178	* Guest Non-Register State.
9179	*/
9180	/* Activity State. */
9181	uint32_t u32ActivityState;
9182	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9183	AssertRCBreak(rc);
9184	HMVMX_CHECK_BREAK( !u32ActivityState
9185	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9186	VMX_IGS_ACTIVITY_STATE_INVALID);
9187	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9188	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9189	uint32_t u32IntrState;
9190	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9191	AssertRCBreak(rc);
9192	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9193	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9194	{
9195	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9196	}
9197
9198	/** @todo Activity state and injecting interrupts. Left as a todo since we
9199	* currently don't use activity states but ACTIVE. */
9200
9201	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9202	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9203
9204	/* Guest interruptibility-state. */
9205	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9206	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9207	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9208	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9209	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9210	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9211	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9212	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9213	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9214	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9215	{
9216	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9217	{
9218	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9219	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9220	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9221	}
9222	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9223	{
9224	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9225	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9226	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9227	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9228	}
9229	}
9230	/** @todo Assumes the processor is not in SMM. */
9231	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9232	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9233	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9234	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9235	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9236	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9237	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9238	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9239	{
9240	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9241	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9242	}
9243
9244	/* Pending debug exceptions. */
9245	if (HMVMX_IS_64BIT_HOST_MODE())
9246	{
9247	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9248	AssertRCBreak(rc);
9249	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9250	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9251	u32Val = u64Val; /* For pending debug exceptions checks below. */
9252	}
9253	else
9254	{
9255	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9256	AssertRCBreak(rc);
9257	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9258	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9259	}
9260
9261	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9262	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9263	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9264	{
9265	if ( (u32Eflags & X86_EFL_TF)
9266	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9267	{
9268	/* Bit 14 is PendingDebug.BS. */
9269	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9270	}
9271	if ( !(u32Eflags & X86_EFL_TF)
9272	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9273	{
9274	/* Bit 14 is PendingDebug.BS. */
9275	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9276	}
9277	}
9278
9279	/* VMCS link pointer. */
9280	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9281	AssertRCBreak(rc);
9282	if (u64Val != UINT64_C(0xffffffffffffffff))
9283	{
9284	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9285	/** @todo Bits beyond the processor's physical-address width MBZ. */
9286	/** @todo 32-bit located in memory referenced by value of this field (as a
9287	* physical address) must contain the processor's VMCS revision ID. */
9288	/** @todo SMM checks. */
9289	}
9290
9291	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9292	* not using Nested Paging? */
9293	if ( pVM->hm.s.fNestedPaging
9294	&& !fLongModeGuest
9295	&& CPUMIsGuestInPAEModeEx(pCtx))
9296	{
9297	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9298	AssertRCBreak(rc);
9299	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9300
9301	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9302	AssertRCBreak(rc);
9303	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9304
9305	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9306	AssertRCBreak(rc);
9307	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9308
9309	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9310	AssertRCBreak(rc);
9311	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9312	}
9313
9314	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9315	if (uError == VMX_IGS_ERROR)
9316	uError = VMX_IGS_REASON_NOT_FOUND;
9317	} while (0);
9318
9319	pVCpu->hm.s.u32HMError = uError;
9320	return uError;
9321
9322	#undef HMVMX_ERROR_BREAK
9323	#undef HMVMX_CHECK_BREAK
9324	}
9325
9326	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9327	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9328	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9329
9330	/** @name VM-exit handlers.
9331	* @{
9332	*/
9333
9334	/**
9335	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9336	*/
9337	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9338	{
9339	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9341	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9342	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9343	return VINF_SUCCESS;
9344	return VINF_EM_RAW_INTERRUPT;
9345	}
9346
9347
9348	/**
9349	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9350	*/
9351	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9352	{
9353	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9354	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9355
9356	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9357	AssertRCReturn(rc, rc);
9358
9359	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9360	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9361	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9362	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9363
9364	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9365	{
9366	/*
9367	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9368	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9369	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9370	*
9371	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9372	*/
9373	VMXDispatchHostNmi();
9374	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9375	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9376	return VINF_SUCCESS;
9377	}
9378
9379	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9380	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9381	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9382	{
9383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9384	return VINF_SUCCESS;
9385	}
9386	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9387	{
9388	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9389	return rc;
9390	}
9391
9392	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9393	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9394	switch (uIntType)
9395	{
9396	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9397	Assert(uVector == X86_XCPT_DB);
9398	/* no break */
9399	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9400	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9401	/* no break */
9402	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9403	{
9404	switch (uVector)
9405	{
9406	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9407	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9408	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9409	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9410	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9411	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9412	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9413	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9414	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9415	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9416	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9417	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9418	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9419	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9420	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9421	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9422	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9423	#endif
9424	default:
9425	{
9426	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9427	AssertRCReturn(rc, rc);
9428
9429	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9430	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9431	{
9432	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9433	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9434	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9435
9436	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9437	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9438	AssertRCReturn(rc, rc);
9439	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9440	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9441	0 /* GCPtrFaultAddress */);
9442	AssertRCReturn(rc, rc);
9443	}
9444	else
9445	{
9446	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9447	pVCpu->hm.s.u32HMError = uVector;
9448	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9449	}
9450	break;
9451	}
9452	}
9453	break;
9454	}
9455
9456	default:
9457	{
9458	pVCpu->hm.s.u32HMError = uExitIntInfo;
9459	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9460	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9461	break;
9462	}
9463	}
9464	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9465	return rc;
9466	}
9467
9468
9469	/**
9470	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9471	*/
9472	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9473	{
9474	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9475
9476	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9477	hmR0VmxClearIntWindowExitVmcs(pVCpu);
9478
9479	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9481	return VINF_SUCCESS;
9482	}
9483
9484
9485	/**
9486	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9487	*/
9488	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9489	{
9490	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9491	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9492	HMVMX_RETURN_UNEXPECTED_EXIT();
9493	}
9494
9495
9496	/**
9497	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9498	*/
9499	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9500	{
9501	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9502	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9503	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9504	}
9505
9506
9507	/**
9508	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9509	*/
9510	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9511	{
9512	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9514	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9515	}
9516
9517
9518	/**
9519	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9520	*/
9521	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9522	{
9523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9524	PVM pVM = pVCpu->CTX_SUFF(pVM);
9525	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9526	if (RT_LIKELY(rc == VINF_SUCCESS))
9527	{
9528	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9529	Assert(pVmxTransient->cbInstr == 2);
9530	}
9531	else
9532	{
9533	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9534	rc = VERR_EM_INTERPRETER;
9535	}
9536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9537	return rc;
9538	}
9539
9540
9541	/**
9542	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9543	*/
9544	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9545	{
9546	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9547	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9548	AssertRCReturn(rc, rc);
9549
9550	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9551	return VINF_EM_RAW_EMULATE_INSTR;
9552
9553	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9554	HMVMX_RETURN_UNEXPECTED_EXIT();
9555	}
9556
9557
9558	/**
9559	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9560	*/
9561	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9562	{
9563	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9564	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9565	AssertRCReturn(rc, rc);
9566
9567	PVM pVM = pVCpu->CTX_SUFF(pVM);
9568	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9569	if (RT_LIKELY(rc == VINF_SUCCESS))
9570	{
9571	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9572	Assert(pVmxTransient->cbInstr == 2);
9573	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9574	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9575	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9576	}
9577	else
9578	{
9579	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9580	rc = VERR_EM_INTERPRETER;
9581	}
9582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9583	return rc;
9584	}
9585
9586
9587	/**
9588	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9589	*/
9590	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9591	{
9592	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9593	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9594	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9595	AssertRCReturn(rc, rc);
9596
9597	PVM pVM = pVCpu->CTX_SUFF(pVM);
9598	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9599	if (RT_LIKELY(rc == VINF_SUCCESS))
9600	{
9601	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9602	Assert(pVmxTransient->cbInstr == 3);
9603	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9604	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9605	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9606	}
9607	else
9608	{
9609	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9610	rc = VERR_EM_INTERPRETER;
9611	}
9612	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9613	return rc;
9614	}
9615
9616
9617	/**
9618	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9619	*/
9620	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9621	{
9622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9623	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9624	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9625	AssertRCReturn(rc, rc);
9626
9627	PVM pVM = pVCpu->CTX_SUFF(pVM);
9628	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9629	if (RT_LIKELY(rc == VINF_SUCCESS))
9630	{
9631	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9632	Assert(pVmxTransient->cbInstr == 2);
9633	}
9634	else
9635	{
9636	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9637	rc = VERR_EM_INTERPRETER;
9638	}
9639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9640	return rc;
9641	}
9642
9643
9644	/**
9645	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9646	*/
9647	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9648	{
9649	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9650	PVM pVM = pVCpu->CTX_SUFF(pVM);
9651	Assert(!pVM->hm.s.fNestedPaging);
9652
9653	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9654	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9655	AssertRCReturn(rc, rc);
9656
9657	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9658	rc = VBOXSTRICTRC_VAL(rc2);
9659	if (RT_LIKELY(rc == VINF_SUCCESS))
9660	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9661	else
9662	{
9663	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9664	pVmxTransient->uExitQualification, rc));
9665	}
9666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9667	return rc;
9668	}
9669
9670
9671	/**
9672	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9673	*/
9674	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9675	{
9676	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9677	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9678	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9679	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9680	AssertRCReturn(rc, rc);
9681
9682	PVM pVM = pVCpu->CTX_SUFF(pVM);
9683	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9684	if (RT_LIKELY(rc == VINF_SUCCESS))
9685	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9686	else
9687	{
9688	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9689	rc = VERR_EM_INTERPRETER;
9690	}
9691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9692	return rc;
9693	}
9694
9695
9696	/**
9697	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9698	*/
9699	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9700	{
9701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9702	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9703	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9704	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9705	AssertRCReturn(rc, rc);
9706
9707	PVM pVM = pVCpu->CTX_SUFF(pVM);
9708	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9709	rc = VBOXSTRICTRC_VAL(rc2);
9710	if (RT_LIKELY( rc == VINF_SUCCESS
9711	\|\| rc == VINF_EM_HALT))
9712	{
9713	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9714	AssertRCReturn(rc3, rc3);
9715
9716	if ( rc == VINF_EM_HALT
9717	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9718	{
9719	rc = VINF_SUCCESS;
9720	}
9721	}
9722	else
9723	{
9724	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9725	rc = VERR_EM_INTERPRETER;
9726	}
9727	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9728	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9730	return rc;
9731	}
9732
9733
9734	/**
9735	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9736	*/
9737	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9738	{
9739	/*
9740	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9741	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9742	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9743	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9744	*/
9745	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9746	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9747	HMVMX_RETURN_UNEXPECTED_EXIT();
9748	}
9749
9750
9751	/**
9752	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9753	*/
9754	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9755	{
9756	/*
9757	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9758	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9759	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9760	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9761	*/
9762	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9763	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9764	HMVMX_RETURN_UNEXPECTED_EXIT();
9765	}
9766
9767
9768	/**
9769	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9770	*/
9771	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9772	{
9773	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9774	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9775	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9776	HMVMX_RETURN_UNEXPECTED_EXIT();
9777	}
9778
9779
9780	/**
9781	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9782	*/
9783	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9784	{
9785	/*
9786	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9787	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9788	* See Intel spec. 25.3 "Other Causes of VM-exits".
9789	*/
9790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9791	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9792	HMVMX_RETURN_UNEXPECTED_EXIT();
9793	}
9794
9795
9796	/**
9797	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9798	* VM-exit.
9799	*/
9800	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9801	{
9802	/*
9803	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9804	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9805	*
9806	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9807	* See Intel spec. "23.8 Restrictions on VMX operation".
9808	*/
9809	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9810	return VINF_SUCCESS;
9811	}
9812
9813
9814	/**
9815	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9816	* VM-exit.
9817	*/
9818	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819	{
9820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821	return VINF_EM_RESET;
9822	}
9823
9824
9825	/**
9826	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9827	*/
9828	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9829	{
9830	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9831	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9832	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9833	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9834	AssertRCReturn(rc, rc);
9835
9836	pMixedCtx->rip++;
9837	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9838	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9839	rc = VINF_SUCCESS;
9840	else
9841	rc = VINF_EM_HALT;
9842
9843	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9844	return rc;
9845	}
9846
9847
9848	/**
9849	* VM-exit handler for instructions that result in a #UD exception delivered to
9850	* the guest.
9851	*/
9852	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9853	{
9854	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9855	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9856	return VINF_SUCCESS;
9857	}
9858
9859
9860	/**
9861	* VM-exit handler for expiry of the VMX preemption timer.
9862	*/
9863	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9864	{
9865	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9866
9867	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9868	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9869
9870	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9871	PVM pVM = pVCpu->CTX_SUFF(pVM);
9872	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9874	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9875	}
9876
9877
9878	/**
9879	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9880	*/
9881	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9882	{
9883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9884
9885	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9886	/** @todo check if XSETBV is supported by the recompiler. */
9887	return VERR_EM_INTERPRETER;
9888	}
9889
9890
9891	/**
9892	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9893	*/
9894	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9895	{
9896	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9897
9898	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9899	/** @todo implement EMInterpretInvpcid() */
9900	return VERR_EM_INTERPRETER;
9901	}
9902
9903
9904	/**
9905	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9906	* Error VM-exit.
9907	*/
9908	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909	{
9910	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9911	AssertRCReturn(rc, rc);
9912
9913	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9914	NOREF(uInvalidReason);
9915
9916	#ifdef VBOX_STRICT
9917	uint32_t uIntrState;
9918	HMVMXHCUINTREG uHCReg;
9919	uint64_t u64Val;
9920	uint32_t u32Val;
9921
9922	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9923	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9924	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9925	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9926	AssertRCReturn(rc, rc);
9927
9928	Log4(("uInvalidReason %u\n", uInvalidReason));
9929	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9930	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9931	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9932	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9933
9934	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9935	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9936	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9937	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9938	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9939	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9940	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9941	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9942	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9943	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9944	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9945	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9946	#else
9947	NOREF(pVmxTransient);
9948	#endif
9949
9950	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9951	return VERR_VMX_INVALID_GUEST_STATE;
9952	}
9953
9954
9955	/**
9956	* VM-exit handler for VM-entry failure due to an MSR-load
9957	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9958	*/
9959	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9960	{
9961	NOREF(pVmxTransient);
9962	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9963	HMVMX_RETURN_UNEXPECTED_EXIT();
9964	}
9965
9966
9967	/**
9968	* VM-exit handler for VM-entry failure due to a machine-check event
9969	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9970	*/
9971	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9972	{
9973	NOREF(pVmxTransient);
9974	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9975	HMVMX_RETURN_UNEXPECTED_EXIT();
9976	}
9977
9978
9979	/**
9980	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9981	* theory.
9982	*/
9983	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9984	{
9985	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9986	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9987	return VERR_VMX_UNDEFINED_EXIT_CODE;
9988	}
9989
9990
9991	/**
9992	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9993	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9994	* Conditional VM-exit.
9995	*/
9996	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9997	{
9998	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9999
10000	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10002	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10003	return VERR_EM_INTERPRETER;
10004	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10005	HMVMX_RETURN_UNEXPECTED_EXIT();
10006	}
10007
10008
10009	/**
10010	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10011	*/
10012	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10013	{
10014	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10015
10016	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10018	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10019	return VERR_EM_INTERPRETER;
10020	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10021	HMVMX_RETURN_UNEXPECTED_EXIT();
10022	}
10023
10024
10025	/**
10026	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10027	*/
10028	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10029	{
10030	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10031
10032	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10033	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10034	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10035	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10036	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10037	{
10038	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10039	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10040	}
10041	AssertRCReturn(rc, rc);
10042	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10043
10044	#ifdef VBOX_STRICT
10045	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10046	{
10047	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10048	{
10049	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10050	HMVMX_RETURN_UNEXPECTED_EXIT();
10051	}
10052	# if HC_ARCH_BITS == 64
10053	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10054	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10055	{
10056	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10057	HMVMX_RETURN_UNEXPECTED_EXIT();
10058	}
10059	# endif
10060	}
10061	#endif
10062
10063	PVM pVM = pVCpu->CTX_SUFF(pVM);
10064	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10065	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
10066	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10067	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10068
10069	if (RT_LIKELY(rc == VINF_SUCCESS))
10070	{
10071	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10072	Assert(pVmxTransient->cbInstr == 2);
10073	}
10074	return rc;
10075	}
10076
10077
10078	/**
10079	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10080	*/
10081	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10082	{
10083	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10084	PVM pVM = pVCpu->CTX_SUFF(pVM);
10085	int rc = VINF_SUCCESS;
10086
10087	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10088	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10089	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10090	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10091	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10092	{
10093	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10094	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10095	}
10096	AssertRCReturn(rc, rc);
10097	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10098
10099	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10100	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10102
10103	if (RT_LIKELY(rc == VINF_SUCCESS))
10104	{
10105	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10106
10107	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10108	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10109	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10110	{
10111	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10112	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10113	EMInterpretWrmsr() changes it. */
10114	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10115	}
10116	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10117	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10118
10119	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10120	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10121	{
10122	switch (pMixedCtx->ecx)
10123	{
10124	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10125	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10126	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10127	case MSR_K8_FS_BASE: /* no break */
10128	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10129	default:
10130	{
10131	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10132	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10133	#if HC_ARCH_BITS == 64
10134	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10135	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10136	#endif
10137	break;
10138	}
10139	}
10140	}
10141	#ifdef VBOX_STRICT
10142	else
10143	{
10144	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10145	switch (pMixedCtx->ecx)
10146	{
10147	case MSR_IA32_SYSENTER_CS:
10148	case MSR_IA32_SYSENTER_EIP:
10149	case MSR_IA32_SYSENTER_ESP:
10150	case MSR_K8_FS_BASE:
10151	case MSR_K8_GS_BASE:
10152	{
10153	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10154	HMVMX_RETURN_UNEXPECTED_EXIT();
10155	}
10156
10157	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10158	default:
10159	{
10160	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10161	{
10162	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10163	pMixedCtx->ecx));
10164	HMVMX_RETURN_UNEXPECTED_EXIT();
10165	}
10166
10167	#if HC_ARCH_BITS == 64
10168	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10169	{
10170	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10171	HMVMX_RETURN_UNEXPECTED_EXIT();
10172	}
10173	#endif
10174	break;
10175	}
10176	}
10177	}
10178	#endif /* VBOX_STRICT */
10179	}
10180	return rc;
10181	}
10182
10183
10184	/**
10185	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10186	*/
10187	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10188	{
10189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10190
10191	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10193	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10194	return VERR_EM_INTERPRETER;
10195	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10196	HMVMX_RETURN_UNEXPECTED_EXIT();
10197	}
10198
10199
10200	/**
10201	* VM-exit handler for when the TPR value is lowered below the specified
10202	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10203	*/
10204	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10205	{
10206	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10207	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10208
10209	/*
10210	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10211	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10212	* resume guest execution.
10213	*/
10214	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10216	return VINF_SUCCESS;
10217	}
10218
10219
10220	/**
10221	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10222	* VM-exit.
10223	*
10224	* @retval VINF_SUCCESS when guest execution can continue.
10225	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10226	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10227	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10228	* recompiler.
10229	*/
10230	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231	{
10232	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10234	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10235	AssertRCReturn(rc, rc);
10236
10237	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10238	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10239	PVM pVM = pVCpu->CTX_SUFF(pVM);
10240	switch (uAccessType)
10241	{
10242	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10243	{
10244	#if 0
10245	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10246	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10247	#else
10248	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10249	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10250	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10251	#endif
10252	AssertRCReturn(rc, rc);
10253
10254	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10255	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10256	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10257	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10258
10259	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10260	{
10261	case 0: /* CR0 */
10262	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10263	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10264	break;
10265	case 2: /* CR2 */
10266	/* Nothing to do here, CR2 it's not part of the VMCS. */
10267	break;
10268	case 3: /* CR3 */
10269	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10270	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10271	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10272	break;
10273	case 4: /* CR4 */
10274	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10275	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10276	break;
10277	case 8: /* CR8 */
10278	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10279	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10280	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10281	break;
10282	default:
10283	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10284	break;
10285	}
10286
10287	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10288	break;
10289	}
10290
10291	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10292	{
10293	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10294	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10295	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10296	AssertRCReturn(rc, rc);
10297	Assert( !pVM->hm.s.fNestedPaging
10298	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10299	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10300
10301	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10302	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10303	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10304
10305	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10306	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10307	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10308	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10310	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10311	break;
10312	}
10313
10314	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10315	{
10316	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10317	AssertRCReturn(rc, rc);
10318	rc = EMInterpretCLTS(pVM, pVCpu);
10319	AssertRCReturn(rc, rc);
10320	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10321	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10322	Log4(("CRX CLTS write rc=%d\n", rc));
10323	break;
10324	}
10325
10326	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10327	{
10328	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10329	AssertRCReturn(rc, rc);
10330	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10331	if (RT_LIKELY(rc == VINF_SUCCESS))
10332	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10334	Log4(("CRX LMSW write rc=%d\n", rc));
10335	break;
10336	}
10337
10338	default:
10339	{
10340	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10341	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10342	}
10343	}
10344
10345	/* Validate possible error codes. */
10346	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10347	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10348	if (RT_SUCCESS(rc))
10349	{
10350	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10351	AssertRCReturn(rc2, rc2);
10352	}
10353
10354	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10355	return rc;
10356	}
10357
10358
10359	/**
10360	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10361	* VM-exit.
10362	*/
10363	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10364	{
10365	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10366	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10367
10368	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10369	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10370	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10371	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10372	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10373	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10374	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10375	AssertRCReturn(rc2, rc2);
10376
10377	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10378	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10379	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10380	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10381	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10382	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10383	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10384	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10385
10386	/* I/O operation lookup arrays. */
10387	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10388	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10389
10390	VBOXSTRICTRC rcStrict;
10391	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10392	const uint32_t cbInstr = pVmxTransient->cbInstr;
10393	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10394	PVM pVM = pVCpu->CTX_SUFF(pVM);
10395	if (fIOString)
10396	{
10397	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10398	/*
10399	* INS/OUTS - I/O String instruction.
10400	*
10401	* Use instruction-information if available, otherwise fall back on
10402	* interpreting the instruction.
10403	*/
10404	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10405	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10406	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10407	{
10408	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10409	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10410	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10411	AssertRCReturn(rc2, rc2);
10412	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10413	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10414	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10415	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10416	if (fIOWrite)
10417	{
10418	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10419	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10420	}
10421	else
10422	{
10423	/*
10424	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10425	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10426	* See Intel Instruction spec. for "INS".
10427	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10428	*/
10429	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10430	}
10431	}
10432	else
10433	{
10434	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10435	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10436	AssertRCReturn(rc2, rc2);
10437	rcStrict = IEMExecOne(pVCpu);
10438	}
10439	/** @todo IEM needs to be setting these flags somehow. */
10440	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10441	fUpdateRipAlready = true;
10442	#else
10443	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10444	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10445	if (RT_SUCCESS(rcStrict))
10446	{
10447	if (fIOWrite)
10448	{
10449	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10450	(DISCPUMODE)pDis->uAddrMode, cbValue);
10451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10452	}
10453	else
10454	{
10455	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10456	(DISCPUMODE)pDis->uAddrMode, cbValue);
10457	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10458	}
10459	}
10460	else
10461	{
10462	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10463	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10464	}
10465	#endif
10466	}
10467	else
10468	{
10469	/*
10470	* IN/OUT - I/O instruction.
10471	*/
10472	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10473	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10474	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10475	if (fIOWrite)
10476	{
10477	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10478	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10479	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10481	}
10482	else
10483	{
10484	uint32_t u32Result = 0;
10485	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10486	if (IOM_SUCCESS(rcStrict))
10487	{
10488	/* Save result of I/O IN instr. in AL/AX/EAX. */
10489	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10490	}
10491	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10492	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10494	}
10495	}
10496
10497	if (IOM_SUCCESS(rcStrict))
10498	{
10499	if (!fUpdateRipAlready)
10500	{
10501	pMixedCtx->rip += cbInstr;
10502	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10503	}
10504
10505	/*
10506	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10507	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10508	*/
10509	if (fIOString)
10510	{
10511	/** @todo Single-step for INS/OUTS with REP prefix? */
10512	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10513	}
10514	else if (fStepping)
10515	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10516
10517	/*
10518	* If any I/O breakpoints are armed, we need to check if one triggered
10519	* and take appropriate action.
10520	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10521	*/
10522	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10523	AssertRCReturn(rc2, rc2);
10524
10525	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10526	* execution engines about whether hyper BPs and such are pending. */
10527	uint32_t const uDr7 = pMixedCtx->dr[7];
10528	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10529	&& X86_DR7_ANY_RW_IO(uDr7)
10530	&& (pMixedCtx->cr4 & X86_CR4_DE))
10531	\|\| DBGFBpIsHwIoArmed(pVM)))
10532	{
10533	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10534
10535	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10536	VMMRZCallRing3Disable(pVCpu);
10537	HM_DISABLE_PREEMPT_IF_NEEDED();
10538
10539	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10540
10541	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10542	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10543	{
10544	/* Raise #DB. */
10545	if (fIsGuestDbgActive)
10546	ASMSetDR6(pMixedCtx->dr[6]);
10547	if (pMixedCtx->dr[7] != uDr7)
10548	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10549
10550	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10551	}
10552	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10553	else if ( rcStrict2 != VINF_SUCCESS
10554	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10555	rcStrict = rcStrict2;
10556
10557	HM_RESTORE_PREEMPT_IF_NEEDED();
10558	VMMRZCallRing3Enable(pVCpu);
10559	}
10560	}
10561
10562	#ifdef DEBUG
10563	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10564	Assert(!fIOWrite);
10565	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10566	Assert(fIOWrite);
10567	else
10568	{
10569	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10570	* statuses, that the VMM device and some others may return. See
10571	* IOM_SUCCESS() for guidance. */
10572	AssertMsg( RT_FAILURE(rcStrict)
10573	\|\| rcStrict == VINF_SUCCESS
10574	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10575	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10576	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10577	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10578	}
10579	#endif
10580
10581	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10582	return VBOXSTRICTRC_TODO(rcStrict);
10583	}
10584
10585
10586	/**
10587	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10588	* VM-exit.
10589	*/
10590	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10591	{
10592	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10593
10594	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10595	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10596	AssertRCReturn(rc, rc);
10597	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10598	{
10599	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10600	AssertRCReturn(rc, rc);
10601	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10602	{
10603	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10604
10605	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10606	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10607	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10608	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10609	{
10610	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10611	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10612
10613	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10614	Assert(!pVCpu->hm.s.Event.fPending);
10615	pVCpu->hm.s.Event.fPending = true;
10616	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10617	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10618	AssertRCReturn(rc, rc);
10619	if (fErrorCodeValid)
10620	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10621	else
10622	pVCpu->hm.s.Event.u32ErrCode = 0;
10623	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10624	&& uVector == X86_XCPT_PF)
10625	{
10626	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10627	}
10628
10629	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10630	}
10631	}
10632	}
10633
10634	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10635	* emulation. */
10636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10637	return VERR_EM_INTERPRETER;
10638	}
10639
10640
10641	/**
10642	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10643	*/
10644	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10645	{
10646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10647	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10648	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10649	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10650	AssertRCReturn(rc, rc);
10651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10652	return VINF_EM_DBG_STEPPED;
10653	}
10654
10655
10656	/**
10657	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10658	*/
10659	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10660	{
10661	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10662
10663	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10664	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10665	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10666	return VINF_SUCCESS;
10667	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10668	return rc;
10669
10670	#if 0
10671	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10672	* just sync the whole thing. */
10673	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10674	#else
10675	/* Aggressive state sync. for now. */
10676	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10677	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10678	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10679	#endif
10680	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10681	AssertRCReturn(rc, rc);
10682
10683	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10684	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10685	switch (uAccessType)
10686	{
10687	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10688	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10689	{
10690	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10691	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10692	{
10693	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10694	}
10695
10696	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10697	GCPhys &= PAGE_BASE_GC_MASK;
10698	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10699	PVM pVM = pVCpu->CTX_SUFF(pVM);
10700	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10701	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10702
10703	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10704	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10705	CPUMCTX2CORE(pMixedCtx), GCPhys);
10706	rc = VBOXSTRICTRC_VAL(rc2);
10707	Log4(("ApicAccess rc=%d\n", rc));
10708	if ( rc == VINF_SUCCESS
10709	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10710	\|\| rc == VERR_PAGE_NOT_PRESENT)
10711	{
10712	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10713	\| HM_CHANGED_GUEST_RSP
10714	\| HM_CHANGED_GUEST_RFLAGS
10715	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10716	rc = VINF_SUCCESS;
10717	}
10718	break;
10719	}
10720
10721	default:
10722	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10723	rc = VINF_EM_RAW_EMULATE_INSTR;
10724	break;
10725	}
10726
10727	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10728	return rc;
10729	}
10730
10731
10732	/**
10733	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10734	* VM-exit.
10735	*/
10736	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10737	{
10738	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10739
10740	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10741	if (pVmxTransient->fWasGuestDebugStateActive)
10742	{
10743	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10744	HMVMX_RETURN_UNEXPECTED_EXIT();
10745	}
10746
10747	int rc = VERR_INTERNAL_ERROR_5;
10748	if ( !DBGFIsStepping(pVCpu)
10749	&& !pVCpu->hm.s.fSingleInstruction
10750	&& !pVmxTransient->fWasHyperDebugStateActive)
10751	{
10752	/* Don't intercept MOV DRx and #DB any more. */
10753	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10754	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10755	AssertRCReturn(rc, rc);
10756
10757	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10758	{
10759	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10760	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10762	AssertRCReturn(rc, rc);
10763	#endif
10764	}
10765
10766	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10767	VMMRZCallRing3Disable(pVCpu);
10768	HM_DISABLE_PREEMPT_IF_NEEDED();
10769
10770	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10771	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10772	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10773
10774	HM_RESTORE_PREEMPT_IF_NEEDED();
10775	VMMRZCallRing3Enable(pVCpu);
10776
10777	#ifdef VBOX_WITH_STATISTICS
10778	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10779	AssertRCReturn(rc, rc);
10780	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10782	else
10783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10784	#endif
10785	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10786	return VINF_SUCCESS;
10787	}
10788
10789	/*
10790	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10791	* Update the segment registers and DR7 from the CPU.
10792	*/
10793	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10794	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10795	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10796	AssertRCReturn(rc, rc);
10797	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10798
10799	PVM pVM = pVCpu->CTX_SUFF(pVM);
10800	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10801	{
10802	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10803	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10804	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10805	if (RT_SUCCESS(rc))
10806	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10808	}
10809	else
10810	{
10811	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10812	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10813	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10815	}
10816
10817	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10818	if (RT_SUCCESS(rc))
10819	{
10820	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10821	AssertRCReturn(rc2, rc2);
10822	}
10823	return rc;
10824	}
10825
10826
10827	/**
10828	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10829	* Conditional VM-exit.
10830	*/
10831	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10832	{
10833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10834	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10835
10836	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10837	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10838	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10839	return VINF_SUCCESS;
10840	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10841	return rc;
10842
10843	RTGCPHYS GCPhys = 0;
10844	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10845
10846	#if 0
10847	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10848	#else
10849	/* Aggressive state sync. for now. */
10850	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10851	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10852	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10853	#endif
10854	AssertRCReturn(rc, rc);
10855
10856	/*
10857	* If we succeed, resume guest execution.
10858	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10859	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10860	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10861	* weird case. See @bugref{6043}.
10862	*/
10863	PVM pVM = pVCpu->CTX_SUFF(pVM);
10864	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10865	rc = VBOXSTRICTRC_VAL(rc2);
10866	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10867	if ( rc == VINF_SUCCESS
10868	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10869	\|\| rc == VERR_PAGE_NOT_PRESENT)
10870	{
10871	/* Successfully handled MMIO operation. */
10872	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10873	\| HM_CHANGED_GUEST_RSP
10874	\| HM_CHANGED_GUEST_RFLAGS
10875	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10876	rc = VINF_SUCCESS;
10877	}
10878	return rc;
10879	}
10880
10881
10882	/**
10883	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10884	* VM-exit.
10885	*/
10886	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10887	{
10888	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10889	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10890
10891	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10892	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10893	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10894	return VINF_SUCCESS;
10895	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10896	return rc;
10897
10898	RTGCPHYS GCPhys = 0;
10899	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10900	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10901	#if 0
10902	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10903	#else
10904	/* Aggressive state sync. for now. */
10905	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10906	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10907	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10908	#endif
10909	AssertRCReturn(rc, rc);
10910
10911	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10912	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10913
10914	RTGCUINT uErrorCode = 0;
10915	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10916	uErrorCode \|= X86_TRAP_PF_ID;
10917	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10918	uErrorCode \|= X86_TRAP_PF_RW;
10919	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10920	uErrorCode \|= X86_TRAP_PF_P;
10921
10922	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10923
10924	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10925	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10926
10927	/* Handle the pagefault trap for the nested shadow table. */
10928	PVM pVM = pVCpu->CTX_SUFF(pVM);
10929	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10930	TRPMResetTrap(pVCpu);
10931
10932	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10933	if ( rc == VINF_SUCCESS
10934	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10935	\|\| rc == VERR_PAGE_NOT_PRESENT)
10936	{
10937	/* Successfully synced our nested page tables. */
10938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10939	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10940	\| HM_CHANGED_GUEST_RSP
10941	\| HM_CHANGED_GUEST_RFLAGS);
10942	return VINF_SUCCESS;
10943	}
10944
10945	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10946	return rc;
10947	}
10948
10949	/** @} */
10950
10951	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10952	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10953	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10954
10955	/** @name VM-exit exception handlers.
10956	* @{
10957	*/
10958
10959	/**
10960	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10961	*/
10962	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10963	{
10964	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10966
10967	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10968	AssertRCReturn(rc, rc);
10969
10970	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10971	{
10972	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10973	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10974
10975	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10976	* provides VM-exit instruction length. If this causes problem later,
10977	* disassemble the instruction like it's done on AMD-V. */
10978	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10979	AssertRCReturn(rc2, rc2);
10980	return rc;
10981	}
10982
10983	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10984	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10985	return rc;
10986	}
10987
10988
10989	/**
10990	* VM-exit exception handler for #BP (Breakpoint exception).
10991	*/
10992	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10993	{
10994	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10995	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10996
10997	/** @todo Try optimize this by not saving the entire guest state unless
10998	* really needed. */
10999	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11000	AssertRCReturn(rc, rc);
11001
11002	PVM pVM = pVCpu->CTX_SUFF(pVM);
11003	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11004	if (rc == VINF_EM_RAW_GUEST_TRAP)
11005	{
11006	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11007	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11008	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11009	AssertRCReturn(rc, rc);
11010
11011	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11012	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11013	}
11014
11015	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
11016	return rc;
11017	}
11018
11019
11020	/**
11021	* VM-exit exception handler for #DB (Debug exception).
11022	*/
11023	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11024	{
11025	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11027	Log6(("XcptDB\n"));
11028
11029	/*
11030	* Get the DR6-like values from the exit qualification and pass it to DBGF
11031	* for processing.
11032	*/
11033	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11034	AssertRCReturn(rc, rc);
11035
11036	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11037	uint64_t uDR6 = X86_DR6_INIT_VAL;
11038	uDR6 \|= ( pVmxTransient->uExitQualification
11039	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
11040
11041	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11042	if (rc == VINF_EM_RAW_GUEST_TRAP)
11043	{
11044	/*
11045	* The exception was for the guest. Update DR6, DR7.GD and
11046	* IA32_DEBUGCTL.LBR before forwarding it.
11047	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11048	*/
11049	VMMRZCallRing3Disable(pVCpu);
11050	HM_DISABLE_PREEMPT_IF_NEEDED();
11051
11052	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11053	pMixedCtx->dr[6] \|= uDR6;
11054	if (CPUMIsGuestDebugStateActive(pVCpu))
11055	ASMSetDR6(pMixedCtx->dr[6]);
11056
11057	HM_RESTORE_PREEMPT_IF_NEEDED();
11058	VMMRZCallRing3Enable(pVCpu);
11059
11060	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11061	AssertRCReturn(rc, rc);
11062
11063	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11064	pMixedCtx->dr[7] &= ~X86_DR7_GD;
11065
11066	/* Paranoia. */
11067	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11068	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
11069
11070	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11071	AssertRCReturn(rc, rc);
11072
11073	/*
11074	* Raise #DB in the guest.
11075	*
11076	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11077	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11078	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11079	*
11080	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11081	*/
11082	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11083	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11084	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11085	AssertRCReturn(rc, rc);
11086	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11087	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11088	return VINF_SUCCESS;
11089	}
11090
11091	/*
11092	* Not a guest trap, must be a hypervisor related debug event then.
11093	* Update DR6 in case someone is interested in it.
11094	*/
11095	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11096	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11097	CPUMSetHyperDR6(pVCpu, uDR6);
11098
11099	return rc;
11100	}
11101
11102
11103	/**
11104	* VM-exit exception handler for #NM (Device-not-available exception: floating
11105	* point exception).
11106	*/
11107	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11108	{
11109	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11110
11111	/* We require CR0 and EFER. EFER is always up-to-date. */
11112	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11113	AssertRCReturn(rc, rc);
11114
11115	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11116	VMMRZCallRing3Disable(pVCpu);
11117	HM_DISABLE_PREEMPT_IF_NEEDED();
11118
11119	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11120	if (pVmxTransient->fWasGuestFPUStateActive)
11121	{
11122	rc = VINF_EM_RAW_GUEST_TRAP;
11123	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11124	}
11125	else
11126	{
11127	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11128	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11129	#endif
11130	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11131	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11132	}
11133
11134	HM_RESTORE_PREEMPT_IF_NEEDED();
11135	VMMRZCallRing3Enable(pVCpu);
11136
11137	if (rc == VINF_SUCCESS)
11138	{
11139	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11140	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11141	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11142	pVCpu->hm.s.fUseGuestFpu = true;
11143	}
11144	else
11145	{
11146	/* Forward #NM to the guest. */
11147	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11148	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11149	AssertRCReturn(rc, rc);
11150	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11151	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11153	}
11154
11155	return VINF_SUCCESS;
11156	}
11157
11158
11159	/**
11160	* VM-exit exception handler for #GP (General-protection exception).
11161	*
11162	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11163	*/
11164	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11165	{
11166	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11168
11169	int rc = VERR_INTERNAL_ERROR_5;
11170	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11171	{
11172	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11173	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11174	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11175	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11176	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11177	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11178	AssertRCReturn(rc, rc);
11179	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11180	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11181	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11182	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11183	return rc;
11184	#else
11185	/* We don't intercept #GP. */
11186	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11187	NOREF(pVmxTransient);
11188	return VERR_VMX_UNEXPECTED_EXCEPTION;
11189	#endif
11190	}
11191
11192	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11193	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11194
11195	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11196	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11197	AssertRCReturn(rc, rc);
11198
11199	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11200	uint32_t cbOp = 0;
11201	PVM pVM = pVCpu->CTX_SUFF(pVM);
11202	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11203	if (RT_SUCCESS(rc))
11204	{
11205	rc = VINF_SUCCESS;
11206	Assert(cbOp == pDis->cbInstr);
11207	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11208	switch (pDis->pCurInstr->uOpcode)
11209	{
11210	case OP_CLI:
11211	{
11212	pMixedCtx->eflags.Bits.u1IF = 0;
11213	pMixedCtx->rip += pDis->cbInstr;
11214	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11215	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11217	break;
11218	}
11219
11220	case OP_STI:
11221	{
11222	pMixedCtx->eflags.Bits.u1IF = 1;
11223	pMixedCtx->rip += pDis->cbInstr;
11224	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11225	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11226	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11227	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11228	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11229	break;
11230	}
11231
11232	case OP_HLT:
11233	{
11234	rc = VINF_EM_HALT;
11235	pMixedCtx->rip += pDis->cbInstr;
11236	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11238	break;
11239	}
11240
11241	case OP_POPF:
11242	{
11243	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11244	uint32_t cbParm;
11245	uint32_t uMask;
11246	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11247	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11248	{
11249	cbParm = 4;
11250	uMask = 0xffffffff;
11251	}
11252	else
11253	{
11254	cbParm = 2;
11255	uMask = 0xffff;
11256	}
11257
11258	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11259	RTGCPTR GCPtrStack = 0;
11260	X86EFLAGS Eflags;
11261	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11262	&GCPtrStack);
11263	if (RT_SUCCESS(rc))
11264	{
11265	Assert(sizeof(Eflags.u32) >= cbParm);
11266	Eflags.u32 = 0;
11267	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11268	}
11269	if (RT_FAILURE(rc))
11270	{
11271	rc = VERR_EM_INTERPRETER;
11272	break;
11273	}
11274	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11275	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11276	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11277	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11278	pMixedCtx->esp += cbParm;
11279	pMixedCtx->esp &= uMask;
11280	pMixedCtx->rip += pDis->cbInstr;
11281	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11282	\| HM_CHANGED_GUEST_RSP
11283	\| HM_CHANGED_GUEST_RFLAGS);
11284	/* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11285	if (fStepping)
11286	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11287
11288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11289	break;
11290	}
11291
11292	case OP_PUSHF:
11293	{
11294	uint32_t cbParm;
11295	uint32_t uMask;
11296	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11297	{
11298	cbParm = 4;
11299	uMask = 0xffffffff;
11300	}
11301	else
11302	{
11303	cbParm = 2;
11304	uMask = 0xffff;
11305	}
11306
11307	/* Get the stack pointer & push the contents of eflags onto the stack. */
11308	RTGCPTR GCPtrStack = 0;
11309	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11310	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11311	if (RT_FAILURE(rc))
11312	{
11313	rc = VERR_EM_INTERPRETER;
11314	break;
11315	}
11316	X86EFLAGS Eflags = pMixedCtx->eflags;
11317	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11318	Eflags.Bits.u1RF = 0;
11319	Eflags.Bits.u1VM = 0;
11320
11321	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11322	if (RT_FAILURE(rc))
11323	{
11324	rc = VERR_EM_INTERPRETER;
11325	break;
11326	}
11327	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11328	pMixedCtx->esp -= cbParm;
11329	pMixedCtx->esp &= uMask;
11330	pMixedCtx->rip += pDis->cbInstr;
11331	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11332	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11334	break;
11335	}
11336
11337	case OP_IRET:
11338	{
11339	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11340	* instruction reference. */
11341	RTGCPTR GCPtrStack = 0;
11342	uint32_t uMask = 0xffff;
11343	bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11344	uint16_t aIretFrame[3];
11345	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11346	{
11347	rc = VERR_EM_INTERPRETER;
11348	break;
11349	}
11350	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11351	&GCPtrStack);
11352	if (RT_SUCCESS(rc))
11353	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11354	if (RT_FAILURE(rc))
11355	{
11356	rc = VERR_EM_INTERPRETER;
11357	break;
11358	}
11359	pMixedCtx->eip = 0;
11360	pMixedCtx->ip = aIretFrame[0];
11361	pMixedCtx->cs.Sel = aIretFrame[1];
11362	pMixedCtx->cs.ValidSel = aIretFrame[1];
11363	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11364	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11365	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11366	pMixedCtx->sp += sizeof(aIretFrame);
11367	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11368	\| HM_CHANGED_GUEST_SEGMENT_REGS
11369	\| HM_CHANGED_GUEST_RSP
11370	\| HM_CHANGED_GUEST_RFLAGS);
11371	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11372	if (fStepping)
11373	hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11374	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11376	break;
11377	}
11378
11379	case OP_INT:
11380	{
11381	uint16_t uVector = pDis->Param1.uValue & 0xff;
11382	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11383	/* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11385	break;
11386	}
11387
11388	case OP_INTO:
11389	{
11390	if (pMixedCtx->eflags.Bits.u1OF)
11391	{
11392	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11393	/* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11395	}
11396	break;
11397	}
11398
11399	default:
11400	{
11401	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11402	EMCODETYPE_SUPERVISOR);
11403	rc = VBOXSTRICTRC_VAL(rc2);
11404	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11405	/** @todo We have to set pending-debug exceptions here when the guest is
11406	* single-stepping depending on the instruction that was interpreted. */
11407	Log4(("#GP rc=%Rrc\n", rc));
11408	break;
11409	}
11410	}
11411	}
11412	else
11413	rc = VERR_EM_INTERPRETER;
11414
11415	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11416	("#GP Unexpected rc=%Rrc\n", rc));
11417	return rc;
11418	}
11419
11420
11421	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11422	/**
11423	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11424	* the exception reported in the VMX transient structure back into the VM.
11425	*
11426	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11427	* up-to-date.
11428	*/
11429	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11430	{
11431	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11432
11433	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11434	hmR0VmxCheckExitDueToEventDelivery(). */
11435	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11436	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11437	AssertRCReturn(rc, rc);
11438	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11439
11440	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11441	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11442	return VINF_SUCCESS;
11443	}
11444	#endif
11445
11446
11447	/**
11448	* VM-exit exception handler for #PF (Page-fault exception).
11449	*/
11450	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11451	{
11452	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11453	PVM pVM = pVCpu->CTX_SUFF(pVM);
11454	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11455	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11456	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11457	AssertRCReturn(rc, rc);
11458
11459	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11460	if (pVM->hm.s.fNestedPaging)
11461	{
11462	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11463	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11464	{
11465	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11466	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11467	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11468	}
11469	else
11470	{
11471	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11472	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11473	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11474	}
11475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11476	return rc;
11477	}
11478	#else
11479	Assert(!pVM->hm.s.fNestedPaging);
11480	NOREF(pVM);
11481	#endif
11482
11483	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11484	AssertRCReturn(rc, rc);
11485
11486	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11487	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11488
11489	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11490	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11491	(RTGCPTR)pVmxTransient->uExitQualification);
11492
11493	Log4(("#PF: rc=%Rrc\n", rc));
11494	if (rc == VINF_SUCCESS)
11495	{
11496	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11497	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11498	* memory? We don't update the whole state here... */
11499	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11500	\| HM_CHANGED_GUEST_RSP
11501	\| HM_CHANGED_GUEST_RFLAGS
11502	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11503	TRPMResetTrap(pVCpu);
11504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11505	return rc;
11506	}
11507	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11508	{
11509	if (!pVmxTransient->fVectoringPF)
11510	{
11511	/* It's a guest page fault and needs to be reflected to the guest. */
11512	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11513	TRPMResetTrap(pVCpu);
11514	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11515	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11516	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11517	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11518	}
11519	else
11520	{
11521	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11522	TRPMResetTrap(pVCpu);
11523	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11524	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11525	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11526	}
11527
11528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11529	return VINF_SUCCESS;
11530	}
11531
11532	TRPMResetTrap(pVCpu);
11533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11534	return rc;
11535	}
11536
11537	/** @} */
11538

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50655

以其他格式下載: