HMR0.cpp@ 93554

最後變更在這個檔案從93554是 93554,由 vboxsync 提交於 3 年前
VMM: Changed PAGE_SIZE -> GUEST_PAGE_SIZE / HOST_PAGE_SIZE, PAGE_SHIFT -> GUEST_PAGE_SHIFT / HOST_PAGE_SHIFT, and PAGE_OFFSET_MASK -> GUEST_PAGE_OFFSET_MASK / HOST_PAGE_OFFSET_MASK. Also removed most usage of ASMMemIsZeroPage and ASMMemZeroPage since the host and guest page size doesn't need to be the same any more. Some work left to do in the page pool code. bugref:9898
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Id Revision`
檔案大小: 68.0 KB

行
1	/* $Id: HMR0.cpp 93554 2022-02-02 22:57:02Z vboxsync $ */
2	/** @file
3	* Hardware Assisted Virtualization Manager (HM) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <VBox/vmm/hm.h>
25	#include <VBox/vmm/pgm.h>
26	#include "HMInternal.h"
27	#include <VBox/vmm/vmcc.h>
28	#include <VBox/vmm/hm_svm.h>
29	#include <VBox/vmm/hmvmxinline.h>
30	#include <VBox/err.h>
31	#include <VBox/log.h>
32	#include <iprt/assert.h>
33	#include <iprt/asm.h>
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/cpuset.h>
36	#include <iprt/mem.h>
37	#include <iprt/memobj.h>
38	#include <iprt/once.h>
39	#include <iprt/param.h>
40	#include <iprt/power.h>
41	#include <iprt/string.h>
42	#include <iprt/thread.h>
43	#include <iprt/x86.h>
44	#include "HMVMXR0.h"
45	#include "HMSVMR0.h"
46
47
48	/*********************************************************************************************************************************
49	* Internal Functions *
50	*********************************************************************************************************************************/
51	static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
52	static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2);
53	static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser);
54	static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData);
55
56
57	/*********************************************************************************************************************************
58	* Structures and Typedefs *
59	*********************************************************************************************************************************/
60	/**
61	* This is used to manage the status code of a RTMpOnAll in HM.
62	*/
63	typedef struct HMR0FIRSTRC
64	{
65	/** The status code. */
66	int32_t volatile rc;
67	/** The ID of the CPU reporting the first failure. */
68	RTCPUID volatile idCpu;
69	} HMR0FIRSTRC;
70	/** Pointer to a first return code structure. */
71	typedef HMR0FIRSTRC *PHMR0FIRSTRC;
72
73	/**
74	* Ring-0 method table for AMD-V and VT-x specific operations.
75	*/
76	typedef struct HMR0VTABLE
77	{
78	DECLR0CALLBACKMEMBER(int, pfnEnterSession, (PVMCPUCC pVCpu));
79	DECLR0CALLBACKMEMBER(void, pfnThreadCtxCallback, (RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit));
80	DECLR0CALLBACKMEMBER(int, pfnAssertionCallback, (PVMCPUCC pVCpu));
81	DECLR0CALLBACKMEMBER(int, pfnExportHostState, (PVMCPUCC pVCpu));
82	DECLR0CALLBACKMEMBER(VBOXSTRICTRC, pfnRunGuestCode, (PVMCPUCC pVCpu));
83	DECLR0CALLBACKMEMBER(int, pfnEnableCpu, (PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
84	bool fEnabledByHost, PCSUPHWVIRTMSRS pHwvirtMsrs));
85	DECLR0CALLBACKMEMBER(int, pfnDisableCpu, (PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage));
86	DECLR0CALLBACKMEMBER(int, pfnInitVM, (PVMCC pVM));
87	DECLR0CALLBACKMEMBER(int, pfnTermVM, (PVMCC pVM));
88	DECLR0CALLBACKMEMBER(int, pfnSetupVM, (PVMCC pVM));
89	} HMR0VTABLE;
90
91
92	/*********************************************************************************************************************************
93	* Global Variables *
94	*********************************************************************************************************************************/
95	/** The active ring-0 HM operations (copied from one of the table at init). */
96	static HMR0VTABLE g_HmR0Ops;
97	/** Indicates whether the host is suspending or not. We'll refuse a few
98	* actions when the host is being suspended to speed up the suspending and
99	* avoid trouble. */
100	static bool volatile g_fHmSuspended;
101	/** If set, VT-x/AMD-V is enabled globally at init time, otherwise it's
102	* enabled and disabled each time it's used to execute guest code. */
103	static bool g_fHmGlobalInit;
104	/** Host kernel flags that HM might need to know (SUPKERNELFEATURES_XXX). */
105	uint32_t g_fHmHostKernelFeatures;
106	/** Maximum allowed ASID/VPID (inclusive).
107	* @todo r=bird: This is exclusive for VT-x according to source code comment.
108	* Couldn't immediately find any docs on AMD-V, but suspect it is
109	* exclusive there as well given how hmR0SvmFlushTaggedTlb() use it. */
110	uint32_t g_uHmMaxAsid;
111
112
113	/** Set if VT-x (VMX) is supported by the CPU. */
114	bool g_fHmVmxSupported = false;
115	/** VMX: Whether we're using the preemption timer or not. */
116	bool g_fHmVmxUsePreemptTimer;
117	/** VMX: The shift mask employed by the VMX-Preemption timer. */
118	uint8_t g_cHmVmxPreemptTimerShift;
119	/** VMX: Set if swapping EFER is supported. */
120	bool g_fHmVmxSupportsVmcsEfer = false;
121	/** VMX: Whether we're using SUPR0EnableVTx or not. */
122	static bool g_fHmVmxUsingSUPR0EnableVTx = false;
123	/** VMX: Set if we've called SUPR0EnableVTx(true) and should disable it during
124	* module termination. */
125	static bool g_fHmVmxCalledSUPR0EnableVTx = false;
126	/** VMX: Host CR4 value (set by ring-0 VMX init) */
127	uint64_t g_uHmVmxHostCr4;
128	/** VMX: Host EFER value (set by ring-0 VMX init) */
129	uint64_t g_uHmVmxHostMsrEfer;
130	/** VMX: Host SMM monitor control (used for logging/diagnostics) */
131	uint64_t g_uHmVmxHostSmmMonitorCtl;
132
133
134	/** Set if AMD-V is supported by the CPU. */
135	bool g_fHmSvmSupported = false;
136	/** SVM revision. */
137	uint32_t g_uHmSvmRev;
138	/** SVM feature bits from cpuid 0x8000000a */
139	uint32_t g_fHmSvmFeatures;
140
141
142	/** MSRs. */
143	SUPHWVIRTMSRS g_HmMsrs;
144
145	/** Last recorded error code during HM ring-0 init. */
146	static int32_t g_rcHmInit = VINF_SUCCESS;
147
148	/** Per CPU globals. */
149	static HMPHYSCPU g_aHmCpuInfo[RTCPUSET_MAX_CPUS];
150
151	/** Whether we've already initialized all CPUs.
152	* @remarks We could check the EnableAllCpusOnce state, but this is
153	* simpler and hopefully easier to understand. */
154	static bool g_fHmEnabled = false;
155	/** Serialize initialization in HMR0EnableAllCpus. */
156	static RTONCE g_HmEnableAllCpusOnce = RTONCE_INITIALIZER;
157
158
159	/** HM ring-0 operations for VT-x. */
160	static HMR0VTABLE const g_HmR0OpsVmx =
161	{
162	/* .pfnEnterSession = */ VMXR0Enter,
163	/* .pfnThreadCtxCallback = */ VMXR0ThreadCtxCallback,
164	/* .pfnAssertionCallback = */ VMXR0AssertionCallback,
165	/* .pfnExportHostState = */ VMXR0ExportHostState,
166	/* .pfnRunGuestCode = */ VMXR0RunGuestCode,
167	/* .pfnEnableCpu = */ VMXR0EnableCpu,
168	/* .pfnDisableCpu = */ VMXR0DisableCpu,
169	/* .pfnInitVM = */ VMXR0InitVM,
170	/* .pfnTermVM = */ VMXR0TermVM,
171	/* .pfnSetupVM = */ VMXR0SetupVM,
172	};
173
174	/** HM ring-0 operations for AMD-V. */
175	static HMR0VTABLE const g_HmR0OpsSvm =
176	{
177	/* .pfnEnterSession = */ SVMR0Enter,
178	/* .pfnThreadCtxCallback = */ SVMR0ThreadCtxCallback,
179	/* .pfnAssertionCallback = */ SVMR0AssertionCallback,
180	/* .pfnExportHostState = */ SVMR0ExportHostState,
181	/* .pfnRunGuestCode = */ SVMR0RunGuestCode,
182	/* .pfnEnableCpu = */ SVMR0EnableCpu,
183	/* .pfnDisableCpu = */ SVMR0DisableCpu,
184	/* .pfnInitVM = */ SVMR0InitVM,
185	/* .pfnTermVM = */ SVMR0TermVM,
186	/* .pfnSetupVM = */ SVMR0SetupVM,
187	};
188
189
190	/** @name Dummy callback handlers for when neither VT-x nor AMD-V is supported.
191	* @{ */
192
193	static DECLCALLBACK(int) hmR0DummyEnter(PVMCPUCC pVCpu)
194	{
195	RT_NOREF(pVCpu);
196	return VINF_SUCCESS;
197	}
198
199	static DECLCALLBACK(void) hmR0DummyThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
200	{
201	RT_NOREF(enmEvent, pVCpu, fGlobalInit);
202	}
203
204	static DECLCALLBACK(int) hmR0DummyEnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage,
205	bool fEnabledBySystem, PCSUPHWVIRTMSRS pHwvirtMsrs)
206	{
207	RT_NOREF(pHostCpu, pVM, pvCpuPage, HCPhysCpuPage, fEnabledBySystem, pHwvirtMsrs);
208	return VINF_SUCCESS;
209	}
210
211	static DECLCALLBACK(int) hmR0DummyDisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
212	{
213	RT_NOREF(pHostCpu, pvCpuPage, HCPhysCpuPage);
214	return VINF_SUCCESS;
215	}
216
217	static DECLCALLBACK(int) hmR0DummyInitVM(PVMCC pVM)
218	{
219	RT_NOREF(pVM);
220	return VINF_SUCCESS;
221	}
222
223	static DECLCALLBACK(int) hmR0DummyTermVM(PVMCC pVM)
224	{
225	RT_NOREF(pVM);
226	return VINF_SUCCESS;
227	}
228
229	static DECLCALLBACK(int) hmR0DummySetupVM(PVMCC pVM)
230	{
231	RT_NOREF(pVM);
232	return VINF_SUCCESS;
233	}
234
235	static DECLCALLBACK(int) hmR0DummyAssertionCallback(PVMCPUCC pVCpu)
236	{
237	RT_NOREF(pVCpu);
238	return VINF_SUCCESS;
239	}
240
241	static DECLCALLBACK(VBOXSTRICTRC) hmR0DummyRunGuestCode(PVMCPUCC pVCpu)
242	{
243	RT_NOREF(pVCpu);
244	return VERR_NOT_SUPPORTED;
245	}
246
247	static DECLCALLBACK(int) hmR0DummyExportHostState(PVMCPUCC pVCpu)
248	{
249	RT_NOREF(pVCpu);
250	return VINF_SUCCESS;
251	}
252
253	/** Dummy ops. */
254	static HMR0VTABLE const g_HmR0OpsDummy =
255	{
256	/* .pfnEnterSession = */ hmR0DummyEnter,
257	/* .pfnThreadCtxCallback = */ hmR0DummyThreadCtxCallback,
258	/* .pfnAssertionCallback = */ hmR0DummyAssertionCallback,
259	/* .pfnExportHostState = */ hmR0DummyExportHostState,
260	/* .pfnRunGuestCode = */ hmR0DummyRunGuestCode,
261	/* .pfnEnableCpu = */ hmR0DummyEnableCpu,
262	/* .pfnDisableCpu = */ hmR0DummyDisableCpu,
263	/* .pfnInitVM = */ hmR0DummyInitVM,
264	/* .pfnTermVM = */ hmR0DummyTermVM,
265	/* .pfnSetupVM = */ hmR0DummySetupVM,
266	};
267
268	/** @} */
269
270
271	/**
272	* Initializes a first return code structure.
273	*
274	* @param pFirstRc The structure to init.
275	*/
276	static void hmR0FirstRcInit(PHMR0FIRSTRC pFirstRc)
277	{
278	pFirstRc->rc = VINF_SUCCESS;
279	pFirstRc->idCpu = NIL_RTCPUID;
280	}
281
282
283	/**
284	* Try set the status code (success ignored).
285	*
286	* @param pFirstRc The first return code structure.
287	* @param rc The status code.
288	*/
289	static void hmR0FirstRcSetStatus(PHMR0FIRSTRC pFirstRc, int rc)
290	{
291	if ( RT_FAILURE(rc)
292	&& ASMAtomicCmpXchgS32(&pFirstRc->rc, rc, VINF_SUCCESS))
293	pFirstRc->idCpu = RTMpCpuId();
294	}
295
296
297	/**
298	* Get the status code of a first return code structure.
299	*
300	* @returns The status code; VINF_SUCCESS or error status, no informational or
301	* warning errors.
302	* @param pFirstRc The first return code structure.
303	*/
304	static int hmR0FirstRcGetStatus(PHMR0FIRSTRC pFirstRc)
305	{
306	return pFirstRc->rc;
307	}
308
309
310	#ifdef VBOX_STRICT
311	# ifndef DEBUG_bird
312	/**
313	* Get the CPU ID on which the failure status code was reported.
314	*
315	* @returns The CPU ID, NIL_RTCPUID if no failure was reported.
316	* @param pFirstRc The first return code structure.
317	*/
318	static RTCPUID hmR0FirstRcGetCpuId(PHMR0FIRSTRC pFirstRc)
319	{
320	return pFirstRc->idCpu;
321	}
322	# endif
323	#endif /* VBOX_STRICT */
324
325
326
327	/**
328	* Worker function used by hmR0PowerCallback() and HMR0Init() to initalize VT-x
329	* on a CPU.
330	*
331	* @param idCpu The identifier for the CPU the function is called on.
332	* @param pvUser1 Pointer to the first RC structure.
333	* @param pvUser2 Ignored.
334	*/
335	static DECLCALLBACK(void) hmR0InitIntelCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
336	{
337	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
338	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
339	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
340	NOREF(idCpu); NOREF(pvUser2);
341
342	int rc = SUPR0GetVmxUsability(NULL /* pfIsSmxModeAmbiguous */);
343	hmR0FirstRcSetStatus(pFirstRc, rc);
344	}
345
346
347	/**
348	* Intel specific initialization code.
349	*
350	* @returns VBox status code (will only fail if out of memory).
351	*/
352	static int hmR0InitIntel(void)
353	{
354	/* Read this MSR now as it may be useful for error reporting when initializing VT-x fails. */
355	g_HmMsrs.u.vmx.u64FeatCtrl = ASMRdMsr(MSR_IA32_FEATURE_CONTROL);
356
357	/*
358	* First try use native kernel API for controlling VT-x.
359	* (This is only supported by some Mac OS X kernels atm.)
360	*/
361	int rc;
362	g_rcHmInit = rc = SUPR0EnableVTx(true /* fEnable */);
363	g_fHmVmxUsingSUPR0EnableVTx = rc != VERR_NOT_SUPPORTED;
364	if (g_fHmVmxUsingSUPR0EnableVTx)
365	{
366	AssertLogRelMsg(rc == VINF_SUCCESS \|\| rc == VERR_VMX_IN_VMX_ROOT_MODE \|\| rc == VERR_VMX_NO_VMX, ("%Rrc\n", rc));
367	if (RT_SUCCESS(rc))
368	{
369	g_fHmVmxSupported = true;
370	rc = SUPR0EnableVTx(false /* fEnable */);
371	AssertLogRelRC(rc);
372	rc = VINF_SUCCESS;
373	}
374	}
375	else
376	{
377	HMR0FIRSTRC FirstRc;
378	hmR0FirstRcInit(&FirstRc);
379	g_rcHmInit = rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
380	if (RT_SUCCESS(rc))
381	g_rcHmInit = rc = hmR0FirstRcGetStatus(&FirstRc);
382	}
383
384	if (RT_SUCCESS(rc))
385	{
386	/* Read CR4 and EFER for logging/diagnostic purposes. */
387	g_uHmVmxHostCr4 = ASMGetCR4();
388	g_uHmVmxHostMsrEfer = ASMRdMsr(MSR_K6_EFER);
389
390	/* Get VMX MSRs (and feature control MSR) for determining VMX features we can ultimately use. */
391	SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_VT_X, false /* fForce */);
392
393	/*
394	* Nested KVM workaround: Intel SDM section 34.15.5 describes that
395	* MSR_IA32_SMM_MONITOR_CTL depends on bit 49 of MSR_IA32_VMX_BASIC while
396	* table 35-2 says that this MSR is available if either VMX or SMX is supported.
397	*/
398	uint64_t const uVmxBasicMsr = g_HmMsrs.u.vmx.u64Basic;
399	if (RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_DUAL_MON))
400	g_uHmVmxHostSmmMonitorCtl = ASMRdMsr(MSR_IA32_SMM_MONITOR_CTL);
401
402	/* Initialize VPID - 16 bits ASID. */
403	g_uHmMaxAsid = 0x10000; /* exclusive */
404
405	/*
406	* If the host OS has not enabled VT-x for us, try enter VMX root mode
407	* to really verify if VT-x is usable.
408	*/
409	if (!g_fHmVmxUsingSUPR0EnableVTx)
410	{
411	/* Allocate a temporary VMXON region. */
412	RTR0MEMOBJ hScatchMemObj;
413	rc = RTR0MemObjAllocCont(&hScatchMemObj, HOST_PAGE_SIZE, false /* fExecutable */);
414	if (RT_FAILURE(rc))
415	{
416	LogRel(("hmR0InitIntel: RTR0MemObjAllocCont(,HOST_PAGE_SIZE,false) -> %Rrc\n", rc));
417	return rc;
418	}
419	void *pvScatchPage = RTR0MemObjAddress(hScatchMemObj);
420	RTHCPHYS const HCPhysScratchPage = RTR0MemObjGetPagePhysAddr(hScatchMemObj, 0);
421	ASMMemZeroPage(pvScatchPage);
422
423	/* Set revision dword at the beginning of the VMXON structure. */
424	(uint32_t )pvScatchPage = RT_BF_GET(uVmxBasicMsr, VMX_BF_BASIC_VMCS_ID);
425
426	/* Make sure we don't get rescheduled to another CPU during this probe. */
427	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
428
429	/* Enable CR4.VMXE if it isn't already set. */
430	RTCCUINTREG const uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
431
432	/*
433	* The only way of checking if we're in VMX root mode or not is to try and enter it.
434	* There is no instruction or control bit that tells us if we're in VMX root mode.
435	* Therefore, try and enter VMX root mode here.
436	*/
437	rc = VMXEnable(HCPhysScratchPage);
438	if (RT_SUCCESS(rc))
439	{
440	g_fHmVmxSupported = true;
441	VMXDisable();
442	}
443	else
444	{
445	/*
446	* KVM leaves the CPU in VMX root mode. Not only is this not allowed,
447	* it will crash the host when we enter raw mode, because:
448	*
449	* (a) clearing X86_CR4_VMXE in CR4 causes a #GP (we no longer modify
450	* this bit), and
451	* (b) turning off paging causes a #GP (unavoidable when switching
452	* from long to 32 bits mode or 32 bits to PAE).
453	*
454	* They should fix their code, but until they do we simply refuse to run.
455	*/
456	g_rcHmInit = VERR_VMX_IN_VMX_ROOT_MODE;
457	Assert(g_fHmVmxSupported == false);
458	}
459
460	/* Restore CR4.VMXE if it wasn't set prior to us setting it above. */
461	if (!(uOldCr4 & X86_CR4_VMXE))
462	SUPR0ChangeCR4(0 /* fOrMask */, ~(uint64_t)X86_CR4_VMXE);
463
464	/* Restore interrupts. */
465	ASMSetFlags(fEFlags);
466
467	RTR0MemObjFree(hScatchMemObj, false);
468	}
469
470	if (g_fHmVmxSupported)
471	{
472	rc = VMXR0GlobalInit();
473	if (RT_SUCCESS(rc))
474	{
475	/*
476	* Install the VT-x methods.
477	*/
478	g_HmR0Ops = g_HmR0OpsVmx;
479
480	/*
481	* Check for the VMX-Preemption Timer and adjust for the "VMX-Preemption
482	* Timer Does Not Count Down at the Rate Specified" CPU erratum.
483	*/
484	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER)
485	{
486	g_fHmVmxUsePreemptTimer = true;
487	g_cHmVmxPreemptTimerShift = RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_PREEMPT_TIMER_TSC);
488	if (HMIsSubjectToVmxPreemptTimerErratum())
489	g_cHmVmxPreemptTimerShift = 0; /* This is about right most of the time here. */
490	}
491	else
492	g_fHmVmxUsePreemptTimer = false;
493
494	/*
495	* Check for EFER swapping support.
496	*/
497	g_fHmVmxSupportsVmcsEfer = (g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
498	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
499	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
500	}
501	else
502	{
503	g_rcHmInit = rc;
504	g_fHmVmxSupported = false;
505	}
506	}
507	}
508	#ifdef LOG_ENABLED
509	else
510	SUPR0Printf("hmR0InitIntelCpu failed with rc=%Rrc\n", g_rcHmInit);
511	#endif
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Worker function used by hmR0PowerCallback() and HMR0Init() to initalize AMD-V
518	* on a CPU.
519	*
520	* @param idCpu The identifier for the CPU the function is called on.
521	* @param pvUser1 Pointer to the first RC structure.
522	* @param pvUser2 Ignored.
523	*/
524	static DECLCALLBACK(void) hmR0InitAmdCpu(RTCPUID idCpu, void pvUser1, void pvUser2)
525	{
526	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser1;
527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
528	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
529	NOREF(idCpu); NOREF(pvUser2);
530
531	int rc = SUPR0GetSvmUsability(true /* fInitSvm */);
532	hmR0FirstRcSetStatus(pFirstRc, rc);
533	}
534
535
536	/**
537	* AMD-specific initialization code.
538	*
539	* @returns VBox status code (will only fail if out of memory).
540	*/
541	static int hmR0InitAmd(void)
542	{
543	/* Call the global AMD-V initialization routine (should only fail in out-of-memory situations). */
544	int rc = SVMR0GlobalInit();
545	if (RT_SUCCESS(rc))
546	{
547	/*
548	* Install the AMD-V methods.
549	*/
550	g_HmR0Ops = g_HmR0OpsSvm;
551
552	/* Query AMD features. */
553	uint32_t u32Dummy;
554	ASMCpuId(0x8000000a, &g_uHmSvmRev, &g_uHmMaxAsid, &u32Dummy, &g_fHmSvmFeatures);
555
556	/*
557	* We need to check if AMD-V has been properly initialized on all CPUs.
558	* Some BIOSes might do a poor job.
559	*/
560	HMR0FIRSTRC FirstRc;
561	hmR0FirstRcInit(&FirstRc);
562	rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
563	AssertRC(rc);
564	if (RT_SUCCESS(rc))
565	rc = hmR0FirstRcGetStatus(&FirstRc);
566	#ifndef DEBUG_bird
567	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_SVM_IN_USE,
568	("hmR0InitAmdCpu failed for cpu %d with rc=%Rrc\n", hmR0FirstRcGetCpuId(&FirstRc), rc));
569	#endif
570	if (RT_SUCCESS(rc))
571	{
572	SUPR0GetHwvirtMsrs(&g_HmMsrs, SUPVTCAPS_AMD_V, false /* fForce */);
573	g_fHmSvmSupported = true;
574	}
575	else
576	{
577	g_rcHmInit = rc;
578	if (rc == VERR_SVM_DISABLED \|\| rc == VERR_SVM_IN_USE)
579	rc = VINF_SUCCESS; /* Don't fail if AMD-V is disabled or in use. */
580	}
581	}
582	else
583	g_rcHmInit = rc;
584	return rc;
585	}
586
587
588	/**
589	* Does global Ring-0 HM initialization (at module init).
590	*
591	* @returns VBox status code.
592	*/
593	VMMR0_INT_DECL(int) HMR0Init(void)
594	{
595	/*
596	* Initialize the globals.
597	*/
598	g_fHmEnabled = false;
599	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
600	{
601	g_aHmCpuInfo[i].idCpu = NIL_RTCPUID;
602	g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
603	g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
604	g_aHmCpuInfo[i].pvMemObj = NULL;
605	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
606	g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
607	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
608	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
609	#endif
610	}
611
612	/* Fill in all callbacks with placeholders. */
613	g_HmR0Ops = g_HmR0OpsDummy;
614
615	/* Default is global VT-x/AMD-V init. */
616	g_fHmGlobalInit = true;
617
618	g_fHmVmxSupported = false;
619	g_fHmSvmSupported = false;
620	g_uHmMaxAsid = 0;
621
622	/*
623	* Get host kernel features that HM might need to know in order
624	* to co-operate and function properly with the host OS (e.g. SMAP).
625	*/
626	g_fHmHostKernelFeatures = SUPR0GetKernelFeatures();
627
628	/*
629	* Make sure aCpuInfo is big enough for all the CPUs on this system.
630	*/
631	if (RTMpGetArraySize() > RT_ELEMENTS(g_aHmCpuInfo))
632	{
633	LogRel(("HM: Too many real CPUs/cores/threads - %u, max %u\n", RTMpGetArraySize(), RT_ELEMENTS(g_aHmCpuInfo)));
634	return VERR_TOO_MANY_CPUS;
635	}
636
637	/*
638	* Check for VT-x or AMD-V support.
639	* Return failure only in out-of-memory situations.
640	*/
641	uint32_t fCaps = 0;
642	int rc = SUPR0GetVTSupport(&fCaps);
643	if (RT_SUCCESS(rc))
644	{
645	if (fCaps & SUPVTCAPS_VT_X)
646	rc = hmR0InitIntel();
647	else
648	{
649	Assert(fCaps & SUPVTCAPS_AMD_V);
650	rc = hmR0InitAmd();
651	}
652	if (RT_SUCCESS(rc))
653	{
654	/*
655	* Register notification callbacks that we can use to disable/enable CPUs
656	* when brought offline/online or suspending/resuming.
657	*/
658	if (!g_fHmVmxUsingSUPR0EnableVTx)
659	{
660	rc = RTMpNotificationRegister(hmR0MpEventCallback, NULL);
661	if (RT_SUCCESS(rc))
662	{
663	rc = RTPowerNotificationRegister(hmR0PowerCallback, NULL);
664	if (RT_FAILURE(rc))
665	RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
666	}
667	if (RT_FAILURE(rc))
668	{
669	/* There shouldn't be any per-cpu allocations at this point,
670	so just have to call SVMR0GlobalTerm and VMXR0GlobalTerm. */
671	if (fCaps & SUPVTCAPS_VT_X)
672	VMXR0GlobalTerm();
673	else
674	SVMR0GlobalTerm();
675	g_HmR0Ops = g_HmR0OpsDummy;
676	g_rcHmInit = rc;
677	g_fHmSvmSupported = false;
678	g_fHmVmxSupported = false;
679	}
680	}
681	}
682	}
683	else
684	{
685	g_rcHmInit = rc;
686	rc = VINF_SUCCESS; /* We return success here because module init shall not fail if HM fails to initialize. */
687	}
688	return rc;
689	}
690
691
692	/**
693	* Does global Ring-0 HM termination (at module termination).
694	*
695	* @returns VBox status code (ignored).
696	*/
697	VMMR0_INT_DECL(int) HMR0Term(void)
698	{
699	int rc;
700	if ( g_fHmVmxSupported
701	&& g_fHmVmxUsingSUPR0EnableVTx)
702	{
703	/*
704	* Simple if the host OS manages VT-x.
705	*/
706	Assert(g_fHmGlobalInit);
707
708	if (g_fHmVmxCalledSUPR0EnableVTx)
709	{
710	rc = SUPR0EnableVTx(false /* fEnable */);
711	g_fHmVmxCalledSUPR0EnableVTx = false;
712	}
713	else
714	rc = VINF_SUCCESS;
715
716	for (unsigned iCpu = 0; iCpu < RT_ELEMENTS(g_aHmCpuInfo); iCpu++)
717	{
718	g_aHmCpuInfo[iCpu].fConfigured = false;
719	Assert(g_aHmCpuInfo[iCpu].hMemObj == NIL_RTR0MEMOBJ);
720	}
721	}
722	else
723	{
724	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
725
726	/* Doesn't really matter if this fails. */
727	RTMpNotificationDeregister(hmR0MpEventCallback, NULL);
728	RTPowerNotificationDeregister(hmR0PowerCallback, NULL);
729	rc = VINF_SUCCESS;
730
731	/*
732	* Disable VT-x/AMD-V on all CPUs if we enabled it before.
733	*/
734	if (g_fHmGlobalInit)
735	{
736	HMR0FIRSTRC FirstRc;
737	hmR0FirstRcInit(&FirstRc);
738	rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
739	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
740	if (RT_SUCCESS(rc))
741	rc = hmR0FirstRcGetStatus(&FirstRc);
742	}
743
744	/*
745	* Free the per-cpu pages used for VT-x and AMD-V.
746	*/
747	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
748	{
749	if (g_aHmCpuInfo[i].hMemObj != NIL_RTR0MEMOBJ)
750	{
751	RTR0MemObjFree(g_aHmCpuInfo[i].hMemObj, false);
752	g_aHmCpuInfo[i].hMemObj = NIL_RTR0MEMOBJ;
753	g_aHmCpuInfo[i].HCPhysMemObj = NIL_RTHCPHYS;
754	g_aHmCpuInfo[i].pvMemObj = NULL;
755	}
756	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
757	if (g_aHmCpuInfo[i].n.svm.hNstGstMsrpm != NIL_RTR0MEMOBJ)
758	{
759	RTR0MemObjFree(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, false);
760	g_aHmCpuInfo[i].n.svm.hNstGstMsrpm = NIL_RTR0MEMOBJ;
761	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = NIL_RTHCPHYS;
762	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = NULL;
763	}
764	#endif
765	}
766	}
767
768	/** @todo This needs cleaning up. There's no matching
769	* hmR0TermIntel()/hmR0TermAmd() and all the VT-x/AMD-V specific bits
770	* should move into their respective modules. */
771	/* Finally, call global VT-x/AMD-V termination. */
772	if (g_fHmVmxSupported)
773	VMXR0GlobalTerm();
774	else if (g_fHmSvmSupported)
775	SVMR0GlobalTerm();
776
777	return rc;
778	}
779
780
781	/**
782	* Enable VT-x or AMD-V on the current CPU
783	*
784	* @returns VBox status code.
785	* @param pVM The cross context VM structure. Can be NULL.
786	* @param idCpu The identifier for the CPU the function is called on.
787	*
788	* @remarks Maybe called with interrupts disabled!
789	*/
790	static int hmR0EnableCpu(PVMCC pVM, RTCPUID idCpu)
791	{
792	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
793
794	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
795	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
796	Assert(!pHostCpu->fConfigured);
797	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799	pHostCpu->idCpu = idCpu;
800	/* Do NOT reset cTlbFlushes here, see @bugref{6255}. */
801
802	int rc;
803	if ( g_fHmVmxSupported
804	&& g_fHmVmxUsingSUPR0EnableVTx)
805	rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, NULL /* pvCpuPage */, NIL_RTHCPHYS, true, &g_HmMsrs);
806	else
807	{
808	AssertLogRelMsgReturn(pHostCpu->hMemObj != NIL_RTR0MEMOBJ, ("hmR0EnableCpu failed idCpu=%u.\n", idCpu), VERR_HM_IPE_1);
809	rc = g_HmR0Ops.pfnEnableCpu(pHostCpu, pVM, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj, false, &g_HmMsrs);
810	}
811	if (RT_SUCCESS(rc))
812	pHostCpu->fConfigured = true;
813	return rc;
814	}
815
816
817	/**
818	* Worker function passed to RTMpOnAll() that is to be called on all CPUs.
819	*
820	* @param idCpu The identifier for the CPU the function is called on.
821	* @param pvUser1 Opaque pointer to the VM (can be NULL!).
822	* @param pvUser2 The 2nd user argument.
823	*/
824	static DECLCALLBACK(void) hmR0EnableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
825	{
826	PVMCC pVM = (PVMCC)pvUser1; /* can be NULL! */
827	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2;
828	AssertReturnVoid(g_fHmGlobalInit);
829	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
830	hmR0FirstRcSetStatus(pFirstRc, hmR0EnableCpu(pVM, idCpu));
831	}
832
833
834	/**
835	* RTOnce callback employed by HMR0EnableAllCpus.
836	*
837	* @returns VBox status code.
838	* @param pvUser Pointer to the VM.
839	*/
840	static DECLCALLBACK(int32_t) hmR0EnableAllCpuOnce(void *pvUser)
841	{
842	PVMCC pVM = (PVMCC)pvUser;
843
844	/*
845	* Indicate that we've initialized.
846	*
847	* Note! There is a potential race between this function and the suspend
848	* notification. Kind of unlikely though, so ignored for now.
849	*/
850	AssertReturn(!g_fHmEnabled, VERR_HM_ALREADY_ENABLED_IPE);
851	ASMAtomicWriteBool(&g_fHmEnabled, true);
852
853	/*
854	* The global init variable is set by the first VM.
855	*/
856	g_fHmGlobalInit = pVM->hm.s.fGlobalInit;
857
858	#ifdef VBOX_STRICT
859	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
860	{
861	Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
862	Assert(g_aHmCpuInfo[i].HCPhysMemObj == NIL_RTHCPHYS);
863	Assert(g_aHmCpuInfo[i].pvMemObj == NULL);
864	Assert(!g_aHmCpuInfo[i].fConfigured);
865	Assert(!g_aHmCpuInfo[i].cTlbFlushes);
866	Assert(!g_aHmCpuInfo[i].uCurrentAsid);
867	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
868	Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
869	Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm == NIL_RTHCPHYS);
870	Assert(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm == NULL);
871	# endif
872	}
873	#endif
874
875	int rc;
876	if ( g_fHmVmxSupported
877	&& g_fHmVmxUsingSUPR0EnableVTx)
878	{
879	/*
880	* Global VT-x initialization API (only darwin for now).
881	*/
882	rc = SUPR0EnableVTx(true /* fEnable */);
883	if (RT_SUCCESS(rc))
884	{
885	g_fHmVmxCalledSUPR0EnableVTx = true;
886	/* If the host provides a VT-x init API, then we'll rely on that for global init. */
887	g_fHmGlobalInit = pVM->hm.s.fGlobalInit = true;
888	}
889	else
890	AssertMsgFailed(("hmR0EnableAllCpuOnce/SUPR0EnableVTx: rc=%Rrc\n", rc));
891	}
892	else
893	{
894	/*
895	* We're doing the job ourselves.
896	*/
897	/* Allocate one page per cpu for the global VT-x and AMD-V pages */
898	for (unsigned i = 0; i < RT_ELEMENTS(g_aHmCpuInfo); i++)
899	{
900	Assert(g_aHmCpuInfo[i].hMemObj == NIL_RTR0MEMOBJ);
901	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
902	Assert(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm == NIL_RTR0MEMOBJ);
903	#endif
904	if (RTMpIsCpuPossible(RTMpCpuIdFromSetIndex(i)))
905	{
906	/** @todo NUMA */
907	rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].hMemObj, HOST_PAGE_SIZE, false /* executable R0 mapping */);
908	AssertLogRelRCReturn(rc, rc);
909
910	g_aHmCpuInfo[i].HCPhysMemObj = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].hMemObj, 0);
911	Assert(g_aHmCpuInfo[i].HCPhysMemObj != NIL_RTHCPHYS);
912	Assert(!(g_aHmCpuInfo[i].HCPhysMemObj & HOST_PAGE_OFFSET_MASK));
913
914	g_aHmCpuInfo[i].pvMemObj = RTR0MemObjAddress(g_aHmCpuInfo[i].hMemObj);
915	AssertPtr(g_aHmCpuInfo[i].pvMemObj);
916	ASMMemZeroPage(g_aHmCpuInfo[i].pvMemObj);
917
918	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
919	rc = RTR0MemObjAllocCont(&g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
920	false /* executable R0 mapping */);
921	AssertLogRelRCReturn(rc, rc);
922
923	g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm = RTR0MemObjGetPagePhysAddr(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm, 0);
924	Assert(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm != NIL_RTHCPHYS);
925	Assert(!(g_aHmCpuInfo[i].n.svm.HCPhysNstGstMsrpm & HOST_PAGE_OFFSET_MASK));
926
927	g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm = RTR0MemObjAddress(g_aHmCpuInfo[i].n.svm.hNstGstMsrpm);
928	AssertPtr(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm);
929	ASMMemFill32(g_aHmCpuInfo[i].n.svm.pvNstGstMsrpm, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
930	#endif
931	}
932	}
933
934	rc = VINF_SUCCESS;
935	}
936
937	if ( RT_SUCCESS(rc)
938	&& g_fHmGlobalInit)
939	{
940	/* First time, so initialize each cpu/core. */
941	HMR0FIRSTRC FirstRc;
942	hmR0FirstRcInit(&FirstRc);
943	rc = RTMpOnAll(hmR0EnableCpuCallback, (void *)pVM, &FirstRc);
944	if (RT_SUCCESS(rc))
945	rc = hmR0FirstRcGetStatus(&FirstRc);
946	}
947
948	return rc;
949	}
950
951
952	/**
953	* Sets up HM on all cpus.
954	*
955	* @returns VBox status code.
956	* @param pVM The cross context VM structure.
957	*/
958	VMMR0_INT_DECL(int) HMR0EnableAllCpus(PVMCC pVM)
959	{
960	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
961	if (ASMAtomicReadBool(&g_fHmSuspended))
962	return VERR_HM_SUSPEND_PENDING;
963
964	return RTOnce(&g_HmEnableAllCpusOnce, hmR0EnableAllCpuOnce, pVM);
965	}
966
967
968	/**
969	* Disable VT-x or AMD-V on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param idCpu The identifier for the CPU this function is called on.
973	*
974	* @remarks Must be called with preemption disabled.
975	*/
976	static int hmR0DisableCpu(RTCPUID idCpu)
977	{
978	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
979
980	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
981	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
982	Assert(idCpu == (RTCPUID)RTMpCpuIdToSetIndex(idCpu)); /** @todo fix idCpu == index assumption (rainy day) */
983	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
984	Assert(!pHostCpu->fConfigured \|\| pHostCpu->hMemObj != NIL_RTR0MEMOBJ);
985	AssertRelease(idCpu == RTMpCpuId());
986
987	if (pHostCpu->hMemObj == NIL_RTR0MEMOBJ)
988	return pHostCpu->fConfigured ? VERR_NO_MEMORY : VINF_SUCCESS /* not initialized. */;
989	AssertPtr(pHostCpu->pvMemObj);
990	Assert(pHostCpu->HCPhysMemObj != NIL_RTHCPHYS);
991
992	int rc;
993	if (pHostCpu->fConfigured)
994	{
995	rc = g_HmR0Ops.pfnDisableCpu(pHostCpu, pHostCpu->pvMemObj, pHostCpu->HCPhysMemObj);
996	AssertRCReturn(rc, rc);
997
998	pHostCpu->fConfigured = false;
999	pHostCpu->idCpu = NIL_RTCPUID;
1000	}
1001	else
1002	rc = VINF_SUCCESS; /* nothing to do */
1003	return rc;
1004	}
1005
1006
1007	/**
1008	* Worker function passed to RTMpOnAll() that is to be called on the target
1009	* CPUs.
1010	*
1011	* @param idCpu The identifier for the CPU the function is called on.
1012	* @param pvUser1 The 1st user argument.
1013	* @param pvUser2 Opaque pointer to the FirstRc.
1014	*/
1015	static DECLCALLBACK(void) hmR0DisableCpuCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1016	{
1017	PHMR0FIRSTRC pFirstRc = (PHMR0FIRSTRC)pvUser2; NOREF(pvUser1);
1018	AssertReturnVoid(g_fHmGlobalInit);
1019	hmR0FirstRcSetStatus(pFirstRc, hmR0DisableCpu(idCpu));
1020	}
1021
1022
1023	/**
1024	* Worker function passed to RTMpOnSpecific() that is to be called on the target
1025	* CPU.
1026	*
1027	* @param idCpu The identifier for the CPU the function is called on.
1028	* @param pvUser1 Null, not used.
1029	* @param pvUser2 Null, not used.
1030	*/
1031	static DECLCALLBACK(void) hmR0DisableCpuOnSpecificCallback(RTCPUID idCpu, void pvUser1, void pvUser2)
1032	{
1033	NOREF(pvUser1);
1034	NOREF(pvUser2);
1035	hmR0DisableCpu(idCpu);
1036	}
1037
1038
1039	/**
1040	* Callback function invoked when a cpu goes online or offline.
1041	*
1042	* @param enmEvent The Mp event.
1043	* @param idCpu The identifier for the CPU the function is called on.
1044	* @param pvData Opaque data (PVMCC pointer).
1045	*/
1046	static DECLCALLBACK(void) hmR0MpEventCallback(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvData)
1047	{
1048	NOREF(pvData);
1049	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1050
1051	/*
1052	* We only care about uninitializing a CPU that is going offline. When a
1053	* CPU comes online, the initialization is done lazily in HMR0Enter().
1054	*/
1055	switch (enmEvent)
1056	{
1057	case RTMPEVENT_OFFLINE:
1058	{
1059	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1060	RTThreadPreemptDisable(&PreemptState);
1061	if (idCpu == RTMpCpuId())
1062	{
1063	int rc = hmR0DisableCpu(idCpu);
1064	AssertRC(rc);
1065	RTThreadPreemptRestore(&PreemptState);
1066	}
1067	else
1068	{
1069	RTThreadPreemptRestore(&PreemptState);
1070	RTMpOnSpecific(idCpu, hmR0DisableCpuOnSpecificCallback, NULL /* pvUser1 /, NULL / pvUser2 */);
1071	}
1072	break;
1073	}
1074
1075	default:
1076	break;
1077	}
1078	}
1079
1080
1081	/**
1082	* Called whenever a system power state change occurs.
1083	*
1084	* @param enmEvent The Power event.
1085	* @param pvUser User argument.
1086	*/
1087	static DECLCALLBACK(void) hmR0PowerCallback(RTPOWEREVENT enmEvent, void *pvUser)
1088	{
1089	NOREF(pvUser);
1090	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1091
1092	#ifdef LOG_ENABLED
1093	if (enmEvent == RTPOWEREVENT_SUSPEND)
1094	SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_SUSPEND\n");
1095	else
1096	SUPR0Printf("hmR0PowerCallback RTPOWEREVENT_RESUME\n");
1097	#endif
1098
1099	if (enmEvent == RTPOWEREVENT_SUSPEND)
1100	ASMAtomicWriteBool(&g_fHmSuspended, true);
1101
1102	if (g_fHmEnabled)
1103	{
1104	int rc;
1105	HMR0FIRSTRC FirstRc;
1106	hmR0FirstRcInit(&FirstRc);
1107
1108	if (enmEvent == RTPOWEREVENT_SUSPEND)
1109	{
1110	if (g_fHmGlobalInit)
1111	{
1112	/* Turn off VT-x or AMD-V on all CPUs. */
1113	rc = RTMpOnAll(hmR0DisableCpuCallback, NULL /* pvUser 1 */, &FirstRc);
1114	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1115	}
1116	/* else nothing to do here for the local init case */
1117	}
1118	else
1119	{
1120	/* Reinit the CPUs from scratch as the suspend state might have
1121	messed with the MSRs. (lousy BIOSes as usual) */
1122	if (g_fHmVmxSupported)
1123	rc = RTMpOnAll(hmR0InitIntelCpu, &FirstRc, NULL);
1124	else
1125	rc = RTMpOnAll(hmR0InitAmdCpu, &FirstRc, NULL);
1126	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1127	if (RT_SUCCESS(rc))
1128	rc = hmR0FirstRcGetStatus(&FirstRc);
1129	#ifdef LOG_ENABLED
1130	if (RT_FAILURE(rc))
1131	SUPR0Printf("hmR0PowerCallback hmR0InitXxxCpu failed with %Rc\n", rc);
1132	#endif
1133	if (g_fHmGlobalInit)
1134	{
1135	/* Turn VT-x or AMD-V back on on all CPUs. */
1136	rc = RTMpOnAll(hmR0EnableCpuCallback, NULL /* pVM /, &FirstRc / output ignored */);
1137	Assert(RT_SUCCESS(rc) \|\| rc == VERR_NOT_SUPPORTED);
1138	}
1139	/* else nothing to do here for the local init case */
1140	}
1141	}
1142
1143	if (enmEvent == RTPOWEREVENT_RESUME)
1144	ASMAtomicWriteBool(&g_fHmSuspended, false);
1145	}
1146
1147
1148	/**
1149	* Does ring-0 per-VM HM initialization.
1150	*
1151	* This will call the CPU specific init. routine which may initialize and allocate
1152	* resources for virtual CPUs.
1153	*
1154	* @returns VBox status code.
1155	* @param pVM The cross context VM structure.
1156	*
1157	* @remarks This is called after HMR3Init(), see vmR3CreateU() and
1158	* vmR3InitRing3().
1159	*/
1160	VMMR0_INT_DECL(int) HMR0InitVM(PVMCC pVM)
1161	{
1162	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1163
1164	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1165	if (ASMAtomicReadBool(&g_fHmSuspended))
1166	return VERR_HM_SUSPEND_PENDING;
1167
1168	/*
1169	* Copy globals to the VM structure.
1170	*/
1171	Assert(!(pVM->hm.s.vmx.fSupported && pVM->hm.s.svm.fSupported));
1172	if (pVM->hm.s.vmx.fSupported)
1173	{
1174	pVM->hmr0.s.vmx.fUsePreemptTimer = pVM->hm.s.vmx.fUsePreemptTimerCfg && g_fHmVmxUsePreemptTimer;
1175	pVM->hm.s.vmx.fUsePreemptTimerCfg = pVM->hmr0.s.vmx.fUsePreemptTimer;
1176	pVM->hm.s.vmx.cPreemptTimerShift = g_cHmVmxPreemptTimerShift;
1177	pVM->hm.s.ForR3.vmx.u64HostCr4 = g_uHmVmxHostCr4;
1178	pVM->hm.s.ForR3.vmx.u64HostMsrEfer = g_uHmVmxHostMsrEfer;
1179	pVM->hm.s.ForR3.vmx.u64HostSmmMonitorCtl = g_uHmVmxHostSmmMonitorCtl;
1180	pVM->hm.s.ForR3.vmx.u64HostFeatCtrl = g_HmMsrs.u.vmx.u64FeatCtrl;
1181	HMGetVmxMsrsFromHwvirtMsrs(&g_HmMsrs, &pVM->hm.s.ForR3.vmx.Msrs);
1182	/* If you need to tweak host MSRs for testing VMX R0 code, do it here. */
1183
1184	/* Enable VPID if supported and configured. */
1185	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VPID)
1186	pVM->hm.s.ForR3.vmx.fVpid = pVM->hmr0.s.vmx.fVpid = pVM->hm.s.vmx.fAllowVpid; /* Can be overridden by CFGM in HMR3Init(). */
1187
1188	/* Use VMCS shadowing if supported. */
1189	pVM->hmr0.s.vmx.fUseVmcsShadowing = pVM->cpum.ro.GuestFeatures.fVmx
1190	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VMCS_SHADOWING);
1191	pVM->hm.s.ForR3.vmx.fUseVmcsShadowing = pVM->hmr0.s.vmx.fUseVmcsShadowing;
1192
1193	/* Use the VMCS controls for swapping the EFER MSR if supported. */
1194	pVM->hm.s.ForR3.vmx.fSupportsVmcsEfer = g_fHmVmxSupportsVmcsEfer;
1195
1196	#if 0
1197	/* Enable APIC register virtualization and virtual-interrupt delivery if supported. */
1198	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT)
1199	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY))
1200	pVM->hm.s.fVirtApicRegs = true;
1201
1202	/* Enable posted-interrupt processing if supported. */
1203	/** @todo Add and query IPRT API for host OS support for posted-interrupt IPI
1204	* here. */
1205	if ( (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT)
1206	&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT))
1207	pVM->hm.s.fPostedIntrs = true;
1208	#endif
1209	}
1210	else if (pVM->hm.s.svm.fSupported)
1211	{
1212	pVM->hm.s.ForR3.svm.u32Rev = g_uHmSvmRev;
1213	pVM->hm.s.ForR3.svm.fFeatures = g_fHmSvmFeatures;
1214	pVM->hm.s.ForR3.svm.u64MsrHwcr = g_HmMsrs.u.svm.u64MsrHwcr;
1215	/* If you need to tweak host MSRs for testing SVM R0 code, do it here. */
1216	}
1217	pVM->hm.s.ForR3.rcInit = g_rcHmInit;
1218	pVM->hm.s.ForR3.uMaxAsid = g_uHmMaxAsid;
1219
1220	/*
1221	* Set default maximum inner loops in ring-0 before returning to ring-3.
1222	* Can be overriden using CFGM.
1223	*/
1224	uint32_t cMaxResumeLoops = pVM->hm.s.cMaxResumeLoopsCfg;
1225	if (!cMaxResumeLoops)
1226	{
1227	cMaxResumeLoops = 1024;
1228	if (RTThreadPreemptIsPendingTrusty())
1229	cMaxResumeLoops = 8192;
1230	}
1231	else if (cMaxResumeLoops > 16384)
1232	cMaxResumeLoops = 16384;
1233	else if (cMaxResumeLoops < 32)
1234	cMaxResumeLoops = 32;
1235	pVM->hm.s.cMaxResumeLoopsCfg = pVM->hmr0.s.cMaxResumeLoops = cMaxResumeLoops;
1236
1237	/*
1238	* Initialize some per-VCPU fields.
1239	*/
1240	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1241	{
1242	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1243	pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1244	pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1245
1246	/* We'll aways increment this the first time (host uses ASID 0). */
1247	AssertReturn(!pVCpu->hmr0.s.uCurrentAsid, VERR_HM_IPE_3);
1248	}
1249
1250	/*
1251	* Configure defences against spectre and other CPU bugs.
1252	*/
1253	uint32_t fWorldSwitcher = 0;
1254	uint32_t cLastStdLeaf = ASMCpuId_EAX(0);
1255	if (cLastStdLeaf >= 0x00000007 && RTX86IsValidStdRange(cLastStdLeaf))
1256	{
1257	uint32_t uEdx = 0;
1258	ASMCpuIdExSlow(0x00000007, 0, 0, 0, NULL, NULL, NULL, &uEdx);
1259
1260	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB)
1261	{
1262	if (pVM->hm.s.fIbpbOnVmExit)
1263	fWorldSwitcher \|= HM_WSF_IBPB_EXIT;
1264	if (pVM->hm.s.fIbpbOnVmEntry)
1265	fWorldSwitcher \|= HM_WSF_IBPB_ENTRY;
1266	}
1267	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD)
1268	{
1269	if (pVM->hm.s.fL1dFlushOnVmEntry)
1270	fWorldSwitcher \|= HM_WSF_L1D_ENTRY;
1271	else if (pVM->hm.s.fL1dFlushOnSched)
1272	fWorldSwitcher \|= HM_WSF_L1D_SCHED;
1273	}
1274	if (uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR)
1275	{
1276	if (pVM->hm.s.fMdsClearOnVmEntry)
1277	fWorldSwitcher \|= HM_WSF_MDS_ENTRY;
1278	else if (pVM->hm.s.fMdsClearOnSched)
1279	fWorldSwitcher \|= HM_WSF_MDS_SCHED;
1280	}
1281	}
1282	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1283	{
1284	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
1285	pVCpu->hmr0.s.fWorldSwitcher = fWorldSwitcher;
1286	}
1287	pVM->hm.s.ForR3.fWorldSwitcher = fWorldSwitcher;
1288
1289
1290	/*
1291	* Call the hardware specific initialization method.
1292	*/
1293	return g_HmR0Ops.pfnInitVM(pVM);
1294	}
1295
1296
1297	/**
1298	* Does ring-0 per VM HM termination.
1299	*
1300	* @returns VBox status code.
1301	* @param pVM The cross context VM structure.
1302	*/
1303	VMMR0_INT_DECL(int) HMR0TermVM(PVMCC pVM)
1304	{
1305	Log(("HMR0TermVM: %p\n", pVM));
1306	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1307
1308	/*
1309	* Call the hardware specific method.
1310	*
1311	* Note! We might be preparing for a suspend, so the pfnTermVM() functions should probably not
1312	* mess with VT-x/AMD-V features on the CPU, currently all they do is free memory so this is safe.
1313	*/
1314	return g_HmR0Ops.pfnTermVM(pVM);
1315	}
1316
1317
1318	/**
1319	* Sets up a VT-x or AMD-V session.
1320	*
1321	* This is mostly about setting up the hardware VM state.
1322	*
1323	* @returns VBox status code.
1324	* @param pVM The cross context VM structure.
1325	*/
1326	VMMR0_INT_DECL(int) HMR0SetupVM(PVMCC pVM)
1327	{
1328	Log(("HMR0SetupVM: %p\n", pVM));
1329	AssertReturn(pVM, VERR_INVALID_PARAMETER);
1330
1331	/* Make sure we don't touch HM after we've disabled HM in preparation of a suspend. */
1332	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1333
1334	/* On first entry we'll sync everything. */
1335	VMCC_FOR_EACH_VMCPU_STMT(pVM, pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
1336
1337	/*
1338	* Call the hardware specific setup VM method. This requires the CPU to be
1339	* enabled for AMD-V/VT-x and preemption to be prevented.
1340	*/
1341	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
1342	RTThreadPreemptDisable(&PreemptState);
1343	RTCPUID const idCpu = RTMpCpuId();
1344
1345	/* Enable VT-x or AMD-V if local init is required. */
1346	int rc;
1347	if (!g_fHmGlobalInit)
1348	{
1349	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1350	rc = hmR0EnableCpu(pVM, idCpu);
1351	if (RT_FAILURE(rc))
1352	{
1353	RTThreadPreemptRestore(&PreemptState);
1354	return rc;
1355	}
1356	}
1357
1358	/* Setup VT-x or AMD-V. */
1359	rc = g_HmR0Ops.pfnSetupVM(pVM);
1360
1361	/* Disable VT-x or AMD-V if local init was done before. */
1362	if (!g_fHmGlobalInit)
1363	{
1364	Assert(!g_fHmVmxSupported \|\| !g_fHmVmxUsingSUPR0EnableVTx);
1365	int rc2 = hmR0DisableCpu(idCpu);
1366	AssertRC(rc2);
1367	}
1368
1369	RTThreadPreemptRestore(&PreemptState);
1370	return rc;
1371	}
1372
1373
1374	/**
1375	* Notification callback before an assertion longjump and guru mediation.
1376	*
1377	* @returns VBox status code.
1378	* @param pVCpu The cross context virtual CPU structure.
1379	* @param pvUser User argument, currently unused, NULL.
1380	*/
1381	static DECLCALLBACK(int) hmR0AssertionCallback(PVMCPUCC pVCpu, void *pvUser)
1382	{
1383	RT_NOREF(pvUser);
1384	Assert(pVCpu);
1385	Assert(g_HmR0Ops.pfnAssertionCallback);
1386	return g_HmR0Ops.pfnAssertionCallback(pVCpu);
1387	}
1388
1389
1390	/**
1391	* Turns on HM on the CPU if necessary and initializes the bare minimum state
1392	* required for entering HM context.
1393	*
1394	* @returns VBox status code.
1395	* @param pVCpu The cross context virtual CPU structure.
1396	*
1397	* @remarks No-long-jump zone!!!
1398	*/
1399	VMMR0_INT_DECL(int) hmR0EnterCpu(PVMCPUCC pVCpu)
1400	{
1401	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1402
1403	int rc = VINF_SUCCESS;
1404	RTCPUID const idCpu = RTMpCpuId();
1405	PHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1406	AssertPtr(pHostCpu);
1407
1408	/* Enable VT-x or AMD-V if local init is required, or enable if it's a freshly onlined CPU. */
1409	if (!pHostCpu->fConfigured)
1410	rc = hmR0EnableCpu(pVCpu->CTX_SUFF(pVM), idCpu);
1411
1412	/* Register a callback to fire prior to performing a longjmp to ring-3 so HM can disable VT-x/AMD-V if needed. */
1413	VMMR0AssertionSetNotification(pVCpu, hmR0AssertionCallback, NULL /pvUser/);
1414
1415	/* Reload host-state (back from ring-3/migrated CPUs) and shared guest/host bits. */
1416	if (g_fHmVmxSupported)
1417	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE;
1418	else
1419	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE;
1420
1421	Assert(pHostCpu->idCpu == idCpu && pHostCpu->idCpu != NIL_RTCPUID);
1422	pVCpu->hmr0.s.idEnteredCpu = idCpu;
1423	return rc;
1424	}
1425
1426
1427	/**
1428	* Enters the VT-x or AMD-V session.
1429	*
1430	* @returns VBox status code.
1431	* @param pVCpu The cross context virtual CPU structure.
1432	*
1433	* @remarks This is called with preemption disabled.
1434	*/
1435	VMMR0_INT_DECL(int) HMR0Enter(PVMCPUCC pVCpu)
1436	{
1437	/* Make sure we can't enter a session after we've disabled HM in preparation of a suspend. */
1438	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1439	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440
1441	/* Load the bare minimum state required for entering HM. */
1442	int rc = hmR0EnterCpu(pVCpu);
1443	if (RT_SUCCESS(rc))
1444	{
1445	if (g_fHmVmxSupported)
1446	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
1447	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
1448	else
1449	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
1450	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
1451
1452	/* Keep track of the CPU owning the VMCS for debugging scheduling weirdness and ring-3 calls. */
1453	rc = g_HmR0Ops.pfnEnterSession(pVCpu);
1454	AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1455
1456	/* Exports the host-state as we may be resuming code after a longjmp and quite
1457	possibly now be scheduled on a different CPU. */
1458	rc = g_HmR0Ops.pfnExportHostState(pVCpu);
1459	AssertMsgRCReturnStmt(rc, ("rc=%Rrc pVCpu=%p\n", rc, pVCpu), pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID, rc);
1460	}
1461	return rc;
1462	}
1463
1464
1465	/**
1466	* Deinitializes the bare minimum state used for HM context and if necessary
1467	* disable HM on the CPU.
1468	*
1469	* @returns VBox status code.
1470	* @param pVCpu The cross context virtual CPU structure.
1471	*
1472	* @remarks No-long-jump zone!!!
1473	*/
1474	VMMR0_INT_DECL(int) HMR0LeaveCpu(PVMCPUCC pVCpu)
1475	{
1476	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1477	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_HM_WRONG_CPU);
1478
1479	RTCPUID const idCpu = RTMpCpuId();
1480	PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[idCpu];
1481
1482	if ( !g_fHmGlobalInit
1483	&& pHostCpu->fConfigured)
1484	{
1485	int rc = hmR0DisableCpu(idCpu);
1486	AssertRCReturn(rc, rc);
1487	Assert(!pHostCpu->fConfigured);
1488	Assert(pHostCpu->idCpu == NIL_RTCPUID);
1489
1490	/* For obtaining a non-zero ASID/VPID on next re-entry. */
1491	pVCpu->hmr0.s.idLastCpu = NIL_RTCPUID;
1492	}
1493
1494	/* Clear it while leaving HM context, hmPokeCpuForTlbFlush() relies on this. */
1495	pVCpu->hmr0.s.idEnteredCpu = NIL_RTCPUID;
1496
1497	/* De-register the longjmp-to-ring 3 callback now that we have reliquished hardware resources. */
1498	VMMR0AssertionRemoveNotification(pVCpu);
1499	return VINF_SUCCESS;
1500	}
1501
1502
1503	/**
1504	* Thread-context hook for HM.
1505	*
1506	* This is used together with RTThreadCtxHookCreate() on platforms which
1507	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
1508	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
1509	*
1510	* @param enmEvent The thread-context event.
1511	* @param pvUser Opaque pointer to the VMCPU.
1512	*/
1513	VMMR0_INT_DECL(void) HMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, void *pvUser)
1514	{
1515	PVMCPUCC pVCpu = (PVMCPUCC)pvUser;
1516	Assert(pVCpu);
1517	Assert(g_HmR0Ops.pfnThreadCtxCallback);
1518
1519	g_HmR0Ops.pfnThreadCtxCallback(enmEvent, pVCpu, g_fHmGlobalInit);
1520	}
1521
1522
1523	/**
1524	* Runs guest code in a hardware accelerated VM.
1525	*
1526	* @returns Strict VBox status code. (VBOXSTRICTRC isn't used because it's
1527	* called from setjmp assembly.)
1528	* @param pVM The cross context VM structure.
1529	* @param pVCpu The cross context virtual CPU structure.
1530	*
1531	* @remarks Can be called with preemption enabled if thread-context hooks are
1532	* used!!!
1533	*/
1534	VMMR0_INT_DECL(int) HMR0RunGuestCode(PVMCC pVM, PVMCPUCC pVCpu)
1535	{
1536	RT_NOREF(pVM);
1537
1538	#ifdef VBOX_STRICT
1539	/* With thread-context hooks we would be running this code with preemption enabled. */
1540	if (!RTThreadPreemptIsEnabled(NIL_RTTHREAD))
1541	{
1542	PCHMPHYSCPU pHostCpu = &g_aHmCpuInfo[RTMpCpuId()];
1543	Assert(!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1544	Assert(pHostCpu->fConfigured);
1545	AssertReturn(!ASMAtomicReadBool(&g_fHmSuspended), VERR_HM_SUSPEND_PENDING);
1546	}
1547	#endif
1548
1549	VBOXSTRICTRC rcStrict = g_HmR0Ops.pfnRunGuestCode(pVCpu);
1550	return VBOXSTRICTRC_VAL(rcStrict);
1551	}
1552
1553
1554	/**
1555	* Notification from CPUM that it has unloaded the guest FPU/SSE/AVX state from
1556	* the host CPU and that guest access to it must be intercepted.
1557	*
1558	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1559	*/
1560	VMMR0_INT_DECL(void) HMR0NotifyCpumUnloadedGuestFpuState(PVMCPUCC pVCpu)
1561	{
1562	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
1563	}
1564
1565
1566	/**
1567	* Notification from CPUM that it has modified the host CR0 (because of FPU).
1568	*
1569	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1570	*/
1571	VMMR0_INT_DECL(void) HMR0NotifyCpumModifiedHostCr0(PVMCPUCC pVCpu)
1572	{
1573	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT);
1574	}
1575
1576
1577	/**
1578	* Returns suspend status of the host.
1579	*
1580	* @returns Suspend pending or not.
1581	*/
1582	VMMR0_INT_DECL(bool) HMR0SuspendPending(void)
1583	{
1584	return ASMAtomicReadBool(&g_fHmSuspended);
1585	}
1586
1587
1588	/**
1589	* Invalidates a guest page from the host TLB.
1590	*
1591	* @param pVCpu The cross context virtual CPU structure.
1592	* @param GCVirt Page to invalidate.
1593	*/
1594	VMMR0_INT_DECL(int) HMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1595	{
1596	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1597	if (pVM->hm.s.vmx.fSupported)
1598	return VMXR0InvalidatePage(pVCpu, GCVirt);
1599	return SVMR0InvalidatePage(pVCpu, GCVirt);
1600	}
1601
1602
1603	/**
1604	* Returns the cpu structure for the current cpu.
1605	* Keep in mind that there is no guarantee it will stay the same (long jumps to ring 3!!!).
1606	*
1607	* @returns The cpu structure pointer.
1608	*/
1609	VMMR0_INT_DECL(PHMPHYSCPU) hmR0GetCurrentCpu(void)
1610	{
1611	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1612	RTCPUID const idCpu = RTMpCpuId();
1613	Assert(idCpu < RT_ELEMENTS(g_aHmCpuInfo));
1614	return &g_aHmCpuInfo[idCpu];
1615	}
1616
1617
1618	/**
1619	* Interface for importing state on demand (used by IEM).
1620	*
1621	* @returns VBox status code.
1622	* @param pVCpu The cross context CPU structure.
1623	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
1624	*/
1625	VMMR0_INT_DECL(int) HMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
1626	{
1627	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
1628	return VMXR0ImportStateOnDemand(pVCpu, fWhat);
1629	return SVMR0ImportStateOnDemand(pVCpu, fWhat);
1630	}
1631
1632	#ifdef VBOX_STRICT
1633
1634	/**
1635	* Dumps a descriptor.
1636	*
1637	* @param pDesc Descriptor to dump.
1638	* @param Sel The selector.
1639	* @param pszSel The name of the selector.
1640	*/
1641	VMMR0_INT_DECL(void) hmR0DumpDescriptor(PCX86DESCHC pDesc, RTSEL Sel, const char *pszSel)
1642	{
1643	/*
1644	* Make variable description string.
1645	*/
1646	static struct
1647	{
1648	unsigned cch;
1649	const char *psz;
1650	} const s_aTypes[32] =
1651	{
1652	# define STRENTRY(str) { sizeof(str) - 1, str }
1653
1654	/* system */
1655	# if HC_ARCH_BITS == 64
1656	STRENTRY("Reserved0 "), /* 0x00 */
1657	STRENTRY("Reserved1 "), /* 0x01 */
1658	STRENTRY("LDT "), /* 0x02 */
1659	STRENTRY("Reserved3 "), /* 0x03 */
1660	STRENTRY("Reserved4 "), /* 0x04 */
1661	STRENTRY("Reserved5 "), /* 0x05 */
1662	STRENTRY("Reserved6 "), /* 0x06 */
1663	STRENTRY("Reserved7 "), /* 0x07 */
1664	STRENTRY("Reserved8 "), /* 0x08 */
1665	STRENTRY("TSS64Avail "), /* 0x09 */
1666	STRENTRY("ReservedA "), /* 0x0a */
1667	STRENTRY("TSS64Busy "), /* 0x0b */
1668	STRENTRY("Call64 "), /* 0x0c */
1669	STRENTRY("ReservedD "), /* 0x0d */
1670	STRENTRY("Int64 "), /* 0x0e */
1671	STRENTRY("Trap64 "), /* 0x0f */
1672	# else
1673	STRENTRY("Reserved0 "), /* 0x00 */
1674	STRENTRY("TSS16Avail "), /* 0x01 */
1675	STRENTRY("LDT "), /* 0x02 */
1676	STRENTRY("TSS16Busy "), /* 0x03 */
1677	STRENTRY("Call16 "), /* 0x04 */
1678	STRENTRY("Task "), /* 0x05 */
1679	STRENTRY("Int16 "), /* 0x06 */
1680	STRENTRY("Trap16 "), /* 0x07 */
1681	STRENTRY("Reserved8 "), /* 0x08 */
1682	STRENTRY("TSS32Avail "), /* 0x09 */
1683	STRENTRY("ReservedA "), /* 0x0a */
1684	STRENTRY("TSS32Busy "), /* 0x0b */
1685	STRENTRY("Call32 "), /* 0x0c */
1686	STRENTRY("ReservedD "), /* 0x0d */
1687	STRENTRY("Int32 "), /* 0x0e */
1688	STRENTRY("Trap32 "), /* 0x0f */
1689	# endif
1690	/* non system */
1691	STRENTRY("DataRO "), /* 0x10 */
1692	STRENTRY("DataRO Accessed "), /* 0x11 */
1693	STRENTRY("DataRW "), /* 0x12 */
1694	STRENTRY("DataRW Accessed "), /* 0x13 */
1695	STRENTRY("DataDownRO "), /* 0x14 */
1696	STRENTRY("DataDownRO Accessed "), /* 0x15 */
1697	STRENTRY("DataDownRW "), /* 0x16 */
1698	STRENTRY("DataDownRW Accessed "), /* 0x17 */
1699	STRENTRY("CodeEO "), /* 0x18 */
1700	STRENTRY("CodeEO Accessed "), /* 0x19 */
1701	STRENTRY("CodeER "), /* 0x1a */
1702	STRENTRY("CodeER Accessed "), /* 0x1b */
1703	STRENTRY("CodeConfEO "), /* 0x1c */
1704	STRENTRY("CodeConfEO Accessed "), /* 0x1d */
1705	STRENTRY("CodeConfER "), /* 0x1e */
1706	STRENTRY("CodeConfER Accessed ") /* 0x1f */
1707	# undef SYSENTRY
1708	};
1709	# define ADD_STR(psz, pszAdd) do { strcpy(psz, pszAdd); psz += strlen(pszAdd); } while (0)
1710	char szMsg[128];
1711	char *psz = &szMsg[0];
1712	unsigned i = pDesc->Gen.u1DescType << 4 \| pDesc->Gen.u4Type;
1713	memcpy(psz, s_aTypes[i].psz, s_aTypes[i].cch);
1714	psz += s_aTypes[i].cch;
1715
1716	if (pDesc->Gen.u1Present)
1717	ADD_STR(psz, "Present ");
1718	else
1719	ADD_STR(psz, "Not-Present ");
1720	# if HC_ARCH_BITS == 64
1721	if (pDesc->Gen.u1Long)
1722	ADD_STR(psz, "64-bit ");
1723	else
1724	ADD_STR(psz, "Comp ");
1725	# else
1726	if (pDesc->Gen.u1Granularity)
1727	ADD_STR(psz, "Page ");
1728	if (pDesc->Gen.u1DefBig)
1729	ADD_STR(psz, "32-bit ");
1730	else
1731	ADD_STR(psz, "16-bit ");
1732	# endif
1733	# undef ADD_STR
1734	*psz = '\0';
1735
1736	/*
1737	* Limit and Base and format the output.
1738	*/
1739	#ifdef LOG_ENABLED
1740	uint32_t u32Limit = X86DESC_LIMIT_G(pDesc);
1741
1742	# if HC_ARCH_BITS == 64
1743	uint64_t const u64Base = X86DESC64_BASE(pDesc);
1744	Log((" %s { %#04x - %#RX64 %#RX64 - base=%#RX64 limit=%#08x dpl=%d } %s\n", pszSel,
1745	Sel, pDesc->au64[0], pDesc->au64[1], u64Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1746	# else
1747	uint32_t const u32Base = X86DESC_BASE(pDesc);
1748	Log((" %s { %#04x - %#08x %#08x - base=%#08x limit=%#08x dpl=%d } %s\n", pszSel,
1749	Sel, pDesc->au32[0], pDesc->au32[1], u32Base, u32Limit, pDesc->Gen.u2Dpl, szMsg));
1750	# endif
1751	#else
1752	NOREF(Sel); NOREF(pszSel);
1753	#endif
1754	}
1755
1756
1757	/**
1758	* Formats a full register dump.
1759	*
1760	* @param pVCpu The cross context virtual CPU structure.
1761	* @param fFlags The dumping flags (HM_DUMP_REG_FLAGS_XXX).
1762	*/
1763	VMMR0_INT_DECL(void) hmR0DumpRegs(PVMCPUCC pVCpu, uint32_t fFlags)
1764	{
1765	/*
1766	* Format the flags.
1767	*/
1768	static struct
1769	{
1770	const char *pszSet;
1771	const char *pszClear;
1772	uint32_t fFlag;
1773	} const s_aFlags[] =
1774	{
1775	{ "vip", NULL, X86_EFL_VIP },
1776	{ "vif", NULL, X86_EFL_VIF },
1777	{ "ac", NULL, X86_EFL_AC },
1778	{ "vm", NULL, X86_EFL_VM },
1779	{ "rf", NULL, X86_EFL_RF },
1780	{ "nt", NULL, X86_EFL_NT },
1781	{ "ov", "nv", X86_EFL_OF },
1782	{ "dn", "up", X86_EFL_DF },
1783	{ "ei", "di", X86_EFL_IF },
1784	{ "tf", NULL, X86_EFL_TF },
1785	{ "nt", "pl", X86_EFL_SF },
1786	{ "nz", "zr", X86_EFL_ZF },
1787	{ "ac", "na", X86_EFL_AF },
1788	{ "po", "pe", X86_EFL_PF },
1789	{ "cy", "nc", X86_EFL_CF },
1790	};
1791	char szEFlags[80];
1792	char *psz = szEFlags;
1793	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1794	uint32_t uEFlags = pCtx->eflags.u32;
1795	for (unsigned i = 0; i < RT_ELEMENTS(s_aFlags); i++)
1796	{
1797	const char *pszAdd = s_aFlags[i].fFlag & uEFlags ? s_aFlags[i].pszSet : s_aFlags[i].pszClear;
1798	if (pszAdd)
1799	{
1800	strcpy(psz, pszAdd);
1801	psz += strlen(pszAdd);
1802	*psz++ = ' ';
1803	}
1804	}
1805	psz[-1] = '\0';
1806
1807	if (fFlags & HM_DUMP_REG_FLAGS_GPRS)
1808	{
1809	/*
1810	* Format the registers.
1811	*/
1812	if (CPUMIsGuestIn64BitCode(pVCpu))
1813	{
1814	Log(("rax=%016RX64 rbx=%016RX64 rcx=%016RX64 rdx=%016RX64\n"
1815	"rsi=%016RX64 rdi=%016RX64 r8 =%016RX64 r9 =%016RX64\n"
1816	"r10=%016RX64 r11=%016RX64 r12=%016RX64 r13=%016RX64\n"
1817	"r14=%016RX64 r15=%016RX64\n"
1818	"rip=%016RX64 rsp=%016RX64 rbp=%016RX64 iopl=%d %*s\n"
1819	"cs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1820	"ds={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1821	"es={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1822	"fs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1823	"gs={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1824	"ss={%04x base=%016RX64 limit=%08x flags=%08x}\n"
1825	"cr0=%016RX64 cr2=%016RX64 cr3=%016RX64 cr4=%016RX64\n"
1826	"dr0=%016RX64 dr1=%016RX64 dr2=%016RX64 dr3=%016RX64\n"
1827	"dr4=%016RX64 dr5=%016RX64 dr6=%016RX64 dr7=%016RX64\n"
1828	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1829	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1830	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1831	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1832	,
1833	pCtx->rax, pCtx->rbx, pCtx->rcx, pCtx->rdx, pCtx->rsi, pCtx->rdi,
1834	pCtx->r8, pCtx->r9, pCtx->r10, pCtx->r11, pCtx->r12, pCtx->r13,
1835	pCtx->r14, pCtx->r15,
1836	pCtx->rip, pCtx->rsp, pCtx->rbp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
1837	pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u,
1838	pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u,
1839	pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u,
1840	pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u,
1841	pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u,
1842	pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u,
1843	pCtx->cr0, pCtx->cr2, pCtx->cr3, pCtx->cr4,
1844	pCtx->dr[0], pCtx->dr[1], pCtx->dr[2], pCtx->dr[3],
1845	pCtx->dr[4], pCtx->dr[5], pCtx->dr[6], pCtx->dr[7],
1846	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
1847	pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1848	pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1849	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1850	}
1851	else
1852	Log(("eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
1853	"eip=%08x esp=%08x ebp=%08x iopl=%d %*s\n"
1854	"cs={%04x base=%016RX64 limit=%08x flags=%08x} dr0=%08RX64 dr1=%08RX64\n"
1855	"ds={%04x base=%016RX64 limit=%08x flags=%08x} dr2=%08RX64 dr3=%08RX64\n"
1856	"es={%04x base=%016RX64 limit=%08x flags=%08x} dr4=%08RX64 dr5=%08RX64\n"
1857	"fs={%04x base=%016RX64 limit=%08x flags=%08x} dr6=%08RX64 dr7=%08RX64\n"
1858	"gs={%04x base=%016RX64 limit=%08x flags=%08x} cr0=%08RX64 cr2=%08RX64\n"
1859	"ss={%04x base=%016RX64 limit=%08x flags=%08x} cr3=%08RX64 cr4=%08RX64\n"
1860	"gdtr=%016RX64:%04x idtr=%016RX64:%04x eflags=%08x\n"
1861	"ldtr={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1862	"tr ={%04x base=%08RX64 limit=%08x flags=%08x}\n"
1863	"SysEnter={cs=%04llx eip=%08llx esp=%08llx}\n"
1864	,
1865	pCtx->eax, pCtx->ebx, pCtx->ecx, pCtx->edx, pCtx->esi, pCtx->edi,
1866	pCtx->eip, pCtx->esp, pCtx->ebp, X86_EFL_GET_IOPL(uEFlags), 31, szEFlags,
1867	pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit, pCtx->cs.Attr.u, pCtx->dr[0], pCtx->dr[1],
1868	pCtx->ds.Sel, pCtx->ds.u64Base, pCtx->ds.u32Limit, pCtx->ds.Attr.u, pCtx->dr[2], pCtx->dr[3],
1869	pCtx->es.Sel, pCtx->es.u64Base, pCtx->es.u32Limit, pCtx->es.Attr.u, pCtx->dr[4], pCtx->dr[5],
1870	pCtx->fs.Sel, pCtx->fs.u64Base, pCtx->fs.u32Limit, pCtx->fs.Attr.u, pCtx->dr[6], pCtx->dr[7],
1871	pCtx->gs.Sel, pCtx->gs.u64Base, pCtx->gs.u32Limit, pCtx->gs.Attr.u, pCtx->cr0, pCtx->cr2,
1872	pCtx->ss.Sel, pCtx->ss.u64Base, pCtx->ss.u32Limit, pCtx->ss.Attr.u, pCtx->cr3, pCtx->cr4,
1873	pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt, pCtx->idtr.pIdt, pCtx->idtr.cbIdt, uEFlags,
1874	pCtx->ldtr.Sel, pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit, pCtx->ldtr.Attr.u,
1875	pCtx->tr.Sel, pCtx->tr.u64Base, pCtx->tr.u32Limit, pCtx->tr.Attr.u,
1876	pCtx->SysEnter.cs, pCtx->SysEnter.eip, pCtx->SysEnter.esp));
1877	}
1878
1879	if (fFlags & HM_DUMP_REG_FLAGS_FPU)
1880	{
1881	PCX86FXSTATE pFpuCtx = &pCtx->XState.x87;
1882	Log(("FPU:\n"
1883	"FCW=%04x FSW=%04x FTW=%02x\n"
1884	"FOP=%04x FPUIP=%08x CS=%04x Rsrvd1=%04x\n"
1885	"FPUDP=%04x DS=%04x Rsvrd2=%04x MXCSR=%08x MXCSR_MASK=%08x\n"
1886	,
1887	pFpuCtx->FCW, pFpuCtx->FSW, pFpuCtx->FTW,
1888	pFpuCtx->FOP, pFpuCtx->FPUIP, pFpuCtx->CS, pFpuCtx->Rsrvd1,
1889	pFpuCtx->FPUDP, pFpuCtx->DS, pFpuCtx->Rsrvd2,
1890	pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK));
1891	NOREF(pFpuCtx);
1892	}
1893
1894	if (fFlags & HM_DUMP_REG_FLAGS_MSRS)
1895	{
1896	Log(("MSR:\n"
1897	"EFER =%016RX64\n"
1898	"PAT =%016RX64\n"
1899	"STAR =%016RX64\n"
1900	"CSTAR =%016RX64\n"
1901	"LSTAR =%016RX64\n"
1902	"SFMASK =%016RX64\n"
1903	"KERNELGSBASE =%016RX64\n",
1904	pCtx->msrEFER,
1905	pCtx->msrPAT,
1906	pCtx->msrSTAR,
1907	pCtx->msrCSTAR,
1908	pCtx->msrLSTAR,
1909	pCtx->msrSFMASK,
1910	pCtx->msrKERNELGSBASE));
1911	}
1912	}
1913
1914	#endif /* VBOX_STRICT */
1915

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp@ 93554

以其他格式下載: