NEMR3Native-darwin.cpp@ 92521

最後變更在這個檔案從92521是 92499,由 vboxsync 提交於 3 年前
VMM/NEMR3Native-darwin.cpp: Fix TPR shadowing configuration (huge speedup for Windows 10 guests) and add some statistics, bugref:9044
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 128.9 KB

行
1	/* $Id: NEMR3Native-darwin.cpp 92499 2021-11-18 16:07:42Z vboxsync $ */
2	/** @file
3	* NEM - Native execution manager, native ring-3 macOS backend using Hypervisor.framework.
4	*
5	* Log group 2: Exit logging.
6	* Log group 3: Log context on exit.
7	* Log group 5: Ring-3 memory management
8	*/
9
10	/*
11	* Copyright (C) 2020 Oracle Corporation
12	*
13	* This file is part of VirtualBox Open Source Edition (OSE), as
14	* available from http://www.alldomusa.eu.org. This file is free software;
15	* you can redistribute it and/or modify it under the terms of the GNU
16	* General Public License (GPL) as published by the Free Software
17	* Foundation, in version 2 as it comes in the "COPYING" file of the
18	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20	*/
21
22
23	/*********************************************************************************************************************************
24	* Header Files *
25	*********************************************************************************************************************************/
26	#define LOG_GROUP LOG_GROUP_NEM
27	#define VMCPU_INCL_CPUM_GST_CTX
28	#include <VBox/vmm/nem.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/em.h>
31	#include <VBox/vmm/apic.h>
32	#include <VBox/vmm/pdm.h>
33	#include <VBox/vmm/hm.h>
34	#include <VBox/vmm/hm_vmx.h>
35	#include <VBox/vmm/dbgftrace.h>
36	#include "VMXInternal.h"
37	#include "NEMInternal.h"
38	#include <VBox/vmm/vmcc.h>
39	#include "dtrace/VBoxVMM.h"
40
41	#include <iprt/asm.h>
42	#include <iprt/ldr.h>
43	#include <iprt/mem.h>
44	#include <iprt/path.h>
45	#include <iprt/string.h>
46	#include <iprt/system.h>
47	#include <iprt/utf16.h>
48
49
50	/*********************************************************************************************************************************
51	* Defined Constants And Macros *
52	*********************************************************************************************************************************/
53	/* No nested hwvirt (for now). */
54	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
55	# undef VBOX_WITH_NESTED_HWVIRT_VMX
56	#endif
57
58
59	/** @name HV return codes.
60	* @{ */
61	/** Operation was successful. */
62	#define HV_SUCCESS 0
63	/** An error occurred during operation. */
64	#define HV_ERROR 0xfae94001
65	/** The operation could not be completed right now, try again. */
66	#define HV_BUSY 0xfae94002
67	/** One of the parameters passed wis invalid. */
68	#define HV_BAD_ARGUMENT 0xfae94003
69	/** Not enough resources left to fulfill the operation. */
70	#define HV_NO_RESOURCES 0xfae94005
71	/** The device could not be found. */
72	#define HV_NO_DEVICE 0xfae94006
73	/** The operation is not supportd on this platform with this configuration. */
74	#define HV_UNSUPPORTED 0xfae94007
75	/** @} */
76
77
78	/** @name HV memory protection flags.
79	* @{ */
80	/** Memory is readable. */
81	#define HV_MEMORY_READ RT_BIT_64(0)
82	/** Memory is writeable. */
83	#define HV_MEMORY_WRITE RT_BIT_64(1)
84	/** Memory is executable. */
85	#define HV_MEMORY_EXEC RT_BIT_64(2)
86	/** @} */
87
88
89	/** @name HV shadow VMCS protection flags.
90	* @{ */
91	/** Shadow VMCS field is not accessible. */
92	#define HV_SHADOW_VMCS_NONE 0
93	/** Shadow VMCS fild is readable. */
94	#define HV_SHADOW_VMCS_READ RT_BIT_64(0)
95	/** Shadow VMCS field is writeable. */
96	#define HV_SHADOW_VMCS_WRITE RT_BIT_64(1)
97	/** @} */
98
99
100	/** Default VM creation flags. */
101	#define HV_VM_DEFAULT 0
102	/** Default guest address space creation flags. */
103	#define HV_VM_SPACE_DEFAULT 0
104	/** Default vCPU creation flags. */
105	#define HV_VCPU_DEFAULT 0
106
107	#define HV_DEADLINE_FOREVER UINT64_MAX
108
109
110	/*********************************************************************************************************************************
111	* Structures and Typedefs *
112	*********************************************************************************************************************************/
113
114	/** HV return code type. */
115	typedef uint32_t hv_return_t;
116	/** HV capability bitmask. */
117	typedef uint64_t hv_capability_t;
118	/** Option bitmask type when creating a VM. */
119	typedef uint64_t hv_vm_options_t;
120	/** Option bitmask when creating a vCPU. */
121	typedef uint64_t hv_vcpu_options_t;
122	/** HV memory protection flags type. */
123	typedef uint64_t hv_memory_flags_t;
124	/** Shadow VMCS protection flags. */
125	typedef uint64_t hv_shadow_flags_t;
126	/** Guest physical address type. */
127	typedef uint64_t hv_gpaddr_t;
128
129
130	/**
131	* VMX Capability enumeration.
132	*/
133	typedef enum
134	{
135	HV_VMX_CAP_PINBASED = 0,
136	HV_VMX_CAP_PROCBASED,
137	HV_VMX_CAP_PROCBASED2,
138	HV_VMX_CAP_ENTRY,
139	HV_VMX_CAP_EXIT,
140	HV_VMX_CAP_BASIC, /* Since 11.0 */
141	HV_VMX_CAP_TRUE_PINBASED, /* Since 11.0 */
142	HV_VMX_CAP_TRUE_PROCBASED, /* Since 11.0 */
143	HV_VMX_CAP_TRUE_ENTRY, /* Since 11.0 */
144	HV_VMX_CAP_TRUE_EXIT, /* Since 11.0 */
145	HV_VMX_CAP_MISC, /* Since 11.0 */
146	HV_VMX_CAP_CR0_FIXED0, /* Since 11.0 */
147	HV_VMX_CAP_CR0_FIXED1, /* Since 11.0 */
148	HV_VMX_CAP_CR4_FIXED0, /* Since 11.0 */
149	HV_VMX_CAP_CR4_FIXED1, /* Since 11.0 */
150	HV_VMX_CAP_VMCS_ENUM, /* Since 11.0 */
151	HV_VMX_CAP_EPT_VPID_CAP, /* Since 11.0 */
152	HV_VMX_CAP_PREEMPTION_TIMER = 32
153	} hv_vmx_capability_t;
154
155
156	/**
157	* HV x86 register enumeration.
158	*/
159	typedef enum
160	{
161	HV_X86_RIP = 0,
162	HV_X86_RFLAGS,
163	HV_X86_RAX,
164	HV_X86_RCX,
165	HV_X86_RDX,
166	HV_X86_RBX,
167	HV_X86_RSI,
168	HV_X86_RDI,
169	HV_X86_RSP,
170	HV_X86_RBP,
171	HV_X86_R8,
172	HV_X86_R9,
173	HV_X86_R10,
174	HV_X86_R11,
175	HV_X86_R12,
176	HV_X86_R13,
177	HV_X86_R14,
178	HV_X86_R15,
179	HV_X86_CS,
180	HV_X86_SS,
181	HV_X86_DS,
182	HV_X86_ES,
183	HV_X86_FS,
184	HV_X86_GS,
185	HV_X86_IDT_BASE,
186	HV_X86_IDT_LIMIT,
187	HV_X86_GDT_BASE,
188	HV_X86_GDT_LIMIT,
189	HV_X86_LDTR,
190	HV_X86_LDT_BASE,
191	HV_X86_LDT_LIMIT,
192	HV_X86_LDT_AR,
193	HV_X86_TR,
194	HV_X86_TSS_BASE,
195	HV_X86_TSS_LIMIT,
196	HV_X86_TSS_AR,
197	HV_X86_CR0,
198	HV_X86_CR1,
199	HV_X86_CR2,
200	HV_X86_CR3,
201	HV_X86_CR4,
202	HV_X86_DR0,
203	HV_X86_DR1,
204	HV_X86_DR2,
205	HV_X86_DR3,
206	HV_X86_DR4,
207	HV_X86_DR5,
208	HV_X86_DR6,
209	HV_X86_DR7,
210	HV_X86_TPR,
211	HV_X86_XCR0,
212	HV_X86_REGISTERS_MAX
213	} hv_x86_reg_t;
214
215
216	typedef hv_return_t FN_HV_CAPABILITY(hv_capability_t capability, uint64_t *valu);
217	typedef hv_return_t FN_HV_VM_CREATE(hv_vm_options_t flags);
218	typedef hv_return_t FN_HV_VM_DESTROY(void);
219	typedef hv_return_t FN_HV_VM_SPACE_CREATE(hv_vm_space_t *asid);
220	typedef hv_return_t FN_HV_VM_SPACE_DESTROY(hv_vm_space_t asid);
221	typedef hv_return_t FN_HV_VM_MAP(const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
222	typedef hv_return_t FN_HV_VM_UNMAP(hv_gpaddr_t gpa, size_t size);
223	typedef hv_return_t FN_HV_VM_PROTECT(hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
224	typedef hv_return_t FN_HV_VM_MAP_SPACE(hv_vm_space_t asid, const void *uva, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
225	typedef hv_return_t FN_HV_VM_UNMAP_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size);
226	typedef hv_return_t FN_HV_VM_PROTECT_SPACE(hv_vm_space_t asid, hv_gpaddr_t gpa, size_t size, hv_memory_flags_t flags);
227	typedef hv_return_t FN_HV_VM_SYNC_TSC(uint64_t tsc);
228
229	typedef hv_return_t FN_HV_VCPU_CREATE(hv_vcpuid_t *vcpu, hv_vcpu_options_t flags);
230	typedef hv_return_t FN_HV_VCPU_DESTROY(hv_vcpuid_t vcpu);
231	typedef hv_return_t FN_HV_VCPU_SET_SPACE(hv_vcpuid_t vcpu, hv_vm_space_t asid);
232	typedef hv_return_t FN_HV_VCPU_READ_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t *value);
233	typedef hv_return_t FN_HV_VCPU_WRITE_REGISTER(hv_vcpuid_t vcpu, hv_x86_reg_t reg, uint64_t value);
234	typedef hv_return_t FN_HV_VCPU_READ_FPSTATE(hv_vcpuid_t vcpu, void *buffer, size_t size);
235	typedef hv_return_t FN_HV_VCPU_WRITE_FPSTATE(hv_vcpuid_t vcpu, const void *buffer, size_t size);
236	typedef hv_return_t FN_HV_VCPU_ENABLE_NATIVE_MSR(hv_vcpuid_t vcpu, uint32_t msr, bool enable);
237	typedef hv_return_t FN_HV_VCPU_READ_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t *value);
238	typedef hv_return_t FN_HV_VCPU_WRITE_MSR(hv_vcpuid_t vcpu, uint32_t msr, uint64_t value);
239	typedef hv_return_t FN_HV_VCPU_FLUSH(hv_vcpuid_t vcpu);
240	typedef hv_return_t FN_HV_VCPU_INVALIDATE_TLB(hv_vcpuid_t vcpu);
241	typedef hv_return_t FN_HV_VCPU_RUN(hv_vcpuid_t vcpu);
242	typedef hv_return_t FN_HV_VCPU_RUN_UNTIL(hv_vcpuid_t vcpu, uint64_t deadline);
243	typedef hv_return_t FN_HV_VCPU_INTERRUPT(hv_vcpuid_t *vcpus, unsigned int vcpu_count);
244	typedef hv_return_t FN_HV_VCPU_GET_EXEC_TIME(hv_vcpuid_t vcpus, uint64_t time);
245
246	typedef hv_return_t FN_HV_VMX_VCPU_READ_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
247	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
248
249	typedef hv_return_t FN_HV_VMX_VCPU_READ_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t *value);
250	typedef hv_return_t FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS(hv_vcpuid_t vcpu, uint32_t field, uint64_t value);
251	typedef hv_return_t FN_HV_VMX_VCPU_SET_SHADOW_ACCESS(hv_vcpuid_t vcpu, uint32_t field, hv_shadow_flags_t flags);
252
253	typedef hv_return_t FN_HV_VMX_READ_CAPABILITY(hv_vmx_capability_t field, uint64_t *value);
254	typedef hv_return_t FN_HV_VMX_VCPU_SET_APIC_ADDRESS(hv_vcpuid_t vcpu, hv_gpaddr_t gpa);
255
256
257	/*********************************************************************************************************************************
258	* Global Variables *
259	*********************************************************************************************************************************/
260	/** NEM_DARWIN_PAGE_STATE_XXX names. */
261	NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
262	/** MSRs. */
263	static SUPHWVIRTMSRS g_HmMsrs;
264	/** VMX: Set if swapping EFER is supported. */
265	static bool g_fHmVmxSupportsVmcsEfer = false;
266	/** @name APIs imported from Hypervisor.framework.
267	* @{ */
268	static FN_HV_CAPABILITY g_pfnHvCapability = NULL; / Since 10.15 */
269	static FN_HV_VM_CREATE g_pfnHvVmCreate = NULL; / Since 10.10 */
270	static FN_HV_VM_DESTROY g_pfnHvVmDestroy = NULL; / Since 10.10 */
271	static FN_HV_VM_SPACE_CREATE g_pfnHvVmSpaceCreate = NULL; / Since 10.15 */
272	static FN_HV_VM_SPACE_DESTROY g_pfnHvVmSpaceDestroy = NULL; / Since 10.15 */
273	static FN_HV_VM_MAP g_pfnHvVmMap = NULL; / Since 10.10 */
274	static FN_HV_VM_UNMAP g_pfnHvVmUnmap = NULL; / Since 10.10 */
275	static FN_HV_VM_PROTECT g_pfnHvVmProtect = NULL; / Since 10.10 */
276	static FN_HV_VM_MAP_SPACE g_pfnHvVmMapSpace = NULL; / Since 10.15 */
277	static FN_HV_VM_UNMAP_SPACE g_pfnHvVmUnmapSpace = NULL; / Since 10.15 */
278	static FN_HV_VM_PROTECT_SPACE g_pfnHvVmProtectSpace = NULL; / Since 10.15 */
279	static FN_HV_VM_SYNC_TSC g_pfnHvVmSyncTsc = NULL; / Since 10.10 */
280
281	static FN_HV_VCPU_CREATE g_pfnHvVCpuCreate = NULL; / Since 10.10 */
282	static FN_HV_VCPU_DESTROY g_pfnHvVCpuDestroy = NULL; / Since 10.10 */
283	static FN_HV_VCPU_SET_SPACE g_pfnHvVCpuSetSpace = NULL; / Since 10.15 */
284	static FN_HV_VCPU_READ_REGISTER g_pfnHvVCpuReadRegister = NULL; / Since 10.10 */
285	static FN_HV_VCPU_WRITE_REGISTER g_pfnHvVCpuWriteRegister = NULL; / Since 10.10 */
286	static FN_HV_VCPU_READ_FPSTATE g_pfnHvVCpuReadFpState = NULL; / Since 10.10 */
287	static FN_HV_VCPU_WRITE_FPSTATE g_pfnHvVCpuWriteFpState = NULL; / Since 10.10 */
288	static FN_HV_VCPU_ENABLE_NATIVE_MSR g_pfnHvVCpuEnableNativeMsr = NULL; / Since 10.10 */
289	static FN_HV_VCPU_READ_MSR g_pfnHvVCpuReadMsr = NULL; / Since 10.10 */
290	static FN_HV_VCPU_WRITE_MSR g_pfnHvVCpuWriteMsr = NULL; / Since 10.10 */
291	static FN_HV_VCPU_FLUSH g_pfnHvVCpuFlush = NULL; / Since 10.10 */
292	static FN_HV_VCPU_INVALIDATE_TLB g_pfnHvVCpuInvalidateTlb = NULL; / Since 10.10 */
293	static FN_HV_VCPU_RUN g_pfnHvVCpuRun = NULL; / Since 10.10 */
294	static FN_HV_VCPU_RUN_UNTIL g_pfnHvVCpuRunUntil = NULL; / Since 10.15 */
295	static FN_HV_VCPU_INTERRUPT g_pfnHvVCpuInterrupt = NULL; / Since 10.10 */
296	static FN_HV_VCPU_GET_EXEC_TIME g_pfnHvVCpuGetExecTime = NULL; / Since 10.10 */
297
298	static FN_HV_VMX_READ_CAPABILITY g_pfnHvVmxReadCapability = NULL; / Since 10.10 */
299	static FN_HV_VMX_VCPU_READ_VMCS g_pfnHvVmxVCpuReadVmcs = NULL; / Since 10.10 */
300	static FN_HV_VMX_VCPU_WRITE_VMCS g_pfnHvVmxVCpuWriteVmcs = NULL; / Since 10.10 */
301	static FN_HV_VMX_VCPU_READ_SHADOW_VMCS g_pfnHvVmxVCpuReadShadowVmcs = NULL; / Since 10.15 */
302	static FN_HV_VMX_VCPU_WRITE_SHADOW_VMCS g_pfnHvVmxVCpuWriteShadowVmcs = NULL; / Since 10.15 */
303	static FN_HV_VMX_VCPU_SET_SHADOW_ACCESS g_pfnHvVmxVCpuSetShadowAccess = NULL; / Since 10.15 */
304	static FN_HV_VMX_VCPU_SET_APIC_ADDRESS g_pfnHvVmxVCpuSetApicAddress = NULL; / Since 10.10 */
305	/** @} */
306
307
308	/**
309	* Import instructions.
310	*/
311	static const struct
312	{
313	bool fOptional; /*< Set if import is optional. /
314	void ppfn; /< The function pointer variable. */
315	const char pszName; /< The function name. /
316	} g_aImports[] =
317	{
318	#define NEM_DARWIN_IMPORT(a_fOptional, a_Pfn, a_Name) { (a_fOptional), (void **)&(a_Pfn), #a_Name }
319	NEM_DARWIN_IMPORT(true, g_pfnHvCapability, hv_capability),
320	NEM_DARWIN_IMPORT(false, g_pfnHvVmCreate, hv_vm_create),
321	NEM_DARWIN_IMPORT(false, g_pfnHvVmDestroy, hv_vm_destroy),
322	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceCreate, hv_vm_space_create),
323	NEM_DARWIN_IMPORT(true, g_pfnHvVmSpaceDestroy, hv_vm_space_destroy),
324	NEM_DARWIN_IMPORT(false, g_pfnHvVmMap, hv_vm_map),
325	NEM_DARWIN_IMPORT(false, g_pfnHvVmUnmap, hv_vm_unmap),
326	NEM_DARWIN_IMPORT(false, g_pfnHvVmProtect, hv_vm_protect),
327	NEM_DARWIN_IMPORT(true, g_pfnHvVmMapSpace, hv_vm_map_space),
328	NEM_DARWIN_IMPORT(true, g_pfnHvVmUnmapSpace, hv_vm_unmap_space),
329	NEM_DARWIN_IMPORT(true, g_pfnHvVmProtectSpace, hv_vm_protect_space),
330	NEM_DARWIN_IMPORT(false, g_pfnHvVmSyncTsc, hv_vm_sync_tsc),
331
332	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuCreate, hv_vcpu_create),
333	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuDestroy, hv_vcpu_destroy),
334	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuSetSpace, hv_vcpu_set_space),
335	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadRegister, hv_vcpu_read_register),
336	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteRegister, hv_vcpu_write_register),
337	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadFpState, hv_vcpu_read_fpstate),
338	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteFpState, hv_vcpu_write_fpstate),
339	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuEnableNativeMsr, hv_vcpu_enable_native_msr),
340	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuReadMsr, hv_vcpu_read_msr),
341	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuWriteMsr, hv_vcpu_write_msr),
342	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuFlush, hv_vcpu_flush),
343	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInvalidateTlb, hv_vcpu_invalidate_tlb),
344	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuRun, hv_vcpu_run),
345	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuRunUntil, hv_vcpu_run_until),
346	NEM_DARWIN_IMPORT(false, g_pfnHvVCpuInterrupt, hv_vcpu_interrupt),
347	NEM_DARWIN_IMPORT(true, g_pfnHvVCpuGetExecTime, hv_vcpu_get_exec_time),
348	NEM_DARWIN_IMPORT(false, g_pfnHvVmxReadCapability, hv_vmx_read_capability),
349	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuReadVmcs, hv_vmx_vcpu_read_vmcs),
350	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuWriteVmcs, hv_vmx_vcpu_write_vmcs),
351	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuReadShadowVmcs, hv_vmx_vcpu_read_shadow_vmcs),
352	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuWriteShadowVmcs, hv_vmx_vcpu_write_shadow_vmcs),
353	NEM_DARWIN_IMPORT(true, g_pfnHvVmxVCpuSetShadowAccess, hv_vmx_vcpu_set_shadow_access),
354	NEM_DARWIN_IMPORT(false, g_pfnHvVmxVCpuSetApicAddress, hv_vmx_vcpu_set_apic_address),
355	#undef NEM_DARWIN_IMPORT
356	};
357
358
359	/*
360	* Let the preprocessor alias the APIs to import variables for better autocompletion.
361	*/
362	#ifndef IN_SLICKEDIT
363	# define hv_capability g_pfnHvCapability
364	# define hv_vm_create g_pfnHvVmCreate
365	# define hv_vm_destroy g_pfnHvVmDestroy
366	# define hv_vm_space_create g_pfnHvVmSpaceCreate
367	# define hv_vm_space_destroy g_pfnHvVmSpaceDestroy
368	# define hv_vm_map g_pfnHvVmMap
369	# define hv_vm_unmap g_pfnHvVmUnmap
370	# define hv_vm_protect g_pfnHvVmProtect
371	# define hv_vm_map_space g_pfnHvVmMapSpace
372	# define hv_vm_unmap_space g_pfnHvVmUnmapSpace
373	# define hv_vm_protect_space g_pfnHvVmProtectSpace
374	# define hv_vm_sync_tsc g_pfnHvVmSyncTsc
375
376	# define hv_vcpu_create g_pfnHvVCpuCreate
377	# define hv_vcpu_destroy g_pfnHvVCpuDestroy
378	# define hv_vcpu_set_space g_pfnHvVCpuSetSpace
379	# define hv_vcpu_read_register g_pfnHvVCpuReadRegister
380	# define hv_vcpu_write_register g_pfnHvVCpuWriteRegister
381	# define hv_vcpu_read_fpstate g_pfnHvVCpuReadFpState
382	# define hv_vcpu_write_fpstate g_pfnHvVCpuWriteFpState
383	# define hv_vcpu_enable_native_msr g_pfnHvVCpuEnableNativeMsr
384	# define hv_vcpu_read_msr g_pfnHvVCpuReadMsr
385	# define hv_vcpu_write_msr g_pfnHvVCpuWriteMsr
386	# define hv_vcpu_flush g_pfnHvVCpuFlush
387	# define hv_vcpu_invalidate_tlb g_pfnHvVCpuInvalidateTlb
388	# define hv_vcpu_run g_pfnHvVCpuRun
389	# define hv_vcpu_run_until g_pfnHvVCpuRunUntil
390	# define hv_vcpu_interrupt g_pfnHvVCpuInterrupt
391	# define hv_vcpu_get_exec_time g_pfnHvVCpuGetExecTime
392
393	# define hv_vmx_read_capability g_pfnHvVmxReadCapability
394	# define hv_vmx_vcpu_read_vmcs g_pfnHvVmxVCpuReadVmcs
395	# define hv_vmx_vcpu_write_vmcs g_pfnHvVmxVCpuWriteVmcs
396	# define hv_vmx_vcpu_read_shadow_vmcs g_pfnHvVmxVCpuReadShadowVmcs
397	# define hv_vmx_vcpu_write_shadow_vmcs g_pfnHvVmxVCpuWriteShadowVmcs
398	# define hv_vmx_vcpu_set_shadow_access g_pfnHvVmxVCpuSetShadowAccess
399	# define hv_vmx_vcpu_set_apic_address g_pfnHvVmxVCpuSetApicAddress
400	#endif
401
402
403	/*********************************************************************************************************************************
404	* Internal Functions *
405	*********************************************************************************************************************************/
406
407	/**
408	* Converts a HV return code to a VBox status code.
409	*
410	* @returns VBox status code.
411	* @param hrc The HV return code to convert.
412	*/
413	DECLINLINE(int) nemR3DarwinHvSts2Rc(hv_return_t hrc)
414	{
415	if (hrc == HV_SUCCESS)
416	return VINF_SUCCESS;
417
418	switch (hrc)
419	{
420	case HV_ERROR: return VERR_INVALID_STATE;
421	case HV_BUSY: return VERR_RESOURCE_BUSY;
422	case HV_BAD_ARGUMENT: return VERR_INVALID_PARAMETER;
423	case HV_NO_RESOURCES: return VERR_OUT_OF_RESOURCES;
424	case HV_NO_DEVICE: return VERR_NOT_FOUND;
425	case HV_UNSUPPORTED: return VERR_NOT_SUPPORTED;
426	}
427
428	return VERR_IPE_UNEXPECTED_STATUS;
429	}
430
431
432	/**
433	* Unmaps the given guest physical address range (page aligned).
434	*
435	* @returns VBox status code.
436	* @param pVM The cross context VM structure.
437	* @param GCPhys The guest physical address to start unmapping at.
438	* @param cb The size of the range to unmap in bytes.
439	*/
440	DECLINLINE(int) nemR3DarwinUnmap(PVM pVM, RTGCPHYS GCPhys, size_t cb)
441	{
442	LogFlowFunc(("Unmapping %RGp LB %zu\n", GCPhys, cb));
443	hv_return_t hrc;
444	if (pVM->nem.s.fCreatedAsid)
445	hrc = hv_vm_unmap_space(pVM->nem.s.uVmAsid, GCPhys, cb);
446	else
447	hrc = hv_vm_unmap(GCPhys, cb);
448	return nemR3DarwinHvSts2Rc(hrc);
449	}
450
451
452	/**
453	* Maps a given guest physical address range backed by the given memory with the given
454	* protection flags.
455	*
456	* @returns VBox status code.
457	* @param pVM The cross context VM structure.
458	* @param GCPhys The guest physical address to start mapping.
459	* @param pvRam The R3 pointer of the memory to back the range with.
460	* @param cb The size of the range, page aligned.
461	* @param fPageProt The page protection flags to use for this range, combination of NEM_PAGE_PROT_XXX
462	*/
463	DECLINLINE(int) nemR3DarwinMap(PVM pVM, RTGCPHYS GCPhys, void *pvRam, size_t cb, uint32_t fPageProt)
464	{
465	LogFlowFunc(("Mapping %RGp LB %zu fProt=%#x\n", GCPhys, cb, fPageProt));
466
467	hv_memory_flags_t fHvMemProt = 0;
468	if (fPageProt & NEM_PAGE_PROT_READ)
469	fHvMemProt \|= HV_MEMORY_READ;
470	if (fPageProt & NEM_PAGE_PROT_WRITE)
471	fHvMemProt \|= HV_MEMORY_WRITE;
472	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
473	fHvMemProt \|= HV_MEMORY_EXEC;
474
475	hv_return_t hrc;
476	if (pVM->nem.s.fCreatedAsid)
477	hrc = hv_vm_map_space(pVM->nem.s.uVmAsid, pvRam, GCPhys, cb, fHvMemProt);
478	else
479	hrc = hv_vm_map(pvRam, GCPhys, cb, fHvMemProt);
480	return nemR3DarwinHvSts2Rc(hrc);
481	}
482
483
484	#if 0 /* unused */
485	DECLINLINE(int) nemR3DarwinProtectPage(PVM pVM, RTGCPHYS GCPhys, size_t cb, uint32_t fPageProt)
486	{
487	hv_memory_flags_t fHvMemProt = 0;
488	if (fPageProt & NEM_PAGE_PROT_READ)
489	fHvMemProt \|= HV_MEMORY_READ;
490	if (fPageProt & NEM_PAGE_PROT_WRITE)
491	fHvMemProt \|= HV_MEMORY_WRITE;
492	if (fPageProt & NEM_PAGE_PROT_EXECUTE)
493	fHvMemProt \|= HV_MEMORY_EXEC;
494
495	if (pVM->nem.s.fCreatedAsid)
496	hrc = hv_vm_protect_space(pVM->nem.s.uVmAsid, GCPhys, cb, fHvMemProt);
497	else
498	hrc = hv_vm_protect(GCPhys, cb, fHvMemProt);
499
500	return nemR3DarwinHvSts2Rc(hrc);
501	}
502	#endif
503
504
505	DECLINLINE(int) nemR3NativeGCPhys2R3PtrReadOnly(PVM pVM, RTGCPHYS GCPhys, const void **ppv)
506	{
507	PGMPAGEMAPLOCK Lock;
508	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhys, ppv, &Lock);
509	if (RT_SUCCESS(rc))
510	PGMPhysReleasePageMappingLock(pVM, &Lock);
511	return rc;
512	}
513
514
515	DECLINLINE(int) nemR3NativeGCPhys2R3PtrWriteable(PVM pVM, RTGCPHYS GCPhys, void **ppv)
516	{
517	PGMPAGEMAPLOCK Lock;
518	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhys, ppv, &Lock);
519	if (RT_SUCCESS(rc))
520	PGMPhysReleasePageMappingLock(pVM, &Lock);
521	return rc;
522	}
523
524
525	/**
526	* Worker that maps pages into Hyper-V.
527	*
528	* This is used by the PGM physical page notifications as well as the memory
529	* access VMEXIT handlers.
530	*
531	* @returns VBox status code.
532	* @param pVM The cross context VM structure.
533	* @param pVCpu The cross context virtual CPU structure of the
534	* calling EMT.
535	* @param GCPhysSrc The source page address.
536	* @param GCPhysDst The hyper-V destination page. This may differ from
537	* GCPhysSrc when A20 is disabled.
538	* @param fPageProt NEM_PAGE_PROT_XXX.
539	* @param pu2State Our page state (input/output).
540	* @param fBackingChanged Set if the page backing is being changed.
541	* @thread EMT(pVCpu)
542	*/
543	NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
544	uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
545	{
546	/*
547	* Looks like we need to unmap a page before we can change the backing
548	* or even modify the protection. This is going to be REALLY efficient.
549	* PGM lends us two bits to keep track of the state here.
550	*/
551	RT_NOREF(pVCpu);
552	uint8_t const u2OldState = *pu2State;
553	uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_DARWIN_PAGE_STATE_WRITABLE
554	: fPageProt & NEM_PAGE_PROT_READ ? NEM_DARWIN_PAGE_STATE_READABLE : NEM_DARWIN_PAGE_STATE_UNMAPPED;
555	if ( fBackingChanged
556	\|\| u2NewState != u2OldState)
557	{
558	if (u2OldState > NEM_DARWIN_PAGE_STATE_UNMAPPED)
559	{
560	int rc = nemR3DarwinUnmap(pVM, GCPhysDst, X86_PAGE_SIZE);
561	if (RT_SUCCESS(rc))
562	{
563	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
564	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
565	if (u2NewState == NEM_DARWIN_PAGE_STATE_UNMAPPED)
566	{
567	Log5(("NEM GPA unmapped/set: %RGp (was %s)\n", GCPhysDst, g_apszPageStates[u2OldState]));
568	return VINF_SUCCESS;
569	}
570	}
571	else
572	{
573	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
574	LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
575	return VERR_NEM_INIT_FAILED;
576	}
577	}
578	}
579
580	/*
581	* Writeable mapping?
582	*/
583	if (fPageProt & NEM_PAGE_PROT_WRITE)
584	{
585	void *pvPage;
586	int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
587	if (RT_SUCCESS(rc))
588	{
589	rc = nemR3DarwinMap(pVM, GCPhysDst, pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
590	if (RT_SUCCESS(rc))
591	{
592	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
593	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
594	Log5(("NEM GPA mapped/set: %RGp %s (was %s)\n", GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState]));
595	return VINF_SUCCESS;
596	}
597	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
598	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst));
599	return VERR_NEM_INIT_FAILED;
600	}
601	LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
602	return rc;
603	}
604
605	if (fPageProt & NEM_PAGE_PROT_READ)
606	{
607	const void *pvPage;
608	int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
609	if (RT_SUCCESS(rc))
610	{
611	rc = nemR3DarwinMap(pVM, GCPhysDst, (void *)pvPage, X86_PAGE_SIZE, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
612	if (RT_SUCCESS(rc))
613	{
614	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
615	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
616	Log5(("NEM GPA mapped/set: %RGp %s (was %s)\n", GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState]));
617	return VINF_SUCCESS;
618	}
619	STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
620	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp rc=%Rrc\n", GCPhysDst, rc));
621	return VERR_NEM_INIT_FAILED;
622	}
623	LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
624	return rc;
625	}
626
627	/* We already unmapped it above. */
628	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
629	return VINF_SUCCESS;
630	}
631
632
633	#ifdef LOG_ENABLED
634	/**
635	* Logs the current CPU state.
636	*/
637	static void nemR3DarwinLogState(PVMCC pVM, PVMCPUCC pVCpu)
638	{
639	if (LogIs3Enabled())
640	{
641	#if 0
642	char szRegs[4096];
643	DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
644	"rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
645	"rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
646	"r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
647	"r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
648	"rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
649	"cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
650	"ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
651	"es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
652	"fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
653	"gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
654	"ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
655	"dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
656	"dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
657	"gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim} idtr=%016VR{idtr_base}:%04VR{idtr_lim} rflags=%08VR{rflags}\n"
658	"ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
659	"tr ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
660	" sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
661	" efer=%016VR{efer}\n"
662	" pat=%016VR{pat}\n"
663	" sf_mask=%016VR{sf_mask}\n"
664	"krnl_gs_base=%016VR{krnl_gs_base}\n"
665	" lstar=%016VR{lstar}\n"
666	" star=%016VR{star} cstar=%016VR{cstar}\n"
667	"fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
668	);
669
670	char szInstr[256];
671	DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
672	DBGF_DISAS_FLAGS_CURRENT_GUEST \| DBGF_DISAS_FLAGS_DEFAULT_MODE,
673	szInstr, sizeof(szInstr), NULL);
674	Log3(("%s%s\n", szRegs, szInstr));
675	#else
676	RT_NOREF(pVM, pVCpu);
677	#endif
678	}
679	}
680	#endif /* LOG_ENABLED */
681
682
683	DECLINLINE(int) nemR3DarwinReadVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t *pData)
684	{
685	uint64_t u64Data;
686	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
687	if (RT_LIKELY(hrc == HV_SUCCESS))
688	{
689	*pData = (uint16_t)u64Data;
690	return VINF_SUCCESS;
691	}
692
693	return nemR3DarwinHvSts2Rc(hrc);
694	}
695
696
697	DECLINLINE(int) nemR3DarwinReadVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t *pData)
698	{
699	uint64_t u64Data;
700	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, &u64Data);
701	if (RT_LIKELY(hrc == HV_SUCCESS))
702	{
703	*pData = (uint32_t)u64Data;
704	return VINF_SUCCESS;
705	}
706
707	return nemR3DarwinHvSts2Rc(hrc);
708	}
709
710
711	DECLINLINE(int) nemR3DarwinReadVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t *pData)
712	{
713	hv_return_t hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, pData);
714	if (RT_LIKELY(hrc == HV_SUCCESS))
715	return VINF_SUCCESS;
716
717	return nemR3DarwinHvSts2Rc(hrc);
718	}
719
720
721	DECLINLINE(int) nemR3DarwinWriteVmcs16(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint16_t u16Val)
722	{
723	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u16Val);
724	if (RT_LIKELY(hrc == HV_SUCCESS))
725	return VINF_SUCCESS;
726
727	return nemR3DarwinHvSts2Rc(hrc);
728	}
729
730
731	DECLINLINE(int) nemR3DarwinWriteVmcs32(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint32_t u32Val)
732	{
733	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u32Val);
734	if (RT_LIKELY(hrc == HV_SUCCESS))
735	return VINF_SUCCESS;
736
737	return nemR3DarwinHvSts2Rc(hrc);
738	}
739
740
741	DECLINLINE(int) nemR3DarwinWriteVmcs64(PVMCPUCC pVCpu, uint32_t uFieldEnc, uint64_t u64Val)
742	{
743	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, uFieldEnc, u64Val);
744	if (RT_LIKELY(hrc == HV_SUCCESS))
745	return VINF_SUCCESS;
746
747	return nemR3DarwinHvSts2Rc(hrc);
748	}
749
750	DECLINLINE(int) nemR3DarwinMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Val)
751	{
752	hv_return_t hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, idMsr, pu64Val);
753	if (RT_LIKELY(hrc == HV_SUCCESS))
754	return VINF_SUCCESS;
755
756	return nemR3DarwinHvSts2Rc(hrc);
757	}
758
759	#if 0 /unused/
760	DECLINLINE(int) nemR3DarwinMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Val)
761	{
762	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, idMsr, u64Val);
763	if (RT_LIKELY(hrc == HV_SUCCESS))
764	return VINF_SUCCESS;
765
766	return nemR3DarwinHvSts2Rc(hrc);
767	}
768	#endif
769
770	static int nemR3DarwinCopyStateFromHv(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
771	{
772	#define READ_GREG(a_GReg, a_Value) \
773	do \
774	{ \
775	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, (a_GReg), &(a_Value)); \
776	if (RT_LIKELY(hrc == HV_SUCCESS)) \
777	{ /* likely */ } \
778	else \
779	return VERR_INTERNAL_ERROR; \
780	} while(0)
781	#define READ_VMCS_FIELD(a_Field, a_Value) \
782	do \
783	{ \
784	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &(a_Value)); \
785	if (RT_LIKELY(hrc == HV_SUCCESS)) \
786	{ /* likely */ } \
787	else \
788	return VERR_INTERNAL_ERROR; \
789	} while(0)
790	#define READ_VMCS16_FIELD(a_Field, a_Value) \
791	do \
792	{ \
793	uint64_t u64Data; \
794	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
795	if (RT_LIKELY(hrc == HV_SUCCESS)) \
796	{ (a_Value) = (uint16_t)u64Data; } \
797	else \
798	return VERR_INTERNAL_ERROR; \
799	} while(0)
800	#define READ_VMCS32_FIELD(a_Field, a_Value) \
801	do \
802	{ \
803	uint64_t u64Data; \
804	hrc = hv_vmx_vcpu_read_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), &u64Data); \
805	if (RT_LIKELY(hrc == HV_SUCCESS)) \
806	{ (a_Value) = (uint32_t)u64Data; } \
807	else \
808	return VERR_INTERNAL_ERROR; \
809	} while(0)
810	#define READ_MSR(a_Msr, a_Value) \
811	do \
812	{ \
813	hrc = hv_vcpu_read_msr(pVCpu->nem.s.hVCpuId, (a_Msr), &(a_Value)); \
814	if (RT_LIKELY(hrc == HV_SUCCESS)) \
815	{ /* likely */ } \
816	else \
817	AssertFailedReturn(VERR_INTERNAL_ERROR); \
818	} while(0)
819
820	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateImport, x);
821
822	RT_NOREF(pVM);
823	fWhat &= pVCpu->cpum.GstCtx.fExtrn;
824
825	/* GPRs */
826	hv_return_t hrc;
827	if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
828	{
829	if (fWhat & CPUMCTX_EXTRN_RAX)
830	READ_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
831	if (fWhat & CPUMCTX_EXTRN_RCX)
832	READ_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
833	if (fWhat & CPUMCTX_EXTRN_RDX)
834	READ_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
835	if (fWhat & CPUMCTX_EXTRN_RBX)
836	READ_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
837	if (fWhat & CPUMCTX_EXTRN_RSP)
838	READ_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
839	if (fWhat & CPUMCTX_EXTRN_RBP)
840	READ_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
841	if (fWhat & CPUMCTX_EXTRN_RSI)
842	READ_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
843	if (fWhat & CPUMCTX_EXTRN_RDI)
844	READ_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
845	if (fWhat & CPUMCTX_EXTRN_R8_R15)
846	{
847	READ_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
848	READ_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
849	READ_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
850	READ_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
851	READ_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
852	READ_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
853	READ_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
854	READ_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
855	}
856	}
857
858	/* RIP & Flags */
859	if (fWhat & CPUMCTX_EXTRN_RIP)
860	READ_GREG(HV_X86_RIP, pVCpu->cpum.GstCtx.rip);
861	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
862	READ_GREG(HV_X86_RFLAGS, pVCpu->cpum.GstCtx.rflags.u);
863
864	/* Segments */
865	#define READ_SEG(a_SReg, a_enmName) \
866	do { \
867	READ_VMCS16_FIELD(VMX_VMCS16_GUEST_ ## a_enmName ## _SEL, (a_SReg).Sel); \
868	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _LIMIT, (a_SReg).u32Limit); \
869	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_ ## a_enmName ## _ACCESS_RIGHTS, (a_SReg).Attr.u); \
870	READ_VMCS_FIELD(VMX_VMCS_GUEST_ ## a_enmName ## _BASE, (a_SReg).u64Base); \
871	(a_SReg).ValidSel = (a_SReg).Sel; \
872	} while (0)
873	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
874	{
875	if (fWhat & CPUMCTX_EXTRN_ES)
876	READ_SEG(pVCpu->cpum.GstCtx.es, ES);
877	if (fWhat & CPUMCTX_EXTRN_CS)
878	READ_SEG(pVCpu->cpum.GstCtx.cs, CS);
879	if (fWhat & CPUMCTX_EXTRN_SS)
880	READ_SEG(pVCpu->cpum.GstCtx.ss, SS);
881	if (fWhat & CPUMCTX_EXTRN_DS)
882	READ_SEG(pVCpu->cpum.GstCtx.ds, DS);
883	if (fWhat & CPUMCTX_EXTRN_FS)
884	READ_SEG(pVCpu->cpum.GstCtx.fs, FS);
885	if (fWhat & CPUMCTX_EXTRN_GS)
886	READ_SEG(pVCpu->cpum.GstCtx.gs, GS);
887	}
888
889	/* Descriptor tables and the task segment. */
890	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
891	{
892	if (fWhat & CPUMCTX_EXTRN_LDTR)
893	READ_SEG(pVCpu->cpum.GstCtx.ldtr, LDTR);
894
895	if (fWhat & CPUMCTX_EXTRN_TR)
896	{
897	/* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY. So,
898	avoid to trigger sanity assertions around the code, always fix this. */
899	READ_SEG(pVCpu->cpum.GstCtx.tr, TR);
900	switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
901	{
902	case X86_SEL_TYPE_SYS_386_TSS_BUSY:
903	case X86_SEL_TYPE_SYS_286_TSS_BUSY:
904	break;
905	case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
906	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
907	break;
908	case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
909	pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
910	break;
911	}
912	}
913	if (fWhat & CPUMCTX_EXTRN_IDTR)
914	{
915	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_IDTR_LIMIT, pVCpu->cpum.GstCtx.idtr.cbIdt);
916	READ_VMCS_FIELD(VMX_VMCS_GUEST_IDTR_BASE, pVCpu->cpum.GstCtx.idtr.pIdt);
917	}
918	if (fWhat & CPUMCTX_EXTRN_GDTR)
919	{
920	READ_VMCS32_FIELD(VMX_VMCS32_GUEST_GDTR_LIMIT, pVCpu->cpum.GstCtx.gdtr.cbGdt);
921	READ_VMCS_FIELD(VMX_VMCS_GUEST_GDTR_BASE, pVCpu->cpum.GstCtx.gdtr.pGdt);
922	}
923	}
924
925	/* Control registers. */
926	bool fMaybeChangedMode = false;
927	bool fUpdateCr3 = false;
928	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
929	{
930	uint64_t u64CrTmp = 0;
931
932	if (fWhat & CPUMCTX_EXTRN_CR0)
933	{
934	READ_GREG(HV_X86_CR0, u64CrTmp);
935	if (pVCpu->cpum.GstCtx.cr0 != u64CrTmp)
936	{
937	CPUMSetGuestCR0(pVCpu, u64CrTmp);
938	fMaybeChangedMode = true;
939	}
940	}
941	if (fWhat & CPUMCTX_EXTRN_CR2)
942	READ_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
943	if (fWhat & CPUMCTX_EXTRN_CR3)
944	{
945	READ_GREG(HV_X86_CR3, u64CrTmp);
946	if (pVCpu->cpum.GstCtx.cr3 != u64CrTmp)
947	{
948	CPUMSetGuestCR3(pVCpu, u64CrTmp);
949	fUpdateCr3 = true;
950	}
951	}
952	if (fWhat & CPUMCTX_EXTRN_CR4)
953	{
954	READ_GREG(HV_X86_CR4, u64CrTmp);
955	u64CrTmp &= ~VMX_V_CR4_FIXED0;
956
957	if (pVCpu->cpum.GstCtx.cr4 != u64CrTmp)
958	{
959	CPUMSetGuestCR4(pVCpu, u64CrTmp);
960	fMaybeChangedMode = true;
961	}
962	}
963	}
964
965	#if 0 /* Always done. */
966	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
967	{
968	uint64_t u64Cr8 = 0;
969
970	READ_GREG(HV_X86_TPR, u64Cr8);
971	APICSetTpr(pVCpu, u64Cr8 << 4);
972	}
973	#endif
974
975	if (fWhat & CPUMCTX_EXTRN_XCRx)
976	READ_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
977
978	/* Debug registers. */
979	if (fWhat & CPUMCTX_EXTRN_DR7)
980	{
981	uint64_t u64Dr7;
982	READ_GREG(HV_X86_DR7, u64Dr7);
983	if (pVCpu->cpum.GstCtx.dr[7] != u64Dr7)
984	CPUMSetGuestDR7(pVCpu, u64Dr7);
985	pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
986	}
987	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
988	{
989	uint64_t u64DrTmp;
990
991	READ_GREG(HV_X86_DR0, u64DrTmp);
992	if (pVCpu->cpum.GstCtx.dr[0] != u64DrTmp)
993	CPUMSetGuestDR0(pVCpu, u64DrTmp);
994	READ_GREG(HV_X86_DR1, u64DrTmp);
995	if (pVCpu->cpum.GstCtx.dr[1] != u64DrTmp)
996	CPUMSetGuestDR1(pVCpu, u64DrTmp);
997	READ_GREG(HV_X86_DR2, u64DrTmp);
998	if (pVCpu->cpum.GstCtx.dr[2] != u64DrTmp)
999	CPUMSetGuestDR2(pVCpu, u64DrTmp);
1000	READ_GREG(HV_X86_DR3, u64DrTmp);
1001	if (pVCpu->cpum.GstCtx.dr[3] != u64DrTmp)
1002	CPUMSetGuestDR3(pVCpu, u64DrTmp);
1003	}
1004	if (fWhat & CPUMCTX_EXTRN_DR6)
1005	{
1006	uint64_t u64Dr6;
1007	READ_GREG(HV_X86_DR6, u64Dr6);
1008	if (pVCpu->cpum.GstCtx.dr[6] != u64Dr6)
1009	CPUMSetGuestDR6(pVCpu, u64Dr6);
1010	}
1011
1012	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX))
1013	{
1014	hrc = hv_vcpu_read_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1015	if (hrc == HV_SUCCESS)
1016	{ /* likely */ }
1017	else
1018	{
1019	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1020	return nemR3DarwinHvSts2Rc(hrc);
1021	}
1022	}
1023
1024	/* MSRs */
1025	if (fWhat & CPUMCTX_EXTRN_EFER)
1026	{
1027	uint64_t u64Efer;
1028
1029	READ_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, u64Efer);
1030	if (u64Efer != pVCpu->cpum.GstCtx.msrEFER)
1031	{
1032	Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, u64Efer));
1033	if ((u64Efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
1034	PGMNotifyNxeChanged(pVCpu, RT_BOOL(u64Efer & MSR_K6_EFER_NXE));
1035	pVCpu->cpum.GstCtx.msrEFER = u64Efer;
1036	fMaybeChangedMode = true;
1037	}
1038	}
1039
1040	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1041	READ_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1042	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1043	{
1044	uint64_t u64Tmp;
1045	READ_MSR(MSR_IA32_SYSENTER_EIP, u64Tmp);
1046	pVCpu->cpum.GstCtx.SysEnter.eip = u64Tmp;
1047	READ_MSR(MSR_IA32_SYSENTER_ESP, u64Tmp);
1048	pVCpu->cpum.GstCtx.SysEnter.esp = u64Tmp;
1049	READ_MSR(MSR_IA32_SYSENTER_CS, u64Tmp);
1050	pVCpu->cpum.GstCtx.SysEnter.cs = u64Tmp;
1051	}
1052	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1053	{
1054	READ_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1055	READ_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1056	READ_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1057	READ_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1058	}
1059	#if 0
1060	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1061	{
1062	Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
1063	const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
1064	if (aValues[iReg].Reg64 != uOldBase)
1065	{
1066	Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
1067	pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
1068	int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
1069	AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
1070	}
1071	iReg++;
1072
1073	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
1074	#if 0 /def LOG_ENABLED/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
1075	GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
1076	#endif
1077	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1078	GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType, WHvX64RegisterMsrMtrrDefType, "MSR MTRR_DEF_TYPE");
1079	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
1080	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
1081	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
1082	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000, WHvX64RegisterMsrMtrrFix4kC0000, "MSR MTRR_FIX_4K_C0000");
1083	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000, WHvX64RegisterMsrMtrrFix4kC8000, "MSR MTRR_FIX_4K_C8000");
1084	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000, WHvX64RegisterMsrMtrrFix4kD0000, "MSR MTRR_FIX_4K_D0000");
1085	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000, WHvX64RegisterMsrMtrrFix4kD8000, "MSR MTRR_FIX_4K_D8000");
1086	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000, WHvX64RegisterMsrMtrrFix4kE0000, "MSR MTRR_FIX_4K_E0000");
1087	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000, WHvX64RegisterMsrMtrrFix4kE8000, "MSR MTRR_FIX_4K_E8000");
1088	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000, WHvX64RegisterMsrMtrrFix4kF0000, "MSR MTRR_FIX_4K_F0000");
1089	GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000, WHvX64RegisterMsrMtrrFix4kF8000, "MSR MTRR_FIX_4K_F8000");
1090	GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
1091	/** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
1092	}
1093	#endif
1094
1095	/* Almost done, just update extrn flags and maybe change PGM mode. */
1096	pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
1097	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1098	pVCpu->cpum.GstCtx.fExtrn = 0;
1099
1100	#ifdef LOG_ENABLED
1101	nemR3DarwinLogState(pVM, pVCpu);
1102	#endif
1103
1104	/* Typical. */
1105	if (!fMaybeChangedMode && !fUpdateCr3)
1106	{
1107	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1108	return VINF_SUCCESS;
1109	}
1110
1111	/*
1112	* Slow.
1113	*/
1114	if (fMaybeChangedMode)
1115	{
1116	int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER);
1117	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
1118	}
1119
1120	if (fUpdateCr3)
1121	{
1122	int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3, false /fPdpesMapped/);
1123	if (rc == VINF_SUCCESS)
1124	{ /* likely */ }
1125	else
1126	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
1127	}
1128
1129	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateImport, x);
1130
1131	return VINF_SUCCESS;
1132	#undef READ_GREG
1133	#undef READ_VMCS_FIELD
1134	#undef READ_VMCS32_FIELD
1135	#undef READ_SEG
1136	#undef READ_MSR
1137	}
1138
1139
1140	/**
1141	* State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
1142	* and nemHCWinHandleMemoryAccessPageCheckerCallback.
1143	*/
1144	typedef struct NEMHCDARWINHMACPCCSTATE
1145	{
1146	/** Input: Write access. */
1147	bool fWriteAccess;
1148	/** Output: Set if we did something. */
1149	bool fDidSomething;
1150	/** Output: Set it we should resume. */
1151	bool fCanResume;
1152	} NEMHCDARWINHMACPCCSTATE;
1153
1154	/**
1155	* @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
1156	* Worker for nemR3WinHandleMemoryAccess; pvUser points to a
1157	* NEMHCDARWINHMACPCCSTATE structure. }
1158	*/
1159	static DECLCALLBACK(int)
1160	nemR3DarwinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
1161	{
1162	NEMHCDARWINHMACPCCSTATE pState = (NEMHCDARWINHMACPCCSTATE )pvUser;
1163	pState->fDidSomething = false;
1164	pState->fCanResume = false;
1165
1166	uint8_t u2State = pInfo->u2NemState;
1167
1168	/*
1169	* Consolidate current page state with actual page protection and access type.
1170	* We don't really consider downgrades here, as they shouldn't happen.
1171	*/
1172	int rc;
1173	switch (u2State)
1174	{
1175	case NEM_DARWIN_PAGE_STATE_UNMAPPED:
1176	case NEM_DARWIN_PAGE_STATE_NOT_SET:
1177	if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
1178	{
1179	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
1180	return VINF_SUCCESS;
1181	}
1182
1183	/* Don't bother remapping it if it's a write request to a non-writable page. */
1184	if ( pState->fWriteAccess
1185	&& !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
1186	{
1187	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
1188	return VINF_SUCCESS;
1189	}
1190
1191	/* Map the page. */
1192	rc = nemHCNativeSetPhysPage(pVM,
1193	pVCpu,
1194	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1195	GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
1196	pInfo->fNemProt,
1197	&u2State,
1198	true /fBackingState/);
1199	pInfo->u2NemState = u2State;
1200	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
1201	GCPhys, g_apszPageStates[u2State], rc));
1202	pState->fDidSomething = true;
1203	pState->fCanResume = true;
1204	return rc;
1205
1206	case NEM_DARWIN_PAGE_STATE_READABLE:
1207	if ( !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1208	&& (pInfo->fNemProt & (NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE)))
1209	{
1210	pState->fCanResume = true;
1211	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
1212	return VINF_SUCCESS;
1213	}
1214	break;
1215
1216	case NEM_DARWIN_PAGE_STATE_WRITABLE:
1217	if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
1218	{
1219	/* We get spurious EPT exit violations when everything is fine (#3a case) but can resume without issues here... */
1220	pState->fCanResume = true;
1221	if (pInfo->u2OldNemState == NEM_DARWIN_PAGE_STATE_WRITABLE)
1222	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
1223	else
1224	Log4(("nemR3DarwinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
1225	GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
1226	return VINF_SUCCESS;
1227	}
1228
1229	break;
1230
1231	default:
1232	AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
1233	}
1234
1235	/*
1236	* Unmap and restart the instruction.
1237	* If this fails, which it does every so often, just unmap everything for now.
1238	*/
1239	rc = nemR3DarwinUnmap(pVM, GCPhys, X86_PAGE_SIZE);
1240	if (RT_SUCCESS(rc))
1241	{
1242	pState->fDidSomething = true;
1243	pState->fCanResume = true;
1244	pInfo->u2NemState = NEM_DARWIN_PAGE_STATE_UNMAPPED;
1245	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
1246	Log5(("NEM GPA unmapped/exit: %RGp (was %s)\n", GCPhys, g_apszPageStates[u2State]));
1247	return VINF_SUCCESS;
1248	}
1249	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
1250	LogRel(("nemR3DarwinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s rc=%Rrc\n",
1251	GCPhys, g_apszPageStates[u2State], rc));
1252	return VERR_NEM_UNMAP_PAGES_FAILED;
1253	}
1254
1255
1256	DECL_FORCE_INLINE(bool) vmxHCShouldSwapEferMsr(PCVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
1257	{
1258	RT_NOREF(pVCpu, pVmxTransient);
1259	return true;
1260	}
1261
1262
1263	DECL_FORCE_INLINE(bool) nemR3DarwinIsUnrestrictedGuest(PCVMCC pVM)
1264	{
1265	RT_NOREF(pVM);
1266	return true;
1267	}
1268
1269
1270	DECL_FORCE_INLINE(bool) nemR3DarwinIsNestedPaging(PCVMCC pVM)
1271	{
1272	RT_NOREF(pVM);
1273	return true;
1274	}
1275
1276
1277	DECL_FORCE_INLINE(bool) nemR3DarwinIsPreemptTimerUsed(PCVMCC pVM)
1278	{
1279	RT_NOREF(pVM);
1280	return false;
1281	}
1282
1283
1284	#if 0 /* unused */
1285	DECL_FORCE_INLINE(bool) nemR3DarwinIsVmxLbr(PCVMCC pVM)
1286	{
1287	RT_NOREF(pVM);
1288	return false;
1289	}
1290	#endif
1291
1292
1293	/*
1294	* Instantiate the code we share with ring-0.
1295	*/
1296	#define IN_NEM_DARWIN
1297	//#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
1298	//#define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
1299	#define VCPU_2_VMXSTATE(a_pVCpu) (a_pVCpu)->nem.s
1300	#define VCPU_2_VMXSTATS(a_pVCpu) (*(a_pVCpu)->nem.s.pVmxStats)
1301
1302	#define VM_IS_VMX_UNRESTRICTED_GUEST(a_pVM) nemR3DarwinIsUnrestrictedGuest((a_pVM))
1303	#define VM_IS_VMX_NESTED_PAGING(a_pVM) nemR3DarwinIsNestedPaging((a_pVM))
1304	#define VM_IS_VMX_PREEMPT_TIMER_USED(a_pVM) nemR3DarwinIsPreemptTimerUsed((a_pVM))
1305	#define VM_IS_VMX_LBR(a_pVM) nemR3DarwinIsVmxLbr((a_pVM))
1306
1307	#define VMX_VMCS_WRITE_16(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs16((a_pVCpu), (a_FieldEnc), (a_Val))
1308	#define VMX_VMCS_WRITE_32(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs32((a_pVCpu), (a_FieldEnc), (a_Val))
1309	#define VMX_VMCS_WRITE_64(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1310	#define VMX_VMCS_WRITE_NW(a_pVCpu, a_FieldEnc, a_Val) nemR3DarwinWriteVmcs64((a_pVCpu), (a_FieldEnc), (a_Val))
1311
1312	#define VMX_VMCS_READ_16(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs16((a_pVCpu), (a_FieldEnc), (a_pVal))
1313	#define VMX_VMCS_READ_32(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs32((a_pVCpu), (a_FieldEnc), (a_pVal))
1314	#define VMX_VMCS_READ_64(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1315	#define VMX_VMCS_READ_NW(a_pVCpu, a_FieldEnc, a_pVal) nemR3DarwinReadVmcs64((a_pVCpu), (a_FieldEnc), (a_pVal))
1316
1317	#include "../VMMAll/VMXAllTemplate.cpp.h"
1318
1319	#undef VMX_VMCS_WRITE_16
1320	#undef VMX_VMCS_WRITE_32
1321	#undef VMX_VMCS_WRITE_64
1322	#undef VMX_VMCS_WRITE_NW
1323
1324	#undef VMX_VMCS_READ_16
1325	#undef VMX_VMCS_READ_32
1326	#undef VMX_VMCS_READ_64
1327	#undef VMX_VMCS_READ_NW
1328
1329	#undef VM_IS_VMX_PREEMPT_TIMER_USED
1330	#undef VM_IS_VMX_NESTED_PAGING
1331	#undef VM_IS_VMX_UNRESTRICTED_GUEST
1332	#undef VCPU_2_VMXSTATS
1333	#undef VCPU_2_VMXSTATE
1334
1335
1336	/**
1337	* Exports the guest GP registers to HV for execution.
1338	*
1339	* @returns VBox status code.
1340	* @param pVCpu The cross context virtual CPU structure of the
1341	* calling EMT.
1342	*/
1343	static int nemR3DarwinExportGuestGprs(PVMCPUCC pVCpu)
1344	{
1345	#define WRITE_GREG(a_GReg, a_Value) \
1346	do \
1347	{ \
1348	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1349	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1350	{ /* likely */ } \
1351	else \
1352	return VERR_INTERNAL_ERROR; \
1353	} while(0)
1354
1355	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->nem.s.fCtxChanged);
1356	if (fCtxChanged & HM_CHANGED_GUEST_GPRS_MASK)
1357	{
1358	if (fCtxChanged & HM_CHANGED_GUEST_RAX)
1359	WRITE_GREG(HV_X86_RAX, pVCpu->cpum.GstCtx.rax);
1360	if (fCtxChanged & HM_CHANGED_GUEST_RCX)
1361	WRITE_GREG(HV_X86_RCX, pVCpu->cpum.GstCtx.rcx);
1362	if (fCtxChanged & HM_CHANGED_GUEST_RDX)
1363	WRITE_GREG(HV_X86_RDX, pVCpu->cpum.GstCtx.rdx);
1364	if (fCtxChanged & HM_CHANGED_GUEST_RBX)
1365	WRITE_GREG(HV_X86_RBX, pVCpu->cpum.GstCtx.rbx);
1366	if (fCtxChanged & HM_CHANGED_GUEST_RSP)
1367	WRITE_GREG(HV_X86_RSP, pVCpu->cpum.GstCtx.rsp);
1368	if (fCtxChanged & HM_CHANGED_GUEST_RBP)
1369	WRITE_GREG(HV_X86_RBP, pVCpu->cpum.GstCtx.rbp);
1370	if (fCtxChanged & HM_CHANGED_GUEST_RSI)
1371	WRITE_GREG(HV_X86_RSI, pVCpu->cpum.GstCtx.rsi);
1372	if (fCtxChanged & HM_CHANGED_GUEST_RDI)
1373	WRITE_GREG(HV_X86_RDI, pVCpu->cpum.GstCtx.rdi);
1374	if (fCtxChanged & HM_CHANGED_GUEST_R8_R15)
1375	{
1376	WRITE_GREG(HV_X86_R8, pVCpu->cpum.GstCtx.r8);
1377	WRITE_GREG(HV_X86_R9, pVCpu->cpum.GstCtx.r9);
1378	WRITE_GREG(HV_X86_R10, pVCpu->cpum.GstCtx.r10);
1379	WRITE_GREG(HV_X86_R11, pVCpu->cpum.GstCtx.r11);
1380	WRITE_GREG(HV_X86_R12, pVCpu->cpum.GstCtx.r12);
1381	WRITE_GREG(HV_X86_R13, pVCpu->cpum.GstCtx.r13);
1382	WRITE_GREG(HV_X86_R14, pVCpu->cpum.GstCtx.r14);
1383	WRITE_GREG(HV_X86_R15, pVCpu->cpum.GstCtx.r15);
1384	}
1385
1386	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_GPRS_MASK);
1387	}
1388
1389	if (fCtxChanged & HM_CHANGED_GUEST_CR2)
1390	{
1391	WRITE_GREG(HV_X86_CR2, pVCpu->cpum.GstCtx.cr2);
1392	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_CR2);
1393	}
1394
1395	return VINF_SUCCESS;
1396	#undef WRITE_GREG
1397	}
1398
1399
1400	/**
1401	* Converts the given CPUM externalized bitmask to the appropriate HM changed bitmask.
1402	*
1403	* @returns Bitmask of HM changed flags.
1404	* @param fCpumExtrn The CPUM extern bitmask.
1405	*/
1406	static uint64_t nemR3DarwinCpumExtrnToHmChanged(uint64_t fCpumExtrn)
1407	{
1408	uint64_t fHmChanged = 0;
1409
1410	/* Invert to gt a mask of things which are kept in CPUM. */
1411	uint64_t fCpumIntern = ~fCpumExtrn;
1412
1413	if (fCpumIntern & CPUMCTX_EXTRN_GPRS_MASK)
1414	{
1415	if (fCpumIntern & CPUMCTX_EXTRN_RAX)
1416	fHmChanged \|= HM_CHANGED_GUEST_RAX;
1417	if (fCpumIntern & CPUMCTX_EXTRN_RCX)
1418	fHmChanged \|= HM_CHANGED_GUEST_RCX;
1419	if (fCpumIntern & CPUMCTX_EXTRN_RDX)
1420	fHmChanged \|= HM_CHANGED_GUEST_RDX;
1421	if (fCpumIntern & CPUMCTX_EXTRN_RBX)
1422	fHmChanged \|= HM_CHANGED_GUEST_RBX;
1423	if (fCpumIntern & CPUMCTX_EXTRN_RSP)
1424	fHmChanged \|= HM_CHANGED_GUEST_RSP;
1425	if (fCpumIntern & CPUMCTX_EXTRN_RBP)
1426	fHmChanged \|= HM_CHANGED_GUEST_RBP;
1427	if (fCpumIntern & CPUMCTX_EXTRN_RSI)
1428	fHmChanged \|= HM_CHANGED_GUEST_RSI;
1429	if (fCpumIntern & CPUMCTX_EXTRN_RDI)
1430	fHmChanged \|= HM_CHANGED_GUEST_RDI;
1431	if (fCpumIntern & CPUMCTX_EXTRN_R8_R15)
1432	fHmChanged \|= HM_CHANGED_GUEST_R8_R15;
1433	}
1434
1435	/* RIP & Flags */
1436	if (fCpumIntern & CPUMCTX_EXTRN_RIP)
1437	fHmChanged \|= HM_CHANGED_GUEST_RIP;
1438	if (fCpumIntern & CPUMCTX_EXTRN_RFLAGS)
1439	fHmChanged \|= HM_CHANGED_GUEST_RFLAGS;
1440
1441	/* Segments */
1442	if (fCpumIntern & CPUMCTX_EXTRN_SREG_MASK)
1443	{
1444	if (fCpumIntern & CPUMCTX_EXTRN_ES)
1445	fHmChanged \|= HM_CHANGED_GUEST_ES;
1446	if (fCpumIntern & CPUMCTX_EXTRN_CS)
1447	fHmChanged \|= HM_CHANGED_GUEST_CS;
1448	if (fCpumIntern & CPUMCTX_EXTRN_SS)
1449	fHmChanged \|= HM_CHANGED_GUEST_SS;
1450	if (fCpumIntern & CPUMCTX_EXTRN_DS)
1451	fHmChanged \|= HM_CHANGED_GUEST_DS;
1452	if (fCpumIntern & CPUMCTX_EXTRN_FS)
1453	fHmChanged \|= HM_CHANGED_GUEST_FS;
1454	if (fCpumIntern & CPUMCTX_EXTRN_GS)
1455	fHmChanged \|= HM_CHANGED_GUEST_GS;
1456	}
1457
1458	/* Descriptor tables & task segment. */
1459	if (fCpumIntern & CPUMCTX_EXTRN_TABLE_MASK)
1460	{
1461	if (fCpumIntern & CPUMCTX_EXTRN_LDTR)
1462	fHmChanged \|= HM_CHANGED_GUEST_LDTR;
1463	if (fCpumIntern & CPUMCTX_EXTRN_TR)
1464	fHmChanged \|= HM_CHANGED_GUEST_TR;
1465	if (fCpumIntern & CPUMCTX_EXTRN_IDTR)
1466	fHmChanged \|= HM_CHANGED_GUEST_IDTR;
1467	if (fCpumIntern & CPUMCTX_EXTRN_GDTR)
1468	fHmChanged \|= HM_CHANGED_GUEST_GDTR;
1469	}
1470
1471	/* Control registers. */
1472	if (fCpumIntern & CPUMCTX_EXTRN_CR_MASK)
1473	{
1474	if (fCpumIntern & CPUMCTX_EXTRN_CR0)
1475	fHmChanged \|= HM_CHANGED_GUEST_CR0;
1476	if (fCpumIntern & CPUMCTX_EXTRN_CR2)
1477	fHmChanged \|= HM_CHANGED_GUEST_CR2;
1478	if (fCpumIntern & CPUMCTX_EXTRN_CR3)
1479	fHmChanged \|= HM_CHANGED_GUEST_CR3;
1480	if (fCpumIntern & CPUMCTX_EXTRN_CR4)
1481	fHmChanged \|= HM_CHANGED_GUEST_CR4;
1482	}
1483	if (fCpumIntern & CPUMCTX_EXTRN_APIC_TPR)
1484	fHmChanged \|= HM_CHANGED_GUEST_APIC_TPR;
1485
1486	/* Debug registers. */
1487	if (fCpumIntern & CPUMCTX_EXTRN_DR0_DR3)
1488	fHmChanged \|= HM_CHANGED_GUEST_DR0_DR3;
1489	if (fCpumIntern & CPUMCTX_EXTRN_DR6)
1490	fHmChanged \|= HM_CHANGED_GUEST_DR6;
1491	if (fCpumIntern & CPUMCTX_EXTRN_DR7)
1492	fHmChanged \|= HM_CHANGED_GUEST_DR7;
1493
1494	/* Floating point state. */
1495	if (fCpumIntern & CPUMCTX_EXTRN_X87)
1496	fHmChanged \|= HM_CHANGED_GUEST_X87;
1497	if (fCpumIntern & CPUMCTX_EXTRN_SSE_AVX)
1498	fHmChanged \|= HM_CHANGED_GUEST_SSE_AVX;
1499	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_XSAVE)
1500	fHmChanged \|= HM_CHANGED_GUEST_OTHER_XSAVE;
1501	if (fCpumIntern & CPUMCTX_EXTRN_XCRx)
1502	fHmChanged \|= HM_CHANGED_GUEST_XCRx;
1503
1504	/* MSRs */
1505	if (fCpumIntern & CPUMCTX_EXTRN_EFER)
1506	fHmChanged \|= HM_CHANGED_GUEST_EFER_MSR;
1507	if (fCpumIntern & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1508	fHmChanged \|= HM_CHANGED_GUEST_KERNEL_GS_BASE;
1509	if (fCpumIntern & CPUMCTX_EXTRN_SYSENTER_MSRS)
1510	fHmChanged \|= HM_CHANGED_GUEST_SYSENTER_MSR_MASK;
1511	if (fCpumIntern & CPUMCTX_EXTRN_SYSCALL_MSRS)
1512	fHmChanged \|= HM_CHANGED_GUEST_SYSCALL_MSRS;
1513	if (fCpumIntern & CPUMCTX_EXTRN_TSC_AUX)
1514	fHmChanged \|= HM_CHANGED_GUEST_TSC_AUX;
1515	if (fCpumIntern & CPUMCTX_EXTRN_OTHER_MSRS)
1516	fHmChanged \|= HM_CHANGED_GUEST_OTHER_MSRS;
1517
1518	return fHmChanged;
1519	}
1520
1521
1522	/**
1523	* Exports the guest state to HV for execution.
1524	*
1525	* @returns VBox status code.
1526	* @param pVM The cross context VM structure.
1527	* @param pVCpu The cross context virtual CPU structure of the
1528	* calling EMT.
1529	* @param pVmxTransient The transient VMX structure.
1530	*/
1531	static int nemR3DarwinExportGuestState(PVMCC pVM, PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
1532	{
1533	#define WRITE_GREG(a_GReg, a_Value) \
1534	do \
1535	{ \
1536	hv_return_t hrc = hv_vcpu_write_register(pVCpu->nem.s.hVCpuId, (a_GReg), (a_Value)); \
1537	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1538	{ /* likely */ } \
1539	else \
1540	return VERR_INTERNAL_ERROR; \
1541	} while(0)
1542	#define WRITE_VMCS_FIELD(a_Field, a_Value) \
1543	do \
1544	{ \
1545	hv_return_t hrc = hv_vmx_vcpu_write_vmcs(pVCpu->nem.s.hVCpuId, (a_Field), (a_Value)); \
1546	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1547	{ /* likely */ } \
1548	else \
1549	return VERR_INTERNAL_ERROR; \
1550	} while(0)
1551	#define WRITE_MSR(a_Msr, a_Value) \
1552	do \
1553	{ \
1554	hv_return_t hrc = hv_vcpu_write_msr(pVCpu->nem.s.hVCpuId, (a_Msr), (a_Value)); \
1555	if (RT_LIKELY(hrc == HV_SUCCESS)) \
1556	{ /* likely */ } \
1557	else \
1558	AssertFailedReturn(VERR_INTERNAL_ERROR); \
1559	} while(0)
1560
1561	RT_NOREF(pVM);
1562
1563	#ifdef LOG_ENABLED
1564	nemR3DarwinLogState(pVM, pVCpu);
1565	#endif
1566
1567	STAM_PROFILE_ADV_START(&pVCpu->nem.s.StatProfGstStateExport, x);
1568
1569	uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL;
1570	if (!fWhat)
1571	return VINF_SUCCESS;
1572
1573	pVCpu->nem.s.fCtxChanged \|= nemR3DarwinCpumExtrnToHmChanged(pVCpu->cpum.GstCtx.fExtrn);
1574
1575	int rc = vmxHCExportGuestEntryExitCtls(pVCpu, pVmxTransient);
1576	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1577
1578	rc = nemR3DarwinExportGuestGprs(pVCpu);
1579	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1580
1581	rc = vmxHCExportGuestCR0(pVCpu, pVmxTransient);
1582	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1583
1584	VBOXSTRICTRC rcStrict = vmxHCExportGuestCR3AndCR4(pVCpu, pVmxTransient);
1585	if (rcStrict == VINF_SUCCESS)
1586	{ /* likely */ }
1587	else
1588	{
1589	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
1590	return VBOXSTRICTRC_VAL(rcStrict);
1591	}
1592
1593	vmxHCExportGuestXcptIntercepts(pVCpu, pVmxTransient);
1594	vmxHCExportGuestRip(pVCpu);
1595	//vmxHCExportGuestRsp(pVCpu);
1596	vmxHCExportGuestRflags(pVCpu, pVmxTransient);
1597
1598	rc = vmxHCExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
1599	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
1600
1601	if (fWhat & CPUMCTX_EXTRN_XCRx)
1602	{
1603	WRITE_GREG(HV_X86_XCR0, pVCpu->cpum.GstCtx.aXcr[0]);
1604	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_XCRx);
1605	}
1606
1607	if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
1608	{
1609	Assert(pVCpu->nem.s.fCtxChanged & HM_CHANGED_GUEST_APIC_TPR);
1610	vmxHCExportGuestApicTpr(pVCpu, pVmxTransient);
1611
1612	rc = APICGetTpr(pVCpu, &pVmxTransient->u8GuestTpr, NULL /pfPending/, NULL /pu8PendingIntr/);
1613	AssertRC(rc);
1614
1615	WRITE_GREG(HV_X86_TPR, pVmxTransient->u8GuestTpr);
1616	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
1617	}
1618
1619	/* Debug registers. */
1620	if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
1621	{
1622	WRITE_GREG(HV_X86_DR0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
1623	WRITE_GREG(HV_X86_DR1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
1624	WRITE_GREG(HV_X86_DR2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
1625	WRITE_GREG(HV_X86_DR3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
1626	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR0_DR3);
1627	}
1628	if (fWhat & CPUMCTX_EXTRN_DR6)
1629	{
1630	WRITE_GREG(HV_X86_DR6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
1631	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR6);
1632	}
1633	if (fWhat & CPUMCTX_EXTRN_DR7)
1634	{
1635	WRITE_GREG(HV_X86_DR7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
1636	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_DR7);
1637	}
1638
1639	if (fWhat & (CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE))
1640	{
1641	hv_return_t hrc = hv_vcpu_write_fpstate(pVCpu->nem.s.hVCpuId, &pVCpu->cpum.GstCtx.XState, sizeof(pVCpu->cpum.GstCtx.XState));
1642	if (hrc == HV_SUCCESS)
1643	{ /* likely */ }
1644	else
1645	return nemR3DarwinHvSts2Rc(hrc);
1646
1647	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(HM_CHANGED_GUEST_X87 \| HM_CHANGED_GUEST_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE));
1648	}
1649
1650	/* MSRs */
1651	if (fWhat & CPUMCTX_EXTRN_EFER)
1652	{
1653	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_EFER_FULL, pVCpu->cpum.GstCtx.msrEFER);
1654	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
1655	}
1656	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
1657	{
1658	WRITE_MSR(MSR_K8_KERNEL_GS_BASE, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
1659	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_KERNEL_GS_BASE);
1660	}
1661	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
1662	{
1663	WRITE_MSR(MSR_IA32_SYSENTER_CS, pVCpu->cpum.GstCtx.SysEnter.cs);
1664	WRITE_MSR(MSR_IA32_SYSENTER_EIP, pVCpu->cpum.GstCtx.SysEnter.eip);
1665	WRITE_MSR(MSR_IA32_SYSENTER_ESP, pVCpu->cpum.GstCtx.SysEnter.esp);
1666	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
1667	}
1668	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
1669	{
1670	WRITE_MSR(MSR_K6_STAR, pVCpu->cpum.GstCtx.msrSTAR);
1671	WRITE_MSR(MSR_K8_LSTAR, pVCpu->cpum.GstCtx.msrLSTAR);
1672	WRITE_MSR(MSR_K8_CSTAR, pVCpu->cpum.GstCtx.msrCSTAR);
1673	WRITE_MSR(MSR_K8_SF_MASK, pVCpu->cpum.GstCtx.msrSFMASK);
1674	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSCALL_MSRS);
1675	}
1676	if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
1677	{
1678	#if 0
1679	hv_return_t hrc = hv_vmx_vcpu_set_apic_address(pVCpu->nem.s.hVCpuId, APICGetBaseMsrNoCheck(pVCpu) & PAGE_BASE_GC_MASK);
1680	if (RT_UNLIKELY(hrc != HV_SUCCESS))
1681	return nemR3DarwinHvSts2Rc(hrc);
1682	#endif
1683
1684	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
1685
1686	#if 0
1687	ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
1688	#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
1689	ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
1690	#endif
1691	PCPUMCTXMSRS pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
1692	ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
1693	ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
1694	ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
1695	ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
1696	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000, pCtxMsrs->msr.MtrrFix4K_C0000);
1697	ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000, pCtxMsrs->msr.MtrrFix4K_C8000);
1698	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000, pCtxMsrs->msr.MtrrFix4K_D0000);
1699	ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000, pCtxMsrs->msr.MtrrFix4K_D8000);
1700	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000, pCtxMsrs->msr.MtrrFix4K_E0000);
1701	ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000, pCtxMsrs->msr.MtrrFix4K_E8000);
1702	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000, pCtxMsrs->msr.MtrrFix4K_F0000);
1703	ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000, pCtxMsrs->msr.MtrrFix4K_F8000);
1704	ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
1705	#if 0 /** @todo these registers aren't available? Might explain something.. .*/
1706	const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
1707	if (enmCpuVendor != CPUMCPUVENDOR_AMD)
1708	{
1709	ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
1710	ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
1711	}
1712	#endif
1713	#endif
1714	}
1715
1716	WRITE_VMCS_FIELD(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0 /MSR_IA32_DEBUGCTL_LBR/);
1717
1718	hv_vcpu_invalidate_tlb(pVCpu->nem.s.hVCpuId);
1719	hv_vcpu_flush(pVCpu->nem.s.hVCpuId);
1720
1721	pVCpu->cpum.GstCtx.fExtrn \|= CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_NEM;
1722
1723	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
1724	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~(
1725	HM_CHANGED_GUEST_TSC_AUX
1726	\| HM_CHANGED_GUEST_HWVIRT
1727	\| HM_CHANGED_VMX_GUEST_AUTO_MSRS
1728	\| HM_CHANGED_VMX_GUEST_LAZY_MSRS
1729	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
1730
1731	STAM_PROFILE_ADV_STOP(&pVCpu->nem.s.StatProfGstStateExport, x);
1732	return VINF_SUCCESS;
1733	#undef WRITE_GREG
1734	#undef WRITE_VMCS_FIELD
1735	}
1736
1737
1738	/**
1739	* Handles an exit from hv_vcpu_run().
1740	*
1741	* @returns VBox strict status code.
1742	* @param pVM The cross context VM structure.
1743	* @param pVCpu The cross context virtual CPU structure of the
1744	* calling EMT.
1745	* @param pVmxTransient The transient VMX structure.
1746	*/
1747	static VBOXSTRICTRC nemR3DarwinHandleExit(PVM pVM, PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1748	{
1749	uint32_t uExitReason;
1750	int rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
1751	AssertRC(rc);
1752	pVmxTransient->fVmcsFieldsRead = 0;
1753	pVmxTransient->fIsNestedGuest = false;
1754	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
1755	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
1756
1757	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed))
1758	AssertLogRelMsgFailedReturn(("Running guest failed for CPU #%u: %#x %u\n",
1759	pVCpu->idCpu, pVmxTransient->uExitReason, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
1760	VERR_NEM_IPE_0);
1761
1762	/** @todo Only copy the state on demand (the R0 VT-x code saves some stuff unconditionally and the VMX template assumes that
1763	* when handling exits). */
1764	rc = nemR3DarwinCopyStateFromHv(pVM, pVCpu, CPUMCTX_EXTRN_ALL);
1765	AssertRCReturn(rc, rc);
1766
1767	STAM_COUNTER_INC(&pVCpu->nem.s.pVmxStats->aStatExitReason[pVmxTransient->uExitReason & MASK_EXITREASON_STAT]);
1768	STAM_REL_COUNTER_INC(&pVCpu->nem.s.pVmxStats->StatExitAll);
1769
1770	#ifndef HMVMX_USE_FUNCTION_TABLE
1771	return vmxHCHandleExit(pVCpu, pVmxTransient);
1772	#else
1773	return g_aVMExitHandlers[pVmxTransient->uExitReason].pfn(pVCpu, pVmxTransient);
1774	#endif
1775	}
1776
1777
1778	/**
1779	* Worker for nemR3NativeInit that loads the Hypervisor.framework shared library.
1780	*
1781	* @returns VBox status code.
1782	* @param fForced Whether the HMForced flag is set and we should
1783	* fail if we cannot initialize.
1784	* @param pErrInfo Where to always return error info.
1785	*/
1786	static int nemR3DarwinLoadHv(bool fForced, PRTERRINFO pErrInfo)
1787	{
1788	RTLDRMOD hMod = NIL_RTLDRMOD;
1789	static const char *s_pszHvPath = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
1790
1791	int rc = RTLdrLoadEx(s_pszHvPath, &hMod, RTLDRLOAD_FLAGS_NO_UNLOAD \| RTLDRLOAD_FLAGS_NO_SUFFIX, pErrInfo);
1792	if (RT_SUCCESS(rc))
1793	{
1794	for (unsigned i = 0; i < RT_ELEMENTS(g_aImports); i++)
1795	{
1796	int rc2 = RTLdrGetSymbol(hMod, g_aImports[i].pszName, (void **)g_aImports[i].ppfn);
1797	if (RT_SUCCESS(rc2))
1798	{
1799	if (g_aImports[i].fOptional)
1800	LogRel(("NEM: info: Found optional import Hypervisor!%s.\n",
1801	g_aImports[i].pszName));
1802	}
1803	else
1804	{
1805	*g_aImports[i].ppfn = NULL;
1806
1807	LogRel(("NEM: %s: Failed to import Hypervisor!%s: %Rrc\n",
1808	g_aImports[i].fOptional ? "info" : fForced ? "fatal" : "error",
1809	g_aImports[i].pszName, rc2));
1810	if (!g_aImports[i].fOptional)
1811	{
1812	if (RTErrInfoIsSet(pErrInfo))
1813	RTErrInfoAddF(pErrInfo, rc2, ", Hypervisor!%s", g_aImports[i].pszName);
1814	else
1815	rc = RTErrInfoSetF(pErrInfo, rc2, "Failed to import: Hypervisor!%s", g_aImports[i].pszName);
1816	Assert(RT_FAILURE(rc));
1817	}
1818	}
1819	}
1820	if (RT_SUCCESS(rc))
1821	{
1822	Assert(!RTErrInfoIsSet(pErrInfo));
1823	}
1824
1825	RTLdrClose(hMod);
1826	}
1827	else
1828	{
1829	RTErrInfoAddF(pErrInfo, rc, "Failed to load Hypervisor.framwork: %s: %Rrc", s_pszHvPath, rc);
1830	rc = VERR_NEM_INIT_FAILED;
1831	}
1832
1833	return rc;
1834	}
1835
1836
1837	/**
1838	* Read and initialize the global capabilities supported by this CPU.
1839	*
1840	* @returns VBox status code.
1841	*/
1842	static int nemR3DarwinCapsInit(void)
1843	{
1844	RT_ZERO(g_HmMsrs);
1845
1846	hv_return_t hrc = hv_vmx_read_capability(HV_VMX_CAP_PINBASED, &g_HmMsrs.u.vmx.PinCtls.u);
1847	if (hrc == HV_SUCCESS)
1848	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED, &g_HmMsrs.u.vmx.ProcCtls.u);
1849	if (hrc == HV_SUCCESS)
1850	hrc = hv_vmx_read_capability(HV_VMX_CAP_ENTRY, &g_HmMsrs.u.vmx.EntryCtls.u);
1851	if (hrc == HV_SUCCESS)
1852	hrc = hv_vmx_read_capability(HV_VMX_CAP_EXIT, &g_HmMsrs.u.vmx.ExitCtls.u);
1853	if (hrc == HV_SUCCESS)
1854	{
1855	hrc = hv_vmx_read_capability(HV_VMX_CAP_BASIC, &g_HmMsrs.u.vmx.u64Basic);
1856	if (hrc == HV_SUCCESS)
1857	{
1858	if (hrc == HV_SUCCESS)
1859	hrc = hv_vmx_read_capability(HV_VMX_CAP_MISC, &g_HmMsrs.u.vmx.u64Misc);
1860	if (hrc == HV_SUCCESS)
1861	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED0, &g_HmMsrs.u.vmx.u64Cr0Fixed0);
1862	if (hrc == HV_SUCCESS)
1863	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR0_FIXED1, &g_HmMsrs.u.vmx.u64Cr0Fixed1);
1864	if (hrc == HV_SUCCESS)
1865	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED0, &g_HmMsrs.u.vmx.u64Cr4Fixed0);
1866	if (hrc == HV_SUCCESS)
1867	hrc = hv_vmx_read_capability(HV_VMX_CAP_CR4_FIXED1, &g_HmMsrs.u.vmx.u64Cr4Fixed1);
1868	if (hrc == HV_SUCCESS)
1869	hrc = hv_vmx_read_capability(HV_VMX_CAP_VMCS_ENUM, &g_HmMsrs.u.vmx.u64VmcsEnum);
1870	if ( hrc == HV_SUCCESS
1871	&& RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_TRUE_CTLS))
1872	{
1873	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PINBASED, &g_HmMsrs.u.vmx.TruePinCtls.u);
1874	if (hrc == HV_SUCCESS)
1875	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_PROCBASED, &g_HmMsrs.u.vmx.TrueProcCtls.u);
1876	if (hrc == HV_SUCCESS)
1877	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_ENTRY, &g_HmMsrs.u.vmx.TrueEntryCtls.u);
1878	if (hrc == HV_SUCCESS)
1879	hrc = hv_vmx_read_capability(HV_VMX_CAP_TRUE_EXIT, &g_HmMsrs.u.vmx.TrueExitCtls.u);
1880	}
1881	}
1882	else
1883	{
1884	/* Likely running on anything < 11.0 (BigSur) so provide some sensible defaults. */
1885	g_HmMsrs.u.vmx.u64Cr0Fixed0 = 0x80000021;
1886	g_HmMsrs.u.vmx.u64Cr0Fixed1 = 0xffffffff;
1887	g_HmMsrs.u.vmx.u64Cr4Fixed0 = 0x2000;
1888	g_HmMsrs.u.vmx.u64Cr4Fixed1 = 0x1767ff;
1889	hrc = HV_SUCCESS;
1890	}
1891	}
1892
1893	if ( hrc == HV_SUCCESS
1894	&& g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1895	{
1896	hrc = hv_vmx_read_capability(HV_VMX_CAP_PROCBASED2, &g_HmMsrs.u.vmx.ProcCtls2.u);
1897
1898	if ( hrc == HV_SUCCESS
1899	&& g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & (VMX_PROC_CTLS2_EPT \| VMX_PROC_CTLS2_VPID))
1900	{
1901	hrc = hv_vmx_read_capability(HV_VMX_CAP_EPT_VPID_CAP, &g_HmMsrs.u.vmx.u64EptVpidCaps);
1902	if (hrc != HV_SUCCESS)
1903	hrc = HV_SUCCESS; /* Probably just outdated OS. */
1904	}
1905
1906	g_HmMsrs.u.vmx.u64VmFunc = 0; /* No way to read that on macOS. */
1907	}
1908
1909	if (hrc == HV_SUCCESS)
1910	{
1911	/*
1912	* Check for EFER swapping support.
1913	*/
1914	g_fHmVmxSupportsVmcsEfer = true; //(g_HmMsrs.u.vmx.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
1915	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1916	//&& (g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR);
1917	}
1918
1919	return nemR3DarwinHvSts2Rc(hrc);
1920	}
1921
1922
1923	/**
1924	* Sets up pin-based VM-execution controls in the VMCS.
1925	*
1926	* @returns VBox status code.
1927	* @param pVCpu The cross context virtual CPU structure.
1928	* @param pVmcsInfo The VMCS info. object.
1929	*/
1930	static int nemR3DarwinVmxSetupVmcsPinCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1931	{
1932	//PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1933	uint32_t fVal = g_HmMsrs.u.vmx.PinCtls.n.allowed0; /* Bits set here must always be set. */
1934	uint32_t const fZap = g_HmMsrs.u.vmx.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1935
1936	if (g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
1937	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
1938
1939	#if 0 /** @todo Use preemption timer */
1940	/* Enable the VMX-preemption timer. */
1941	if (pVM->hmr0.s.vmx.fUsePreemptTimer)
1942	{
1943	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
1944	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
1945	}
1946
1947	/* Enable posted-interrupt processing. */
1948	if (pVM->hm.s.fPostedIntrs)
1949	{
1950	Assert(g_HmMsrs.u.vmx.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
1951	Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
1952	fVal \|= VMX_PIN_CTLS_POSTED_INT;
1953	}
1954	#endif
1955
1956	if ((fVal & fZap) != fVal)
1957	{
1958	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
1959	g_HmMsrs.u.vmx.PinCtls.n.allowed0, fVal, fZap));
1960	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1961	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1962	}
1963
1964	/* Commit it to the VMCS and update our cache. */
1965	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, fVal);
1966	AssertRC(rc);
1967	pVmcsInfo->u32PinCtls = fVal;
1968
1969	return VINF_SUCCESS;
1970	}
1971
1972
1973	/**
1974	* Sets up secondary processor-based VM-execution controls in the VMCS.
1975	*
1976	* @returns VBox status code.
1977	* @param pVCpu The cross context virtual CPU structure.
1978	* @param pVmcsInfo The VMCS info. object.
1979	*/
1980	static int nemR3DarwinVmxSetupVmcsProcCtls2(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
1981	{
1982	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1983	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
1984	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1985
1986	/* WBINVD causes a VM-exit. */
1987	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
1988	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
1989
1990	/* Enable the INVPCID instruction if we expose it to the guest and is supported
1991	by the hardware. Without this, guest executing INVPCID would cause a #UD. */
1992	if ( pVM->cpum.ro.GuestFeatures.fInvpcid
1993	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID))
1994	fVal \|= VMX_PROC_CTLS2_INVPCID;
1995
1996	#if 0 /** @todo */
1997	/* Enable VPID. */
1998	if (pVM->hmr0.s.vmx.fVpid)
1999	fVal \|= VMX_PROC_CTLS2_VPID;
2000
2001	if (pVM->hm.s.fVirtApicRegs)
2002	{
2003	/* Enable APIC-register virtualization. */
2004	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2005	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2006
2007	/* Enable virtual-interrupt delivery. */
2008	Assert(g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2009	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2010	}
2011
2012	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is
2013	where the TPR shadow resides. */
2014	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2015	* done dynamically. */
2016	if (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2017	{
2018	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
2019	hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
2020	}
2021	#endif
2022
2023	/* Enable the RDTSCP instruction if we expose it to the guest and is supported
2024	by the hardware. Without this, guest executing RDTSCP would cause a #UD. */
2025	if ( pVM->cpum.ro.GuestFeatures.fRdTscP
2026	&& (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP))
2027	fVal \|= VMX_PROC_CTLS2_RDTSCP;
2028
2029	#if 0
2030	/* Enable Pause-Loop exiting. */
2031	if ( (g_HmMsrs.u.vmx.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
2032	&& pVM->hm.s.vmx.cPleGapTicks
2033	&& pVM->hm.s.vmx.cPleWindowTicks)
2034	{
2035	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2036
2037	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks); AssertRC(rc);
2038	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks); AssertRC(rc);
2039	}
2040	#endif
2041
2042	if ((fVal & fZap) != fVal)
2043	{
2044	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2045	g_HmMsrs.u.vmx.ProcCtls2.n.allowed0, fVal, fZap));
2046	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2047	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2048	}
2049
2050	/* Commit it to the VMCS and update our cache. */
2051	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2052	AssertRC(rc);
2053	pVmcsInfo->u32ProcCtls2 = fVal;
2054
2055	return VINF_SUCCESS;
2056	}
2057
2058
2059	/**
2060	* Enables native access for the given MSR.
2061	*
2062	* @returns VBox status code.
2063	* @param pVCpu The cross context virtual CPU structure.
2064	* @param idMsr The MSR to enable native access for.
2065	*/
2066	static int nemR3DarwinMsrSetNative(PVMCPUCC pVCpu, uint32_t idMsr)
2067	{
2068	hv_return_t hrc = hv_vcpu_enable_native_msr(pVCpu->nem.s.hVCpuId, idMsr, true /enable/);
2069	if (hrc == HV_SUCCESS)
2070	return VINF_SUCCESS;
2071
2072	return nemR3DarwinHvSts2Rc(hrc);
2073	}
2074
2075
2076	/**
2077	* Sets up the MSR permissions which don't change through the lifetime of the VM.
2078	*
2079	* @returns VBox status code.
2080	* @param pVCpu The cross context virtual CPU structure.
2081	* @param pVmcsInfo The VMCS info. object.
2082	*/
2083	static int nemR3DarwinSetupVmcsMsrPermissions(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2084	{
2085	RT_NOREF(pVmcsInfo);
2086
2087	/*
2088	* The guest can access the following MSRs (read, write) without causing
2089	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2090	*/
2091	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2092	int rc;
2093	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_CS); AssertRCReturn(rc, rc);
2094	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_ESP); AssertRCReturn(rc, rc);
2095	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SYSENTER_EIP); AssertRCReturn(rc, rc);
2096	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_GS_BASE); AssertRCReturn(rc, rc);
2097	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_FS_BASE); AssertRCReturn(rc, rc);
2098
2099	/*
2100	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2101	* associated with then. We never need to intercept access (writes need to be
2102	* executed without causing a VM-exit, reads will #GP fault anyway).
2103	*
2104	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2105	* read/write them. We swap the guest/host MSR value using the
2106	* auto-load/store MSR area.
2107	*/
2108	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2109	{
2110	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_PRED_CMD);
2111	AssertRCReturn(rc, rc);
2112	}
2113	#if 0 /* Doesn't work. */
2114	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2115	{
2116	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_FLUSH_CMD);
2117	AssertRCReturn(rc, rc);
2118	}
2119	#endif
2120	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2121	{
2122	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_IA32_SPEC_CTRL);
2123	AssertRCReturn(rc, rc);
2124	}
2125
2126	/*
2127	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2128	* required for 64-bit guests.
2129	*/
2130	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_LSTAR); AssertRCReturn(rc, rc);
2131	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K6_STAR); AssertRCReturn(rc, rc);
2132	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_SF_MASK); AssertRCReturn(rc, rc);
2133	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_KERNEL_GS_BASE); AssertRCReturn(rc, rc);
2134
2135	/* Required for enabling the RDTSCP instruction. */
2136	rc = nemR3DarwinMsrSetNative(pVCpu, MSR_K8_TSC_AUX); AssertRCReturn(rc, rc);
2137
2138	return VINF_SUCCESS;
2139	}
2140
2141
2142	/**
2143	* Sets up processor-based VM-execution controls in the VMCS.
2144	*
2145	* @returns VBox status code.
2146	* @param pVCpu The cross context virtual CPU structure.
2147	* @param pVmcsInfo The VMCS info. object.
2148	*/
2149	static int nemR3DarwinVmxSetupVmcsProcCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2150	{
2151	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2152	uint32_t fVal = g_HmMsrs.u.vmx.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
2153	uint32_t const fZap = g_HmMsrs.u.vmx.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2154
2155	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2156	// \| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2157	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2158	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2159	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2160	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2161	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2162
2163	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2164	if ( !(g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2165	\|\| (g_HmMsrs.u.vmx.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2166	{
2167	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2168	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2169	}
2170
2171	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2172	if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2173	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2174
2175	if ((fVal & fZap) != fVal)
2176	{
2177	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2178	g_HmMsrs.u.vmx.ProcCtls.n.allowed0, fVal, fZap));
2179	pVCpu->nem.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2180	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2181	}
2182
2183	/* Commit it to the VMCS and update our cache. */
2184	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2185	AssertRC(rc);
2186	pVmcsInfo->u32ProcCtls = fVal;
2187
2188	/* Set up MSR permissions that don't change through the lifetime of the VM. */
2189	rc = nemR3DarwinSetupVmcsMsrPermissions(pVCpu, pVmcsInfo);
2190	AssertRCReturn(rc, rc);
2191
2192	/*
2193	* Set up secondary processor-based VM-execution controls
2194	* (we assume the CPU to always support it as we rely on unrestricted guest execution support).
2195	*/
2196	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
2197	return nemR3DarwinVmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
2198	}
2199
2200
2201	/**
2202	* Sets up miscellaneous (everything other than Pin, Processor and secondary
2203	* Processor-based VM-execution) control fields in the VMCS.
2204	*
2205	* @returns VBox status code.
2206	* @param pVCpu The cross context virtual CPU structure.
2207	* @param pVmcsInfo The VMCS info. object.
2208	*/
2209	static int nemR3DarwinVmxSetupVmcsMiscCtls(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2210	{
2211	int rc = VINF_SUCCESS;
2212	//rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo); TODO
2213	if (RT_SUCCESS(rc))
2214	{
2215	uint64_t const u64Cr0Mask = vmxHCGetFixedCr0Mask(pVCpu);
2216	uint64_t const u64Cr4Mask = vmxHCGetFixedCr4Mask(pVCpu);
2217
2218	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask); AssertRC(rc);
2219	rc = nemR3DarwinWriteVmcs64(pVCpu, VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask); AssertRC(rc);
2220
2221	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
2222	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
2223
2224	#if 0 /** @todo */
2225	if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fLbr)
2226	{
2227	rc = VMXWriteVmcsNw(VMX_VMCS64_GUEST_DEBUGCTL_FULL, MSR_IA32_DEBUGCTL_LBR);
2228	AssertRC(rc);
2229	}
2230	#endif
2231	return VINF_SUCCESS;
2232	}
2233	else
2234	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
2235	return rc;
2236	}
2237
2238
2239	/**
2240	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2241	*
2242	* We shall setup those exception intercepts that don't change during the
2243	* lifetime of the VM here. The rest are done dynamically while loading the
2244	* guest state.
2245	*
2246	* @param pVCpu The cross context virtual CPU structure.
2247	* @param pVmcsInfo The VMCS info. object.
2248	*/
2249	static void nemR3DarwinVmxSetupVmcsXcptBitmap(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
2250	{
2251	/*
2252	* The following exceptions are always intercepted:
2253	*
2254	* #AC - To prevent the guest from hanging the CPU and for dealing with
2255	* split-lock detecting host configs.
2256	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
2257	* recursive #DBs can cause a CPU hang.
2258	*/
2259	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
2260	\| RT_BIT(X86_XCPT_DB);
2261
2262	/* Commit it to the VMCS. */
2263	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2264	AssertRC(rc);
2265
2266	/* Update our cache of the exception bitmap. */
2267	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
2268	}
2269
2270
2271	/**
2272	* Initialize the VMCS information field for the given vCPU.
2273	*
2274	* @returns VBox status code.
2275	* @param pVCpu The cross context virtual CPU structure of the
2276	* calling EMT.
2277	*/
2278	static int nemR3DarwinInitVmcs(PVMCPU pVCpu)
2279	{
2280	int rc = nemR3DarwinVmxSetupVmcsPinCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2281	if (RT_SUCCESS(rc))
2282	{
2283	rc = nemR3DarwinVmxSetupVmcsProcCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2284	if (RT_SUCCESS(rc))
2285	{
2286	rc = nemR3DarwinVmxSetupVmcsMiscCtls(pVCpu, &pVCpu->nem.s.VmcsInfo);
2287	if (RT_SUCCESS(rc))
2288	{
2289	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &pVCpu->nem.s.VmcsInfo.u32EntryCtls);
2290	if (RT_SUCCESS(rc))
2291	{
2292	rc = nemR3DarwinReadVmcs32(pVCpu, VMX_VMCS32_CTRL_EXIT, &pVCpu->nem.s.VmcsInfo.u32ExitCtls);
2293	if (RT_SUCCESS(rc))
2294	{
2295	nemR3DarwinVmxSetupVmcsXcptBitmap(pVCpu, &pVCpu->nem.s.VmcsInfo);
2296	return VINF_SUCCESS;
2297	}
2298	else
2299	LogRelFunc(("Failed to read the exit controls. rc=%Rrc\n", rc));
2300	}
2301	else
2302	LogRelFunc(("Failed to read the entry controls. rc=%Rrc\n", rc));
2303	}
2304	else
2305	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
2306	}
2307	else
2308	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
2309	}
2310	else
2311	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
2312
2313	return rc;
2314	}
2315
2316
2317	/**
2318	* Registers statistics for the given vCPU.
2319	*
2320	* @returns VBox status code.
2321	* @param pVM The cross context VM structure.
2322	* @param idCpu The CPU ID.
2323	* @param pNemCpu The NEM CPU structure.
2324	*/
2325	static int nemR3DarwinStatisticsRegister(PVM pVM, VMCPUID idCpu, PNEMCPU pNemCpu)
2326	{
2327	#define NEM_REG_STAT(a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szNmFmt, a_szDesc) do { \
2328	int rc = STAMR3RegisterF(pVM, a_pVar, a_enmType, s_enmVisibility, a_enmUnit, a_szDesc, a_szNmFmt, idCpu); \
2329	AssertRC(rc); \
2330	} while (0)
2331	#define NEM_REG_PROFILE(a_pVar, a_szNmFmt, a_szDesc) \
2332	NEM_REG_STAT(a_pVar, STAMTYPE_PROFILE, STAMVISIBILITY_USED, STAMUNIT_TICKS_PER_CALL, a_szNmFmt, a_szDesc)
2333	#define NEM_REG_COUNTER(a, b, desc) NEM_REG_STAT(a, STAMTYPE_COUNTER, STAMVISIBILITY_ALWAYS, STAMUNIT_OCCURENCES, b, desc)
2334
2335	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR0Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR0", "CR0 read.");
2336	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR2Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR2", "CR2 read.");
2337	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR3Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR3", "CR3 read.");
2338	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR4Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR4", "CR4 read.");
2339	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR8Read, "/NEM/CPU%u/Exit/Instr/CR-Read/CR8", "CR8 read.");
2340	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR0Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR0", "CR0 write.");
2341	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR2Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR2", "CR2 write.");
2342	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR3Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR3", "CR3 write.");
2343	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR4Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR4", "CR4 write.");
2344	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitCR8Write, "/NEM/CPU%u/Exit/Instr/CR-Write/CR8", "CR8 write.");
2345
2346	NEM_REG_COUNTER(&pNemCpu->pVmxStats->StatExitAll, "/NEM/CPU%u/Exit/All", "Total exits (including nested-guest exits).");
2347
2348	#ifdef VBOX_WITH_STATISTICS
2349	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateImport, "/NEM/CPU%u/ImportGuestState", "Profiling of importing guest state from hardware after VM-exit.");
2350	NEM_REG_PROFILE(&pNemCpu->StatProfGstStateExport, "/NEM/CPU%u/ExportGuestState", "Profiling of exporting guest state from hardware after VM-exit.");
2351
2352	for (int j = 0; j < MAX_EXITREASON_STAT; j++)
2353	{
2354	const char *pszExitName = HMGetVmxExitName(j);
2355	if (pszExitName)
2356	{
2357	int rc = STAMR3RegisterF(pVM, &pNemCpu->pVmxStats->aStatExitReason[j], STAMTYPE_COUNTER, STAMVISIBILITY_USED,
2358	STAMUNIT_OCCURENCES, pszExitName, "/NEM/CPU%u/Exit/Reason/%02x", idCpu, j);
2359	AssertRCReturn(rc, rc);
2360	}
2361	}
2362	#endif
2363
2364	return VINF_SUCCESS;
2365
2366	#undef NEM_REG_COUNTER
2367	#undef NEM_REG_PROFILE
2368	#undef NEM_REG_STAT
2369	}
2370
2371
2372	/**
2373	* Try initialize the native API.
2374	*
2375	* This may only do part of the job, more can be done in
2376	* nemR3NativeInitAfterCPUM() and nemR3NativeInitCompleted().
2377	*
2378	* @returns VBox status code.
2379	* @param pVM The cross context VM structure.
2380	* @param fFallback Whether we're in fallback mode or use-NEM mode. In
2381	* the latter we'll fail if we cannot initialize.
2382	* @param fForced Whether the HMForced flag is set and we should
2383	* fail if we cannot initialize.
2384	*/
2385	int nemR3NativeInit(PVM pVM, bool fFallback, bool fForced)
2386	{
2387	AssertReturn(!pVM->nem.s.fCreatedVm, VERR_WRONG_ORDER);
2388
2389	/*
2390	* Some state init.
2391	*/
2392
2393	/*
2394	* Error state.
2395	* The error message will be non-empty on failure and 'rc' will be set too.
2396	*/
2397	RTERRINFOSTATIC ErrInfo;
2398	PRTERRINFO pErrInfo = RTErrInfoInitStatic(&ErrInfo);
2399	int rc = nemR3DarwinLoadHv(fForced, pErrInfo);
2400	if (RT_SUCCESS(rc))
2401	{
2402	hv_return_t hrc = hv_vm_create(HV_VM_DEFAULT);
2403	if (hrc == HV_SUCCESS)
2404	{
2405	if (hv_vm_space_create)
2406	{
2407	hrc = hv_vm_space_create(&pVM->nem.s.uVmAsid);
2408	if (hrc == HV_SUCCESS)
2409	{
2410	LogRel(("NEM: Successfully created ASID: %u\n", pVM->nem.s.uVmAsid));
2411	pVM->nem.s.fCreatedAsid = true;
2412	}
2413	else
2414	LogRel(("NEM: Failed to create ASID for VM (hrc=%#x), continuing...\n", pVM->nem.s.uVmAsid));
2415	}
2416	pVM->nem.s.fCreatedVm = true;
2417
2418	/* Register release statistics */
2419	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2420	{
2421	PNEMCPU pNemCpu = &pVM->apCpusR3[idCpu]->nem.s;
2422	PVMXSTATISTICS pVmxStats = (PVMXSTATISTICS)RTMemAllocZ(sizeof(*pVmxStats));
2423	if (RT_LIKELY(pVmxStats))
2424	{
2425	pNemCpu->pVmxStats = pVmxStats;
2426	rc = nemR3DarwinStatisticsRegister(pVM, idCpu, pNemCpu);
2427	AssertRC(rc);
2428	}
2429	else
2430	{
2431	rc = VERR_NO_MEMORY;
2432	break;
2433	}
2434	}
2435
2436	if (RT_SUCCESS(rc))
2437	{
2438	VM_SET_MAIN_EXECUTION_ENGINE(pVM, VM_EXEC_ENGINE_NATIVE_API);
2439	Log(("NEM: Marked active!\n"));
2440	PGMR3EnableNemMode(pVM);
2441	}
2442	}
2443	else
2444	rc = RTErrInfoSetF(pErrInfo, VERR_NEM_INIT_FAILED,
2445	"hv_vm_create() failed: %#x", hrc);
2446	}
2447
2448	/*
2449	* We only fail if in forced mode, otherwise just log the complaint and return.
2450	*/
2451	Assert(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API \|\| RTErrInfoIsSet(pErrInfo));
2452	if ( (fForced \|\| !fFallback)
2453	&& pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NATIVE_API)
2454	return VMSetError(pVM, RT_SUCCESS_NP(rc) ? VERR_NEM_NOT_AVAILABLE : rc, RT_SRC_POS, "%s", pErrInfo->pszMsg);
2455
2456	if (RTErrInfoIsSet(pErrInfo))
2457	LogRel(("NEM: Not available: %s\n", pErrInfo->pszMsg));
2458	return VINF_SUCCESS;
2459	}
2460
2461
2462	/**
2463	* Worker to create the vCPU handle on the EMT running it later on (as required by HV).
2464	*
2465	* @returns VBox status code
2466	* @param pVM The VM handle.
2467	* @param pVCpu The vCPU handle.
2468	* @param idCpu ID of the CPU to create.
2469	*/
2470	static DECLCALLBACK(int) nemR3DarwinNativeInitVCpuOnEmt(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
2471	{
2472	hv_return_t hrc = hv_vcpu_create(&pVCpu->nem.s.hVCpuId, HV_VCPU_DEFAULT);
2473	if (hrc != HV_SUCCESS)
2474	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
2475	"Call to hv_vcpu_create failed on vCPU %u: %#x (%Rrc)", idCpu, hrc, nemR3DarwinHvSts2Rc(hrc));
2476
2477	if (idCpu == 0)
2478	{
2479	/* First call initializs the MSR structure holding the capabilities of the host CPU. */
2480	int rc = nemR3DarwinCapsInit();
2481	AssertRCReturn(rc, rc);
2482	}
2483
2484	int rc = nemR3DarwinInitVmcs(pVCpu);
2485	AssertRCReturn(rc, rc);
2486
2487	if (pVM->nem.s.fCreatedAsid)
2488	{
2489	hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, pVM->nem.s.uVmAsid);
2490	AssertReturn(hrc == HV_SUCCESS, VERR_NEM_VM_CREATE_FAILED);
2491	}
2492
2493	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2494
2495	return VINF_SUCCESS;
2496	}
2497
2498
2499	/**
2500	* Worker to destroy the vCPU handle on the EMT running it later on (as required by HV).
2501	*
2502	* @returns VBox status code
2503	* @param pVCpu The vCPU handle.
2504	*/
2505	static DECLCALLBACK(int) nemR3DarwinNativeTermVCpuOnEmt(PVMCPU pVCpu)
2506	{
2507	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
2508	Assert(hrc == HV_SUCCESS);
2509
2510	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2511	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2512	return VINF_SUCCESS;
2513	}
2514
2515
2516	/**
2517	* Worker to setup the TPR shadowing feature if available on the CPU and the VM has an APIC enabled.
2518	*
2519	* @returns VBox status code
2520	* @param pVM The VM handle.
2521	* @param pVCpu The vCPU handle.
2522	* @param idCpu ID of the CPU to create.
2523	*/
2524	static DECLCALLBACK(int) nemR3DarwinNativeInitTprShadowing(PVM pVM, PVMCPU pVCpu, VMCPUID idCpu)
2525	{
2526	PVMXVMCSINFO pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
2527	uint32_t fVal = pVmcsInfo->u32ProcCtls;
2528
2529	/* Use TPR shadowing if supported by the CPU. */
2530	if ( PDMHasApic(pVM)
2531	&& (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
2532	{
2533	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2534	/* CR8 writes cause a VM-exit based on TPR threshold. */
2535	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2536	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2537	}
2538	else
2539	{
2540	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2541	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2542	}
2543
2544	/* Commit it to the VMCS and update our cache. */
2545	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2546	AssertRC(rc);
2547	pVmcsInfo->u32ProcCtls = fVal;
2548
2549	return VINF_SUCCESS;
2550	}
2551
2552
2553	/**
2554	* This is called after CPUMR3Init is done.
2555	*
2556	* @returns VBox status code.
2557	* @param pVM The VM handle..
2558	*/
2559	int nemR3NativeInitAfterCPUM(PVM pVM)
2560	{
2561	/*
2562	* Validate sanity.
2563	*/
2564	AssertReturn(!pVM->nem.s.fCreatedEmts, VERR_WRONG_ORDER);
2565	AssertReturn(pVM->bMainExecutionEngine == VM_EXEC_ENGINE_NATIVE_API, VERR_WRONG_ORDER);
2566
2567	/*
2568	* Setup the EMTs.
2569	*/
2570	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2571	{
2572	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2573
2574	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitVCpuOnEmt, 3, pVM, pVCpu, idCpu);
2575	if (RT_FAILURE(rc))
2576	{
2577	/* Rollback. */
2578	while (idCpu--)
2579	VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeTermVCpuOnEmt, 1, pVCpu);
2580
2581	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
2582	}
2583	}
2584
2585	pVM->nem.s.fCreatedEmts = true;
2586	return VINF_SUCCESS;
2587	}
2588
2589
2590	int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
2591	{
2592	if (enmWhat == VMINITCOMPLETED_RING3)
2593	{
2594	/* Now that PDM is initialized the APIC state is known in order to enable the TPR shadowing feature on all EMTs. */
2595	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
2596	{
2597	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2598
2599	int rc = VMR3ReqCallWait(pVM, idCpu, (PFNRT)nemR3DarwinNativeInitTprShadowing, 3, pVM, pVCpu, idCpu);
2600	if (RT_FAILURE(rc))
2601	return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS, "Call to hv_vcpu_create failed: %Rrc", rc);
2602	}
2603	}
2604	return VINF_SUCCESS;
2605	}
2606
2607
2608	int nemR3NativeTerm(PVM pVM)
2609	{
2610	/*
2611	* Delete the VM.
2612	*/
2613
2614	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu--)
2615	{
2616	PVMCPU pVCpu = pVM->apCpusR3[idCpu];
2617
2618	/*
2619	* Need to do this or hv_vm_space_destroy() fails later on (on 10.15 at least). Could've been documented in
2620	* API reference so I wouldn't have to decompile the kext to find this out but we are talking
2621	* about Apple here unfortunately, API documentation is not their strong suit...
2622	* Would have been of course even better to just automatically drop the address space reference when the vCPU
2623	* gets destroyed.
2624	*/
2625	hv_return_t hrc = hv_vcpu_set_space(pVCpu->nem.s.hVCpuId, 0 /asid/);
2626	Assert(hrc == HV_SUCCESS);
2627
2628	/*
2629	* Apple's documentation states that the vCPU should be destroyed
2630	* on the thread running the vCPU but as all the other EMTs are gone
2631	* at this point, destroying the VM would hang.
2632	*
2633	* We seem to be at luck here though as destroying apparently works
2634	* from EMT(0) as well.
2635	*/
2636	hrc = hv_vcpu_destroy(pVCpu->nem.s.hVCpuId);
2637	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2638
2639	if (pVCpu->nem.s.pVmxStats)
2640	{
2641	RTMemFree(pVCpu->nem.s.pVmxStats);
2642	pVCpu->nem.s.pVmxStats = NULL;
2643	}
2644	}
2645
2646	pVM->nem.s.fCreatedEmts = false;
2647
2648	if (pVM->nem.s.fCreatedAsid)
2649	{
2650	hv_return_t hrc = hv_vm_space_destroy(pVM->nem.s.uVmAsid);
2651	Assert(hrc == HV_SUCCESS); RT_NOREF(hrc);
2652	pVM->nem.s.fCreatedAsid = false;
2653	}
2654
2655	if (pVM->nem.s.fCreatedVm)
2656	{
2657	hv_return_t hrc = hv_vm_destroy();
2658	if (hrc != HV_SUCCESS)
2659	LogRel(("NEM: hv_vm_destroy() failed with %#x\n", hrc));
2660
2661	pVM->nem.s.fCreatedVm = false;
2662	}
2663	return VINF_SUCCESS;
2664	}
2665
2666
2667	/**
2668	* VM reset notification.
2669	*
2670	* @param pVM The cross context VM structure.
2671	*/
2672	void nemR3NativeReset(PVM pVM)
2673	{
2674	RT_NOREF(pVM);
2675	}
2676
2677
2678	/**
2679	* Reset CPU due to INIT IPI or hot (un)plugging.
2680	*
2681	* @param pVCpu The cross context virtual CPU structure of the CPU being
2682	* reset.
2683	* @param fInitIpi Whether this is the INIT IPI or hot (un)plugging case.
2684	*/
2685	void nemR3NativeResetCpu(PVMCPU pVCpu, bool fInitIpi)
2686	{
2687	RT_NOREF(fInitIpi);
2688	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2689	}
2690
2691
2692	VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
2693	{
2694	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags));
2695	#ifdef LOG_ENABLED
2696	if (LogIs3Enabled())
2697	nemR3DarwinLogState(pVM, pVCpu);
2698	#endif
2699
2700	/*
2701	* Try switch to NEM runloop state.
2702	*/
2703	if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
2704	{ /* likely */ }
2705	else
2706	{
2707	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2708	LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
2709	return VINF_SUCCESS;
2710	}
2711
2712	/*
2713	* The run loop.
2714	*
2715	* Current approach to state updating to use the sledgehammer and sync
2716	* everything every time. This will be optimized later.
2717	*/
2718
2719	VMXTRANSIENT VmxTransient;
2720	RT_ZERO(VmxTransient);
2721	VmxTransient.pVmcsInfo = &pVCpu->nem.s.VmcsInfo;
2722
2723	const bool fSingleStepping = DBGFIsStepping(pVCpu);
2724	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
2725	for (unsigned iLoop = 0;; iLoop++)
2726	{
2727	/*
2728	* Check and process force flag actions, some of which might require us to go back to ring-3.
2729	*/
2730	rcStrict = vmxHCCheckForceFlags(pVCpu, false /fIsNestedGuest/, fSingleStepping);
2731	if (rcStrict == VINF_SUCCESS)
2732	{ /likely / }
2733	else
2734	break;
2735
2736	/*
2737	* Evaluate events to be injected into the guest.
2738	*
2739	* Events in TRPM can be injected without inspecting the guest state.
2740	* If any new events (interrupts/NMI) are pending currently, we try to set up the
2741	* guest to cause a VM-exit the next time they are ready to receive the event.
2742	*/
2743	if (TRPMHasTrap(pVCpu))
2744	vmxHCTrpmTrapToPendingEvent(pVCpu);
2745
2746	uint32_t fIntrState;
2747	rcStrict = vmxHCEvaluatePendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, &fIntrState);
2748
2749	/*
2750	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
2751	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
2752	* also result in triple-faulting the VM.
2753	*
2754	* With nested-guests, the above does not apply since unrestricted guest execution is a
2755	* requirement. Regardless, we do this here to avoid duplicating code elsewhere.
2756	*/
2757	rcStrict = vmxHCInjectPendingEvent(pVCpu, &pVCpu->nem.s.VmcsInfo, false /fIsNestedGuest/, fIntrState, fSingleStepping);
2758	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
2759	{ /* likely */ }
2760	else
2761	{
2762	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fSingleStepping),
2763	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
2764	break;
2765	}
2766
2767	int rc = nemR3DarwinExportGuestState(pVM, pVCpu, &VmxTransient);
2768	AssertRCReturn(rc, rc);
2769
2770	/*
2771	* Poll timers and run for a bit.
2772	*/
2773	/** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
2774	* the whole polling job when timers have changed... */
2775	uint64_t offDeltaIgnored;
2776	uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
2777	if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
2778	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2779	{
2780	LogFlowFunc(("Running vCPU\n"));
2781	pVCpu->nem.s.Event.fPending = false;
2782
2783	TMNotifyStartOfExecution(pVM, pVCpu);
2784
2785	Assert(!pVCpu->nem.s.fCtxChanged);
2786	hv_return_t hrc;
2787	if (hv_vcpu_run_until)
2788	hrc = hv_vcpu_run_until(pVCpu->nem.s.hVCpuId, HV_DEADLINE_FOREVER);
2789	else
2790	hrc = hv_vcpu_run(pVCpu->nem.s.hVCpuId);
2791
2792	TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
2793
2794	/*
2795	* Sync the TPR shadow with our APIC state.
2796	*/
2797	if ( !VmxTransient.fIsNestedGuest
2798	&& (pVCpu->nem.s.VmcsInfo.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
2799	{
2800	uint64_t u64Tpr;
2801	hrc = hv_vcpu_read_register(pVCpu->nem.s.hVCpuId, HV_X86_TPR, &u64Tpr);
2802	Assert(hrc == HV_SUCCESS);
2803
2804	if (VmxTransient.u8GuestTpr != (uint8_t)u64Tpr)
2805	{
2806	rc = APICSetTpr(pVCpu, (uint8_t)u64Tpr);
2807	AssertRC(rc);
2808	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
2809	}
2810	}
2811
2812	if (hrc == HV_SUCCESS)
2813	{
2814	/*
2815	* Deal with the message.
2816	*/
2817	rcStrict = nemR3DarwinHandleExit(pVM, pVCpu, &VmxTransient);
2818	if (rcStrict == VINF_SUCCESS)
2819	{ /* hopefully likely */ }
2820	else
2821	{
2822	LogFlow(("NEM/%u: breaking: nemR3DarwinHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
2823	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
2824	break;
2825	}
2826	//Assert(!pVCpu->cpum.GstCtx.fExtrn);
2827	}
2828	else
2829	{
2830	AssertLogRelMsgFailedReturn(("hv_vcpu_run()) failed for CPU #%u: %#x %u\n",
2831	pVCpu->idCpu, hrc, vmxHCCheckGuestState(pVCpu, &pVCpu->nem.s.VmcsInfo)),
2832	VERR_NEM_IPE_0);
2833	}
2834
2835	/*
2836	* If no relevant FFs are pending, loop.
2837	*/
2838	if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
2839	&& !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
2840	continue;
2841
2842	/** @todo Try handle pending flags, not just return to EM loops. Take care
2843	* not to set important RCs here unless we've handled a message. */
2844	LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
2845	pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
2846	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
2847	}
2848	else
2849	{
2850	LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
2851	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
2852	}
2853	break;
2854	} /* the run loop */
2855
2856
2857	/*
2858	* Convert any pending HM events back to TRPM due to premature exits.
2859	*
2860	* This is because execution may continue from IEM and we would need to inject
2861	* the event from there (hence place it back in TRPM).
2862	*/
2863	if (pVCpu->nem.s.Event.fPending)
2864	{
2865	vmxHCPendingEventToTrpmTrap(pVCpu);
2866	Assert(!pVCpu->nem.s.Event.fPending);
2867
2868	/* Clear the events from the VMCS. */
2869	int rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
2870	rc = nemR3DarwinWriteVmcs32(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0); AssertRC(rc);
2871	}
2872
2873
2874	if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
2875	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
2876
2877	if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL))
2878	{
2879	/* Try anticipate what we might need. */
2880	uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK;
2881	if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
2882	\|\| RT_FAILURE(rcStrict))
2883	fImport = CPUMCTX_EXTRN_ALL;
2884	else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC \| VMCPU_FF_INTERRUPT_APIC
2885	\| VMCPU_FF_INTERRUPT_NMI \| VMCPU_FF_INTERRUPT_SMI))
2886	fImport \|= IEM_CPUMCTX_EXTRN_XCPT_MASK;
2887
2888	if (pVCpu->cpum.GstCtx.fExtrn & fImport)
2889	{
2890	/* Only import what is external currently. */
2891	int rc2 = nemR3DarwinCopyStateFromHv(pVM, pVCpu, fImport);
2892	if (RT_SUCCESS(rc2))
2893	pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
2894	else if (RT_SUCCESS(rcStrict))
2895	rcStrict = rc2;
2896	if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
2897	{
2898	pVCpu->cpum.GstCtx.fExtrn = 0;
2899	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2900	}
2901	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
2902	}
2903	else
2904	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2905	}
2906	else
2907	{
2908	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
2909	pVCpu->cpum.GstCtx.fExtrn = 0;
2910	ASMAtomicUoOrU64(&pVCpu->nem.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
2911	}
2912
2913	LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n",
2914	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, VBOXSTRICTRC_VAL(rcStrict) ));
2915	return rcStrict;
2916	}
2917
2918
2919	VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
2920	{
2921	NOREF(pVM);
2922	return PGMPhysIsA20Enabled(pVCpu);
2923	}
2924
2925
2926	bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
2927	{
2928	NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
2929	return false;
2930	}
2931
2932
2933	/**
2934	* Forced flag notification call from VMEmt.h.
2935	*
2936	* This is only called when pVCpu is in the VMCPUSTATE_STARTED_EXEC_NEM state.
2937	*
2938	* @param pVM The cross context VM structure.
2939	* @param pVCpu The cross context virtual CPU structure of the CPU
2940	* to be notified.
2941	* @param fFlags Notification flags, VMNOTIFYFF_FLAGS_XXX.
2942	*/
2943	void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
2944	{
2945	LogFlowFunc(("pVM=%p pVCpu=%p fFlags=%#x\n", pVM, pVCpu, fFlags));
2946
2947	RT_NOREF(pVM, fFlags);
2948
2949	hv_return_t hrc = hv_vcpu_interrupt(&pVCpu->nem.s.hVCpuId, 1);
2950	if (hrc != HV_SUCCESS)
2951	LogRel(("NEM: hv_vcpu_interrupt(%u, 1) failed with %#x\n", pVCpu->nem.s.hVCpuId, hrc));
2952	}
2953
2954
2955	VMMR3_INT_DECL(int) NEMR3NotifyPhysRamRegister(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvR3,
2956	uint8_t pu2State, uint32_t puNemRange)
2957	{
2958	RT_NOREF(pVM, puNemRange);
2959
2960	Log5(("NEMR3NotifyPhysRamRegister: %RGp LB %RGp, pvR3=%p\n", GCPhys, cb, pvR3));
2961	#if defined(VBOX_WITH_PGM_NEM_MODE)
2962	if (pvR3)
2963	{
2964	int rc = nemR3DarwinMap(pVM, GCPhys, pvR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
2965	if (RT_SUCCESS(rc))
2966	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
2967	else
2968	{
2969	LogRel(("NEMR3NotifyPhysRamRegister: GCPhys=%RGp LB %RGp pvR3=%p rc=%Rrc\n", GCPhys, cb, pvR3, rc));
2970	return VERR_NEM_MAP_PAGES_FAILED;
2971	}
2972	}
2973	return VINF_SUCCESS;
2974	#else
2975	RT_NOREF(pVM, GCPhys, cb, pvR3);
2976	return VERR_NEM_MAP_PAGES_FAILED;
2977	#endif
2978	}
2979
2980
2981	VMMR3_INT_DECL(bool) NEMR3IsMmio2DirtyPageTrackingSupported(PVM pVM)
2982	{
2983	RT_NOREF(pVM);
2984	return false;
2985	}
2986
2987
2988	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
2989	void pvRam, void pvMmio2, uint8_t pu2State, uint32_t puNemRange)
2990	{
2991	RT_NOREF(pVM, puNemRange, pvRam, fFlags);
2992
2993	Log5(("NEMR3NotifyPhysMmioExMapEarly: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p (%d)\n",
2994	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, *pu2State));
2995
2996	#if defined(VBOX_WITH_PGM_NEM_MODE)
2997	/*
2998	* Unmap the RAM we're replacing.
2999	*/
3000	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
3001	{
3002	int rc = nemR3DarwinUnmap(pVM, GCPhys, cb);
3003	if (RT_SUCCESS(rc))
3004	{ /* likely */ }
3005	else if (pvMmio2)
3006	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rc(ignored)\n",
3007	GCPhys, cb, fFlags, rc));
3008	else
3009	{
3010	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
3011	GCPhys, cb, fFlags, rc));
3012	return VERR_NEM_UNMAP_PAGES_FAILED;
3013	}
3014	}
3015
3016	/*
3017	* Map MMIO2 if any.
3018	*/
3019	if (pvMmio2)
3020	{
3021	Assert(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2);
3022	int rc = nemR3DarwinMap(pVM, GCPhys, pvMmio2, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3023	if (RT_SUCCESS(rc))
3024	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3025	else
3026	{
3027	LogRel(("NEMR3NotifyPhysMmioExMapEarly: GCPhys=%RGp LB %RGp fFlags=%#x pvMmio2=%p: Map -> rc=%Rrc\n",
3028	GCPhys, cb, fFlags, pvMmio2, rc));
3029	return VERR_NEM_MAP_PAGES_FAILED;
3030	}
3031	}
3032	else
3033	{
3034	Assert(!(fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2));
3035	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3036	}
3037
3038	#else
3039	RT_NOREF(pVM, GCPhys, cb, pvRam, pvMmio2);
3040	*pu2State = (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE) ? UINT8_MAX : NEM_DARWIN_PAGE_STATE_UNMAPPED;
3041	#endif
3042	return VINF_SUCCESS;
3043	}
3044
3045
3046	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExMapLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags,
3047	void pvRam, void pvMmio2, uint32_t *puNemRange)
3048	{
3049	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, puNemRange);
3050	return VINF_SUCCESS;
3051	}
3052
3053
3054	VMMR3_INT_DECL(int) NEMR3NotifyPhysMmioExUnmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t fFlags, void *pvRam,
3055	void pvMmio2, uint8_t pu2State, uint32_t *puNemRange)
3056	{
3057	RT_NOREF(pVM, puNemRange);
3058
3059	Log5(("NEMR3NotifyPhysMmioExUnmap: %RGp LB %RGp fFlags=%#x pvRam=%p pvMmio2=%p pu2State=%p uNemRange=%#x (%#x)\n",
3060	GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State, puNemRange, *puNemRange));
3061
3062	int rc = VINF_SUCCESS;
3063	#if defined(VBOX_WITH_PGM_NEM_MODE)
3064	/*
3065	* Unmap the MMIO2 pages.
3066	*/
3067	/** @todo If we implement aliasing (MMIO2 page aliased into MMIO range),
3068	* we may have more stuff to unmap even in case of pure MMIO... */
3069	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2)
3070	{
3071	rc = nemR3DarwinUnmap(pVM, GCPhys, cb);
3072	if (RT_FAILURE(rc))
3073	{
3074	LogRel2(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp fFlags=%#x: Unmap -> rc=%Rrc\n",
3075	GCPhys, cb, fFlags, rc));
3076	rc = VERR_NEM_UNMAP_PAGES_FAILED;
3077	}
3078	}
3079
3080	/*
3081	* Restore the RAM we replaced.
3082	*/
3083	if (fFlags & NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE)
3084	{
3085	AssertPtr(pvRam);
3086	rc = nemR3DarwinMap(pVM, GCPhys, pvRam, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3087	if (RT_SUCCESS(rc))
3088	{ /* likely */ }
3089	else
3090	{
3091	LogRel(("NEMR3NotifyPhysMmioExUnmap: GCPhys=%RGp LB %RGp pvMmio2=%p rc=%Rrc\n", GCPhys, cb, pvMmio2, rc));
3092	rc = VERR_NEM_MAP_PAGES_FAILED;
3093	}
3094	if (pu2State)
3095	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3096	}
3097	/* Mark the pages as unmapped if relevant. */
3098	else if (pu2State)
3099	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3100
3101	RT_NOREF(pvMmio2);
3102	#else
3103	RT_NOREF(pVM, GCPhys, cb, fFlags, pvRam, pvMmio2, pu2State);
3104	if (pu2State)
3105	*pu2State = UINT8_MAX;
3106	rc = VERR_NEM_UNMAP_PAGES_FAILED;
3107	#endif
3108	return rc;
3109	}
3110
3111
3112	VMMR3_INT_DECL(int) NEMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, uint32_t uNemRange,
3113	void *pvBitmap, size_t cbBitmap)
3114	{
3115	RT_NOREF(pVM, GCPhys, cb, uNemRange, pvBitmap, cbBitmap);
3116	AssertFailed();
3117	return VERR_NOT_IMPLEMENTED;
3118	}
3119
3120
3121	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterEarly(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages, uint32_t fFlags,
3122	uint8_t pu2State, uint32_t puNemRange)
3123	{
3124	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
3125
3126	Log5(("nemR3NativeNotifyPhysRomRegisterEarly: %RGp LB %RGp pvPages=%p fFlags=%#x\n", GCPhys, cb, pvPages, fFlags));
3127	*pu2State = UINT8_MAX;
3128	*puNemRange = 0;
3129	return VINF_SUCCESS;
3130	}
3131
3132
3133	VMMR3_INT_DECL(int) NEMR3NotifyPhysRomRegisterLate(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, void *pvPages,
3134	uint32_t fFlags, uint8_t pu2State, uint32_t puNemRange)
3135	{
3136	Log5(("nemR3NativeNotifyPhysRomRegisterLate: %RGp LB %RGp pvPages=%p fFlags=%#x pu2State=%p (%d) puNemRange=%p (%#x)\n",
3137	GCPhys, cb, pvPages, fFlags, pu2State, pu2State, puNemRange, puNemRange));
3138	*pu2State = UINT8_MAX;
3139
3140	#if defined(VBOX_WITH_PGM_NEM_MODE)
3141	/*
3142	* (Re-)map readonly.
3143	*/
3144	AssertPtrReturn(pvPages, VERR_INVALID_POINTER);
3145	int rc = nemR3DarwinMap(pVM, GCPhys, pvPages, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_EXECUTE);
3146	if (RT_SUCCESS(rc))
3147	*pu2State = NEM_DARWIN_PAGE_STATE_READABLE;
3148	else
3149	{
3150	LogRel(("nemR3NativeNotifyPhysRomRegisterLate: GCPhys=%RGp LB %RGp pvPages=%p fFlags=%#x rc=%Rrc\n",
3151	GCPhys, cb, pvPages, fFlags, rc));
3152	return VERR_NEM_MAP_PAGES_FAILED;
3153	}
3154	RT_NOREF(pVM, fFlags, puNemRange);
3155	return VINF_SUCCESS;
3156	#else
3157	RT_NOREF(pVM, GCPhys, cb, pvPages, fFlags, puNemRange);
3158	return VERR_NEM_MAP_PAGES_FAILED;
3159	#endif
3160	}
3161
3162
3163	VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
3164	RTR3PTR pvMemR3, uint8_t *pu2State)
3165	{
3166	RT_NOREF(pVM);
3167
3168	Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
3169	GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
3170
3171	*pu2State = UINT8_MAX;
3172	#if defined(VBOX_WITH_PGM_NEM_MODE)
3173	if (pvMemR3)
3174	{
3175	int rc = nemR3DarwinMap(pVM, GCPhys, pvMemR3, cb, NEM_PAGE_PROT_READ \| NEM_PAGE_PROT_WRITE \| NEM_PAGE_PROT_EXECUTE);
3176	if (RT_SUCCESS(rc))
3177	*pu2State = NEM_DARWIN_PAGE_STATE_WRITABLE;
3178	else
3179	AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: nemR3DarwinMap(,%p,%RGp,%RGp,) -> %Rrc\n",
3180	pvMemR3, GCPhys, cb, rc));
3181	}
3182	RT_NOREF(enmKind);
3183	#else
3184	RT_NOREF(pVM, enmKind, GCPhys, cb, pvMemR3);
3185	AssertFailed();
3186	#endif
3187	}
3188
3189
3190	static int nemHCJustUnmapPage(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
3191	{
3192	if (*pu2State <= NEM_DARWIN_PAGE_STATE_UNMAPPED)
3193	{
3194	Log5(("nemHCJustUnmapPage: %RGp == unmapped\n", GCPhysDst));
3195	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3196	return VINF_SUCCESS;
3197	}
3198
3199	int rc = nemR3DarwinUnmap(pVM, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
3200	if (RT_SUCCESS(rc))
3201	{
3202	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
3203	*pu2State = NEM_DARWIN_PAGE_STATE_UNMAPPED;
3204	Log5(("nemHCJustUnmapPage: %RGp => unmapped\n", GCPhysDst));
3205	return VINF_SUCCESS;
3206	}
3207	STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
3208	LogRel(("nemHCJustUnmapPage(%RGp): failed! rc=%Rrc\n",
3209	GCPhysDst, rc));
3210	return VERR_NEM_IPE_6;
3211	}
3212
3213
3214	/**
3215	* Called when the A20 state changes.
3216	*
3217	* @param pVCpu The CPU the A20 state changed on.
3218	* @param fEnabled Whether it was enabled (true) or disabled.
3219	*/
3220	VMMR3_INT_DECL(void) NEMR3NotifySetA20(PVMCPU pVCpu, bool fEnabled)
3221	{
3222	Log(("NEMR3NotifySetA20: fEnabled=%RTbool\n", fEnabled));
3223	RT_NOREF(pVCpu, fEnabled);
3224	}
3225
3226
3227	void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
3228	{
3229	Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
3230	NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
3231	}
3232
3233
3234	void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
3235	RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
3236	{
3237	Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
3238	GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
3239	NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
3240	}
3241
3242
3243	int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
3244	PGMPAGETYPE enmType, uint8_t *pu2State)
3245	{
3246	Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3247	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3248	RT_NOREF(HCPhys, fPageProt, enmType);
3249
3250	return nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3251	}
3252
3253
3254	VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
3255	PGMPAGETYPE enmType, uint8_t *pu2State)
3256	{
3257	Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3258	GCPhys, HCPhys, fPageProt, enmType, *pu2State));
3259	RT_NOREF(HCPhys, pvR3, fPageProt, enmType)
3260
3261	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3262	}
3263
3264
3265	VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
3266	RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
3267	{
3268	Log5(("NEMHCNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
3269	GCPhys, HCPhysPrev, HCPhysNew, fPageProt, enmType, *pu2State));
3270	RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType);
3271
3272	nemHCJustUnmapPage(pVM, GCPhys, pu2State);
3273	}
3274
3275
3276	/**
3277	* Interface for importing state on demand (used by IEM).
3278	*
3279	* @returns VBox status code.
3280	* @param pVCpu The cross context CPU structure.
3281	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
3282	*/
3283	VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
3284	{
3285	LogFlowFunc(("pVCpu=%p fWhat=%RX64\n", pVCpu, fWhat));
3286	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
3287
3288	return nemR3DarwinCopyStateFromHv(pVCpu->pVMR3, pVCpu, fWhat);
3289	}
3290
3291
3292	/**
3293	* Query the CPU tick counter and optionally the TSC_AUX MSR value.
3294	*
3295	* @returns VBox status code.
3296	* @param pVCpu The cross context CPU structure.
3297	* @param pcTicks Where to return the CPU tick count.
3298	* @param puAux Where to return the TSC_AUX register value.
3299	*/
3300	VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t pcTicks, uint32_t puAux)
3301	{
3302	LogFlowFunc(("pVCpu=%p pcTicks=%RX64 puAux=%RX32\n", pVCpu, pcTicks, puAux));
3303	STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
3304
3305	int rc = nemR3DarwinMsrRead(pVCpu, MSR_IA32_TSC, pcTicks);
3306	if ( RT_SUCCESS(rc)
3307	&& puAux)
3308	{
3309	if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX)
3310	{
3311	/** @todo Why the heck is puAux a uint32_t?. */
3312	uint64_t u64Aux;
3313	rc = nemR3DarwinMsrRead(pVCpu, MSR_K8_TSC_AUX, &u64Aux);
3314	if (RT_SUCCESS(rc))
3315	*puAux = (uint32_t)u64Aux;
3316	}
3317	else
3318	*puAux = CPUMGetGuestTscAux(pVCpu);
3319	}
3320
3321	return rc;
3322	}
3323
3324
3325	/**
3326	* Resumes CPU clock (TSC) on all virtual CPUs.
3327	*
3328	* This is called by TM when the VM is started, restored, resumed or similar.
3329	*
3330	* @returns VBox status code.
3331	* @param pVM The cross context VM structure.
3332	* @param pVCpu The cross context CPU structure of the calling EMT.
3333	* @param uPausedTscValue The TSC value at the time of pausing.
3334	*/
3335	VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
3336	{
3337	LogFlowFunc(("pVM=%p pVCpu=%p uPausedTscValue=%RX64\n", pVCpu, uPausedTscValue));
3338	VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
3339	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
3340
3341	hv_return_t hrc = hv_vm_sync_tsc(uPausedTscValue);
3342	if (RT_LIKELY(hrc == HV_SUCCESS))
3343	{
3344	ASMAtomicUoAndU64(&pVCpu->nem.s.fCtxChanged, ~HM_CHANGED_GUEST_TSC_AUX);
3345	return VINF_SUCCESS;
3346	}
3347
3348	return nemR3DarwinHvSts2Rc(hrc);
3349	}
3350
3351
3352	/**
3353	* Returns features supported by the NEM backend.
3354	*
3355	* @returns Flags of features supported by the native NEM backend.
3356	* @param pVM The cross context VM structure.
3357	*/
3358	VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
3359	{
3360	RT_NOREF(pVM);
3361	/*
3362	* Apple's Hypervisor.framework is not supported if the CPU doesn't support nested paging
3363	* and unrestricted guest execution support so we can safely return these flags here always.
3364	*/
3365	return NEM_FEAT_F_NESTED_PAGING \| NEM_FEAT_F_FULL_GST_EXEC \| NEM_FEAT_F_XSAVE_XRSTOR;
3366	}
3367
3368
3369	/** @page pg_nem_darwin NEM/darwin - Native Execution Manager, macOS.
3370	*
3371	* @todo Add notes as the implementation progresses...
3372	*/
3373

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-darwin.cpp@ 92521

以其他格式下載: