PGMAll.cpp@ 80317

最後變更在這個檔案從80317是 80281,由 vboxsync 提交於 6 年前
VMM,++: Refactoring code to use VMMC & VMMCPUCC. bugref:9217
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Id Revision`
檔案大小: 132.1 KB

行
1	/* $Id: PGMAll.cpp 80281 2019-08-15 07:29:37Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define VBOX_BUGREF_9217_PART_I
23	#define LOG_GROUP LOG_GROUP_PGM
24	#include <VBox/vmm/pgm.h>
25	#include <VBox/vmm/cpum.h>
26	#include <VBox/vmm/selm.h>
27	#include <VBox/vmm/iem.h>
28	#include <VBox/vmm/iom.h>
29	#include <VBox/sup.h>
30	#include <VBox/vmm/mm.h>
31	#include <VBox/vmm/stam.h>
32	#include <VBox/vmm/trpm.h>
33	#ifdef VBOX_WITH_REM
34	# include <VBox/vmm/rem.h>
35	#endif
36	#include <VBox/vmm/em.h>
37	#include <VBox/vmm/hm.h>
38	#include <VBox/vmm/hm_vmx.h>
39	#include "PGMInternal.h"
40	#include <VBox/vmm/vmcc.h>
41	#include "PGMInline.h"
42	#include <iprt/assert.h>
43	#include <iprt/asm-amd64-x86.h>
44	#include <iprt/string.h>
45	#include <VBox/log.h>
46	#include <VBox/param.h>
47	#include <VBox/err.h>
48
49
50	/*********************************************************************************************************************************
51	* Internal Functions *
52	*********************************************************************************************************************************/
53	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
54	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
55	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
56	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
57
58
59	/*
60	* Shadow - 32-bit mode
61	*/
62	#define PGM_SHW_TYPE PGM_TYPE_32BIT
63	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
64	#include "PGMAllShw.h"
65
66	/* Guest - real mode */
67	#define PGM_GST_TYPE PGM_TYPE_REAL
68	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
69	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
70	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
71	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
72	#include "PGMGstDefs.h"
73	#include "PGMAllGst.h"
74	#include "PGMAllBth.h"
75	#undef BTH_PGMPOOLKIND_PT_FOR_PT
76	#undef BTH_PGMPOOLKIND_ROOT
77	#undef PGM_BTH_NAME
78	#undef PGM_GST_TYPE
79	#undef PGM_GST_NAME
80
81	/* Guest - protected mode */
82	#define PGM_GST_TYPE PGM_TYPE_PROT
83	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
84	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
85	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
86	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
87	#include "PGMGstDefs.h"
88	#include "PGMAllGst.h"
89	#include "PGMAllBth.h"
90	#undef BTH_PGMPOOLKIND_PT_FOR_PT
91	#undef BTH_PGMPOOLKIND_ROOT
92	#undef PGM_BTH_NAME
93	#undef PGM_GST_TYPE
94	#undef PGM_GST_NAME
95
96	/* Guest - 32-bit mode */
97	#define PGM_GST_TYPE PGM_TYPE_32BIT
98	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
99	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
100	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
101	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
102	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
103	#include "PGMGstDefs.h"
104	#include "PGMAllGst.h"
105	#include "PGMAllBth.h"
106	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
107	#undef BTH_PGMPOOLKIND_PT_FOR_PT
108	#undef BTH_PGMPOOLKIND_ROOT
109	#undef PGM_BTH_NAME
110	#undef PGM_GST_TYPE
111	#undef PGM_GST_NAME
112
113	#undef PGM_SHW_TYPE
114	#undef PGM_SHW_NAME
115
116
117	/*
118	* Shadow - PAE mode
119	*/
120	#define PGM_SHW_TYPE PGM_TYPE_PAE
121	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
122	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
123	#include "PGMAllShw.h"
124
125	/* Guest - real mode */
126	#define PGM_GST_TYPE PGM_TYPE_REAL
127	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
128	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
129	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
130	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
131	#include "PGMGstDefs.h"
132	#include "PGMAllBth.h"
133	#undef BTH_PGMPOOLKIND_PT_FOR_PT
134	#undef BTH_PGMPOOLKIND_ROOT
135	#undef PGM_BTH_NAME
136	#undef PGM_GST_TYPE
137	#undef PGM_GST_NAME
138
139	/* Guest - protected mode */
140	#define PGM_GST_TYPE PGM_TYPE_PROT
141	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
142	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
143	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
144	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
145	#include "PGMGstDefs.h"
146	#include "PGMAllBth.h"
147	#undef BTH_PGMPOOLKIND_PT_FOR_PT
148	#undef BTH_PGMPOOLKIND_ROOT
149	#undef PGM_BTH_NAME
150	#undef PGM_GST_TYPE
151	#undef PGM_GST_NAME
152
153	/* Guest - 32-bit mode */
154	#define PGM_GST_TYPE PGM_TYPE_32BIT
155	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
156	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
157	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
158	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
159	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
160	#include "PGMGstDefs.h"
161	#include "PGMAllBth.h"
162	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
163	#undef BTH_PGMPOOLKIND_PT_FOR_PT
164	#undef BTH_PGMPOOLKIND_ROOT
165	#undef PGM_BTH_NAME
166	#undef PGM_GST_TYPE
167	#undef PGM_GST_NAME
168
169
170	/* Guest - PAE mode */
171	#define PGM_GST_TYPE PGM_TYPE_PAE
172	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
173	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
174	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
175	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
176	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
177	#include "PGMGstDefs.h"
178	#include "PGMAllGst.h"
179	#include "PGMAllBth.h"
180	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
181	#undef BTH_PGMPOOLKIND_PT_FOR_PT
182	#undef BTH_PGMPOOLKIND_ROOT
183	#undef PGM_BTH_NAME
184	#undef PGM_GST_TYPE
185	#undef PGM_GST_NAME
186
187	#undef PGM_SHW_TYPE
188	#undef PGM_SHW_NAME
189
190
191	/*
192	* Shadow - AMD64 mode
193	*/
194	#define PGM_SHW_TYPE PGM_TYPE_AMD64
195	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
196	#include "PGMAllShw.h"
197
198	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
199	/** @todo retire this hack. */
200	#define PGM_GST_TYPE PGM_TYPE_PROT
201	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
202	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
203	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
204	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
205	#include "PGMGstDefs.h"
206	#include "PGMAllBth.h"
207	#undef BTH_PGMPOOLKIND_PT_FOR_PT
208	#undef BTH_PGMPOOLKIND_ROOT
209	#undef PGM_BTH_NAME
210	#undef PGM_GST_TYPE
211	#undef PGM_GST_NAME
212
213	#ifdef VBOX_WITH_64_BITS_GUESTS
214	/* Guest - AMD64 mode */
215	# define PGM_GST_TYPE PGM_TYPE_AMD64
216	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
217	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
218	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
219	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
220	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
221	# include "PGMGstDefs.h"
222	# include "PGMAllGst.h"
223	# include "PGMAllBth.h"
224	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
225	# undef BTH_PGMPOOLKIND_PT_FOR_PT
226	# undef BTH_PGMPOOLKIND_ROOT
227	# undef PGM_BTH_NAME
228	# undef PGM_GST_TYPE
229	# undef PGM_GST_NAME
230	#endif /* VBOX_WITH_64_BITS_GUESTS */
231
232	#undef PGM_SHW_TYPE
233	#undef PGM_SHW_NAME
234
235
236	/*
237	* Shadow - 32-bit nested paging mode.
238	*/
239	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
240	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
241	#include "PGMAllShw.h"
242
243	/* Guest - real mode */
244	#define PGM_GST_TYPE PGM_TYPE_REAL
245	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
246	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
247	#include "PGMGstDefs.h"
248	#include "PGMAllBth.h"
249	#undef PGM_BTH_NAME
250	#undef PGM_GST_TYPE
251	#undef PGM_GST_NAME
252
253	/* Guest - protected mode */
254	#define PGM_GST_TYPE PGM_TYPE_PROT
255	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
256	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
257	#include "PGMGstDefs.h"
258	#include "PGMAllBth.h"
259	#undef PGM_BTH_NAME
260	#undef PGM_GST_TYPE
261	#undef PGM_GST_NAME
262
263	/* Guest - 32-bit mode */
264	#define PGM_GST_TYPE PGM_TYPE_32BIT
265	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
266	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
267	#include "PGMGstDefs.h"
268	#include "PGMAllBth.h"
269	#undef PGM_BTH_NAME
270	#undef PGM_GST_TYPE
271	#undef PGM_GST_NAME
272
273	/* Guest - PAE mode */
274	#define PGM_GST_TYPE PGM_TYPE_PAE
275	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
276	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
277	#include "PGMGstDefs.h"
278	#include "PGMAllBth.h"
279	#undef PGM_BTH_NAME
280	#undef PGM_GST_TYPE
281	#undef PGM_GST_NAME
282
283	#ifdef VBOX_WITH_64_BITS_GUESTS
284	/* Guest - AMD64 mode */
285	# define PGM_GST_TYPE PGM_TYPE_AMD64
286	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
287	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
288	# include "PGMGstDefs.h"
289	# include "PGMAllBth.h"
290	# undef PGM_BTH_NAME
291	# undef PGM_GST_TYPE
292	# undef PGM_GST_NAME
293	#endif /* VBOX_WITH_64_BITS_GUESTS */
294
295	#undef PGM_SHW_TYPE
296	#undef PGM_SHW_NAME
297
298
299	/*
300	* Shadow - PAE nested paging mode.
301	*/
302	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
303	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
304	#include "PGMAllShw.h"
305
306	/* Guest - real mode */
307	#define PGM_GST_TYPE PGM_TYPE_REAL
308	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
309	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
310	#include "PGMGstDefs.h"
311	#include "PGMAllBth.h"
312	#undef PGM_BTH_NAME
313	#undef PGM_GST_TYPE
314	#undef PGM_GST_NAME
315
316	/* Guest - protected mode */
317	#define PGM_GST_TYPE PGM_TYPE_PROT
318	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
319	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
320	#include "PGMGstDefs.h"
321	#include "PGMAllBth.h"
322	#undef PGM_BTH_NAME
323	#undef PGM_GST_TYPE
324	#undef PGM_GST_NAME
325
326	/* Guest - 32-bit mode */
327	#define PGM_GST_TYPE PGM_TYPE_32BIT
328	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
329	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
330	#include "PGMGstDefs.h"
331	#include "PGMAllBth.h"
332	#undef PGM_BTH_NAME
333	#undef PGM_GST_TYPE
334	#undef PGM_GST_NAME
335
336	/* Guest - PAE mode */
337	#define PGM_GST_TYPE PGM_TYPE_PAE
338	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
339	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
340	#include "PGMGstDefs.h"
341	#include "PGMAllBth.h"
342	#undef PGM_BTH_NAME
343	#undef PGM_GST_TYPE
344	#undef PGM_GST_NAME
345
346	#ifdef VBOX_WITH_64_BITS_GUESTS
347	/* Guest - AMD64 mode */
348	# define PGM_GST_TYPE PGM_TYPE_AMD64
349	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
350	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
351	# include "PGMGstDefs.h"
352	# include "PGMAllBth.h"
353	# undef PGM_BTH_NAME
354	# undef PGM_GST_TYPE
355	# undef PGM_GST_NAME
356	#endif /* VBOX_WITH_64_BITS_GUESTS */
357
358	#undef PGM_SHW_TYPE
359	#undef PGM_SHW_NAME
360
361
362	/*
363	* Shadow - AMD64 nested paging mode.
364	*/
365	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
366	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
367	#include "PGMAllShw.h"
368
369	/* Guest - real mode */
370	#define PGM_GST_TYPE PGM_TYPE_REAL
371	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
372	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
373	#include "PGMGstDefs.h"
374	#include "PGMAllBth.h"
375	#undef PGM_BTH_NAME
376	#undef PGM_GST_TYPE
377	#undef PGM_GST_NAME
378
379	/* Guest - protected mode */
380	#define PGM_GST_TYPE PGM_TYPE_PROT
381	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
382	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
383	#include "PGMGstDefs.h"
384	#include "PGMAllBth.h"
385	#undef PGM_BTH_NAME
386	#undef PGM_GST_TYPE
387	#undef PGM_GST_NAME
388
389	/* Guest - 32-bit mode */
390	#define PGM_GST_TYPE PGM_TYPE_32BIT
391	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
392	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
393	#include "PGMGstDefs.h"
394	#include "PGMAllBth.h"
395	#undef PGM_BTH_NAME
396	#undef PGM_GST_TYPE
397	#undef PGM_GST_NAME
398
399	/* Guest - PAE mode */
400	#define PGM_GST_TYPE PGM_TYPE_PAE
401	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
402	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
403	#include "PGMGstDefs.h"
404	#include "PGMAllBth.h"
405	#undef PGM_BTH_NAME
406	#undef PGM_GST_TYPE
407	#undef PGM_GST_NAME
408
409	#ifdef VBOX_WITH_64_BITS_GUESTS
410	/* Guest - AMD64 mode */
411	# define PGM_GST_TYPE PGM_TYPE_AMD64
412	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
413	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
414	# include "PGMGstDefs.h"
415	# include "PGMAllBth.h"
416	# undef PGM_BTH_NAME
417	# undef PGM_GST_TYPE
418	# undef PGM_GST_NAME
419	#endif /* VBOX_WITH_64_BITS_GUESTS */
420
421	#undef PGM_SHW_TYPE
422	#undef PGM_SHW_NAME
423
424
425	/*
426	* Shadow - EPT.
427	*/
428	#define PGM_SHW_TYPE PGM_TYPE_EPT
429	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
430	#include "PGMAllShw.h"
431
432	/* Guest - real mode */
433	#define PGM_GST_TYPE PGM_TYPE_REAL
434	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
435	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
436	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
437	#include "PGMGstDefs.h"
438	#include "PGMAllBth.h"
439	#undef BTH_PGMPOOLKIND_PT_FOR_PT
440	#undef PGM_BTH_NAME
441	#undef PGM_GST_TYPE
442	#undef PGM_GST_NAME
443
444	/* Guest - protected mode */
445	#define PGM_GST_TYPE PGM_TYPE_PROT
446	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
447	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
448	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
449	#include "PGMGstDefs.h"
450	#include "PGMAllBth.h"
451	#undef BTH_PGMPOOLKIND_PT_FOR_PT
452	#undef PGM_BTH_NAME
453	#undef PGM_GST_TYPE
454	#undef PGM_GST_NAME
455
456	/* Guest - 32-bit mode */
457	#define PGM_GST_TYPE PGM_TYPE_32BIT
458	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
459	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
460	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
461	#include "PGMGstDefs.h"
462	#include "PGMAllBth.h"
463	#undef BTH_PGMPOOLKIND_PT_FOR_PT
464	#undef PGM_BTH_NAME
465	#undef PGM_GST_TYPE
466	#undef PGM_GST_NAME
467
468	/* Guest - PAE mode */
469	#define PGM_GST_TYPE PGM_TYPE_PAE
470	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
471	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
472	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
473	#include "PGMGstDefs.h"
474	#include "PGMAllBth.h"
475	#undef BTH_PGMPOOLKIND_PT_FOR_PT
476	#undef PGM_BTH_NAME
477	#undef PGM_GST_TYPE
478	#undef PGM_GST_NAME
479
480	#ifdef VBOX_WITH_64_BITS_GUESTS
481	/* Guest - AMD64 mode */
482	# define PGM_GST_TYPE PGM_TYPE_AMD64
483	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
484	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
485	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
486	# include "PGMGstDefs.h"
487	# include "PGMAllBth.h"
488	# undef BTH_PGMPOOLKIND_PT_FOR_PT
489	# undef PGM_BTH_NAME
490	# undef PGM_GST_TYPE
491	# undef PGM_GST_NAME
492	#endif /* VBOX_WITH_64_BITS_GUESTS */
493
494	#undef PGM_SHW_TYPE
495	#undef PGM_SHW_NAME
496
497
498	/*
499	* Shadow - NEM / None.
500	*/
501	#define PGM_SHW_TYPE PGM_TYPE_NONE
502	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
503	#include "PGMAllShw.h"
504
505	/* Guest - real mode */
506	#define PGM_GST_TYPE PGM_TYPE_REAL
507	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
508	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
509	#include "PGMGstDefs.h"
510	#include "PGMAllBth.h"
511	#undef PGM_BTH_NAME
512	#undef PGM_GST_TYPE
513	#undef PGM_GST_NAME
514
515	/* Guest - protected mode */
516	#define PGM_GST_TYPE PGM_TYPE_PROT
517	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
518	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
519	#include "PGMGstDefs.h"
520	#include "PGMAllBth.h"
521	#undef PGM_BTH_NAME
522	#undef PGM_GST_TYPE
523	#undef PGM_GST_NAME
524
525	/* Guest - 32-bit mode */
526	#define PGM_GST_TYPE PGM_TYPE_32BIT
527	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
528	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
529	#include "PGMGstDefs.h"
530	#include "PGMAllBth.h"
531	#undef PGM_BTH_NAME
532	#undef PGM_GST_TYPE
533	#undef PGM_GST_NAME
534
535	/* Guest - PAE mode */
536	#define PGM_GST_TYPE PGM_TYPE_PAE
537	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
538	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
539	#include "PGMGstDefs.h"
540	#include "PGMAllBth.h"
541	#undef PGM_BTH_NAME
542	#undef PGM_GST_TYPE
543	#undef PGM_GST_NAME
544
545	#ifdef VBOX_WITH_64_BITS_GUESTS
546	/* Guest - AMD64 mode */
547	# define PGM_GST_TYPE PGM_TYPE_AMD64
548	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
549	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
550	# include "PGMGstDefs.h"
551	# include "PGMAllBth.h"
552	# undef PGM_BTH_NAME
553	# undef PGM_GST_TYPE
554	# undef PGM_GST_NAME
555	#endif /* VBOX_WITH_64_BITS_GUESTS */
556
557	#undef PGM_SHW_TYPE
558	#undef PGM_SHW_NAME
559
560
561
562	/**
563	* Guest mode data array.
564	*/
565	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
566	{
567	{ UINT32_MAX, NULL, NULL, NULL, NULL, NULL }, /* 0 */
568	{
569	PGM_TYPE_REAL,
570	PGM_GST_NAME_REAL(GetPage),
571	PGM_GST_NAME_REAL(ModifyPage),
572	PGM_GST_NAME_REAL(GetPDE),
573	PGM_GST_NAME_REAL(Enter),
574	PGM_GST_NAME_REAL(Exit),
575	#ifdef IN_RING3
576	PGM_GST_NAME_REAL(Relocate),
577	#endif
578	},
579	{
580	PGM_TYPE_PROT,
581	PGM_GST_NAME_PROT(GetPage),
582	PGM_GST_NAME_PROT(ModifyPage),
583	PGM_GST_NAME_PROT(GetPDE),
584	PGM_GST_NAME_PROT(Enter),
585	PGM_GST_NAME_PROT(Exit),
586	#ifdef IN_RING3
587	PGM_GST_NAME_PROT(Relocate),
588	#endif
589	},
590	{
591	PGM_TYPE_32BIT,
592	PGM_GST_NAME_32BIT(GetPage),
593	PGM_GST_NAME_32BIT(ModifyPage),
594	PGM_GST_NAME_32BIT(GetPDE),
595	PGM_GST_NAME_32BIT(Enter),
596	PGM_GST_NAME_32BIT(Exit),
597	#ifdef IN_RING3
598	PGM_GST_NAME_32BIT(Relocate),
599	#endif
600	},
601	{
602	PGM_TYPE_PAE,
603	PGM_GST_NAME_PAE(GetPage),
604	PGM_GST_NAME_PAE(ModifyPage),
605	PGM_GST_NAME_PAE(GetPDE),
606	PGM_GST_NAME_PAE(Enter),
607	PGM_GST_NAME_PAE(Exit),
608	#ifdef IN_RING3
609	PGM_GST_NAME_PAE(Relocate),
610	#endif
611	},
612	#ifdef VBOX_WITH_64_BITS_GUESTS
613	{
614	PGM_TYPE_AMD64,
615	PGM_GST_NAME_AMD64(GetPage),
616	PGM_GST_NAME_AMD64(ModifyPage),
617	PGM_GST_NAME_AMD64(GetPDE),
618	PGM_GST_NAME_AMD64(Enter),
619	PGM_GST_NAME_AMD64(Exit),
620	# ifdef IN_RING3
621	PGM_GST_NAME_AMD64(Relocate),
622	# endif
623	},
624	#endif
625	};
626
627
628	/**
629	* The shadow mode data array.
630	*/
631	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
632	{
633	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
634	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
635	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
636	{
637	PGM_TYPE_32BIT,
638	PGM_SHW_NAME_32BIT(GetPage),
639	PGM_SHW_NAME_32BIT(ModifyPage),
640	PGM_SHW_NAME_32BIT(Enter),
641	PGM_SHW_NAME_32BIT(Exit),
642	#ifdef IN_RING3
643	PGM_SHW_NAME_32BIT(Relocate),
644	#endif
645	},
646	{
647	PGM_TYPE_PAE,
648	PGM_SHW_NAME_PAE(GetPage),
649	PGM_SHW_NAME_PAE(ModifyPage),
650	PGM_SHW_NAME_PAE(Enter),
651	PGM_SHW_NAME_PAE(Exit),
652	#ifdef IN_RING3
653	PGM_SHW_NAME_PAE(Relocate),
654	#endif
655	},
656	{
657	PGM_TYPE_AMD64,
658	PGM_SHW_NAME_AMD64(GetPage),
659	PGM_SHW_NAME_AMD64(ModifyPage),
660	PGM_SHW_NAME_AMD64(Enter),
661	PGM_SHW_NAME_AMD64(Exit),
662	#ifdef IN_RING3
663	PGM_SHW_NAME_AMD64(Relocate),
664	#endif
665	},
666	{
667	PGM_TYPE_NESTED_32BIT,
668	PGM_SHW_NAME_NESTED_32BIT(GetPage),
669	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
670	PGM_SHW_NAME_NESTED_32BIT(Enter),
671	PGM_SHW_NAME_NESTED_32BIT(Exit),
672	#ifdef IN_RING3
673	PGM_SHW_NAME_NESTED_32BIT(Relocate),
674	#endif
675	},
676	{
677	PGM_TYPE_NESTED_PAE,
678	PGM_SHW_NAME_NESTED_PAE(GetPage),
679	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
680	PGM_SHW_NAME_NESTED_PAE(Enter),
681	PGM_SHW_NAME_NESTED_PAE(Exit),
682	#ifdef IN_RING3
683	PGM_SHW_NAME_NESTED_PAE(Relocate),
684	#endif
685	},
686	{
687	PGM_TYPE_NESTED_AMD64,
688	PGM_SHW_NAME_NESTED_AMD64(GetPage),
689	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
690	PGM_SHW_NAME_NESTED_AMD64(Enter),
691	PGM_SHW_NAME_NESTED_AMD64(Exit),
692	#ifdef IN_RING3
693	PGM_SHW_NAME_NESTED_AMD64(Relocate),
694	#endif
695	},
696	{
697	PGM_TYPE_EPT,
698	PGM_SHW_NAME_EPT(GetPage),
699	PGM_SHW_NAME_EPT(ModifyPage),
700	PGM_SHW_NAME_EPT(Enter),
701	PGM_SHW_NAME_EPT(Exit),
702	#ifdef IN_RING3
703	PGM_SHW_NAME_EPT(Relocate),
704	#endif
705	},
706	{
707	PGM_TYPE_NONE,
708	PGM_SHW_NAME_NONE(GetPage),
709	PGM_SHW_NAME_NONE(ModifyPage),
710	PGM_SHW_NAME_NONE(Enter),
711	PGM_SHW_NAME_NONE(Exit),
712	#ifdef IN_RING3
713	PGM_SHW_NAME_NONE(Relocate),
714	#endif
715	},
716	};
717
718
719	/**
720	* The guest+shadow mode data array.
721	*/
722	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
723	{
724	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
725	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
726	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
727	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler) }
728
729	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
730	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
731	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
732	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(AssertCR3) }
733
734	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
735	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
736	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
737	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
738
739	#elif defined(IN_RING3) && defined(VBOX_STRICT)
740	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
741	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
742	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
743
744	#else
745	# error "Misconfig."
746	#endif
747
748	/* 32-bit shadow paging mode: */
749	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
750	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
751	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
752	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
753	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
754	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
755	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
756	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
757	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
758	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
759	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
760
761	/* PAE shadow paging mode: */
762	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
763	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
764	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
765	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
766	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
767	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
768	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
769	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
770	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
771	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
772	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
773
774	/* AMD64 shadow paging mode: */
775	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
776	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
777	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
778	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
779	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
780	#ifdef VBOX_WITH_64_BITS_GUESTS
781	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
782	#else
783	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
784	#endif
785	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
786	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
787	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
788	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
789	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
790
791	/* 32-bit nested paging mode: */
792	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
793	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
794	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
795	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
796	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
797	#ifdef VBOX_WITH_64_BITS_GUESTS
798	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
799	#else
800	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
801	#endif
802	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
803	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
804	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
805	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
806	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
807
808	/* PAE nested paging mode: */
809	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
810	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
811	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
812	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
813	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
814	#ifdef VBOX_WITH_64_BITS_GUESTS
815	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
816	#else
817	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
818	#endif
819	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
820	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
821	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
822	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
823	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
824
825	/* AMD64 nested paging mode: */
826	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
827	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
828	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
829	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
830	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
831	#ifdef VBOX_WITH_64_BITS_GUESTS
832	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
833	#else
834	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
835	#endif
836	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
837	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
838	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
839	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
840	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
841
842	/* EPT nested paging mode: */
843	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
844	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
845	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
846	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
847	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
848	#ifdef VBOX_WITH_64_BITS_GUESTS
849	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
850	#else
851	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
852	#endif
853	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
854	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
855	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
856	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
857	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
858
859	/* NONE / NEM: */
860	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
861	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
862	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
863	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
864	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
865	#ifdef VBOX_WITH_64_BITS_GUESTS
866	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
867	#else
868	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
869	#endif
870	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
871	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
872	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
873	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
874	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
875
876
877	#undef PGMMODEDATABTH_ENTRY
878	#undef PGMMODEDATABTH_NULL_ENTRY
879	};
880
881
882	#ifdef IN_RING0
883	/**
884	* #PF Handler.
885	*
886	* @returns VBox status code (appropriate for trap handling and GC return).
887	* @param pVCpu The cross context virtual CPU structure.
888	* @param uErr The trap error code.
889	* @param pRegFrame Trap register frame.
890	* @param pvFault The fault address.
891	*/
892	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
893	{
894	PVM pVM = pVCpu->CTX_SUFF(pVM);
895
896	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
897	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, a);
898	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = NULL; } );
899
900
901	# ifdef VBOX_WITH_STATISTICS
902	/*
903	* Error code stats.
904	*/
905	if (uErr & X86_TRAP_PF_US)
906	{
907	if (!(uErr & X86_TRAP_PF_P))
908	{
909	if (uErr & X86_TRAP_PF_RW)
910	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentWrite);
911	else
912	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNotPresentRead);
913	}
914	else if (uErr & X86_TRAP_PF_RW)
915	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSWrite);
916	else if (uErr & X86_TRAP_PF_RSVD)
917	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSReserved);
918	else if (uErr & X86_TRAP_PF_ID)
919	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSNXE);
920	else
921	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eUSRead);
922	}
923	else
924	{ /* Supervisor */
925	if (!(uErr & X86_TRAP_PF_P))
926	{
927	if (uErr & X86_TRAP_PF_RW)
928	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentWrite);
929	else
930	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVNotPresentRead);
931	}
932	else if (uErr & X86_TRAP_PF_RW)
933	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVWrite);
934	else if (uErr & X86_TRAP_PF_ID)
935	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSNXE);
936	else if (uErr & X86_TRAP_PF_RSVD)
937	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eSVReserved);
938	}
939	# endif /* VBOX_WITH_STATISTICS */
940
941	/*
942	* Call the worker.
943	*/
944	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
945	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
946	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
947	bool fLockTaken = false;
948	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
949	if (fLockTaken)
950	{
951	PGM_LOCK_ASSERT_OWNER(pVM);
952	pgmUnlock(pVM);
953	}
954	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
955
956	/*
957	* Return code tweaks.
958	*/
959	if (rc != VINF_SUCCESS)
960	{
961	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
962	rc = VINF_SUCCESS;
963
964	/* Note: hack alert for difficult to reproduce problem. */
965	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
966	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
967	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
968	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
969	{
970	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
971	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
972	rc = VINF_SUCCESS;
973	}
974	}
975
976	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPF); });
977	STAM_STATS({ if (!pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution))
978	pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Misc; });
979	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0e, pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution), a);
980	return rc;
981	}
982	#endif /* IN_RING0 */
983
984
985	/**
986	* Prefetch a page
987	*
988	* Typically used to sync commonly used pages before entering raw mode
989	* after a CR3 reload.
990	*
991	* @returns VBox status code suitable for scheduling.
992	* @retval VINF_SUCCESS on success.
993	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
994	* @param pVCpu The cross context virtual CPU structure.
995	* @param GCPtrPage Page to invalidate.
996	*/
997	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
998	{
999	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
1000
1001	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1002	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1003	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
1004	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
1005
1006	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,Prefetch), a);
1007	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1008	return rc;
1009	}
1010
1011
1012	#ifndef PGM_WITHOUT_MAPPINGS
1013	/**
1014	* Gets the mapping corresponding to the specified address (if any).
1015	*
1016	* @returns Pointer to the mapping.
1017	* @returns NULL if not
1018	*
1019	* @param pVM The cross context VM structure.
1020	* @param GCPtr The guest context pointer.
1021	*/
1022	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
1023	{
1024	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
1025	while (pMapping)
1026	{
1027	if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
1028	break;
1029	if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
1030	return pMapping;
1031	pMapping = pMapping->CTX_SUFF(pNext);
1032	}
1033	return NULL;
1034	}
1035	#endif
1036
1037
1038	/**
1039	* Verifies a range of pages for read or write access
1040	*
1041	* Only checks the guest's page tables
1042	*
1043	* @returns VBox status code.
1044	* @param pVCpu The cross context virtual CPU structure.
1045	* @param Addr Guest virtual address to check
1046	* @param cbSize Access size
1047	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1048	* @remarks Current not in use.
1049	*/
1050	VMMDECL(int) PGMIsValidAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1051	{
1052	/*
1053	* Validate input.
1054	*/
1055	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
1056	{
1057	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
1058	return VERR_INVALID_PARAMETER;
1059	}
1060
1061	uint64_t fPage;
1062	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
1063	if (RT_FAILURE(rc))
1064	{
1065	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
1066	return VINF_EM_RAW_GUEST_TRAP;
1067	}
1068
1069	/*
1070	* Check if the access would cause a page fault
1071	*
1072	* Note that hypervisor page directories are not present in the guest's tables, so this check
1073	* is sufficient.
1074	*/
1075	bool fWrite = !!(fAccess & X86_PTE_RW);
1076	bool fUser = !!(fAccess & X86_PTE_US);
1077	if ( !(fPage & X86_PTE_P)
1078	\|\| (fWrite && !(fPage & X86_PTE_RW))
1079	\|\| (fUser && !(fPage & X86_PTE_US)) )
1080	{
1081	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
1082	return VINF_EM_RAW_GUEST_TRAP;
1083	}
1084	if ( RT_SUCCESS(rc)
1085	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
1086	return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
1087	return rc;
1088	}
1089
1090
1091	/**
1092	* Verifies a range of pages for read or write access
1093	*
1094	* Supports handling of pages marked for dirty bit tracking and CSAM
1095	*
1096	* @returns VBox status code.
1097	* @param pVCpu The cross context virtual CPU structure.
1098	* @param Addr Guest virtual address to check
1099	* @param cbSize Access size
1100	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1101	*/
1102	VMMDECL(int) PGMVerifyAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1103	{
1104	PVM pVM = pVCpu->CTX_SUFF(pVM);
1105
1106	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
1107
1108	/*
1109	* Get going.
1110	*/
1111	uint64_t fPageGst;
1112	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
1113	if (RT_FAILURE(rc))
1114	{
1115	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
1116	return VINF_EM_RAW_GUEST_TRAP;
1117	}
1118
1119	/*
1120	* Check if the access would cause a page fault
1121	*
1122	* Note that hypervisor page directories are not present in the guest's tables, so this check
1123	* is sufficient.
1124	*/
1125	const bool fWrite = !!(fAccess & X86_PTE_RW);
1126	const bool fUser = !!(fAccess & X86_PTE_US);
1127	if ( !(fPageGst & X86_PTE_P)
1128	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
1129	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
1130	{
1131	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
1132	return VINF_EM_RAW_GUEST_TRAP;
1133	}
1134
1135	if (!pVM->pgm.s.fNestedPaging)
1136	{
1137	/*
1138	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
1139	*/
1140	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
1141	if ( rc == VERR_PAGE_NOT_PRESENT
1142	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
1143	{
1144	/*
1145	* Page is not present in our page tables.
1146	* Try to sync it!
1147	*/
1148	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
1149	uint32_t const uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
1150	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1151	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1152	AssertReturn(g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
1153	rc = g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage(pVCpu, Addr, fPageGst, uErr);
1154	if (rc != VINF_SUCCESS)
1155	return rc;
1156	}
1157	else
1158	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
1159	}
1160
1161	#if 0 /* def VBOX_STRICT; triggers too often now */
1162	/*
1163	* This check is a bit paranoid, but useful.
1164	*/
1165	/* Note! This will assert when writing to monitored pages (a bit annoying actually). */
1166	uint64_t fPageShw;
1167	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
1168	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
1169	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
1170	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
1171	{
1172	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
1173	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
1174	return VINF_EM_RAW_GUEST_TRAP;
1175	}
1176	#endif
1177
1178	if ( RT_SUCCESS(rc)
1179	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
1180	\|\| Addr + cbSize < Addr))
1181	{
1182	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
1183	for (;;)
1184	{
1185	Addr += PAGE_SIZE;
1186	if (cbSize > PAGE_SIZE)
1187	cbSize -= PAGE_SIZE;
1188	else
1189	cbSize = 1;
1190	rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
1191	if (rc != VINF_SUCCESS)
1192	break;
1193	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
1194	break;
1195	}
1196	}
1197	return rc;
1198	}
1199
1200
1201	/**
1202	* Emulation of the invlpg instruction (HC only actually).
1203	*
1204	* @returns Strict VBox status code, special care required.
1205	* @retval VINF_PGM_SYNC_CR3 - handled.
1206	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1207	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1208	*
1209	* @param pVCpu The cross context virtual CPU structure.
1210	* @param GCPtrPage Page to invalidate.
1211	*
1212	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1213	* safe, but there could be edge cases!
1214	*
1215	* @todo Flush page or page directory only if necessary!
1216	* @todo VBOXSTRICTRC
1217	*/
1218	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1219	{
1220	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1221	int rc;
1222	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1223
1224	#if !defined(IN_RING3) && defined(VBOX_WITH_REM)
1225	/*
1226	* Notify the recompiler so it can record this instruction.
1227	*/
1228	REMNotifyInvalidatePage(pVM, GCPtrPage);
1229	#endif
1230	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1231
1232	/*
1233	* Call paging mode specific worker.
1234	*/
1235	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1236	pgmLock(pVM);
1237
1238	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1239	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), pgmUnlock(pVM), VERR_PGM_MODE_IPE);
1240	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, pgmUnlock(pVM), VERR_PGM_MODE_IPE);
1241	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1242
1243	pgmUnlock(pVM);
1244	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage), a);
1245
1246	#ifdef IN_RING3
1247	/*
1248	* Check if we have a pending update of the CR3 monitoring.
1249	*/
1250	if ( RT_SUCCESS(rc)
1251	&& (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
1252	{
1253	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1254	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
1255	}
1256	#endif /* IN_RING3 */
1257
1258	/* Ignore all irrelevant error codes. */
1259	if ( rc == VERR_PAGE_NOT_PRESENT
1260	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1261	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1262	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1263	rc = VINF_SUCCESS;
1264
1265	return rc;
1266	}
1267
1268
1269	/**
1270	* Executes an instruction using the interpreter.
1271	*
1272	* @returns VBox status code (appropriate for trap handling and GC return).
1273	* @param pVM The cross context VM structure.
1274	* @param pVCpu The cross context virtual CPU structure.
1275	* @param pRegFrame Register frame.
1276	* @param pvFault Fault address.
1277	*/
1278	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1279	{
1280	NOREF(pVM);
1281	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1282	if (rc == VERR_EM_INTERPRETER)
1283	rc = VINF_EM_RAW_EMULATE_INSTR;
1284	if (rc != VINF_SUCCESS)
1285	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1286	return rc;
1287	}
1288
1289
1290	/**
1291	* Gets effective page information (from the VMM page directory).
1292	*
1293	* @returns VBox status code.
1294	* @param pVCpu The cross context virtual CPU structure.
1295	* @param GCPtr Guest Context virtual address of the page.
1296	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1297	* @param pHCPhys Where to store the HC physical address of the page.
1298	* This is page aligned.
1299	* @remark You should use PGMMapGetPage() for pages in a mapping.
1300	*/
1301	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1302	{
1303	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1304	pgmLock(pVM);
1305
1306	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1307	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1308	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1309	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1310
1311	pgmUnlock(pVM);
1312	return rc;
1313	}
1314
1315
1316	/**
1317	* Modify page flags for a range of pages in the shadow context.
1318	*
1319	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1320	*
1321	* @returns VBox status code.
1322	* @param pVCpu The cross context virtual CPU structure.
1323	* @param GCPtr Virtual address of the first page in the range.
1324	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1325	* @param fMask The AND mask - page flags X86_PTE_*.
1326	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1327	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1328	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1329	*/
1330	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1331	{
1332	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1333	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1334
1335	GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
1336
1337	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1338	pgmLock(pVM);
1339
1340	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1341	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1342	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1343	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
1344
1345	pgmUnlock(pVM);
1346	return rc;
1347	}
1348
1349
1350	/**
1351	* Changing the page flags for a single page in the shadow page tables so as to
1352	* make it read-only.
1353	*
1354	* @returns VBox status code.
1355	* @param pVCpu The cross context virtual CPU structure.
1356	* @param GCPtr Virtual address of the first page in the range.
1357	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1358	*/
1359	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1360	{
1361	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1362	}
1363
1364
1365	/**
1366	* Changing the page flags for a single page in the shadow page tables so as to
1367	* make it writable.
1368	*
1369	* The call must know with 101% certainty that the guest page tables maps this
1370	* as writable too. This function will deal shared, zero and write monitored
1371	* pages.
1372	*
1373	* @returns VBox status code.
1374	* @param pVCpu The cross context virtual CPU structure.
1375	* @param GCPtr Virtual address of the first page in the range.
1376	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1377	*/
1378	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1379	{
1380	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1381	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1382	return VINF_SUCCESS;
1383	}
1384
1385
1386	/**
1387	* Changing the page flags for a single page in the shadow page tables so as to
1388	* make it not present.
1389	*
1390	* @returns VBox status code.
1391	* @param pVCpu The cross context virtual CPU structure.
1392	* @param GCPtr Virtual address of the first page in the range.
1393	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1394	*/
1395	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1396	{
1397	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1398	}
1399
1400
1401	/**
1402	* Changing the page flags for a single page in the shadow page tables so as to
1403	* make it supervisor and writable.
1404	*
1405	* This if for dealing with CR0.WP=0 and readonly user pages.
1406	*
1407	* @returns VBox status code.
1408	* @param pVCpu The cross context virtual CPU structure.
1409	* @param GCPtr Virtual address of the first page in the range.
1410	* @param fBigPage Whether or not this is a big page. If it is, we have to
1411	* change the shadow PDE as well. If it isn't, the caller
1412	* has checked that the shadow PDE doesn't need changing.
1413	* We ASSUME 4KB pages backing the big page here!
1414	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1415	*/
1416	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1417	{
1418	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1419	if (rc == VINF_SUCCESS && fBigPage)
1420	{
1421	/* this is a bit ugly... */
1422	switch (pVCpu->pgm.s.enmShadowMode)
1423	{
1424	case PGMMODE_32_BIT:
1425	{
1426	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1427	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1428	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1429	pPde->n.u1Write = 1;
1430	Log(("-> PDE=%#llx (32)\n", pPde->u));
1431	break;
1432	}
1433	case PGMMODE_PAE:
1434	case PGMMODE_PAE_NX:
1435	{
1436	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1437	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1438	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1439	pPde->n.u1Write = 1;
1440	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1441	break;
1442	}
1443	default:
1444	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1445	}
1446	}
1447	return rc;
1448	}
1449
1450
1451	/**
1452	* Gets the shadow page directory for the specified address, PAE.
1453	*
1454	* @returns Pointer to the shadow PD.
1455	* @param pVCpu The cross context virtual CPU structure.
1456	* @param GCPtr The address.
1457	* @param uGstPdpe Guest PDPT entry. Valid.
1458	* @param ppPD Receives address of page directory
1459	*/
1460	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1461	{
1462	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1463	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1464	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1465	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1466	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1467	PPGMPOOLPAGE pShwPage;
1468	int rc;
1469
1470	PGM_LOCK_ASSERT_OWNER(pVM);
1471
1472	/* Allocate page directory if not present. */
1473	if ( !pPdpe->n.u1Present
1474	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1475	{
1476	RTGCPTR64 GCPdPt;
1477	PGMPOOLKIND enmKind;
1478
1479	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1480	{
1481	/* AMD-V nested paging or real/protected mode without paging. */
1482	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1483	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1484	}
1485	else
1486	{
1487	if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1488	{
1489	if (!(uGstPdpe & X86_PDPE_P))
1490	{
1491	/* PD not present; guest must reload CR3 to change it.
1492	* No need to monitor anything in this case.
1493	*/
1494	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1495
1496	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1497	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1498	uGstPdpe \|= X86_PDPE_P;
1499	}
1500	else
1501	{
1502	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1503	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1504	}
1505	}
1506	else
1507	{
1508	GCPdPt = CPUMGetGuestCR3(pVCpu);
1509	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1510	}
1511	}
1512
1513	/* Create a reference back to the PDPT by using the index in its shadow page. */
1514	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1515	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1516	&pShwPage);
1517	AssertRCReturn(rc, rc);
1518
1519	/* The PD was cached or created; hook it up now. */
1520	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A));
1521	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1522	}
1523	else
1524	{
1525	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1526	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1527	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1528
1529	pgmPoolCacheUsed(pPool, pShwPage);
1530	}
1531	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1532	return VINF_SUCCESS;
1533	}
1534
1535
1536	/**
1537	* Gets the pointer to the shadow page directory entry for an address, PAE.
1538	*
1539	* @returns Pointer to the PDE.
1540	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1541	* @param GCPtr The address.
1542	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1543	*/
1544	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1545	{
1546	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1547	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1548	PVM pVM = pVCpu->CTX_SUFF(pVM);
1549
1550	PGM_LOCK_ASSERT_OWNER(pVM);
1551
1552	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1553	if (!pPdpt->a[iPdPt].n.u1Present)
1554	{
1555	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, pPdpt->a[iPdPt].u));
1556	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1557	}
1558	AssertMsg(pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1559
1560	/* Fetch the pgm pool shadow descriptor. */
1561	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1562	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1563
1564	*ppShwPde = pShwPde;
1565	return VINF_SUCCESS;
1566	}
1567
1568
1569	/**
1570	* Syncs the SHADOW page directory pointer for the specified address.
1571	*
1572	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1573	*
1574	* The caller is responsible for making sure the guest has a valid PD before
1575	* calling this function.
1576	*
1577	* @returns VBox status code.
1578	* @param pVCpu The cross context virtual CPU structure.
1579	* @param GCPtr The address.
1580	* @param uGstPml4e Guest PML4 entry (valid).
1581	* @param uGstPdpe Guest PDPT entry (valid).
1582	* @param ppPD Receives address of page directory
1583	*/
1584	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1585	{
1586	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1587	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1588	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1589	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1590	bool fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1591	PPGMPOOLPAGE pShwPage;
1592	int rc;
1593
1594	PGM_LOCK_ASSERT_OWNER(pVM);
1595
1596	/* Allocate page directory pointer table if not present. */
1597	if ( !pPml4e->n.u1Present
1598	&& !(pPml4e->u & X86_PML4E_PG_MASK))
1599	{
1600	RTGCPTR64 GCPml4;
1601	PGMPOOLKIND enmKind;
1602
1603	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1604
1605	if (fNestedPagingOrNoGstPaging)
1606	{
1607	/* AMD-V nested paging or real/protected mode without paging */
1608	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1609	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1610	}
1611	else
1612	{
1613	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1614	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1615	}
1616
1617	/* Create a reference back to the PDPT by using the index in its shadow page. */
1618	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1619	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1620	&pShwPage);
1621	AssertRCReturn(rc, rc);
1622	}
1623	else
1624	{
1625	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1626	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1627
1628	pgmPoolCacheUsed(pPool, pShwPage);
1629	}
1630	/* The PDPT was cached or created; hook it up now. */
1631	pPml4e->u \|= pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask);
1632
1633	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1634	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1635	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1636
1637	/* Allocate page directory if not present. */
1638	if ( !pPdpe->n.u1Present
1639	&& !(pPdpe->u & X86_PDPE_PG_MASK))
1640	{
1641	RTGCPTR64 GCPdPt;
1642	PGMPOOLKIND enmKind;
1643
1644	if (fNestedPagingOrNoGstPaging)
1645	{
1646	/* AMD-V nested paging or real/protected mode without paging */
1647	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1648	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1649	}
1650	else
1651	{
1652	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1653	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1654	}
1655
1656	/* Create a reference back to the PDPT by using the index in its shadow page. */
1657	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1658	pShwPage->idx, iPdPt, false /fLockPage/,
1659	&pShwPage);
1660	AssertRCReturn(rc, rc);
1661	}
1662	else
1663	{
1664	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & X86_PDPE_PG_MASK);
1665	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1666
1667	pgmPoolCacheUsed(pPool, pShwPage);
1668	}
1669	/* The PD was cached or created; hook it up now. */
1670	pPdpe->u \|= pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask);
1671
1672	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1673	return VINF_SUCCESS;
1674	}
1675
1676
1677	/**
1678	* Gets the SHADOW page directory pointer for the specified address (long mode).
1679	*
1680	* @returns VBox status code.
1681	* @param pVCpu The cross context virtual CPU structure.
1682	* @param GCPtr The address.
1683	* @param ppPml4e Receives the address of the page map level 4 entry.
1684	* @param ppPdpt Receives the address of the page directory pointer table.
1685	* @param ppPD Receives the address of the page directory.
1686	*/
1687	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1688	{
1689	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1690	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1691
1692	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1693
1694	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1695	if (ppPml4e)
1696	*ppPml4e = (PX86PML4E)pPml4e;
1697
1698	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, pPml4e->u));
1699
1700	if (!pPml4e->n.u1Present)
1701	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1702
1703	PVM pVM = pVCpu->CTX_SUFF(pVM);
1704	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1705	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPml4e->u & X86_PML4E_PG_MASK);
1706	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1707
1708	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1709	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1710	if (!pPdpt->a[iPdPt].n.u1Present)
1711	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1712
1713	pShwPage = pgmPoolGetPage(pPool, pPdpt->a[iPdPt].u & X86_PDPE_PG_MASK);
1714	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1715
1716	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1717	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1718	return VINF_SUCCESS;
1719	}
1720
1721
1722	/**
1723	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1724	* backing pages in case the PDPT or PML4 entry is missing.
1725	*
1726	* @returns VBox status code.
1727	* @param pVCpu The cross context virtual CPU structure.
1728	* @param GCPtr The address.
1729	* @param ppPdpt Receives address of pdpt
1730	* @param ppPD Receives address of page directory
1731	*/
1732	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1733	{
1734	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1735	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1736	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1737	PEPTPML4 pPml4;
1738	PEPTPML4E pPml4e;
1739	PPGMPOOLPAGE pShwPage;
1740	int rc;
1741
1742	Assert(pVM->pgm.s.fNestedPaging);
1743	PGM_LOCK_ASSERT_OWNER(pVM);
1744
1745	pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1746	Assert(pPml4);
1747
1748	/* Allocate page directory pointer table if not present. */
1749	pPml4e = &pPml4->a[iPml4];
1750	if ( !pPml4e->n.u1Present
1751	&& !(pPml4e->u & EPT_PML4E_PG_MASK))
1752	{
1753	Assert(!(pPml4e->u & EPT_PML4E_PG_MASK));
1754	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1755
1756	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1757	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1758	&pShwPage);
1759	AssertRCReturn(rc, rc);
1760	}
1761	else
1762	{
1763	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1764	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1765
1766	pgmPoolCacheUsed(pPool, pShwPage);
1767	}
1768	/* The PDPT was cached or created; hook it up now and fill with the default value. */
1769	pPml4e->u = pShwPage->Core.Key;
1770	pPml4e->n.u1Present = 1;
1771	pPml4e->n.u1Write = 1;
1772	pPml4e->n.u1Execute = 1;
1773
1774	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1775	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1776	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1777
1778	if (ppPdpt)
1779	*ppPdpt = pPdpt;
1780
1781	/* Allocate page directory if not present. */
1782	if ( !pPdpe->n.u1Present
1783	&& !(pPdpe->u & EPT_PDPTE_PG_MASK))
1784	{
1785	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1786	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1787	pShwPage->idx, iPdPt, false /fLockPage/,
1788	&pShwPage);
1789	AssertRCReturn(rc, rc);
1790	}
1791	else
1792	{
1793	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1794	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1795
1796	pgmPoolCacheUsed(pPool, pShwPage);
1797	}
1798	/* The PD was cached or created; hook it up now and fill with the default value. */
1799	pPdpe->u = pShwPage->Core.Key;
1800	pPdpe->n.u1Present = 1;
1801	pPdpe->n.u1Write = 1;
1802	pPdpe->n.u1Execute = 1;
1803
1804	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1805	return VINF_SUCCESS;
1806	}
1807
1808
1809	#ifdef IN_RING0
1810	/**
1811	* Synchronizes a range of nested page table entries.
1812	*
1813	* The caller must own the PGM lock.
1814	*
1815	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1816	* @param GCPhys Where to start.
1817	* @param cPages How many pages which entries should be synced.
1818	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1819	* host paging mode for AMD-V).
1820	*/
1821	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1822	{
1823	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1824
1825	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1826	int rc;
1827	switch (enmShwPagingMode)
1828	{
1829	case PGMMODE_32_BIT:
1830	{
1831	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1832	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1833	break;
1834	}
1835
1836	case PGMMODE_PAE:
1837	case PGMMODE_PAE_NX:
1838	{
1839	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1840	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1841	break;
1842	}
1843
1844	case PGMMODE_AMD64:
1845	case PGMMODE_AMD64_NX:
1846	{
1847	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1848	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1849	break;
1850	}
1851
1852	case PGMMODE_EPT:
1853	{
1854	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1855	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1856	break;
1857	}
1858
1859	default:
1860	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1861	}
1862	return rc;
1863	}
1864	#endif /* IN_RING0 */
1865
1866
1867	/**
1868	* Gets effective Guest OS page information.
1869	*
1870	* When GCPtr is in a big page, the function will return as if it was a normal
1871	* 4KB page. If the need for distinguishing between big and normal page becomes
1872	* necessary at a later point, a PGMGstGetPage() will be created for that
1873	* purpose.
1874	*
1875	* @returns VBox status code.
1876	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1877	* @param GCPtr Guest Context virtual address of the page.
1878	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1879	* @param pGCPhys Where to store the GC physical address of the page.
1880	* This is page aligned. The fact that the
1881	*/
1882	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1883	{
1884	VMCPU_ASSERT_EMT(pVCpu);
1885	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1886	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1887	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1888	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
1889	}
1890
1891
1892	/**
1893	* Performs a guest page table walk.
1894	*
1895	* The guest should be in paged protect mode or long mode when making a call to
1896	* this function.
1897	*
1898	* @returns VBox status code.
1899	* @retval VINF_SUCCESS on success.
1900	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1901	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1902	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1903	*
1904	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1905	* @param GCPtr The guest virtual address to walk by.
1906	* @param pWalk Where to return the walk result. This is valid for some
1907	* error codes as well.
1908	*/
1909	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1910	{
1911	VMCPU_ASSERT_EMT(pVCpu);
1912	switch (pVCpu->pgm.s.enmGuestMode)
1913	{
1914	case PGMMODE_32_BIT:
1915	pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1916	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
1917
1918	case PGMMODE_PAE:
1919	case PGMMODE_PAE_NX:
1920	pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1921	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
1922
1923	case PGMMODE_AMD64:
1924	case PGMMODE_AMD64_NX:
1925	pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1926	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
1927
1928	case PGMMODE_REAL:
1929	case PGMMODE_PROTECTED:
1930	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1931	return VERR_PGM_NOT_USED_IN_MODE;
1932
1933	case PGMMODE_NESTED_32BIT:
1934	case PGMMODE_NESTED_PAE:
1935	case PGMMODE_NESTED_AMD64:
1936	case PGMMODE_EPT:
1937	default:
1938	AssertFailed();
1939	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1940	return VERR_PGM_NOT_USED_IN_MODE;
1941	}
1942	}
1943
1944
1945	/**
1946	* Tries to continue the previous walk.
1947	*
1948	* @note Requires the caller to hold the PGM lock from the first
1949	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1950	* we cannot use the pointers.
1951	*
1952	* @returns VBox status code.
1953	* @retval VINF_SUCCESS on success.
1954	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1955	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1956	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1957	*
1958	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1959	* @param GCPtr The guest virtual address to walk by.
1960	* @param pWalk Pointer to the previous walk result and where to return
1961	* the result of this walk. This is valid for some error
1962	* codes as well.
1963	*/
1964	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1965	{
1966	/*
1967	* We can only handle successfully walks.
1968	* We also limit ourselves to the next page.
1969	*/
1970	if ( pWalk->u.Core.fSucceeded
1971	&& GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
1972	{
1973	Assert(pWalk->u.Core.uLevel == 0);
1974	if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
1975	{
1976	/*
1977	* AMD64
1978	*/
1979	if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
1980	{
1981	/*
1982	* We fall back to full walk if the PDE table changes, if any
1983	* reserved bits are set, or if the effective page access changes.
1984	*/
1985	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
1986	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
1987	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
1988	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
1989
1990	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
1991	{
1992	if (pWalk->u.Amd64.pPte)
1993	{
1994	X86PTEPAE Pte;
1995	Pte.u = pWalk->u.Amd64.pPte[1].u;
1996	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
1997	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
1998	{
1999
2000	pWalk->u.Core.GCPtr = GCPtr;
2001	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2002	pWalk->u.Amd64.Pte.u = Pte.u;
2003	pWalk->u.Amd64.pPte++;
2004	return VINF_SUCCESS;
2005	}
2006	}
2007	}
2008	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
2009	{
2010	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2011	if (pWalk->u.Amd64.pPde)
2012	{
2013	X86PDEPAE Pde;
2014	Pde.u = pWalk->u.Amd64.pPde[1].u;
2015	if ( (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
2016	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2017	{
2018	/* Get the new PTE and check out the first entry. */
2019	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2020	&pWalk->u.Amd64.pPt);
2021	if (RT_SUCCESS(rc))
2022	{
2023	pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
2024	X86PTEPAE Pte;
2025	Pte.u = pWalk->u.Amd64.pPte->u;
2026	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2027	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2028	{
2029	pWalk->u.Core.GCPtr = GCPtr;
2030	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2031	pWalk->u.Amd64.Pte.u = Pte.u;
2032	pWalk->u.Amd64.Pde.u = Pde.u;
2033	pWalk->u.Amd64.pPde++;
2034	return VINF_SUCCESS;
2035	}
2036	}
2037	}
2038	}
2039	}
2040	}
2041	else if (!pWalk->u.Core.fGigantPage)
2042	{
2043	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
2044	{
2045	pWalk->u.Core.GCPtr = GCPtr;
2046	pWalk->u.Core.GCPhys += PAGE_SIZE;
2047	return VINF_SUCCESS;
2048	}
2049	}
2050	else
2051	{
2052	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
2053	{
2054	pWalk->u.Core.GCPtr = GCPtr;
2055	pWalk->u.Core.GCPhys += PAGE_SIZE;
2056	return VINF_SUCCESS;
2057	}
2058	}
2059	}
2060	}
2061	/* Case we don't handle. Do full walk. */
2062	return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
2063	}
2064
2065
2066	/**
2067	* Checks if the page is present.
2068	*
2069	* @returns true if the page is present.
2070	* @returns false if the page is not present.
2071	* @param pVCpu The cross context virtual CPU structure.
2072	* @param GCPtr Address within the page.
2073	*/
2074	VMMDECL(bool) PGMGstIsPagePresent(PVMCPUCC pVCpu, RTGCPTR GCPtr)
2075	{
2076	VMCPU_ASSERT_EMT(pVCpu);
2077	int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
2078	return RT_SUCCESS(rc);
2079	}
2080
2081
2082	/**
2083	* Sets (replaces) the page flags for a range of pages in the guest's tables.
2084	*
2085	* @returns VBox status code.
2086	* @param pVCpu The cross context virtual CPU structure.
2087	* @param GCPtr The address of the first page.
2088	* @param cb The size of the range in bytes.
2089	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
2090	*/
2091	VMMDECL(int) PGMGstSetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
2092	{
2093	VMCPU_ASSERT_EMT(pVCpu);
2094	return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
2095	}
2096
2097
2098	/**
2099	* Modify page flags for a range of pages in the guest's tables
2100	*
2101	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2102	*
2103	* @returns VBox status code.
2104	* @param pVCpu The cross context virtual CPU structure.
2105	* @param GCPtr Virtual address of the first page in the range.
2106	* @param cb Size (in bytes) of the range to apply the modification to.
2107	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2108	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2109	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2110	*/
2111	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2112	{
2113	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2114	VMCPU_ASSERT_EMT(pVCpu);
2115
2116	/*
2117	* Validate input.
2118	*/
2119	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2120	Assert(cb);
2121
2122	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2123
2124	/*
2125	* Adjust input.
2126	*/
2127	cb += GCPtr & PAGE_OFFSET_MASK;
2128	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
2129	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
2130
2131	/*
2132	* Call worker.
2133	*/
2134	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2135	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2136	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2137	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2138
2139	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,GstModifyPage), a);
2140	return rc;
2141	}
2142
2143
2144	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2145
2146	/**
2147	* Performs the lazy mapping of the 32-bit guest PD.
2148	*
2149	* @returns VBox status code.
2150	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2151	* @param ppPd Where to return the pointer to the mapping. This is
2152	* always set.
2153	*/
2154	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2155	{
2156	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2157	pgmLock(pVM);
2158
2159	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2160
2161	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2162	PPGMPAGE pPage;
2163	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2164	if (RT_SUCCESS(rc))
2165	{
2166	RTHCPTR HCPtrGuestCR3;
2167	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2168	if (RT_SUCCESS(rc))
2169	{
2170	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
2171	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2172	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
2173	# endif
2174	*ppPd = (PX86PD)HCPtrGuestCR3;
2175
2176	pgmUnlock(pVM);
2177	return VINF_SUCCESS;
2178	}
2179
2180	AssertRC(rc);
2181	}
2182	pgmUnlock(pVM);
2183
2184	*ppPd = NULL;
2185	return rc;
2186	}
2187
2188
2189	/**
2190	* Performs the lazy mapping of the PAE guest PDPT.
2191	*
2192	* @returns VBox status code.
2193	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2194	* @param ppPdpt Where to return the pointer to the mapping. This is
2195	* always set.
2196	*/
2197	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2198	{
2199	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2200	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2201	pgmLock(pVM);
2202
2203	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2204	PPGMPAGE pPage;
2205	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2206	if (RT_SUCCESS(rc))
2207	{
2208	RTHCPTR HCPtrGuestCR3;
2209	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2210	if (RT_SUCCESS(rc))
2211	{
2212	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2213	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2214	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
2215	# endif
2216	*ppPdpt = (PX86PDPT)HCPtrGuestCR3;
2217
2218	pgmUnlock(pVM);
2219	return VINF_SUCCESS;
2220	}
2221
2222	AssertRC(rc);
2223	}
2224
2225	pgmUnlock(pVM);
2226	*ppPdpt = NULL;
2227	return rc;
2228	}
2229
2230
2231	/**
2232	* Performs the lazy mapping / updating of a PAE guest PD.
2233	*
2234	* @returns Pointer to the mapping.
2235	* @returns VBox status code.
2236	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2237	* @param iPdpt Which PD entry to map (0..3).
2238	* @param ppPd Where to return the pointer to the mapping. This is
2239	* always set.
2240	*/
2241	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2242	{
2243	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2244	pgmLock(pVM);
2245
2246	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2247	Assert(pGuestPDPT);
2248	Assert(pGuestPDPT->a[iPdpt].n.u1Present);
2249	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2250	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2251
2252	PPGMPAGE pPage;
2253	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2254	if (RT_SUCCESS(rc))
2255	{
2256	RTHCPTR HCPtr = NIL_RTHCPTR;
2257	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2258	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, &HCPtr);
2259	AssertRC(rc);
2260	# endif
2261	if (RT_SUCCESS(rc))
2262	{
2263	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = (R3PTRTYPE(PX86PDPAE))HCPtr;
2264	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2265	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = (R0PTRTYPE(PX86PDPAE))HCPtr;
2266	# endif
2267	if (fChanged)
2268	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2269
2270	*ppPd = pVCpu->pgm.s.CTX_SUFF(apGstPaePDs)[iPdpt];
2271	pgmUnlock(pVM);
2272	return VINF_SUCCESS;
2273	}
2274	}
2275
2276	/* Invalid page or some failure, invalidate the entry. */
2277	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2278	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = 0;
2279	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2280	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = 0;
2281	# endif
2282
2283	pgmUnlock(pVM);
2284	return rc;
2285	}
2286
2287
2288	/**
2289	* Performs the lazy mapping of the 32-bit guest PD.
2290	*
2291	* @returns VBox status code.
2292	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2293	* @param ppPml4 Where to return the pointer to the mapping. This will
2294	* always be set.
2295	*/
2296	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2297	{
2298	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2299	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2300	pgmLock(pVM);
2301
2302	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2303	PPGMPAGE pPage;
2304	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2305	if (RT_SUCCESS(rc))
2306	{
2307	RTHCPTR HCPtrGuestCR3;
2308	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)&HCPtrGuestCR3);
2309	if (RT_SUCCESS(rc))
2310	{
2311	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
2312	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2313	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
2314	# endif
2315	*ppPml4 = (PX86PML4)HCPtrGuestCR3;
2316
2317	pgmUnlock(pVM);
2318	return VINF_SUCCESS;
2319	}
2320	}
2321
2322	pgmUnlock(pVM);
2323	*ppPml4 = NULL;
2324	return rc;
2325	}
2326
2327	#endif /* !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
2328
2329
2330	/**
2331	* Gets the PAE PDPEs values cached by the CPU.
2332	*
2333	* @returns VBox status code.
2334	* @param pVCpu The cross context virtual CPU structure.
2335	* @param paPdpes Where to return the four PDPEs. The array
2336	* pointed to must have 4 entries.
2337	*/
2338	VMM_INT_DECL(int) PGMGstGetPaePdpes(PVMCPUCC pVCpu, PX86PDPE paPdpes)
2339	{
2340	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2341
2342	paPdpes[0] = pVCpu->pgm.s.aGstPaePdpeRegs[0];
2343	paPdpes[1] = pVCpu->pgm.s.aGstPaePdpeRegs[1];
2344	paPdpes[2] = pVCpu->pgm.s.aGstPaePdpeRegs[2];
2345	paPdpes[3] = pVCpu->pgm.s.aGstPaePdpeRegs[3];
2346	return VINF_SUCCESS;
2347	}
2348
2349
2350	/**
2351	* Sets the PAE PDPEs values cached by the CPU.
2352	*
2353	* @remarks This must be called AFTER PGMUpdateCR3.
2354	*
2355	* @param pVCpu The cross context virtual CPU structure.
2356	* @param paPdpes The four PDPE values. The array pointed to must
2357	* have exactly 4 entries.
2358	*
2359	* @remarks No-long-jump zone!!!
2360	*/
2361	VMM_INT_DECL(void) PGMGstUpdatePaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPdpes)
2362	{
2363	Assert(pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2364
2365	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGstPaePdpeRegs); i++)
2366	{
2367	if (pVCpu->pgm.s.aGstPaePdpeRegs[i].u != paPdpes[i].u)
2368	{
2369	pVCpu->pgm.s.aGstPaePdpeRegs[i] = paPdpes[i];
2370
2371	/* Force lazy remapping if it changed in any way. */
2372	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2373	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2374	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2375	#endif
2376	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2377	}
2378	}
2379
2380	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
2381	}
2382
2383
2384	/**
2385	* Gets the current CR3 register value for the shadow memory context.
2386	* @returns CR3 value.
2387	* @param pVCpu The cross context virtual CPU structure.
2388	*/
2389	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2390	{
2391	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2392	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2393	return pPoolPage->Core.Key;
2394	}
2395
2396
2397	/**
2398	* Performs and schedules necessary updates following a CR3 load or reload.
2399	*
2400	* This will normally involve mapping the guest PD or nPDPT
2401	*
2402	* @returns VBox status code.
2403	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2404	* safely be ignored and overridden since the FF will be set too then.
2405	* @param pVCpu The cross context virtual CPU structure.
2406	* @param cr3 The new cr3.
2407	* @param fGlobal Indicates whether this is a global flush or not.
2408	*/
2409	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
2410	{
2411	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2412	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2413
2414	VMCPU_ASSERT_EMT(pVCpu);
2415
2416	/*
2417	* Always flag the necessary updates; necessary for hardware acceleration
2418	*/
2419	/** @todo optimize this, it shouldn't always be necessary. */
2420	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2421	if (fGlobal)
2422	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2423	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
2424
2425	/*
2426	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2427	*/
2428	int rc = VINF_SUCCESS;
2429	RTGCPHYS GCPhysCR3;
2430	switch (pVCpu->pgm.s.enmGuestMode)
2431	{
2432	case PGMMODE_PAE:
2433	case PGMMODE_PAE_NX:
2434	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2435	break;
2436	case PGMMODE_AMD64:
2437	case PGMMODE_AMD64_NX:
2438	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2439	break;
2440	default:
2441	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2442	break;
2443	}
2444	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2445
2446	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2447	if (GCPhysOldCR3 != GCPhysCR3)
2448	{
2449	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2450	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2451	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2452
2453	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2454	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2455	if (RT_LIKELY(rc == VINF_SUCCESS))
2456	{
2457	if (pgmMapAreMappingsFloating(pVM))
2458	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2459	}
2460	else
2461	{
2462	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2463	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2464	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2465	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2466	if (pgmMapAreMappingsFloating(pVM))
2467	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MONITOR_CR3;
2468	}
2469
2470	if (fGlobal)
2471	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2472	else
2473	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBNewCR3));
2474	}
2475	else
2476	{
2477	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2478	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2479	if (pPool->cDirtyPages)
2480	{
2481	pgmLock(pVM);
2482	pgmPoolResetDirtyPages(pVM);
2483	pgmUnlock(pVM);
2484	}
2485	#endif
2486	/*
2487	* Check if we have a pending update of the CR3 monitoring.
2488	*/
2489	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2490	{
2491	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2492	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
2493	}
2494	if (fGlobal)
2495	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2496	else
2497	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLBSameCR3));
2498	}
2499
2500	IEMTlbInvalidateAll(pVCpu, false /fVmm/);
2501	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,FlushTLB), a);
2502	return rc;
2503	}
2504
2505
2506	/**
2507	* Performs and schedules necessary updates following a CR3 load or reload when
2508	* using nested or extended paging.
2509	*
2510	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2511	* TLB and triggering a SyncCR3.
2512	*
2513	* This will normally involve mapping the guest PD or nPDPT
2514	*
2515	* @returns VBox status code.
2516	* @retval VINF_SUCCESS.
2517	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2518	* paging modes). This can safely be ignored and overridden since the
2519	* FF will be set too then.
2520	* @param pVCpu The cross context virtual CPU structure.
2521	* @param cr3 The new cr3.
2522	*/
2523	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
2524	{
2525	VMCPU_ASSERT_EMT(pVCpu);
2526	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
2527
2528	/* We assume we're only called in nested paging mode. */
2529	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2530	Assert(!pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2531	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
2532
2533	/*
2534	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2535	*/
2536	int rc = VINF_SUCCESS;
2537	RTGCPHYS GCPhysCR3;
2538	switch (pVCpu->pgm.s.enmGuestMode)
2539	{
2540	case PGMMODE_PAE:
2541	case PGMMODE_PAE_NX:
2542	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2543	break;
2544	case PGMMODE_AMD64:
2545	case PGMMODE_AMD64_NX:
2546	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2547	break;
2548	default:
2549	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2550	break;
2551	}
2552	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2553
2554	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2555	{
2556	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2557	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2558	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2559
2560	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2561	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2562
2563	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2564	}
2565
2566	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Synchronize the paging structures.
2573	*
2574	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2575	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2576	* in several places, most importantly whenever the CR3 is loaded.
2577	*
2578	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2579	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2580	* the VMM into guest context.
2581	* @param pVCpu The cross context virtual CPU structure.
2582	* @param cr0 Guest context CR0 register
2583	* @param cr3 Guest context CR3 register
2584	* @param cr4 Guest context CR4 register
2585	* @param fGlobal Including global page directories or not
2586	*/
2587	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2588	{
2589	int rc;
2590
2591	VMCPU_ASSERT_EMT(pVCpu);
2592
2593	/*
2594	* The pool may have pending stuff and even require a return to ring-3 to
2595	* clear the whole thing.
2596	*/
2597	rc = pgmPoolSyncCR3(pVCpu);
2598	if (rc != VINF_SUCCESS)
2599	return rc;
2600
2601	/*
2602	* We might be called when we shouldn't.
2603	*
2604	* The mode switching will ensure that the PD is resynced after every mode
2605	* switch. So, if we find ourselves here when in protected or real mode
2606	* we can safely clear the FF and return immediately.
2607	*/
2608	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2609	{
2610	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2611	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2612	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2613	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2614	return VINF_SUCCESS;
2615	}
2616
2617	/* If global pages are not supported, then all flushes are global. */
2618	if (!(cr4 & X86_CR4_PGE))
2619	fGlobal = true;
2620	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2621	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2622
2623	/*
2624	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2625	* This should be done before SyncCR3.
2626	*/
2627	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2628	{
2629	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2630
2631	RTGCPHYS GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
2632	RTGCPHYS GCPhysCR3;
2633	switch (pVCpu->pgm.s.enmGuestMode)
2634	{
2635	case PGMMODE_PAE:
2636	case PGMMODE_PAE_NX:
2637	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAE_PAGE_MASK);
2638	break;
2639	case PGMMODE_AMD64:
2640	case PGMMODE_AMD64_NX:
2641	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_AMD64_PAGE_MASK);
2642	break;
2643	default:
2644	GCPhysCR3 = (RTGCPHYS)(cr3 & X86_CR3_PAGE_MASK);
2645	break;
2646	}
2647	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2648
2649	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2650	{
2651	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2652	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2653	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2654	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2655	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
2656	}
2657
2658	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2659	if ( rc == VINF_PGM_SYNC_CR3
2660	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2661	{
2662	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2663	#ifdef IN_RING3
2664	rc = pgmPoolSyncCR3(pVCpu);
2665	#else
2666	if (rc == VINF_PGM_SYNC_CR3)
2667	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
2668	return VINF_PGM_SYNC_CR3;
2669	#endif
2670	}
2671	AssertRCReturn(rc, rc);
2672	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2673	}
2674
2675	/*
2676	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2677	*/
2678	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2679
2680	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2681	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2682	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2683	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2684
2685	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
2686	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2687	if (rc == VINF_SUCCESS)
2688	{
2689	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2690	{
2691	/* Go back to ring 3 if a pgm pool sync is again pending. */
2692	return VINF_PGM_SYNC_CR3;
2693	}
2694
2695	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2696	{
2697	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2698	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2699	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2700	}
2701
2702	/*
2703	* Check if we have a pending update of the CR3 monitoring.
2704	*/
2705	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2706	{
2707	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2708	Assert(!pVCpu->CTX_SUFF(pVM)->pgm.s.fMappingsFixed);
2709	Assert(pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2710	}
2711	}
2712
2713	/*
2714	* Now flush the CR3 (guest context).
2715	*/
2716	if (rc == VINF_SUCCESS)
2717	PGM_INVL_VCPU_TLBS(pVCpu);
2718	return rc;
2719	}
2720
2721
2722	/**
2723	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2724	*
2725	* @returns VBox status code, with the following informational code for
2726	* VM scheduling.
2727	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2728	* @retval VINF_PGM_CHANGE_MODE if we're in RC the mode changes. This will
2729	* NOT be returned in ring-3 or ring-0.
2730	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2731	*
2732	* @param pVCpu The cross context virtual CPU structure.
2733	* @param cr0 The new cr0.
2734	* @param cr4 The new cr4.
2735	* @param efer The new extended feature enable register.
2736	*/
2737	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
2738	{
2739	VMCPU_ASSERT_EMT(pVCpu);
2740
2741	/*
2742	* Calc the new guest mode.
2743	*
2744	* Note! We check PG before PE and without requiring PE because of the
2745	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2746	*/
2747	PGMMODE enmGuestMode;
2748	if (cr0 & X86_CR0_PG)
2749	{
2750	if (!(cr4 & X86_CR4_PAE))
2751	{
2752	bool const fPse = !!(cr4 & X86_CR4_PSE);
2753	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2754	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2755	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2756	enmGuestMode = PGMMODE_32_BIT;
2757	}
2758	else if (!(efer & MSR_K6_EFER_LME))
2759	{
2760	if (!(efer & MSR_K6_EFER_NXE))
2761	enmGuestMode = PGMMODE_PAE;
2762	else
2763	enmGuestMode = PGMMODE_PAE_NX;
2764	}
2765	else
2766	{
2767	if (!(efer & MSR_K6_EFER_NXE))
2768	enmGuestMode = PGMMODE_AMD64;
2769	else
2770	enmGuestMode = PGMMODE_AMD64_NX;
2771	}
2772	}
2773	else if (!(cr0 & X86_CR0_PE))
2774	enmGuestMode = PGMMODE_REAL;
2775	else
2776	enmGuestMode = PGMMODE_PROTECTED;
2777
2778	/*
2779	* Did it change?
2780	*/
2781	if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2782	return VINF_SUCCESS;
2783
2784	/* Flush the TLB */
2785	PGM_INVL_VCPU_TLBS(pVCpu);
2786	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
2787	}
2788
2789
2790	/**
2791	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
2792	*
2793	* @returns PGM_TYPE_*.
2794	* @param pgmMode The mode value to convert.
2795	*/
2796	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
2797	{
2798	switch (pgmMode)
2799	{
2800	case PGMMODE_REAL: return PGM_TYPE_REAL;
2801	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
2802	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
2803	case PGMMODE_PAE:
2804	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
2805	case PGMMODE_AMD64:
2806	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
2807	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
2808	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
2809	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
2810	case PGMMODE_EPT: return PGM_TYPE_EPT;
2811	case PGMMODE_NONE: return PGM_TYPE_NONE;
2812	default:
2813	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
2814	}
2815	}
2816
2817
2818	/**
2819	* Calculates the shadow paging mode.
2820	*
2821	* @returns The shadow paging mode.
2822	* @param pVM The cross context VM structure.
2823	* @param enmGuestMode The guest mode.
2824	* @param enmHostMode The host mode.
2825	* @param enmShadowMode The current shadow mode.
2826	*/
2827	static PGMMODE pgmCalcShadowMode(PVM pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
2828	{
2829	switch (enmGuestMode)
2830	{
2831	/*
2832	* When switching to real or protected mode we don't change
2833	* anything since it's likely that we'll switch back pretty soon.
2834	*
2835	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
2836	* and is supposed to determine which shadow paging and switcher to
2837	* use during init.
2838	*/
2839	case PGMMODE_REAL:
2840	case PGMMODE_PROTECTED:
2841	if ( enmShadowMode != PGMMODE_INVALID
2842	&& VM_IS_RAW_MODE_ENABLED(pVM) /* always switch in hm and nem modes! */)
2843	break; /* (no change) */
2844
2845	switch (enmHostMode)
2846	{
2847	case SUPPAGINGMODE_32_BIT:
2848	case SUPPAGINGMODE_32_BIT_GLOBAL:
2849	enmShadowMode = PGMMODE_32_BIT;
2850	break;
2851
2852	case SUPPAGINGMODE_PAE:
2853	case SUPPAGINGMODE_PAE_NX:
2854	case SUPPAGINGMODE_PAE_GLOBAL:
2855	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2856	enmShadowMode = PGMMODE_PAE;
2857	break;
2858
2859	case SUPPAGINGMODE_AMD64:
2860	case SUPPAGINGMODE_AMD64_GLOBAL:
2861	case SUPPAGINGMODE_AMD64_NX:
2862	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2863	enmShadowMode = PGMMODE_PAE;
2864	break;
2865
2866	default:
2867	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2868	}
2869	break;
2870
2871	case PGMMODE_32_BIT:
2872	switch (enmHostMode)
2873	{
2874	case SUPPAGINGMODE_32_BIT:
2875	case SUPPAGINGMODE_32_BIT_GLOBAL:
2876	enmShadowMode = PGMMODE_32_BIT;
2877	break;
2878
2879	case SUPPAGINGMODE_PAE:
2880	case SUPPAGINGMODE_PAE_NX:
2881	case SUPPAGINGMODE_PAE_GLOBAL:
2882	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2883	enmShadowMode = PGMMODE_PAE;
2884	break;
2885
2886	case SUPPAGINGMODE_AMD64:
2887	case SUPPAGINGMODE_AMD64_GLOBAL:
2888	case SUPPAGINGMODE_AMD64_NX:
2889	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2890	enmShadowMode = PGMMODE_PAE;
2891	break;
2892
2893	default:
2894	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2895	}
2896	break;
2897
2898	case PGMMODE_PAE:
2899	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
2900	switch (enmHostMode)
2901	{
2902	case SUPPAGINGMODE_32_BIT:
2903	case SUPPAGINGMODE_32_BIT_GLOBAL:
2904	enmShadowMode = PGMMODE_PAE;
2905	break;
2906
2907	case SUPPAGINGMODE_PAE:
2908	case SUPPAGINGMODE_PAE_NX:
2909	case SUPPAGINGMODE_PAE_GLOBAL:
2910	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2911	enmShadowMode = PGMMODE_PAE;
2912	break;
2913
2914	case SUPPAGINGMODE_AMD64:
2915	case SUPPAGINGMODE_AMD64_GLOBAL:
2916	case SUPPAGINGMODE_AMD64_NX:
2917	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2918	enmShadowMode = PGMMODE_PAE;
2919	break;
2920
2921	default:
2922	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2923	}
2924	break;
2925
2926	case PGMMODE_AMD64:
2927	case PGMMODE_AMD64_NX:
2928	switch (enmHostMode)
2929	{
2930	case SUPPAGINGMODE_32_BIT:
2931	case SUPPAGINGMODE_32_BIT_GLOBAL:
2932	enmShadowMode = PGMMODE_AMD64;
2933	break;
2934
2935	case SUPPAGINGMODE_PAE:
2936	case SUPPAGINGMODE_PAE_NX:
2937	case SUPPAGINGMODE_PAE_GLOBAL:
2938	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2939	enmShadowMode = PGMMODE_AMD64;
2940	break;
2941
2942	case SUPPAGINGMODE_AMD64:
2943	case SUPPAGINGMODE_AMD64_GLOBAL:
2944	case SUPPAGINGMODE_AMD64_NX:
2945	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2946	enmShadowMode = PGMMODE_AMD64;
2947	break;
2948
2949	default:
2950	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2951	}
2952	break;
2953
2954	default:
2955	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
2956	}
2957
2958	/*
2959	* Override the shadow mode when NEM or nested paging is active.
2960	*/
2961	if (VM_IS_NEM_ENABLED(pVM))
2962	{
2963	pVM->pgm.s.fNestedPaging = true;
2964	enmShadowMode = PGMMODE_NONE;
2965	}
2966	else
2967	{
2968	bool fNestedPaging = HMIsNestedPagingActive(pVM);
2969	pVM->pgm.s.fNestedPaging = fNestedPaging;
2970	if (fNestedPaging)
2971	{
2972	if (HMIsVmxActive(pVM))
2973	enmShadowMode = PGMMODE_EPT;
2974	else
2975	{
2976	/* The nested SVM paging depends on the host one. */
2977	Assert(HMIsSvmActive(pVM));
2978	if ( enmGuestMode == PGMMODE_AMD64
2979	\|\| enmGuestMode == PGMMODE_AMD64_NX)
2980	enmShadowMode = PGMMODE_NESTED_AMD64;
2981	else
2982	switch (pVM->pgm.s.enmHostMode)
2983	{
2984	case SUPPAGINGMODE_32_BIT:
2985	case SUPPAGINGMODE_32_BIT_GLOBAL:
2986	enmShadowMode = PGMMODE_NESTED_32BIT;
2987	break;
2988
2989	case SUPPAGINGMODE_PAE:
2990	case SUPPAGINGMODE_PAE_GLOBAL:
2991	case SUPPAGINGMODE_PAE_NX:
2992	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2993	enmShadowMode = PGMMODE_NESTED_PAE;
2994	break;
2995
2996	case SUPPAGINGMODE_AMD64:
2997	case SUPPAGINGMODE_AMD64_GLOBAL:
2998	case SUPPAGINGMODE_AMD64_NX:
2999	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3000	enmShadowMode = PGMMODE_NESTED_AMD64;
3001	break;
3002
3003	default:
3004	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3005	}
3006	}
3007	}
3008	}
3009
3010	return enmShadowMode;
3011	}
3012
3013
3014	/**
3015	* Performs the actual mode change.
3016	* This is called by PGMChangeMode and pgmR3InitPaging().
3017	*
3018	* @returns VBox status code. May suspend or power off the VM on error, but this
3019	* will trigger using FFs and not informational status codes.
3020	*
3021	* @param pVM The cross context VM structure.
3022	* @param pVCpu The cross context virtual CPU structure.
3023	* @param enmGuestMode The new guest mode. This is assumed to be different from
3024	* the current mode.
3025	*/
3026	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode)
3027	{
3028	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3029	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3030
3031	/*
3032	* Calc the shadow mode and switcher.
3033	*/
3034	PGMMODE enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3035
3036	/*
3037	* Exit old mode(s).
3038	*/
3039	/* shadow */
3040	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3041	{
3042	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3043	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3044	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3045	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3046	{
3047	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3048	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3049	}
3050	}
3051	else
3052	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3053
3054	/* guest */
3055	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3056	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3057	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3058	{
3059	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3060	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3061	}
3062	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3063
3064	/*
3065	* Change the paging mode data indexes.
3066	*/
3067	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3068	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3069	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3070	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3071	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3072	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPDE, VERR_PGM_MODE_IPE);
3073	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3074	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3075	#ifdef IN_RING3
3076	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3077	#endif
3078
3079	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3080	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3081	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3082	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3083	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3084	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3085	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3086	#ifdef IN_RING3
3087	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3088	#endif
3089
3090	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3091	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3092	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3093	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3094	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3095	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3096	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3097	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3098	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3099	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3100	#ifdef VBOX_STRICT
3101	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3102	#endif
3103
3104	/*
3105	* Enter new shadow mode (if changed).
3106	*/
3107	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3108	{
3109	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3110	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
3111	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3112	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3113	}
3114
3115	/*
3116	* Always flag the necessary updates
3117	*/
3118	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3119
3120	/*
3121	* Enter the new guest and shadow+guest modes.
3122	*/
3123	/* Calc the new CR3 value. */
3124	RTGCPHYS GCPhysCR3;
3125	switch (enmGuestMode)
3126	{
3127	case PGMMODE_REAL:
3128	case PGMMODE_PROTECTED:
3129	GCPhysCR3 = NIL_RTGCPHYS;
3130	break;
3131
3132	case PGMMODE_32_BIT:
3133	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3134	break;
3135
3136	case PGMMODE_PAE_NX:
3137	case PGMMODE_PAE:
3138	if (!pVM->cpum.ro.GuestFeatures.fPae)
3139	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3140	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3141	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3142	break;
3143
3144	#ifdef VBOX_WITH_64_BITS_GUESTS
3145	case PGMMODE_AMD64_NX:
3146	case PGMMODE_AMD64:
3147	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3148	break;
3149	#endif
3150	default:
3151	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3152	}
3153
3154	/* Enter the new guest mode. */
3155	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3156	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3157	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3158
3159	/* Set the new guest CR3. */
3160	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3161
3162	/* status codes. */
3163	AssertRC(rc);
3164	AssertRC(rc2);
3165	if (RT_SUCCESS(rc))
3166	{
3167	rc = rc2;
3168	if (RT_SUCCESS(rc)) /* no informational status codes. */
3169	rc = VINF_SUCCESS;
3170	}
3171
3172	/*
3173	* Notify HM.
3174	*/
3175	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3176	return rc;
3177	}
3178
3179
3180	/**
3181	* Called by CPUM or REM when CR0.WP changes to 1.
3182	*
3183	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3184	* @thread EMT
3185	*/
3186	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3187	{
3188	/*
3189	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3190	*
3191	* Use the counter to judge whether there might be pool pages with active
3192	* hacks in them. If there are, we will be running the risk of messing up
3193	* the guest by allowing it to write to read-only pages. Thus, we have to
3194	* clear the page pool ASAP if there is the slightest chance.
3195	*/
3196	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3197	{
3198	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3199
3200	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3201	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3202	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3203	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3204	}
3205	}
3206
3207
3208	/**
3209	* Gets the current guest paging mode.
3210	*
3211	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3212	*
3213	* @returns The current paging mode.
3214	* @param pVCpu The cross context virtual CPU structure.
3215	*/
3216	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3217	{
3218	return pVCpu->pgm.s.enmGuestMode;
3219	}
3220
3221
3222	/**
3223	* Gets the current shadow paging mode.
3224	*
3225	* @returns The current paging mode.
3226	* @param pVCpu The cross context virtual CPU structure.
3227	*/
3228	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3229	{
3230	return pVCpu->pgm.s.enmShadowMode;
3231	}
3232
3233
3234	/**
3235	* Gets the current host paging mode.
3236	*
3237	* @returns The current paging mode.
3238	* @param pVM The cross context VM structure.
3239	*/
3240	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3241	{
3242	switch (pVM->pgm.s.enmHostMode)
3243	{
3244	case SUPPAGINGMODE_32_BIT:
3245	case SUPPAGINGMODE_32_BIT_GLOBAL:
3246	return PGMMODE_32_BIT;
3247
3248	case SUPPAGINGMODE_PAE:
3249	case SUPPAGINGMODE_PAE_GLOBAL:
3250	return PGMMODE_PAE;
3251
3252	case SUPPAGINGMODE_PAE_NX:
3253	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3254	return PGMMODE_PAE_NX;
3255
3256	case SUPPAGINGMODE_AMD64:
3257	case SUPPAGINGMODE_AMD64_GLOBAL:
3258	return PGMMODE_AMD64;
3259
3260	case SUPPAGINGMODE_AMD64_NX:
3261	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3262	return PGMMODE_AMD64_NX;
3263
3264	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3265	}
3266
3267	return PGMMODE_INVALID;
3268	}
3269
3270
3271	/**
3272	* Get mode name.
3273	*
3274	* @returns read-only name string.
3275	* @param enmMode The mode which name is desired.
3276	*/
3277	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3278	{
3279	switch (enmMode)
3280	{
3281	case PGMMODE_REAL: return "Real";
3282	case PGMMODE_PROTECTED: return "Protected";
3283	case PGMMODE_32_BIT: return "32-bit";
3284	case PGMMODE_PAE: return "PAE";
3285	case PGMMODE_PAE_NX: return "PAE+NX";
3286	case PGMMODE_AMD64: return "AMD64";
3287	case PGMMODE_AMD64_NX: return "AMD64+NX";
3288	case PGMMODE_NESTED_32BIT: return "Nested-32";
3289	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3290	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3291	case PGMMODE_EPT: return "EPT";
3292	case PGMMODE_NONE: return "None";
3293	default: return "unknown mode value";
3294	}
3295	}
3296
3297
3298	/**
3299	* Gets the physical address represented in the guest CR3 as PGM sees it.
3300	*
3301	* This is mainly for logging and debugging.
3302	*
3303	* @returns PGM's guest CR3 value.
3304	* @param pVCpu The cross context virtual CPU structure.
3305	*/
3306	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3307	{
3308	return pVCpu->pgm.s.GCPhysCR3;
3309	}
3310
3311
3312
3313	/**
3314	* Notification from CPUM that the EFER.NXE bit has changed.
3315	*
3316	* @param pVCpu The cross context virtual CPU structure of the CPU for
3317	* which EFER changed.
3318	* @param fNxe The new NXE state.
3319	*/
3320	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3321	{
3322	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3323	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3324
3325	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3326	if (fNxe)
3327	{
3328	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3329	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3330	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3331	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3332	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3333	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3334	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3335	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3336	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3337	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3338	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3339
3340	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3341	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3342	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3343	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3344	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3345	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3346	}
3347	else
3348	{
3349	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3350	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3351	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3352	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3353	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3354	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3355	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3356	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3357	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3358	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3359	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3360
3361	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3362	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3363	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3364	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3365	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3366	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3367	}
3368	}
3369
3370
3371	/**
3372	* Check if any pgm pool pages are marked dirty (not monitored)
3373	*
3374	* @returns bool locked/not locked
3375	* @param pVM The cross context VM structure.
3376	*/
3377	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3378	{
3379	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3380	}
3381
3382
3383	/**
3384	* Check if this VCPU currently owns the PGM lock.
3385	*
3386	* @returns bool owner/not owner
3387	* @param pVM The cross context VM structure.
3388	*/
3389	VMMDECL(bool) PGMIsLockOwner(PVM pVM)
3390	{
3391	return PDMCritSectIsOwner(&pVM->pgm.s.CritSectX);
3392	}
3393
3394
3395	/**
3396	* Enable or disable large page usage
3397	*
3398	* @returns VBox status code.
3399	* @param pVM The cross context VM structure.
3400	* @param fUseLargePages Use/not use large pages
3401	*/
3402	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3403	{
3404	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3405
3406	pVM->fUseLargePages = fUseLargePages;
3407	return VINF_SUCCESS;
3408	}
3409
3410
3411	/**
3412	* Acquire the PGM lock.
3413	*
3414	* @returns VBox status code
3415	* @param pVM The cross context VM structure.
3416	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3417	*/
3418	#if (defined(VBOX_STRICT) && defined(IN_RING3)) \|\| defined(DOXYGEN_RUNNING)
3419	int pgmLockDebug(PVMCC pVM, RT_SRC_POS_DECL)
3420	#else
3421	int pgmLock(PVMCC pVM)
3422	#endif
3423	{
3424	#if defined(VBOX_STRICT) && defined(IN_RING3)
3425	int rc = PDMCritSectEnterDebug(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3426	#else
3427	int rc = PDMCritSectEnter(&pVM->pgm.s.CritSectX, VERR_SEM_BUSY);
3428	#endif
3429	#ifdef IN_RING0
3430	if (rc == VERR_SEM_BUSY)
3431	rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_LOCK, 0);
3432	#endif
3433	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
3434	return rc;
3435	}
3436
3437
3438	/**
3439	* Release the PGM lock.
3440	*
3441	* @returns VBox status code
3442	* @param pVM The cross context VM structure.
3443	*/
3444	void pgmUnlock(PVM pVM)
3445	{
3446	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3447	pVM->pgm.s.cDeprecatedPageLocks = 0;
3448	int rc = PDMCritSectLeave(&pVM->pgm.s.CritSectX);
3449	if (rc == VINF_SEM_NESTED)
3450	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3451	}
3452
3453	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3454
3455	/**
3456	* Common worker for pgmRZDynMapGCPageOffInlined and pgmRZDynMapGCPageV2Inlined.
3457	*
3458	* @returns VBox status code.
3459	* @param pVM The cross context VM structure.
3460	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3461	* @param GCPhys The guest physical address of the page to map. The
3462	* offset bits are not ignored.
3463	* @param ppv Where to return the address corresponding to @a GCPhys.
3464	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3465	*/
3466	int pgmRZDynMapGCPageCommon(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, void **ppv RTLOG_COMMA_SRC_POS_DECL)
3467	{
3468	pgmLock(pVM);
3469
3470	/*
3471	* Convert it to a writable page and it on to the dynamic mapper.
3472	*/
3473	int rc;
3474	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
3475	if (RT_LIKELY(pPage))
3476	{
3477	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3478	if (RT_SUCCESS(rc))
3479	{
3480	void *pv;
3481	rc = pgmRZDynMapHCPageInlined(pVCpu, PGM_PAGE_GET_HCPHYS(pPage), &pv RTLOG_COMMA_SRC_POS_ARGS);
3482	if (RT_SUCCESS(rc))
3483	ppv = (void )((uintptr_t)pv \| ((uintptr_t)GCPhys & PAGE_OFFSET_MASK));
3484	}
3485	else
3486	AssertRC(rc);
3487	}
3488	else
3489	{
3490	AssertMsgFailed(("Invalid physical address %RGp!\n", GCPhys));
3491	rc = VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3492	}
3493
3494	pgmUnlock(pVM);
3495	return rc;
3496	}
3497
3498	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3499	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3500
3501	/** Format handler for PGMPAGE.
3502	* @copydoc FNRTSTRFORMATTYPE */
3503	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3504	const char pszType, void const pvValue,
3505	int cchWidth, int cchPrecision, unsigned fFlags,
3506	void *pvUser)
3507	{
3508	size_t cch;
3509	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3510	if (RT_VALID_PTR(pPage))
3511	{
3512	char szTmp[64+80];
3513
3514	cch = 0;
3515
3516	/* The single char state stuff. */
3517	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3518	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3519
3520	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3521	if (IS_PART_INCLUDED(5))
3522	{
3523	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3524	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3525	}
3526
3527	/* The type. */
3528	if (IS_PART_INCLUDED(4))
3529	{
3530	szTmp[cch++] = ':';
3531	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3532	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3533	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3534	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3535	}
3536
3537	/* The numbers. */
3538	if (IS_PART_INCLUDED(3))
3539	{
3540	szTmp[cch++] = ':';
3541	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3542	}
3543
3544	if (IS_PART_INCLUDED(2))
3545	{
3546	szTmp[cch++] = ':';
3547	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3548	}
3549
3550	if (IS_PART_INCLUDED(6))
3551	{
3552	szTmp[cch++] = ':';
3553	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3554	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3555	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3556	}
3557	# undef IS_PART_INCLUDED
3558
3559	cch = pfnOutput(pvArgOutput, szTmp, cch);
3560	}
3561	else
3562	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3563	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3564	return cch;
3565	}
3566
3567
3568	/** Format handler for PGMRAMRANGE.
3569	* @copydoc FNRTSTRFORMATTYPE */
3570	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3571	const char pszType, void const pvValue,
3572	int cchWidth, int cchPrecision, unsigned fFlags,
3573	void *pvUser)
3574	{
3575	size_t cch;
3576	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3577	if (VALID_PTR(pRam))
3578	{
3579	char szTmp[80];
3580	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3581	cch = pfnOutput(pvArgOutput, szTmp, cch);
3582	}
3583	else
3584	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3585	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3586	return cch;
3587	}
3588
3589	/** Format type andlers to be registered/deregistered. */
3590	static const struct
3591	{
3592	char szType[24];
3593	PFNRTSTRFORMATTYPE pfnHandler;
3594	} g_aPgmFormatTypes[] =
3595	{
3596	{ "pgmpage", pgmFormatTypeHandlerPage },
3597	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3598	};
3599
3600	#endif /* !IN_R0 \|\| LOG_ENABLED */
3601
3602	/**
3603	* Registers the global string format types.
3604	*
3605	* This should be called at module load time or in some other manner that ensure
3606	* that it's called exactly one time.
3607	*
3608	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3609	*/
3610	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3611	{
3612	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3613	int rc = VINF_SUCCESS;
3614	unsigned i;
3615	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3616	{
3617	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3618	# ifdef IN_RING0
3619	if (rc == VERR_ALREADY_EXISTS)
3620	{
3621	/* in case of cleanup failure in ring-0 */
3622	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3623	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3624	}
3625	# endif
3626	}
3627	if (RT_FAILURE(rc))
3628	while (i-- > 0)
3629	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3630
3631	return rc;
3632	#else
3633	return VINF_SUCCESS;
3634	#endif
3635	}
3636
3637
3638	/**
3639	* Deregisters the global string format types.
3640	*
3641	* This should be called at module unload time or in some other manner that
3642	* ensure that it's called exactly one time.
3643	*/
3644	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3645	{
3646	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3647	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3648	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3649	#endif
3650	}
3651
3652	#ifdef VBOX_STRICT
3653
3654	# ifndef PGM_WITHOUT_MAPPINGS
3655	/**
3656	* Asserts that there are no mapping conflicts.
3657	*
3658	* @returns Number of conflicts.
3659	* @param pVM The cross context VM structure.
3660	*/
3661	VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
3662	{
3663	unsigned cErrors = 0;
3664
3665	/* Only applies to raw mode -> 1 VPCU */
3666	Assert(pVM->cCpus == 1);
3667	PVMCPU pVCpu = &VMCC_GET_CPU_0(pVM);
3668
3669	/*
3670	* Check for mapping conflicts.
3671	*/
3672	for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3673	pMapping;
3674	pMapping = pMapping->CTX_SUFF(pNext))
3675	{
3676	/** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
3677	for (RTGCPTR GCPtr = pMapping->GCPtr; GCPtr <= pMapping->GCPtrLast; GCPtr += PAGE_SIZE)
3678	{
3679	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)GCPtr, NULL, NULL);
3680	if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
3681	{
3682	AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
3683	cErrors++;
3684	break;
3685	}
3686	}
3687	}
3688
3689	return cErrors;
3690	}
3691	# endif /* !PGM_WITHOUT_MAPPINGS */
3692
3693
3694	/**
3695	* Asserts that everything related to the guest CR3 is correctly shadowed.
3696	*
3697	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3698	* and assert the correctness of the guest CR3 mapping before asserting that the
3699	* shadow page tables is in sync with the guest page tables.
3700	*
3701	* @returns Number of conflicts.
3702	* @param pVM The cross context VM structure.
3703	* @param pVCpu The cross context virtual CPU structure.
3704	* @param cr3 The current guest CR3 register value.
3705	* @param cr4 The current guest CR4 register value.
3706	*/
3707	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3708	{
3709	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3710
3711	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3712	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3713	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3714
3715	pgmLock(pVM);
3716	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3717	pgmUnlock(pVM);
3718
3719	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3), a);
3720	return cErrors;
3721	}
3722
3723	#endif /* VBOX_STRICT */
3724

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 80317

以其他格式下載: