PGMInternal.h@ 20125

最後變更在這個檔案從20125是 20125,由 vboxsync 提交於 16 年前
Prevent reuse of cached large pages with different access attributes.
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Id`
檔案大小: 186.1 KB

行
1	/* $Id: PGMInternal.h 20125 2009-05-28 15:44:30Z vboxsync $ */
2	/** @file
3	* PGM - Internal header file.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22	#ifndef ___PGMInternal_h
23	#define ___PGMInternal_h
24
25	#include <VBox/cdefs.h>
26	#include <VBox/types.h>
27	#include <VBox/err.h>
28	#include <VBox/stam.h>
29	#include <VBox/param.h>
30	#include <VBox/vmm.h>
31	#include <VBox/mm.h>
32	#include <VBox/pdmcritsect.h>
33	#include <VBox/pdmapi.h>
34	#include <VBox/dis.h>
35	#include <VBox/dbgf.h>
36	#include <VBox/log.h>
37	#include <VBox/gmm.h>
38	#include <VBox/hwaccm.h>
39	#include <iprt/avl.h>
40	#include <iprt/asm.h>
41	#include <iprt/assert.h>
42	#include <iprt/critsect.h>
43
44
45
46	/** @defgroup grp_pgm_int Internals
47	* @ingroup grp_pgm
48	* @internal
49	* @{
50	*/
51
52
53	/** @name PGM Compile Time Config
54	* @{
55	*/
56
57	/**
58	* Solve page is out of sync issues inside Guest Context (in PGMGC.cpp).
59	* Comment it if it will break something.
60	*/
61	#define PGM_OUT_OF_SYNC_IN_GC
62
63	/**
64	* Check and skip global PDEs for non-global flushes
65	*/
66	#define PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
67
68	/**
69	* Sync N pages instead of a whole page table
70	*/
71	#define PGM_SYNC_N_PAGES
72
73	/**
74	* Number of pages to sync during a page fault
75	*
76	* When PGMPOOL_WITH_GCPHYS_TRACKING is enabled using high values here
77	* causes a lot of unnecessary extents and also is slower than taking more \#PFs.
78	*/
79	#define PGM_SYNC_NR_PAGES 8
80
81	/**
82	* Number of PGMPhysRead/Write cache entries (must be <= sizeof(uint64_t))
83	*/
84	#define PGM_MAX_PHYSCACHE_ENTRIES 64
85	#define PGM_MAX_PHYSCACHE_ENTRIES_MASK (PGM_MAX_PHYSCACHE_ENTRIES-1)
86
87	/**
88	* Enable caching of PGMR3PhysRead/WriteByte/Word/Dword
89	*/
90	#define PGM_PHYSMEMACCESS_CACHING
91
92	/** @def PGMPOOL_WITH_CACHE
93	* Enable agressive caching using the page pool.
94	*
95	* This requires PGMPOOL_WITH_USER_TRACKING and PGMPOOL_WITH_MONITORING.
96	*/
97	#define PGMPOOL_WITH_CACHE
98
99	/** @def PGMPOOL_WITH_MIXED_PT_CR3
100	* When defined, we'll deal with 'uncachable' pages.
101	*/
102	#ifdef PGMPOOL_WITH_CACHE
103	# define PGMPOOL_WITH_MIXED_PT_CR3
104	#endif
105
106	/** @def PGMPOOL_WITH_MONITORING
107	* Monitor the guest pages which are shadowed.
108	* When this is enabled, PGMPOOL_WITH_CACHE or PGMPOOL_WITH_GCPHYS_TRACKING must
109	* be enabled as well.
110	* @remark doesn't really work without caching now. (Mixed PT/CR3 change.)
111	*/
112	#ifdef PGMPOOL_WITH_CACHE
113	# define PGMPOOL_WITH_MONITORING
114	#endif
115
116	/** @def PGMPOOL_WITH_GCPHYS_TRACKING
117	* Tracking the of shadow pages mapping guest physical pages.
118	*
119	* This is very expensive, the current cache prototype is trying to figure out
120	* whether it will be acceptable with an agressive caching policy.
121	*/
122	#if defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
123	# define PGMPOOL_WITH_GCPHYS_TRACKING
124	#endif
125
126	/** @def PGMPOOL_WITH_USER_TRACKING
127	* Tracking users of shadow pages. This is required for the linking of shadow page
128	* tables and physical guest addresses.
129	*/
130	#if defined(PGMPOOL_WITH_GCPHYS_TRACKING) \|\| defined(PGMPOOL_WITH_CACHE) \|\| defined(PGMPOOL_WITH_MONITORING)
131	# define PGMPOOL_WITH_USER_TRACKING
132	#endif
133
134	/** @def PGMPOOL_CFG_MAX_GROW
135	* The maximum number of pages to add to the pool in one go.
136	*/
137	#define PGMPOOL_CFG_MAX_GROW (_256K >> PAGE_SHIFT)
138
139	/** @def VBOX_STRICT_PGM_HANDLER_VIRTUAL
140	* Enables some extra assertions for virtual handlers (mainly phys2virt related).
141	*/
142	#ifdef VBOX_STRICT
143	# define VBOX_STRICT_PGM_HANDLER_VIRTUAL
144	#endif
145
146	/** @def VBOX_WITH_NEW_LAZY_PAGE_ALLOC
147	* Enables the experimental lazy page allocation code. */
148	/# define VBOX_WITH_NEW_LAZY_PAGE_ALLOC /
149
150	/** @} */
151
152
153	/** @name PDPT and PML4 flags.
154	* These are placed in the three bits available for system programs in
155	* the PDPT and PML4 entries.
156	* @{ */
157	/** The entry is a permanent one and it's must always be present.
158	* Never free such an entry. */
159	#define PGM_PLXFLAGS_PERMANENT RT_BIT_64(10)
160	/** Mapping (hypervisor allocated pagetable). */
161	#define PGM_PLXFLAGS_MAPPING RT_BIT_64(11)
162	/** @} */
163
164	/** @name Page directory flags.
165	* These are placed in the three bits available for system programs in
166	* the page directory entries.
167	* @{ */
168	/** Mapping (hypervisor allocated pagetable). */
169	#define PGM_PDFLAGS_MAPPING RT_BIT_64(10)
170	/** Made read-only to facilitate dirty bit tracking. */
171	#define PGM_PDFLAGS_TRACK_DIRTY RT_BIT_64(11)
172	/** @} */
173
174	/** @name Page flags.
175	* These are placed in the three bits available for system programs in
176	* the page entries.
177	* @{ */
178	/** Made read-only to facilitate dirty bit tracking. */
179	#define PGM_PTFLAGS_TRACK_DIRTY RT_BIT_64(9)
180
181	#ifndef PGM_PTFLAGS_CSAM_VALIDATED
182	/** Scanned and approved by CSAM (tm).
183	* NOTE: Must be identical to the one defined in CSAMInternal.h!!
184	* @todo Move PGM_PTFLAGS_* and PGM_PDFLAGS_* to VBox/pgm.h. */
185	#define PGM_PTFLAGS_CSAM_VALIDATED RT_BIT_64(11)
186	#endif
187
188	/** @} */
189
190	/** @name PGM pool physical address flags
191	*
192	* These are bit markers for PGM pool physical addresses.
193	* @{ */
194	#define PGMPOOL_PHYS_NON_PAGED RT_BIT_64(63)
195	#define PGMPOOL_PHYS_ACCESS_USER RT_BIT_64(62) /* set = user, cleared = supervisor */
196	#define PGMPOOL_PHYS_ACCESS_RW RT_BIT_64(61) /* set = read/write, cleared = read-only. */
197	/** @} */
198
199	/** @name Defines used to indicate the shadow and guest paging in the templates.
200	* @{ */
201	#define PGM_TYPE_REAL 1
202	#define PGM_TYPE_PROT 2
203	#define PGM_TYPE_32BIT 3
204	#define PGM_TYPE_PAE 4
205	#define PGM_TYPE_AMD64 5
206	#define PGM_TYPE_NESTED 6
207	#define PGM_TYPE_EPT 7
208	#define PGM_TYPE_MAX PGM_TYPE_EPT
209	/** @} */
210
211	/** Macro for checking if the guest is using paging.
212	* @param uGstType PGM_TYPE_*
213	* @param uShwType PGM_TYPE_*
214	* @remark ASSUMES certain order of the PGM_TYPE_* values.
215	*/
216	#define PGM_WITH_PAGING(uGstType, uShwType) \
217	( (uGstType) >= PGM_TYPE_32BIT \
218	&& (uShwType) != PGM_TYPE_NESTED \
219	&& (uShwType) != PGM_TYPE_EPT)
220
221	/** Macro for checking if the guest supports the NX bit.
222	* @param uGstType PGM_TYPE_*
223	* @param uShwType PGM_TYPE_*
224	* @remark ASSUMES certain order of the PGM_TYPE_* values.
225	*/
226	#define PGM_WITH_NX(uGstType, uShwType) \
227	( (uGstType) >= PGM_TYPE_PAE \
228	&& (uShwType) != PGM_TYPE_NESTED \
229	&& (uShwType) != PGM_TYPE_EPT)
230
231
232	/** @def PGM_HCPHYS_2_PTR
233	* Maps a HC physical page pool address to a virtual address.
234	*
235	* @returns VBox status code.
236	* @param pVM The VM handle.
237	* @param HCPhys The HC physical address to map to a virtual one.
238	* @param ppv Where to store the virtual address. No need to cast this.
239	*
240	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
241	* small page window employeed by that function. Be careful.
242	* @remark There is no need to assert on the result.
243	*/
244	#ifdef IN_RC
245	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
246	PGMDynMapHCPage(pVM, HCPhys, (void **)(ppv))
247	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
248	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
249	pgmR0DynMapHCPageInlined(&(pVM)->pgm.s, HCPhys, (void **)(ppv))
250	#else
251	# define PGM_HCPHYS_2_PTR(pVM, HCPhys, ppv) \
252	MMPagePhys2PageEx(pVM, HCPhys, (void **)(ppv))
253	#endif
254
255	/** @def PGM_HCPHYS_2_PTR_BY_PGM
256	* Maps a HC physical page pool address to a virtual address.
257	*
258	* @returns VBox status code.
259	* @param pPGM The PGM instance data.
260	* @param HCPhys The HC physical address to map to a virtual one.
261	* @param ppv Where to store the virtual address. No need to cast this.
262	*
263	* @remark In GC this uses PGMGCDynMapHCPage(), so it will consume of the
264	* small page window employeed by that function. Be careful.
265	* @remark There is no need to assert on the result.
266	*/
267	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
268	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
269	pgmR0DynMapHCPageInlined(pPGM, HCPhys, (void **)(ppv))
270	#else
271	# define PGM_HCPHYS_2_PTR_BY_PGM(pPGM, HCPhys, ppv) \
272	PGM_HCPHYS_2_PTR(PGM2VM(pPGM), HCPhys, (void **)(ppv))
273	#endif
274
275	/** @def PGM_GCPHYS_2_PTR
276	* Maps a GC physical page address to a virtual address.
277	*
278	* @returns VBox status code.
279	* @param pVM The VM handle.
280	* @param GCPhys The GC physical address to map to a virtual one.
281	* @param ppv Where to store the virtual address. No need to cast this.
282	*
283	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
284	* small page window employeed by that function. Be careful.
285	* @remark There is no need to assert on the result.
286	*/
287	#ifdef IN_RC
288	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
289	PGMDynMapGCPage(pVM, GCPhys, (void **)(ppv))
290	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
291	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
292	pgmR0DynMapGCPageInlined(&(pVM)->pgm.s, GCPhys, (void **)(ppv))
293	#else
294	# define PGM_GCPHYS_2_PTR(pVM, GCPhys, ppv) \
295	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
296	#endif
297
298	/** @def PGM_GCPHYS_2_PTR_BY_PGMCPU
299	* Maps a GC physical page address to a virtual address.
300	*
301	* @returns VBox status code.
302	* @param pPGM Pointer to the PGM instance data.
303	* @param GCPhys The GC physical address to map to a virtual one.
304	* @param ppv Where to store the virtual address. No need to cast this.
305	*
306	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
307	* small page window employeed by that function. Be careful.
308	* @remark There is no need to assert on the result.
309	*/
310	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
311	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
312	pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), GCPhys, (void **)(ppv))
313	#else
314	# define PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, GCPhys, ppv) \
315	PGM_GCPHYS_2_PTR(PGMCPU2VM(pPGM), GCPhys, ppv)
316	#endif
317
318	/** @def PGM_GCPHYS_2_PTR_EX
319	* Maps a unaligned GC physical page address to a virtual address.
320	*
321	* @returns VBox status code.
322	* @param pVM The VM handle.
323	* @param GCPhys The GC physical address to map to a virtual one.
324	* @param ppv Where to store the virtual address. No need to cast this.
325	*
326	* @remark In GC this uses PGMGCDynMapGCPage(), so it will consume of the
327	* small page window employeed by that function. Be careful.
328	* @remark There is no need to assert on the result.
329	*/
330	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
331	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
332	PGMDynMapGCPageOff(pVM, GCPhys, (void **)(ppv))
333	#else
334	# define PGM_GCPHYS_2_PTR_EX(pVM, GCPhys, ppv) \
335	PGMPhysGCPhys2R3Ptr(pVM, GCPhys, 1 /* one page only /, (PRTR3PTR)(ppv)) /* @todo this isn't asserting, use PGMRamGCPhys2HCPtr! */
336	#endif
337
338	/** @def PGM_INVL_PG
339	* Invalidates a page.
340	*
341	* @param pVCpu The VMCPU handle.
342	* @param GCVirt The virtual address of the page to invalidate.
343	*/
344	#ifdef IN_RC
345	# define PGM_INVL_PG(pVCpu, GCVirt) ASMInvalidatePage((void *)(GCVirt))
346	#elif defined(IN_RING0)
347	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
348	#else
349	# define PGM_INVL_PG(pVCpu, GCVirt) HWACCMInvalidatePage(pVCpu, (RTGCPTR)(GCVirt))
350	#endif
351
352	/** @def PGM_INVL_PG
353	* Invalidates a page on all VCPUs
354	*
355	* @param pVM The VM handle.
356	* @param GCVirt The virtual address of the page to invalidate.
357	*/
358	#ifdef IN_RC
359	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) ASMInvalidatePage((void *)(GCVirt))
360	#elif defined(IN_RING0)
361	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
362	#else
363	# define PGM_INVL_ALL_VCPU_PG(pVM, GCVirt) HWACCMInvalidatePageOnAllVCpus(pVM, (RTGCPTR)(GCVirt))
364	#endif
365
366	/** @def PGM_INVL_BIG_PG
367	* Invalidates a 4MB page directory entry.
368	*
369	* @param pVCpu The VMCPU handle.
370	* @param GCVirt The virtual address within the page directory to invalidate.
371	*/
372	#ifdef IN_RC
373	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) ASMReloadCR3()
374	#elif defined(IN_RING0)
375	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
376	#else
377	# define PGM_INVL_BIG_PG(pVCpu, GCVirt) HWACCMFlushTLB(pVCpu)
378	#endif
379
380	/** @def PGM_INVL_VCPU_TLBS()
381	* Invalidates the TLBs of the specified VCPU
382	*
383	* @param pVCpu The VMCPU handle.
384	*/
385	#ifdef IN_RC
386	# define PGM_INVL_VCPU_TLBS(pVCpu) ASMReloadCR3()
387	#elif defined(IN_RING0)
388	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
389	#else
390	# define PGM_INVL_VCPU_TLBS(pVCpu) HWACCMFlushTLB(pVCpu)
391	#endif
392
393	/** @def PGM_INVL_ALL_VCPU_TLBS()
394	* Invalidates the TLBs of all VCPUs
395	*
396	* @param pVM The VM handle.
397	*/
398	#ifdef IN_RC
399	# define PGM_INVL_ALL_VCPU_TLBS(pVM) ASMReloadCR3()
400	#elif defined(IN_RING0)
401	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
402	#else
403	# define PGM_INVL_ALL_VCPU_TLBS(pVM) HWACCMFlushTLBOnAllVCpus(pVM)
404	#endif
405
406	/** Size of the GCPtrConflict array in PGMMAPPING.
407	* @remarks Must be a power of two. */
408	#define PGMMAPPING_CONFLICT_MAX 8
409
410	/**
411	* Structure for tracking GC Mappings.
412	*
413	* This structure is used by linked list in both GC and HC.
414	*/
415	typedef struct PGMMAPPING
416	{
417	/** Pointer to next entry. */
418	R3PTRTYPE(struct PGMMAPPING *) pNextR3;
419	/** Pointer to next entry. */
420	R0PTRTYPE(struct PGMMAPPING *) pNextR0;
421	/** Pointer to next entry. */
422	RCPTRTYPE(struct PGMMAPPING *) pNextRC;
423	/** Indicate whether this entry is finalized. */
424	bool fFinalized;
425	/** Start Virtual address. */
426	RTGCPTR GCPtr;
427	/** Last Virtual address (inclusive). */
428	RTGCPTR GCPtrLast;
429	/** Range size (bytes). */
430	RTGCPTR cb;
431	/** Pointer to relocation callback function. */
432	R3PTRTYPE(PFNPGMRELOCATE) pfnRelocate;
433	/** User argument to the callback. */
434	R3PTRTYPE(void *) pvUser;
435	/** Mapping description / name. For easing debugging. */
436	R3PTRTYPE(const char *) pszDesc;
437	/** Last 8 addresses that caused conflicts. */
438	RTGCPTR aGCPtrConflicts[PGMMAPPING_CONFLICT_MAX];
439	/** Number of conflicts for this hypervisor mapping. */
440	uint32_t cConflicts;
441	/** Number of page tables. */
442	uint32_t cPTs;
443
444	/** Array of page table mapping data. Each entry
445	* describes one page table. The array can be longer
446	* than the declared length.
447	*/
448	struct
449	{
450	/** The HC physical address of the page table. */
451	RTHCPHYS HCPhysPT;
452	/** The HC physical address of the first PAE page table. */
453	RTHCPHYS HCPhysPaePT0;
454	/** The HC physical address of the second PAE page table. */
455	RTHCPHYS HCPhysPaePT1;
456	/** The HC virtual address of the 32-bit page table. */
457	R3PTRTYPE(PX86PT) pPTR3;
458	/** The HC virtual address of the two PAE page table. (i.e 1024 entries instead of 512) */
459	R3PTRTYPE(PX86PTPAE) paPaePTsR3;
460	/** The GC virtual address of the 32-bit page table. */
461	RCPTRTYPE(PX86PT) pPTRC;
462	/** The GC virtual address of the two PAE page table. */
463	RCPTRTYPE(PX86PTPAE) paPaePTsRC;
464	/** The GC virtual address of the 32-bit page table. */
465	R0PTRTYPE(PX86PT) pPTR0;
466	/** The GC virtual address of the two PAE page table. */
467	R0PTRTYPE(PX86PTPAE) paPaePTsR0;
468	} aPTs[1];
469	} PGMMAPPING;
470	/** Pointer to structure for tracking GC Mappings. */
471	typedef struct PGMMAPPING *PPGMMAPPING;
472
473
474	/**
475	* Physical page access handler structure.
476	*
477	* This is used to keep track of physical address ranges
478	* which are being monitored in some kind of way.
479	*/
480	typedef struct PGMPHYSHANDLER
481	{
482	AVLROGCPHYSNODECORE Core;
483	/** Access type. */
484	PGMPHYSHANDLERTYPE enmType;
485	/** Number of pages to update. */
486	uint32_t cPages;
487	/** Pointer to R3 callback function. */
488	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnHandlerR3;
489	/** User argument for R3 handlers. */
490	R3PTRTYPE(void *) pvUserR3;
491	/** Pointer to R0 callback function. */
492	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnHandlerR0;
493	/** User argument for R0 handlers. */
494	R0PTRTYPE(void *) pvUserR0;
495	/** Pointer to GC callback function. */
496	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnHandlerRC;
497	/** User argument for RC handlers. */
498	RCPTRTYPE(void *) pvUserRC;
499	/** Description / Name. For easing debugging. */
500	R3PTRTYPE(const char *) pszDesc;
501	#ifdef VBOX_WITH_STATISTICS
502	/** Profiling of this handler. */
503	STAMPROFILE Stat;
504	#endif
505	} PGMPHYSHANDLER;
506	/** Pointer to a physical page access handler structure. */
507	typedef PGMPHYSHANDLER *PPGMPHYSHANDLER;
508
509
510	/**
511	* Cache node for the physical addresses covered by a virtual handler.
512	*/
513	typedef struct PGMPHYS2VIRTHANDLER
514	{
515	/** Core node for the tree based on physical ranges. */
516	AVLROGCPHYSNODECORE Core;
517	/** Offset from this struct to the PGMVIRTHANDLER structure. */
518	int32_t offVirtHandler;
519	/** Offset of the next alias relative to this one.
520	* Bit 0 is used for indicating whether we're in the tree.
521	* Bit 1 is used for indicating that we're the head node.
522	*/
523	int32_t offNextAlias;
524	} PGMPHYS2VIRTHANDLER;
525	/** Pointer to a phys to virtual handler structure. */
526	typedef PGMPHYS2VIRTHANDLER *PPGMPHYS2VIRTHANDLER;
527
528	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
529	* node is in the tree. */
530	#define PGMPHYS2VIRTHANDLER_IN_TREE RT_BIT(0)
531	/** The bit in PGMPHYS2VIRTHANDLER::offNextAlias used to indicate that the
532	* node is in the head of an alias chain.
533	* The PGMPHYS2VIRTHANDLER_IN_TREE is always set if this bit is set. */
534	#define PGMPHYS2VIRTHANDLER_IS_HEAD RT_BIT(1)
535	/** The mask to apply to PGMPHYS2VIRTHANDLER::offNextAlias to get the offset. */
536	#define PGMPHYS2VIRTHANDLER_OFF_MASK (~(int32_t)3)
537
538
539	/**
540	* Virtual page access handler structure.
541	*
542	* This is used to keep track of virtual address ranges
543	* which are being monitored in some kind of way.
544	*/
545	typedef struct PGMVIRTHANDLER
546	{
547	/** Core node for the tree based on virtual ranges. */
548	AVLROGCPTRNODECORE Core;
549	/** Size of the range (in bytes). */
550	RTGCPTR cb;
551	/** Number of cache pages. */
552	uint32_t cPages;
553	/** Access type. */
554	PGMVIRTHANDLERTYPE enmType;
555	/** Pointer to the RC callback function. */
556	RCPTRTYPE(PFNPGMRCVIRTHANDLER) pfnHandlerRC;
557	#if HC_ARCH_BITS == 64
558	RTRCPTR padding;
559	#endif
560	/** Pointer to the R3 callback function for invalidation. */
561	R3PTRTYPE(PFNPGMR3VIRTINVALIDATE) pfnInvalidateR3;
562	/** Pointer to the R3 callback function. */
563	R3PTRTYPE(PFNPGMR3VIRTHANDLER) pfnHandlerR3;
564	/** Description / Name. For easing debugging. */
565	R3PTRTYPE(const char *) pszDesc;
566	#ifdef VBOX_WITH_STATISTICS
567	/** Profiling of this handler. */
568	STAMPROFILE Stat;
569	#endif
570	/** Array of cached physical addresses for the monitored ranged. */
571	PGMPHYS2VIRTHANDLER aPhysToVirt[HC_ARCH_BITS == 32 ? 1 : 2];
572	} PGMVIRTHANDLER;
573	/** Pointer to a virtual page access handler structure. */
574	typedef PGMVIRTHANDLER *PPGMVIRTHANDLER;
575
576
577	/**
578	* Page type.
579	*
580	* @remarks This enum has to fit in a 3-bit field (see PGMPAGE::u3Type).
581	* @remarks This is used in the saved state, so changes to it requires bumping
582	* the saved state version.
583	* @todo So, convert to \#defines!
584	*/
585	typedef enum PGMPAGETYPE
586	{
587	/** The usual invalid zero entry. */
588	PGMPAGETYPE_INVALID = 0,
589	/** RAM page. (RWX) */
590	PGMPAGETYPE_RAM,
591	/** MMIO2 page. (RWX) */
592	PGMPAGETYPE_MMIO2,
593	/** MMIO2 page aliased over an MMIO page. (RWX)
594	* See PGMHandlerPhysicalPageAlias(). */
595	PGMPAGETYPE_MMIO2_ALIAS_MMIO,
596	/** Shadowed ROM. (RWX) */
597	PGMPAGETYPE_ROM_SHADOW,
598	/** ROM page. (R-X) */
599	PGMPAGETYPE_ROM,
600	/** MMIO page. (---) */
601	PGMPAGETYPE_MMIO,
602	/** End of valid entries. */
603	PGMPAGETYPE_END
604	} PGMPAGETYPE;
605	AssertCompile(PGMPAGETYPE_END <= 7);
606
607	/** @name Page type predicates.
608	* @{ */
609	#define PGMPAGETYPE_IS_READABLE(type) ( (type) <= PGMPAGETYPE_ROM )
610	#define PGMPAGETYPE_IS_WRITEABLE(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
611	#define PGMPAGETYPE_IS_RWX(type) ( (type) <= PGMPAGETYPE_ROM_SHADOW )
612	#define PGMPAGETYPE_IS_ROX(type) ( (type) == PGMPAGETYPE_ROM )
613	#define PGMPAGETYPE_IS_NP(type) ( (type) == PGMPAGETYPE_MMIO )
614	/** @} */
615
616
617	/**
618	* A Physical Guest Page tracking structure.
619	*
620	* The format of this structure is complicated because we have to fit a lot
621	* of information into as few bits as possible. The format is also subject
622	* to change (there is one comming up soon). Which means that for we'll be
623	* using PGM_PAGE_GET_, PGM_PAGE_IS_ and PGM_PAGE_SET_ macros for all
624	* accessess to the structure.
625	*/
626	typedef struct PGMPAGE
627	{
628	/** The physical address and a whole lot of other stuff. All bits are used! */
629	RTHCPHYS HCPhysX;
630	/** The page state. */
631	uint32_t u2StateX : 2;
632	/** Flag indicating that a write monitored page was written to when set. */
633	uint32_t fWrittenToX : 1;
634	/** For later. */
635	uint32_t fSomethingElse : 1;
636	/** The Page ID.
637	* @todo Merge with HCPhysX once we've liberated HCPhysX of its stuff.
638	* The HCPhysX will then be 100% static. */
639	uint32_t idPageX : 28;
640	/** The page type (PGMPAGETYPE). */
641	uint32_t u3Type : 3;
642	/** The physical handler state (PGM_PAGE_HNDL_PHYS_STATE) /
643	uint32_t u2HandlerPhysStateX : 2;
644	/** The virtual handler state (PGM_PAGE_HNDL_VIRT_STATE) /
645	uint32_t u2HandlerVirtStateX : 2;
646	uint32_t u29B : 25;
647	} PGMPAGE;
648	AssertCompileSize(PGMPAGE, 16);
649	/** Pointer to a physical guest page. */
650	typedef PGMPAGE *PPGMPAGE;
651	/** Pointer to a const physical guest page. */
652	typedef const PGMPAGE *PCPGMPAGE;
653	/** Pointer to a physical guest page pointer. */
654	typedef PPGMPAGE *PPPGMPAGE;
655
656
657	/**
658	* Clears the page structure.
659	* @param pPage Pointer to the physical guest page tracking structure.
660	*/
661	#define PGM_PAGE_CLEAR(pPage) \
662	do { \
663	(pPage)->HCPhysX = 0; \
664	(pPage)->u2StateX = 0; \
665	(pPage)->fWrittenToX = 0; \
666	(pPage)->fSomethingElse = 0; \
667	(pPage)->idPageX = 0; \
668	(pPage)->u3Type = 0; \
669	(pPage)->u29B = 0; \
670	} while (0)
671
672	/**
673	* Initializes the page structure.
674	* @param pPage Pointer to the physical guest page tracking structure.
675	*/
676	#define PGM_PAGE_INIT(pPage, _HCPhys, _idPage, _uType, _uState) \
677	do { \
678	(pPage)->HCPhysX = (_HCPhys); \
679	(pPage)->u2StateX = (_uState); \
680	(pPage)->fWrittenToX = 0; \
681	(pPage)->fSomethingElse = 0; \
682	(pPage)->idPageX = (_idPage); \
683	/(pPage)->u3Type = (_uType); - later / \
684	PGM_PAGE_SET_TYPE(pPage, _uType); \
685	(pPage)->u29B = 0; \
686	} while (0)
687
688	/**
689	* Initializes the page structure of a ZERO page.
690	* @param pPage Pointer to the physical guest page tracking structure.
691	*/
692	#define PGM_PAGE_INIT_ZERO(pPage, pVM, _uType) \
693	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
694	/** Temporary hack. Replaced by PGM_PAGE_INIT_ZERO once the old code is kicked out. */
695	# define PGM_PAGE_INIT_ZERO_REAL(pPage, pVM, _uType) \
696	PGM_PAGE_INIT(pPage, (pVM)->pgm.s.HCPhysZeroPg, NIL_GMM_PAGEID, (_uType), PGM_PAGE_STATE_ZERO)
697
698
699	/** @name The Page state, PGMPAGE::u2StateX.
700	* @{ */
701	/** The zero page.
702	* This is a per-VM page that's never ever mapped writable. */
703	#define PGM_PAGE_STATE_ZERO 0
704	/** A allocated page.
705	* This is a per-VM page allocated from the page pool (or wherever
706	* we get MMIO2 pages from if the type is MMIO2).
707	*/
708	#define PGM_PAGE_STATE_ALLOCATED 1
709	/** A allocated page that's being monitored for writes.
710	* The shadow page table mappings are read-only. When a write occurs, the
711	* fWrittenTo member is set, the page remapped as read-write and the state
712	* moved back to allocated. */
713	#define PGM_PAGE_STATE_WRITE_MONITORED 2
714	/** The page is shared, aka. copy-on-write.
715	* This is a page that's shared with other VMs. */
716	#define PGM_PAGE_STATE_SHARED 3
717	/** @} */
718
719
720	/**
721	* Gets the page state.
722	* @returns page state (PGM_PAGE_STATE_*).
723	* @param pPage Pointer to the physical guest page tracking structure.
724	*/
725	#define PGM_PAGE_GET_STATE(pPage) ( (pPage)->u2StateX )
726
727	/**
728	* Sets the page state.
729	* @param pPage Pointer to the physical guest page tracking structure.
730	* @param _uState The new page state.
731	*/
732	#define PGM_PAGE_SET_STATE(pPage, _uState) \
733	do { (pPage)->u2StateX = (_uState); } while (0)
734
735
736	/**
737	* Gets the host physical address of the guest page.
738	* @returns host physical address (RTHCPHYS).
739	* @param pPage Pointer to the physical guest page tracking structure.
740	*/
741	#define PGM_PAGE_GET_HCPHYS(pPage) ( (pPage)->HCPhysX & UINT64_C(0x0000fffffffff000) )
742
743	/**
744	* Sets the host physical address of the guest page.
745	* @param pPage Pointer to the physical guest page tracking structure.
746	* @param _HCPhys The new host physical address.
747	*/
748	#define PGM_PAGE_SET_HCPHYS(pPage, _HCPhys) \
749	do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0xffff000000000fff)) \
750	\| ((_HCPhys) & UINT64_C(0x0000fffffffff000)); } while (0)
751
752	/**
753	* Get the Page ID.
754	* @returns The Page ID; NIL_GMM_PAGEID if it's a ZERO page.
755	* @param pPage Pointer to the physical guest page tracking structure.
756	*/
757	#define PGM_PAGE_GET_PAGEID(pPage) ( (pPage)->idPageX )
758	/* later:
759	#define PGM_PAGE_GET_PAGEID(pPage) ( ((uint32_t)(pPage)->HCPhysX >> (48 - 12))
760	\| ((uint32_t)(pPage)->HCPhysX & 0xfff) )
761	*/
762	/**
763	* Sets the Page ID.
764	* @param pPage Pointer to the physical guest page tracking structure.
765	*/
766	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->idPageX = (_idPage); } while (0)
767	/* later:
768	#define PGM_PAGE_SET_PAGEID(pPage, _idPage) do { (pPage)->HCPhysX = (((pPage)->HCPhysX) & UINT64_C(0x0000fffffffff000)) \
769	\| ((_idPage) & 0xfff) \
770	\| (((_idPage) & 0x0ffff000) << (48-12)); } while (0)
771	*/
772
773	/**
774	* Get the Chunk ID.
775	* @returns The Chunk ID; NIL_GMM_CHUNKID if it's a ZERO page.
776	* @param pPage Pointer to the physical guest page tracking structure.
777	*/
778	#define PGM_PAGE_GET_CHUNKID(pPage) ( (pPage)->idPageX >> GMM_CHUNKID_SHIFT )
779	/* later:
780	#if GMM_CHUNKID_SHIFT == 12
781	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> 48) )
782	#elif GMM_CHUNKID_SHIFT > 12
783	# define PGM_PAGE_GET_CHUNKID(pPage) ( (uint32_t)((pPage)->HCPhysX >> (48 + (GMM_CHUNKID_SHIFT - 12)) )
784	#elif GMM_CHUNKID_SHIFT < 12
785	# define PGM_PAGE_GET_CHUNKID(pPage) ( ( (uint32_t)((pPage)->HCPhysX >> 48) << (12 - GMM_CHUNKID_SHIFT) ) \
786	\| ( (uint32_t)((pPage)->HCPhysX & 0xfff) >> GMM_CHUNKID_SHIFT ) )
787	#else
788	# error "GMM_CHUNKID_SHIFT isn't defined or something."
789	#endif
790	*/
791
792	/**
793	* Get the index of the page within the allocaiton chunk.
794	* @returns The page index.
795	* @param pPage Pointer to the physical guest page tracking structure.
796	*/
797	#define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (pPage)->idPageX & GMM_PAGEID_IDX_MASK )
798	/* later:
799	#if GMM_CHUNKID_SHIFT <= 12
800	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & GMM_PAGEID_IDX_MASK) )
801	#else
802	# define PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) ( (uint32_t)((pPage)->HCPhysX & 0xfff) \
803	\| ( (uint32_t)((pPage)->HCPhysX >> 48) & (RT_BIT_32(GMM_CHUNKID_SHIFT - 12) - 1) ) )
804	#endif
805	*/
806
807
808	/**
809	* Gets the page type.
810	* @returns The page type.
811	* @param pPage Pointer to the physical guest page tracking structure.
812	*/
813	#define PGM_PAGE_GET_TYPE(pPage) (pPage)->u3Type
814
815	/**
816	* Sets the page type.
817	* @param pPage Pointer to the physical guest page tracking structure.
818	* @param _enmType The new page type (PGMPAGETYPE).
819	*/
820	#define PGM_PAGE_SET_TYPE(pPage, _enmType) \
821	do { (pPage)->u3Type = (_enmType); } while (0)
822
823	/**
824	* Checks if the page is marked for MMIO.
825	* @returns true/false.
826	* @param pPage Pointer to the physical guest page tracking structure.
827	*/
828	#define PGM_PAGE_IS_MMIO(pPage) ( (pPage)->u3Type == PGMPAGETYPE_MMIO )
829
830	/**
831	* Checks if the page is backed by the ZERO page.
832	* @returns true/false.
833	* @param pPage Pointer to the physical guest page tracking structure.
834	*/
835	#define PGM_PAGE_IS_ZERO(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_ZERO )
836
837	/**
838	* Checks if the page is backed by a SHARED page.
839	* @returns true/false.
840	* @param pPage Pointer to the physical guest page tracking structure.
841	*/
842	#define PGM_PAGE_IS_SHARED(pPage) ( (pPage)->u2StateX == PGM_PAGE_STATE_SHARED )
843
844
845	/**
846	* Marks the paget as written to (for GMM change monitoring).
847	* @param pPage Pointer to the physical guest page tracking structure.
848	*/
849	#define PGM_PAGE_SET_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 1; } while (0)
850
851	/**
852	* Clears the written-to indicator.
853	* @param pPage Pointer to the physical guest page tracking structure.
854	*/
855	#define PGM_PAGE_CLEAR_WRITTEN_TO(pPage) do { (pPage)->fWrittenToX = 0; } while (0)
856
857	/**
858	* Checks if the page was marked as written-to.
859	* @returns true/false.
860	* @param pPage Pointer to the physical guest page tracking structure.
861	*/
862	#define PGM_PAGE_IS_WRITTEN_TO(pPage) ( (pPage)->fWrittenToX )
863
864
865	/** @name Physical Access Handler State values (PGMPAGE::u2HandlerPhysStateX).
866	*
867	* @remarks The values are assigned in order of priority, so we can calculate
868	* the correct state for a page with different handlers installed.
869	* @{ */
870	/** No handler installed. */
871	#define PGM_PAGE_HNDL_PHYS_STATE_NONE 0
872	/** Monitoring is temporarily disabled. */
873	#define PGM_PAGE_HNDL_PHYS_STATE_DISABLED 1
874	/** Write access is monitored. */
875	#define PGM_PAGE_HNDL_PHYS_STATE_WRITE 2
876	/** All access is monitored. */
877	#define PGM_PAGE_HNDL_PHYS_STATE_ALL 3
878	/** @} */
879
880	/**
881	* Gets the physical access handler state of a page.
882	* @returns PGM_PAGE_HNDL_PHYS_STATE_* value.
883	* @param pPage Pointer to the physical guest page tracking structure.
884	*/
885	#define PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) ( (pPage)->u2HandlerPhysStateX )
886
887	/**
888	* Sets the physical access handler state of a page.
889	* @param pPage Pointer to the physical guest page tracking structure.
890	* @param _uState The new state value.
891	*/
892	#define PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, _uState) \
893	do { (pPage)->u2HandlerPhysStateX = (_uState); } while (0)
894
895	/**
896	* Checks if the page has any physical access handlers, including temporariliy disabled ones.
897	* @returns true/false
898	* @param pPage Pointer to the physical guest page tracking structure.
899	*/
900	#define PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE )
901
902	/**
903	* Checks if the page has any active physical access handlers.
904	* @returns true/false
905	* @param pPage Pointer to the physical guest page tracking structure.
906	*/
907	#define PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage) ( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE )
908
909
910	/** @name Virtual Access Handler State values (PGMPAGE::u2HandlerVirtStateX).
911	*
912	* @remarks The values are assigned in order of priority, so we can calculate
913	* the correct state for a page with different handlers installed.
914	* @{ */
915	/** No handler installed. */
916	#define PGM_PAGE_HNDL_VIRT_STATE_NONE 0
917	/* 1 is reserved so the lineup is identical with the physical ones. */
918	/** Write access is monitored. */
919	#define PGM_PAGE_HNDL_VIRT_STATE_WRITE 2
920	/** All access is monitored. */
921	#define PGM_PAGE_HNDL_VIRT_STATE_ALL 3
922	/** @} */
923
924	/**
925	* Gets the virtual access handler state of a page.
926	* @returns PGM_PAGE_HNDL_VIRT_STATE_* value.
927	* @param pPage Pointer to the physical guest page tracking structure.
928	*/
929	#define PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) ( (pPage)->u2HandlerVirtStateX )
930
931	/**
932	* Sets the virtual access handler state of a page.
933	* @param pPage Pointer to the physical guest page tracking structure.
934	* @param _uState The new state value.
935	*/
936	#define PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, _uState) \
937	do { (pPage)->u2HandlerVirtStateX = (_uState); } while (0)
938
939	/**
940	* Checks if the page has any virtual access handlers.
941	* @returns true/false
942	* @param pPage Pointer to the physical guest page tracking structure.
943	*/
944	#define PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage) ( (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
945
946	/**
947	* Same as PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS - can't disable pages in
948	* virtual handlers.
949	* @returns true/false
950	* @param pPage Pointer to the physical guest page tracking structure.
951	*/
952	#define PGM_PAGE_HAS_ACTIVE_VIRTUAL_HANDLERS(pPage) PGM_PAGE_HAS_ANY_VIRTUAL_HANDLERS(pPage)
953
954
955
956	/**
957	* Checks if the page has any access handlers, including temporarily disabled ones.
958	* @returns true/false
959	* @param pPage Pointer to the physical guest page tracking structure.
960	*/
961	#define PGM_PAGE_HAS_ANY_HANDLERS(pPage) \
962	( (pPage)->u2HandlerPhysStateX != PGM_PAGE_HNDL_PHYS_STATE_NONE \
963	\|\| (pPage)->u2HandlerVirtStateX != PGM_PAGE_HNDL_VIRT_STATE_NONE )
964
965	/**
966	* Checks if the page has any active access handlers.
967	* @returns true/false
968	* @param pPage Pointer to the physical guest page tracking structure.
969	*/
970	#define PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) \
971	( (pPage)->u2HandlerPhysStateX >= PGM_PAGE_HNDL_PHYS_STATE_WRITE \
972	\|\| (pPage)->u2HandlerVirtStateX >= PGM_PAGE_HNDL_VIRT_STATE_WRITE )
973
974	/**
975	* Checks if the page has any active access handlers catching all accesses.
976	* @returns true/false
977	* @param pPage Pointer to the physical guest page tracking structure.
978	*/
979	#define PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage) \
980	( (pPage)->u2HandlerPhysStateX == PGM_PAGE_HNDL_PHYS_STATE_ALL \
981	\|\| (pPage)->u2HandlerVirtStateX == PGM_PAGE_HNDL_VIRT_STATE_ALL )
982
983
984
985
986	/** @def PGM_PAGE_GET_TRACKING
987	* Gets the packed shadow page pool tracking data associated with a guest page.
988	* @returns uint16_t containing the data.
989	* @param pPage Pointer to the physical guest page tracking structure.
990	*/
991	#define PGM_PAGE_GET_TRACKING(pPage) \
992	( ((uint16_t )&(pPage)->HCPhysX + 3) )
993
994	/** @def PGM_PAGE_SET_TRACKING
995	* Sets the packed shadow page pool tracking data associated with a guest page.
996	* @param pPage Pointer to the physical guest page tracking structure.
997	* @param u16TrackingData The tracking data to store.
998	*/
999	#define PGM_PAGE_SET_TRACKING(pPage, u16TrackingData) \
1000	do { ((uint16_t )&(pPage)->HCPhysX + 3) = (u16TrackingData); } while (0)
1001
1002	/** @def PGM_PAGE_GET_TD_CREFS
1003	* Gets the @a cRefs tracking data member.
1004	* @returns cRefs.
1005	* @param pPage Pointer to the physical guest page tracking structure.
1006	*/
1007	#define PGM_PAGE_GET_TD_CREFS(pPage) \
1008	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK)
1009
1010	#define PGM_PAGE_GET_TD_IDX(pPage) \
1011	((PGM_PAGE_GET_TRACKING(pPage) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK)
1012
1013	/**
1014	* Ram range for GC Phys to HC Phys conversion.
1015	*
1016	* Can be used for HC Virt to GC Phys and HC Virt to HC Phys
1017	* conversions too, but we'll let MM handle that for now.
1018	*
1019	* This structure is used by linked lists in both GC and HC.
1020	*/
1021	typedef struct PGMRAMRANGE
1022	{
1023	/** Start of the range. Page aligned. */
1024	RTGCPHYS GCPhys;
1025	/** Size of the range. (Page aligned of course). */
1026	RTGCPHYS cb;
1027	/** Pointer to the next RAM range - for R3. */
1028	R3PTRTYPE(struct PGMRAMRANGE *) pNextR3;
1029	/** Pointer to the next RAM range - for R0. */
1030	R0PTRTYPE(struct PGMRAMRANGE *) pNextR0;
1031	/** Pointer to the next RAM range - for RC. */
1032	RCPTRTYPE(struct PGMRAMRANGE *) pNextRC;
1033	/** PGM_RAM_RANGE_FLAGS_* flags. */
1034	uint32_t fFlags;
1035	/** Last address in the range (inclusive). Page aligned (-1). */
1036	RTGCPHYS GCPhysLast;
1037	/** Start of the HC mapping of the range. This is only used for MMIO2. */
1038	R3PTRTYPE(void *) pvR3;
1039	/** The range description. */
1040	R3PTRTYPE(const char *) pszDesc;
1041	/** Pointer to self - R0 pointer. */
1042	R0PTRTYPE(struct PGMRAMRANGE *) pSelfR0;
1043	/** Pointer to self - RC pointer. */
1044	RCPTRTYPE(struct PGMRAMRANGE *) pSelfRC;
1045	/** Padding to make aPage aligned on sizeof(PGMPAGE). */
1046	uint32_t au32Alignment2[HC_ARCH_BITS == 32 ? 2 : 1];
1047	/** Array of physical guest page tracking structures. */
1048	PGMPAGE aPages[1];
1049	} PGMRAMRANGE;
1050	/** Pointer to Ram range for GC Phys to HC Phys conversion. */
1051	typedef PGMRAMRANGE *PPGMRAMRANGE;
1052
1053	/** @name PGMRAMRANGE::fFlags
1054	* @{ */
1055	/** The RAM range is floating around as an independent guest mapping. */
1056	#define PGM_RAM_RANGE_FLAGS_FLOATING RT_BIT(20)
1057	/** @} */
1058
1059
1060	/**
1061	* Per page tracking structure for ROM image.
1062	*
1063	* A ROM image may have a shadow page, in which case we may have
1064	* two pages backing it. This structure contains the PGMPAGE for
1065	* both while PGMRAMRANGE have a copy of the active one. It is
1066	* important that these aren't out of sync in any regard other
1067	* than page pool tracking data.
1068	*/
1069	typedef struct PGMROMPAGE
1070	{
1071	/** The page structure for the virgin ROM page. */
1072	PGMPAGE Virgin;
1073	/** The page structure for the shadow RAM page. */
1074	PGMPAGE Shadow;
1075	/** The current protection setting. */
1076	PGMROMPROT enmProt;
1077	/** Pad the structure size to a multiple of 8. */
1078	uint32_t u32Padding;
1079	} PGMROMPAGE;
1080	/** Pointer to a ROM page tracking structure. */
1081	typedef PGMROMPAGE *PPGMROMPAGE;
1082
1083
1084	/**
1085	* A registered ROM image.
1086	*
1087	* This is needed to keep track of ROM image since they generally
1088	* intrude into a PGMRAMRANGE. It also keeps track of additional
1089	* info like the two page sets (read-only virgin and read-write shadow),
1090	* the current state of each page.
1091	*
1092	* Because access handlers cannot easily be executed in a different
1093	* context, the ROM ranges needs to be accessible and in all contexts.
1094	*/
1095	typedef struct PGMROMRANGE
1096	{
1097	/** Pointer to the next range - R3. */
1098	R3PTRTYPE(struct PGMROMRANGE *) pNextR3;
1099	/** Pointer to the next range - R0. */
1100	R0PTRTYPE(struct PGMROMRANGE *) pNextR0;
1101	/** Pointer to the next range - RC. */
1102	RCPTRTYPE(struct PGMROMRANGE *) pNextRC;
1103	/** Pointer alignment */
1104	RTRCPTR GCPtrAlignment;
1105	/** Address of the range. */
1106	RTGCPHYS GCPhys;
1107	/** Address of the last byte in the range. */
1108	RTGCPHYS GCPhysLast;
1109	/** Size of the range. */
1110	RTGCPHYS cb;
1111	/** The flags (PGMPHYS_ROM_FLAG_). /
1112	uint32_t fFlags;
1113	/** Alignment padding ensuring that aPages is sizeof(PGMROMPAGE) aligned. */
1114	uint32_t au32Alignemnt[HC_ARCH_BITS == 32 ? 7 : 3];
1115	/** Pointer to the original bits when PGMPHYS_ROM_FLAGS_PERMANENT_BINARY was specified.
1116	* This is used for strictness checks. */
1117	R3PTRTYPE(const void *) pvOriginal;
1118	/** The ROM description. */
1119	R3PTRTYPE(const char *) pszDesc;
1120	/** The per page tracking structures. */
1121	PGMROMPAGE aPages[1];
1122	} PGMROMRANGE;
1123	/** Pointer to a ROM range. */
1124	typedef PGMROMRANGE *PPGMROMRANGE;
1125
1126
1127	/**
1128	* A registered MMIO2 (= Device RAM) range.
1129	*
1130	* There are a few reason why we need to keep track of these
1131	* registrations. One of them is the deregistration & cleanup
1132	* stuff, while another is that the PGMRAMRANGE associated with
1133	* such a region may have to be removed from the ram range list.
1134	*
1135	* Overlapping with a RAM range has to be 100% or none at all. The
1136	* pages in the existing RAM range must not be ROM nor MMIO. A guru
1137	* meditation will be raised if a partial overlap or an overlap of
1138	* ROM pages is encountered. On an overlap we will free all the
1139	* existing RAM pages and put in the ram range pages instead.
1140	*/
1141	typedef struct PGMMMIO2RANGE
1142	{
1143	/** The owner of the range. (a device) */
1144	PPDMDEVINSR3 pDevInsR3;
1145	/** Pointer to the ring-3 mapping of the allocation. */
1146	RTR3PTR pvR3;
1147	/** Pointer to the next range - R3. */
1148	R3PTRTYPE(struct PGMMMIO2RANGE *) pNextR3;
1149	/** Whether it's mapped or not. */
1150	bool fMapped;
1151	/** Whether it's overlapping or not. */
1152	bool fOverlapping;
1153	/** The PCI region number.
1154	* @remarks This ASSUMES that nobody will ever really need to have multiple
1155	* PCI devices with matching MMIO region numbers on a single device. */
1156	uint8_t iRegion;
1157	/** Alignment padding for putting the ram range on a PGMPAGE alignment boundrary. */
1158	uint8_t abAlignemnt[HC_ARCH_BITS == 32 ? 1 : 5];
1159	/** The associated RAM range. */
1160	PGMRAMRANGE RamRange;
1161	} PGMMMIO2RANGE;
1162	/** Pointer to a MMIO2 range. */
1163	typedef PGMMMIO2RANGE *PPGMMMIO2RANGE;
1164
1165
1166
1167
1168	/**
1169	* PGMPhysRead/Write cache entry
1170	*/
1171	typedef struct PGMPHYSCACHEENTRY
1172	{
1173	/** R3 pointer to physical page. */
1174	R3PTRTYPE(uint8_t *) pbR3;
1175	/** GC Physical address for cache entry */
1176	RTGCPHYS GCPhys;
1177	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
1178	RTGCPHYS u32Padding0; /*< alignment padding. /
1179	#endif
1180	} PGMPHYSCACHEENTRY;
1181
1182	/**
1183	* PGMPhysRead/Write cache to reduce REM memory access overhead
1184	*/
1185	typedef struct PGMPHYSCACHE
1186	{
1187	/** Bitmap of valid cache entries */
1188	uint64_t aEntries;
1189	/** Cache entries */
1190	PGMPHYSCACHEENTRY Entry[PGM_MAX_PHYSCACHE_ENTRIES];
1191	} PGMPHYSCACHE;
1192
1193
1194	/** Pointer to an allocation chunk ring-3 mapping. */
1195	typedef struct PGMCHUNKR3MAP *PPGMCHUNKR3MAP;
1196	/** Pointer to an allocation chunk ring-3 mapping pointer. */
1197	typedef PPGMCHUNKR3MAP *PPPGMCHUNKR3MAP;
1198
1199	/**
1200	* Ring-3 tracking structore for an allocation chunk ring-3 mapping.
1201	*
1202	* The primary tree (Core) uses the chunk id as key.
1203	* The secondary tree (AgeCore) is used for ageing and uses ageing sequence number as key.
1204	*/
1205	typedef struct PGMCHUNKR3MAP
1206	{
1207	/** The key is the chunk id. */
1208	AVLU32NODECORE Core;
1209	/** The key is the ageing sequence number. */
1210	AVLLU32NODECORE AgeCore;
1211	/** The current age thingy. */
1212	uint32_t iAge;
1213	/** The current reference count. */
1214	uint32_t volatile cRefs;
1215	/** The current permanent reference count. */
1216	uint32_t volatile cPermRefs;
1217	/** The mapping address. */
1218	void *pv;
1219	} PGMCHUNKR3MAP;
1220
1221	/**
1222	* Allocation chunk ring-3 mapping TLB entry.
1223	*/
1224	typedef struct PGMCHUNKR3MAPTLBE
1225	{
1226	/** The chunk id. */
1227	uint32_t volatile idChunk;
1228	#if HC_ARCH_BITS == 64
1229	uint32_t u32Padding; /*< alignment padding. /
1230	#endif
1231	/** The chunk map. */
1232	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1233	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1234	#else
1235	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pChunk;
1236	#endif
1237	} PGMCHUNKR3MAPTLBE;
1238	/** Pointer to the an allocation chunk ring-3 mapping TLB entry. */
1239	typedef PGMCHUNKR3MAPTLBE *PPGMCHUNKR3MAPTLBE;
1240
1241	/** The number of TLB entries in PGMCHUNKR3MAPTLB.
1242	* @remark Must be a power of two value. */
1243	#define PGM_CHUNKR3MAPTLB_ENTRIES 32
1244
1245	/**
1246	* Allocation chunk ring-3 mapping TLB.
1247	*
1248	* @remarks We use a TLB to speed up lookups by avoiding walking the AVL.
1249	* At first glance this might look kinda odd since AVL trees are
1250	* supposed to give the most optimial lookup times of all trees
1251	* due to their balancing. However, take a tree with 1023 nodes
1252	* in it, that's 10 levels, meaning that most searches has to go
1253	* down 9 levels before they find what they want. This isn't fast
1254	* compared to a TLB hit. There is the factor of cache misses,
1255	* and of course the problem with trees and branch prediction.
1256	* This is why we use TLBs in front of most of the trees.
1257	*
1258	* @todo Generalize this TLB + AVL stuff, shouldn't be all that
1259	* difficult when we switch to the new inlined AVL trees (from kStuff).
1260	*/
1261	typedef struct PGMCHUNKR3MAPTLB
1262	{
1263	/** The TLB entries. */
1264	PGMCHUNKR3MAPTLBE aEntries[PGM_CHUNKR3MAPTLB_ENTRIES];
1265	} PGMCHUNKR3MAPTLB;
1266
1267	/**
1268	* Calculates the index of a guest page in the Ring-3 Chunk TLB.
1269	* @returns Chunk TLB index.
1270	* @param idChunk The Chunk ID.
1271	*/
1272	#define PGM_CHUNKR3MAPTLB_IDX(idChunk) ( (idChunk) & (PGM_CHUNKR3MAPTLB_ENTRIES - 1) )
1273
1274
1275	/**
1276	* Ring-3 guest page mapping TLB entry.
1277	* @remarks used in ring-0 as well at the moment.
1278	*/
1279	typedef struct PGMPAGER3MAPTLBE
1280	{
1281	/** Address of the page. */
1282	RTGCPHYS volatile GCPhys;
1283	/** The guest page. */
1284	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1285	R3PTRTYPE(PPGMPAGE) volatile pPage;
1286	#else
1287	R3R0PTRTYPE(PPGMPAGE) volatile pPage;
1288	#endif
1289	/** Pointer to the page mapping tracking structure, PGMCHUNKR3MAP. */
1290	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1291	R3PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1292	#else
1293	R3R0PTRTYPE(PPGMCHUNKR3MAP) volatile pMap;
1294	#endif
1295	/** The address */
1296	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1297	R3PTRTYPE(void *) volatile pv;
1298	#else
1299	R3R0PTRTYPE(void *) volatile pv;
1300	#endif
1301	#if HC_ARCH_BITS == 32
1302	uint32_t u32Padding; /*< alignment padding. /
1303	#endif
1304	} PGMPAGER3MAPTLBE;
1305	/** Pointer to an entry in the HC physical TLB. */
1306	typedef PGMPAGER3MAPTLBE *PPGMPAGER3MAPTLBE;
1307
1308
1309	/** The number of entries in the ring-3 guest page mapping TLB.
1310	* @remarks The value must be a power of two. */
1311	#define PGM_PAGER3MAPTLB_ENTRIES 64
1312
1313	/**
1314	* Ring-3 guest page mapping TLB.
1315	* @remarks used in ring-0 as well at the moment.
1316	*/
1317	typedef struct PGMPAGER3MAPTLB
1318	{
1319	/** The TLB entries. */
1320	PGMPAGER3MAPTLBE aEntries[PGM_PAGER3MAPTLB_ENTRIES];
1321	} PGMPAGER3MAPTLB;
1322	/** Pointer to the ring-3 guest page mapping TLB. */
1323	typedef PGMPAGER3MAPTLB *PPGMPAGER3MAPTLB;
1324
1325	/**
1326	* Calculates the index of the TLB entry for the specified guest page.
1327	* @returns Physical TLB index.
1328	* @param GCPhys The guest physical address.
1329	*/
1330	#define PGM_PAGER3MAPTLB_IDX(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGM_PAGER3MAPTLB_ENTRIES - 1) )
1331
1332
1333	/**
1334	* Mapping cache usage set entry.
1335	*
1336	* @remarks 16-bit ints was choosen as the set is not expected to be used beyond
1337	* the dynamic ring-0 and (to some extent) raw-mode context mapping
1338	* cache. If it's extended to include ring-3, well, then something will
1339	* have be changed here...
1340	*/
1341	typedef struct PGMMAPSETENTRY
1342	{
1343	/** The mapping cache index. */
1344	uint16_t iPage;
1345	/** The number of references.
1346	* The max is UINT16_MAX - 1. */
1347	uint16_t cRefs;
1348	#if HC_ARCH_BITS == 64
1349	uint32_t alignment;
1350	#endif
1351	/** Pointer to the page. */
1352	RTR0PTR pvPage;
1353	/** The physical address for this entry. */
1354	RTHCPHYS HCPhys;
1355	} PGMMAPSETENTRY;
1356	/** Pointer to a mapping cache usage set entry. */
1357	typedef PGMMAPSETENTRY *PPGMMAPSETENTRY;
1358
1359	/**
1360	* Mapping cache usage set.
1361	*
1362	* This is used in ring-0 and the raw-mode context to track dynamic mappings
1363	* done during exits / traps. The set is
1364	*/
1365	typedef struct PGMMAPSET
1366	{
1367	/** The number of occupied entries.
1368	* This is PGMMAPSET_CLOSED if the set is closed and we're not supposed to do
1369	* dynamic mappings. */
1370	uint32_t cEntries;
1371	/** The start of the current subset.
1372	* This is UINT32_MAX if no subset is currently open. */
1373	uint32_t iSubset;
1374	/** The index of the current CPU, only valid if the set is open. */
1375	int32_t iCpu;
1376	#if HC_ARCH_BITS == 64
1377	uint32_t alignment;
1378	#endif
1379	/** The entries. */
1380	PGMMAPSETENTRY aEntries[64];
1381	/** HCPhys -> iEntry fast lookup table.
1382	* Use PGMMAPSET_HASH for hashing.
1383	* The entries may or may not be valid, check against cEntries. */
1384	uint8_t aiHashTable[128];
1385	} PGMMAPSET;
1386	/** Pointer to the mapping cache set. */
1387	typedef PGMMAPSET *PPGMMAPSET;
1388
1389	/** PGMMAPSET::cEntries value for a closed set. */
1390	#define PGMMAPSET_CLOSED UINT32_C(0xdeadc0fe)
1391
1392	/** Hash function for aiHashTable. */
1393	#define PGMMAPSET_HASH(HCPhys) (((HCPhys) >> PAGE_SHIFT) & 127)
1394
1395	/** The max fill size (strict builds). */
1396	#define PGMMAPSET_MAX_FILL (64U * 80U / 100U)
1397
1398
1399	/** @name Context neutrual page mapper TLB.
1400	*
1401	* Hoping to avoid some code and bug duplication parts of the GCxxx->CCPtr
1402	* code is writting in a kind of context neutrual way. Time will show whether
1403	* this actually makes sense or not...
1404	*
1405	* @todo this needs to be reconsidered and dropped/redone since the ring-0
1406	* context ends up using a global mapping cache on some platforms
1407	* (darwin).
1408	*
1409	* @{ */
1410	/** @typedef PPGMPAGEMAPTLB
1411	* The page mapper TLB pointer type for the current context. */
1412	/** @typedef PPGMPAGEMAPTLB
1413	* The page mapper TLB entry pointer type for the current context. */
1414	/** @typedef PPGMPAGEMAPTLB
1415	* The page mapper TLB entry pointer pointer type for the current context. */
1416	/** @def PGM_PAGEMAPTLB_ENTRIES
1417	* The number of TLB entries in the page mapper TLB for the current context. */
1418	/** @def PGM_PAGEMAPTLB_IDX
1419	* Calculate the TLB index for a guest physical address.
1420	* @returns The TLB index.
1421	* @param GCPhys The guest physical address. */
1422	/** @typedef PPGMPAGEMAP
1423	* Pointer to a page mapper unit for current context. */
1424	/** @typedef PPPGMPAGEMAP
1425	* Pointer to a page mapper unit pointer for current context. */
1426	#ifdef IN_RC
1427	// typedef PPGMPAGEGCMAPTLB PPGMPAGEMAPTLB;
1428	// typedef PPGMPAGEGCMAPTLBE PPGMPAGEMAPTLBE;
1429	// typedef PPGMPAGEGCMAPTLBE *PPPGMPAGEMAPTLBE;
1430	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGEGCMAPTLB_ENTRIES
1431	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGEGCMAPTLB_IDX(GCPhys)
1432	typedef void * PPGMPAGEMAP;
1433	typedef void ** PPPGMPAGEMAP;
1434	//#elif IN_RING0
1435	// typedef PPGMPAGER0MAPTLB PPGMPAGEMAPTLB;
1436	// typedef PPGMPAGER0MAPTLBE PPGMPAGEMAPTLBE;
1437	// typedef PPGMPAGER0MAPTLBE *PPPGMPAGEMAPTLBE;
1438	//# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER0MAPTLB_ENTRIES
1439	//# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER0MAPTLB_IDX(GCPhys)
1440	// typedef PPGMCHUNKR0MAP PPGMPAGEMAP;
1441	// typedef PPPGMCHUNKR0MAP PPPGMPAGEMAP;
1442	#else
1443	typedef PPGMPAGER3MAPTLB PPGMPAGEMAPTLB;
1444	typedef PPGMPAGER3MAPTLBE PPGMPAGEMAPTLBE;
1445	typedef PPGMPAGER3MAPTLBE *PPPGMPAGEMAPTLBE;
1446	# define PGM_PAGEMAPTLB_ENTRIES PGM_PAGER3MAPTLB_ENTRIES
1447	# define PGM_PAGEMAPTLB_IDX(GCPhys) PGM_PAGER3MAPTLB_IDX(GCPhys)
1448	typedef PPGMCHUNKR3MAP PPGMPAGEMAP;
1449	typedef PPPGMCHUNKR3MAP PPPGMPAGEMAP;
1450	#endif
1451	/** @} */
1452
1453
1454	/** @name PGM Pool Indexes.
1455	* Aka. the unique shadow page identifier.
1456	* @{ */
1457	/** NIL page pool IDX. */
1458	#define NIL_PGMPOOL_IDX 0
1459	/** The first normal index. */
1460	#define PGMPOOL_IDX_FIRST_SPECIAL 1
1461	/** Page directory (32-bit root). */
1462	#define PGMPOOL_IDX_PD 1
1463	/** Page Directory Pointer Table (PAE root). */
1464	#define PGMPOOL_IDX_PDPT 2
1465	/** AMD64 CR3 level index.*/
1466	#define PGMPOOL_IDX_AMD64_CR3 3
1467	/** Nested paging root.*/
1468	#define PGMPOOL_IDX_NESTED_ROOT 4
1469	/** The first normal index. */
1470	#define PGMPOOL_IDX_FIRST 5
1471	/** The last valid index. (inclusive, 14 bits) */
1472	#define PGMPOOL_IDX_LAST 0x3fff
1473	/** @} */
1474
1475	/** The NIL index for the parent chain. */
1476	#define NIL_PGMPOOL_USER_INDEX ((uint16_t)0xffff)
1477
1478	/**
1479	* Node in the chain linking a shadowed page to it's parent (user).
1480	*/
1481	#pragma pack(1)
1482	typedef struct PGMPOOLUSER
1483	{
1484	/** The index to the next item in the chain. NIL_PGMPOOL_USER_INDEX is no next. */
1485	uint16_t iNext;
1486	/** The user page index. */
1487	uint16_t iUser;
1488	/** Index into the user table. */
1489	uint32_t iUserTable;
1490	} PGMPOOLUSER, *PPGMPOOLUSER;
1491	typedef const PGMPOOLUSER *PCPGMPOOLUSER;
1492	#pragma pack()
1493
1494
1495	/** The NIL index for the phys ext chain. */
1496	#define NIL_PGMPOOL_PHYSEXT_INDEX ((uint16_t)0xffff)
1497
1498	/**
1499	* Node in the chain of physical cross reference extents.
1500	* @todo Calling this an 'extent' is not quite right, find a better name.
1501	*/
1502	#pragma pack(1)
1503	typedef struct PGMPOOLPHYSEXT
1504	{
1505	/** The index to the next item in the chain. NIL_PGMPOOL_PHYSEXT_INDEX is no next. */
1506	uint16_t iNext;
1507	/** The user page index. */
1508	uint16_t aidx[3];
1509	} PGMPOOLPHYSEXT, *PPGMPOOLPHYSEXT;
1510	typedef const PGMPOOLPHYSEXT *PCPGMPOOLPHYSEXT;
1511	#pragma pack()
1512
1513
1514	/**
1515	* The kind of page that's being shadowed.
1516	*/
1517	typedef enum PGMPOOLKIND
1518	{
1519	/** The virtual invalid 0 entry. */
1520	PGMPOOLKIND_INVALID = 0,
1521	/** The entry is free (=unused). */
1522	PGMPOOLKIND_FREE,
1523
1524	/** Shw: 32-bit page table; Gst: no paging */
1525	PGMPOOLKIND_32BIT_PT_FOR_PHYS,
1526	/** Shw: 32-bit page table; Gst: 32-bit page table. */
1527	PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT,
1528	/** Shw: 32-bit page table; Gst: 4MB page. */
1529	PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB,
1530	/** Shw: PAE page table; Gst: no paging */
1531	PGMPOOLKIND_PAE_PT_FOR_PHYS,
1532	/** Shw: PAE page table; Gst: 32-bit page table. */
1533	PGMPOOLKIND_PAE_PT_FOR_32BIT_PT,
1534	/** Shw: PAE page table; Gst: Half of a 4MB page. */
1535	PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB,
1536	/** Shw: PAE page table; Gst: PAE page table. */
1537	PGMPOOLKIND_PAE_PT_FOR_PAE_PT,
1538	/** Shw: PAE page table; Gst: 2MB page. */
1539	PGMPOOLKIND_PAE_PT_FOR_PAE_2MB,
1540
1541	/** Shw: 32-bit page directory. Gst: 32-bit page directory. */
1542	PGMPOOLKIND_32BIT_PD,
1543	/** Shw: 32-bit page directory. Gst: no paging. */
1544	PGMPOOLKIND_32BIT_PD_PHYS,
1545	/** Shw: PAE page directory 0; Gst: 32-bit page directory. */
1546	PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD,
1547	/** Shw: PAE page directory 1; Gst: 32-bit page directory. */
1548	PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD,
1549	/** Shw: PAE page directory 2; Gst: 32-bit page directory. */
1550	PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD,
1551	/** Shw: PAE page directory 3; Gst: 32-bit page directory. */
1552	PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
1553	/** Shw: PAE page directory; Gst: PAE page directory. */
1554	PGMPOOLKIND_PAE_PD_FOR_PAE_PD,
1555	/** Shw: PAE page directory; Gst: no paging. */
1556	PGMPOOLKIND_PAE_PD_PHYS,
1557
1558	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst 32 bits paging. */
1559	PGMPOOLKIND_PAE_PDPT_FOR_32BIT,
1560	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst PAE PDPT. */
1561	PGMPOOLKIND_PAE_PDPT,
1562	/** Shw: PAE page directory pointer table (legacy, 4 entries); Gst: no paging. */
1563	PGMPOOLKIND_PAE_PDPT_PHYS,
1564
1565	/** Shw: 64-bit page directory pointer table; Gst: 64-bit page directory pointer table. */
1566	PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT,
1567	/** Shw: 64-bit page directory pointer table; Gst: no paging */
1568	PGMPOOLKIND_64BIT_PDPT_FOR_PHYS,
1569	/** Shw: 64-bit page directory table; Gst: 64-bit page directory table. */
1570	PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD,
1571	/** Shw: 64-bit page directory table; Gst: no paging */
1572	PGMPOOLKIND_64BIT_PD_FOR_PHYS, /* 22 */
1573
1574	/** Shw: 64-bit PML4; Gst: 64-bit PML4. */
1575	PGMPOOLKIND_64BIT_PML4,
1576
1577	/** Shw: EPT page directory pointer table; Gst: no paging */
1578	PGMPOOLKIND_EPT_PDPT_FOR_PHYS,
1579	/** Shw: EPT page directory table; Gst: no paging */
1580	PGMPOOLKIND_EPT_PD_FOR_PHYS,
1581	/** Shw: EPT page table; Gst: no paging */
1582	PGMPOOLKIND_EPT_PT_FOR_PHYS,
1583
1584	/** Shw: Root Nested paging table. */
1585	PGMPOOLKIND_ROOT_NESTED,
1586
1587	/** The last valid entry. */
1588	PGMPOOLKIND_LAST = PGMPOOLKIND_ROOT_NESTED
1589	} PGMPOOLKIND;
1590
1591
1592	/**
1593	* The tracking data for a page in the pool.
1594	*/
1595	typedef struct PGMPOOLPAGE
1596	{
1597	/** AVL node code with the (R3) physical address of this page. */
1598	AVLOHCPHYSNODECORE Core;
1599	/** Pointer to the R3 mapping of the page. */
1600	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
1601	R3PTRTYPE(void *) pvPageR3;
1602	#else
1603	R3R0PTRTYPE(void *) pvPageR3;
1604	#endif
1605	/** The guest physical address. */
1606	#if HC_ARCH_BITS == 32 && GC_ARCH_BITS == 64
1607	uint32_t Alignment0;
1608	#endif
1609	RTGCPHYS GCPhys;
1610	/** The kind of page we're shadowing. (This is really a PGMPOOLKIND enum.) */
1611	uint8_t enmKind;
1612	uint8_t bPadding;
1613	/** The index of this page. */
1614	uint16_t idx;
1615	/** The next entry in the list this page currently resides in.
1616	* It's either in the free list or in the GCPhys hash. */
1617	uint16_t iNext;
1618	#ifdef PGMPOOL_WITH_USER_TRACKING
1619	/** Head of the user chain. NIL_PGMPOOL_USER_INDEX if not currently in use. */
1620	uint16_t iUserHead;
1621	/** The number of present entries. */
1622	uint16_t cPresent;
1623	/** The first entry in the table which is present. */
1624	uint16_t iFirstPresent;
1625	#endif
1626	#ifdef PGMPOOL_WITH_MONITORING
1627	/** The number of modifications to the monitored page. */
1628	uint16_t cModifications;
1629	/** The next modified page. NIL_PGMPOOL_IDX if tail. */
1630	uint16_t iModifiedNext;
1631	/** The previous modified page. NIL_PGMPOOL_IDX if head. */
1632	uint16_t iModifiedPrev;
1633	/** The next page sharing access handler. NIL_PGMPOOL_IDX if tail. */
1634	uint16_t iMonitoredNext;
1635	/** The previous page sharing access handler. NIL_PGMPOOL_IDX if head. */
1636	uint16_t iMonitoredPrev;
1637	#endif
1638	#ifdef PGMPOOL_WITH_CACHE
1639	/** The next page in the age list. */
1640	uint16_t iAgeNext;
1641	/** The previous page in the age list. */
1642	uint16_t iAgePrev;
1643	#endif /* PGMPOOL_WITH_CACHE */
1644	/** Used to indicate that the page is zeroed. */
1645	bool fZeroed;
1646	/** Used to indicate that a PT has non-global entries. */
1647	bool fSeenNonGlobal;
1648	/** Used to indicate that we're monitoring writes to the guest page. */
1649	bool fMonitored;
1650	/** Used to indicate that the page is in the cache (e.g. in the GCPhys hash).
1651	* (All pages are in the age list.) */
1652	bool fCached;
1653	/** This is used by the R3 access handlers when invoked by an async thread.
1654	* It's a hack required because of REMR3NotifyHandlerPhysicalDeregister. */
1655	bool volatile fReusedFlushPending;
1656	bool bPadding1;
1657
1658	/** Used to indicate that this page can't be flushed. Important for cr3 root pages or shadow pae pd pages). */
1659	uint32_t cLocked;
1660	uint32_t bPadding2;
1661	} PGMPOOLPAGE, PPGMPOOLPAGE, *PPPGMPOOLPAGE;
1662	/** Pointer to a const pool page. */
1663	typedef PGMPOOLPAGE const *PCPGMPOOLPAGE;
1664
1665
1666	#ifdef PGMPOOL_WITH_CACHE
1667	/** The hash table size. */
1668	# define PGMPOOL_HASH_SIZE 0x40
1669	/** The hash function. */
1670	# define PGMPOOL_HASH(GCPhys) ( ((GCPhys) >> PAGE_SHIFT) & (PGMPOOL_HASH_SIZE - 1) )
1671	#endif
1672
1673
1674	/**
1675	* The shadow page pool instance data.
1676	*
1677	* It's all one big allocation made at init time, except for the
1678	* pages that is. The user nodes follows immediatly after the
1679	* page structures.
1680	*/
1681	typedef struct PGMPOOL
1682	{
1683	/** The VM handle - R3 Ptr. */
1684	PVMR3 pVMR3;
1685	/** The VM handle - R0 Ptr. */
1686	PVMR0 pVMR0;
1687	/** The VM handle - RC Ptr. */
1688	PVMRC pVMRC;
1689	/** The max pool size. This includes the special IDs. */
1690	uint16_t cMaxPages;
1691	/** The current pool size. */
1692	uint16_t cCurPages;
1693	/** The head of the free page list. */
1694	uint16_t iFreeHead;
1695	/* Padding. */
1696	uint16_t u16Padding;
1697	#ifdef PGMPOOL_WITH_USER_TRACKING
1698	/** Head of the chain of free user nodes. */
1699	uint16_t iUserFreeHead;
1700	/** The number of user nodes we've allocated. */
1701	uint16_t cMaxUsers;
1702	/** The number of present page table entries in the entire pool. */
1703	uint32_t cPresent;
1704	/** Pointer to the array of user nodes - RC pointer. */
1705	RCPTRTYPE(PPGMPOOLUSER) paUsersRC;
1706	/** Pointer to the array of user nodes - R3 pointer. */
1707	R3PTRTYPE(PPGMPOOLUSER) paUsersR3;
1708	/** Pointer to the array of user nodes - R0 pointer. */
1709	R0PTRTYPE(PPGMPOOLUSER) paUsersR0;
1710	#endif /* PGMPOOL_WITH_USER_TRACKING */
1711	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1712	/** Head of the chain of free phys ext nodes. */
1713	uint16_t iPhysExtFreeHead;
1714	/** The number of user nodes we've allocated. */
1715	uint16_t cMaxPhysExts;
1716	/** Pointer to the array of physical xref extent - RC pointer. */
1717	RCPTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsRC;
1718	/** Pointer to the array of physical xref extent nodes - R3 pointer. */
1719	R3PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR3;
1720	/** Pointer to the array of physical xref extent nodes - R0 pointer. */
1721	R0PTRTYPE(PPGMPOOLPHYSEXT) paPhysExtsR0;
1722	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
1723	#ifdef PGMPOOL_WITH_CACHE
1724	/** Hash table for GCPhys addresses. */
1725	uint16_t aiHash[PGMPOOL_HASH_SIZE];
1726	/** The head of the age list. */
1727	uint16_t iAgeHead;
1728	/** The tail of the age list. */
1729	uint16_t iAgeTail;
1730	/** Set if the cache is enabled. */
1731	bool fCacheEnabled;
1732	#endif /* PGMPOOL_WITH_CACHE */
1733	#ifdef PGMPOOL_WITH_MONITORING
1734	/** Head of the list of modified pages. */
1735	uint16_t iModifiedHead;
1736	/** The current number of modified pages. */
1737	uint16_t cModifiedPages;
1738	/** Access handler, RC. */
1739	RCPTRTYPE(PFNPGMRCPHYSHANDLER) pfnAccessHandlerRC;
1740	/** Access handler, R0. */
1741	R0PTRTYPE(PFNPGMR0PHYSHANDLER) pfnAccessHandlerR0;
1742	/** Access handler, R3. */
1743	R3PTRTYPE(PFNPGMR3PHYSHANDLER) pfnAccessHandlerR3;
1744	/** The access handler description (HC ptr). */
1745	R3PTRTYPE(const char *) pszAccessHandler;
1746	#endif /* PGMPOOL_WITH_MONITORING */
1747	/** The number of pages currently in use. */
1748	uint16_t cUsedPages;
1749	#ifdef VBOX_WITH_STATISTICS
1750	/** The high wather mark for cUsedPages. */
1751	uint16_t cUsedPagesHigh;
1752	uint32_t Alignment1; /*< Align the next member on a 64-bit boundrary. /
1753	/** Profiling pgmPoolAlloc(). */
1754	STAMPROFILEADV StatAlloc;
1755	/** Profiling pgmPoolClearAll(). */
1756	STAMPROFILE StatClearAll;
1757	/** Profiling pgmPoolFlushAllInt(). */
1758	STAMPROFILE StatFlushAllInt;
1759	/** Profiling pgmPoolFlushPage(). */
1760	STAMPROFILE StatFlushPage;
1761	/** Profiling pgmPoolFree(). */
1762	STAMPROFILE StatFree;
1763	/** Profiling time spent zeroing pages. */
1764	STAMPROFILE StatZeroPage;
1765	# ifdef PGMPOOL_WITH_USER_TRACKING
1766	/** Profiling of pgmPoolTrackDeref. */
1767	STAMPROFILE StatTrackDeref;
1768	/** Profiling pgmTrackFlushGCPhysPT. */
1769	STAMPROFILE StatTrackFlushGCPhysPT;
1770	/** Profiling pgmTrackFlushGCPhysPTs. */
1771	STAMPROFILE StatTrackFlushGCPhysPTs;
1772	/** Profiling pgmTrackFlushGCPhysPTsSlow. */
1773	STAMPROFILE StatTrackFlushGCPhysPTsSlow;
1774	/** Number of times we've been out of user records. */
1775	STAMCOUNTER StatTrackFreeUpOneUser;
1776	# endif
1777	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
1778	/** Profiling deref activity related tracking GC physical pages. */
1779	STAMPROFILE StatTrackDerefGCPhys;
1780	/** Number of linear searches for a HCPhys in the ram ranges. */
1781	STAMCOUNTER StatTrackLinearRamSearches;
1782	/** The number of failing pgmPoolTrackPhysExtAlloc calls. */
1783	STAMCOUNTER StamTrackPhysExtAllocFailures;
1784	# endif
1785	# ifdef PGMPOOL_WITH_MONITORING
1786	/** Profiling the RC/R0 access handler. */
1787	STAMPROFILE StatMonitorRZ;
1788	/** Times we've failed interpreting the instruction. */
1789	STAMCOUNTER StatMonitorRZEmulateInstr;
1790	/** Profiling the pgmPoolFlushPage calls made from the RC/R0 access handler. */
1791	STAMPROFILE StatMonitorRZFlushPage;
1792	/** Times we've detected fork(). */
1793	STAMCOUNTER StatMonitorRZFork;
1794	/** Profiling the RC/R0 access we've handled (except REP STOSD). */
1795	STAMPROFILE StatMonitorRZHandled;
1796	/** Times we've failed interpreting a patch code instruction. */
1797	STAMCOUNTER StatMonitorRZIntrFailPatch1;
1798	/** Times we've failed interpreting a patch code instruction during flushing. */
1799	STAMCOUNTER StatMonitorRZIntrFailPatch2;
1800	/** The number of times we've seen rep prefixes we can't handle. */
1801	STAMCOUNTER StatMonitorRZRepPrefix;
1802	/** Profiling the REP STOSD cases we've handled. */
1803	STAMPROFILE StatMonitorRZRepStosd;
1804
1805	/** Profiling the R3 access handler. */
1806	STAMPROFILE StatMonitorR3;
1807	/** Times we've failed interpreting the instruction. */
1808	STAMCOUNTER StatMonitorR3EmulateInstr;
1809	/** Profiling the pgmPoolFlushPage calls made from the R3 access handler. */
1810	STAMPROFILE StatMonitorR3FlushPage;
1811	/** Times we've detected fork(). */
1812	STAMCOUNTER StatMonitorR3Fork;
1813	/** Profiling the R3 access we've handled (except REP STOSD). */
1814	STAMPROFILE StatMonitorR3Handled;
1815	/** The number of times we've seen rep prefixes we can't handle. */
1816	STAMCOUNTER StatMonitorR3RepPrefix;
1817	/** Profiling the REP STOSD cases we've handled. */
1818	STAMPROFILE StatMonitorR3RepStosd;
1819	/** The number of times we're called in an async thread an need to flush. */
1820	STAMCOUNTER StatMonitorR3Async;
1821	/** The high wather mark for cModifiedPages. */
1822	uint16_t cModifiedPagesHigh;
1823	uint16_t Alignment2[3]; /*< Align the next member on a 64-bit boundrary. /
1824	# endif
1825	# ifdef PGMPOOL_WITH_CACHE
1826	/** The number of cache hits. */
1827	STAMCOUNTER StatCacheHits;
1828	/** The number of cache misses. */
1829	STAMCOUNTER StatCacheMisses;
1830	/** The number of times we've got a conflict of 'kind' in the cache. */
1831	STAMCOUNTER StatCacheKindMismatches;
1832	/** Number of times we've been out of pages. */
1833	STAMCOUNTER StatCacheFreeUpOne;
1834	/** The number of cacheable allocations. */
1835	STAMCOUNTER StatCacheCacheable;
1836	/** The number of uncacheable allocations. */
1837	STAMCOUNTER StatCacheUncacheable;
1838	# endif
1839	#elif HC_ARCH_BITS == 64
1840	uint32_t Alignment3; /*< Align the next member on a 64-bit boundrary. /
1841	#endif
1842	/** The AVL tree for looking up a page by its HC physical address. */
1843	AVLOHCPHYSTREE HCPhysTree;
1844	uint32_t Alignment4; /*< Align the next member on a 64-bit boundrary. /
1845	/** Array of pages. (cMaxPages in length)
1846	* The Id is the index into thist array.
1847	*/
1848	PGMPOOLPAGE aPages[PGMPOOL_IDX_FIRST];
1849	} PGMPOOL, PPGMPOOL, *PPPGMPOOL;
1850
1851
1852	/** @def PGMPOOL_PAGE_2_PTR
1853	* Maps a pool page pool into the current context.
1854	*
1855	* @returns VBox status code.
1856	* @param pVM The VM handle.
1857	* @param pPage The pool page.
1858	*
1859	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1860	* small page window employeed by that function. Be careful.
1861	* @remark There is no need to assert on the result.
1862	*/
1863	#if defined(IN_RC)
1864	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1865	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1866	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageInlined(&(pVM)->pgm.s, (pPage))
1867	#elif defined(VBOX_STRICT)
1868	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) pgmPoolMapPageStrict(pPage)
1869	DECLINLINE(void *) pgmPoolMapPageStrict(PPGMPOOLPAGE pPage)
1870	{
1871	Assert(pPage && pPage->pvPageR3);
1872	return pPage->pvPageR3;
1873	}
1874	#else
1875	# define PGMPOOL_PAGE_2_PTR(pVM, pPage) ((pPage)->pvPageR3)
1876	#endif
1877
1878	/** @def PGMPOOL_PAGE_2_PTR_BY_PGM
1879	* Maps a pool page pool into the current context.
1880	*
1881	* @returns VBox status code.
1882	* @param pPGM Pointer to the PGM instance data.
1883	* @param pPage The pool page.
1884	*
1885	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1886	* small page window employeed by that function. Be careful.
1887	* @remark There is no need to assert on the result.
1888	*/
1889	#if defined(IN_RC)
1890	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1891	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1892	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) pgmPoolMapPageInlined(pPGM, (pPage))
1893	#else
1894	# define PGMPOOL_PAGE_2_PTR_BY_PGM(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGM2VM(pPGM), pPage)
1895	#endif
1896
1897	/** @def PGMPOOL_PAGE_2_PTR_BY_PGMCPU
1898	* Maps a pool page pool into the current context.
1899	*
1900	* @returns VBox status code.
1901	* @param pPGM Pointer to the PGMCPU instance data.
1902	* @param pPage The pool page.
1903	*
1904	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
1905	* small page window employeed by that function. Be careful.
1906	* @remark There is no need to assert on the result.
1907	*/
1908	#if defined(IN_RC)
1909	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1910	#elif defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
1911	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) pgmPoolMapPageInlined(PGMCPU2PGM(pPGM), (pPage))
1912	#else
1913	# define PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPage) PGMPOOL_PAGE_2_PTR(PGMCPU2VM(pPGM), pPage)
1914	#endif
1915
1916
1917	/** @name Per guest page tracking data.
1918	* This is currently as a 16-bit word in the PGMPAGE structure, the idea though
1919	* is to use more bits for it and split it up later on. But for now we'll play
1920	* safe and change as little as possible.
1921	*
1922	* The 16-bit word has two parts:
1923	*
1924	* The first 14-bit forms the @a idx field. It is either the index of a page in
1925	* the shadow page pool, or and index into the extent list.
1926	*
1927	* The 2 topmost bits makes up the @a cRefs field, which counts the number of
1928	* shadow page pool references to the page. If cRefs equals
1929	* PGMPOOL_CREFS_PHYSEXT, then the @a idx field is an indext into the extent
1930	* (misnomer) table and not the shadow page pool.
1931	*
1932	* See PGM_PAGE_GET_TRACKING and PGM_PAGE_SET_TRACKING for how to get and set
1933	* the 16-bit word.
1934	*
1935	* @{ */
1936	/** The shift count for getting to the cRefs part. */
1937	#define PGMPOOL_TD_CREFS_SHIFT 14
1938	/** The mask applied after shifting the tracking data down by
1939	* PGMPOOL_TD_CREFS_SHIFT. */
1940	#define PGMPOOL_TD_CREFS_MASK 0x3
1941	/** The cRef value used to indiciate that the idx is the head of a
1942	* physical cross reference list. */
1943	#define PGMPOOL_TD_CREFS_PHYSEXT PGMPOOL_TD_CREFS_MASK
1944	/** The shift used to get idx. */
1945	#define PGMPOOL_TD_IDX_SHIFT 0
1946	/** The mask applied to the idx after shifting down by PGMPOOL_TD_IDX_SHIFT. */
1947	#define PGMPOOL_TD_IDX_MASK 0x3fff
1948	/** The idx value when we're out of of PGMPOOLPHYSEXT entries or/and there are
1949	* simply too many mappings of this page. */
1950	#define PGMPOOL_TD_IDX_OVERFLOWED PGMPOOL_TD_IDX_MASK
1951
1952	/** @def PGMPOOL_TD_MAKE
1953	* Makes a 16-bit tracking data word.
1954	*
1955	* @returns tracking data.
1956	* @param cRefs The @a cRefs field. Must be within bounds!
1957	* @param idx The @a idx field. Must also be within bounds! */
1958	#define PGMPOOL_TD_MAKE(cRefs, idx) ( ((cRefs) << PGMPOOL_TD_CREFS_SHIFT) \| (idx) )
1959
1960	/** @def PGMPOOL_TD_GET_CREFS
1961	* Get the @a cRefs field from a tracking data word.
1962	*
1963	* @returns The @a cRefs field
1964	* @param u16 The tracking data word. */
1965	#define PGMPOOL_TD_GET_CREFS(u16) ( ((u16) >> PGMPOOL_TD_CREFS_SHIFT) & PGMPOOL_TD_CREFS_MASK )
1966
1967	/** @def PGMPOOL_TD_GET_IDX
1968	* Get the @a idx field from a tracking data word.
1969	*
1970	* @returns The @a idx field
1971	* @param u16 The tracking data word. */
1972	#define PGMPOOL_TD_GET_IDX(u16) ( ((u16) >> PGMPOOL_TD_IDX_SHIFT) & PGMPOOL_TD_IDX_MASK )
1973	/** @} */
1974
1975
1976	/**
1977	* Trees are using self relative offsets as pointers.
1978	* So, all its data, including the root pointer, must be in the heap for HC and GC
1979	* to have the same layout.
1980	*/
1981	typedef struct PGMTREES
1982	{
1983	/** Physical access handlers (AVL range+offsetptr tree). */
1984	AVLROGCPHYSTREE PhysHandlers;
1985	/** Virtual access handlers (AVL range + GC ptr tree). */
1986	AVLROGCPTRTREE VirtHandlers;
1987	/** Virtual access handlers (Phys range AVL range + offsetptr tree). */
1988	AVLROGCPHYSTREE PhysToVirtHandlers;
1989	/** Virtual access handlers for the hypervisor (AVL range + GC ptr tree). */
1990	AVLROGCPTRTREE HyperVirtHandlers;
1991	} PGMTREES;
1992	/** Pointer to PGM trees. */
1993	typedef PGMTREES *PPGMTREES;
1994
1995
1996	/** @name Paging mode macros
1997	* @{ */
1998	#ifdef IN_RC
1999	# define PGM_CTX(a,b) a##RC##b
2000	# define PGM_CTX_STR(a,b) a "GC" b
2001	# define PGM_CTX_DECL(type) VMMRCDECL(type)
2002	#else
2003	# ifdef IN_RING3
2004	# define PGM_CTX(a,b) a##R3##b
2005	# define PGM_CTX_STR(a,b) a "R3" b
2006	# define PGM_CTX_DECL(type) DECLCALLBACK(type)
2007	# else
2008	# define PGM_CTX(a,b) a##R0##b
2009	# define PGM_CTX_STR(a,b) a "R0" b
2010	# define PGM_CTX_DECL(type) VMMDECL(type)
2011	# endif
2012	#endif
2013
2014	#define PGM_GST_NAME_REAL(name) PGM_CTX(pgm,GstReal##name)
2015	#define PGM_GST_NAME_RC_REAL_STR(name) "pgmRCGstReal" #name
2016	#define PGM_GST_NAME_R0_REAL_STR(name) "pgmR0GstReal" #name
2017	#define PGM_GST_NAME_PROT(name) PGM_CTX(pgm,GstProt##name)
2018	#define PGM_GST_NAME_RC_PROT_STR(name) "pgmRCGstProt" #name
2019	#define PGM_GST_NAME_R0_PROT_STR(name) "pgmR0GstProt" #name
2020	#define PGM_GST_NAME_32BIT(name) PGM_CTX(pgm,Gst32Bit##name)
2021	#define PGM_GST_NAME_RC_32BIT_STR(name) "pgmRCGst32Bit" #name
2022	#define PGM_GST_NAME_R0_32BIT_STR(name) "pgmR0Gst32Bit" #name
2023	#define PGM_GST_NAME_PAE(name) PGM_CTX(pgm,GstPAE##name)
2024	#define PGM_GST_NAME_RC_PAE_STR(name) "pgmRCGstPAE" #name
2025	#define PGM_GST_NAME_R0_PAE_STR(name) "pgmR0GstPAE" #name
2026	#define PGM_GST_NAME_AMD64(name) PGM_CTX(pgm,GstAMD64##name)
2027	#define PGM_GST_NAME_RC_AMD64_STR(name) "pgmRCGstAMD64" #name
2028	#define PGM_GST_NAME_R0_AMD64_STR(name) "pgmR0GstAMD64" #name
2029	#define PGM_GST_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Gst##name))
2030	#define PGM_GST_DECL(type, name) PGM_CTX_DECL(type) PGM_GST_NAME(name)
2031
2032	#define PGM_SHW_NAME_32BIT(name) PGM_CTX(pgm,Shw32Bit##name)
2033	#define PGM_SHW_NAME_RC_32BIT_STR(name) "pgmRCShw32Bit" #name
2034	#define PGM_SHW_NAME_R0_32BIT_STR(name) "pgmR0Shw32Bit" #name
2035	#define PGM_SHW_NAME_PAE(name) PGM_CTX(pgm,ShwPAE##name)
2036	#define PGM_SHW_NAME_RC_PAE_STR(name) "pgmRCShwPAE" #name
2037	#define PGM_SHW_NAME_R0_PAE_STR(name) "pgmR0ShwPAE" #name
2038	#define PGM_SHW_NAME_AMD64(name) PGM_CTX(pgm,ShwAMD64##name)
2039	#define PGM_SHW_NAME_RC_AMD64_STR(name) "pgmRCShwAMD64" #name
2040	#define PGM_SHW_NAME_R0_AMD64_STR(name) "pgmR0ShwAMD64" #name
2041	#define PGM_SHW_NAME_NESTED(name) PGM_CTX(pgm,ShwNested##name)
2042	#define PGM_SHW_NAME_RC_NESTED_STR(name) "pgmRCShwNested" #name
2043	#define PGM_SHW_NAME_R0_NESTED_STR(name) "pgmR0ShwNested" #name
2044	#define PGM_SHW_NAME_EPT(name) PGM_CTX(pgm,ShwEPT##name)
2045	#define PGM_SHW_NAME_RC_EPT_STR(name) "pgmRCShwEPT" #name
2046	#define PGM_SHW_NAME_R0_EPT_STR(name) "pgmR0ShwEPT" #name
2047	#define PGM_SHW_DECL(type, name) PGM_CTX_DECL(type) PGM_SHW_NAME(name)
2048	#define PGM_SHW_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Shw##name))
2049
2050	/* Shw_Gst */
2051	#define PGM_BTH_NAME_32BIT_REAL(name) PGM_CTX(pgm,Bth32BitReal##name)
2052	#define PGM_BTH_NAME_32BIT_PROT(name) PGM_CTX(pgm,Bth32BitProt##name)
2053	#define PGM_BTH_NAME_32BIT_32BIT(name) PGM_CTX(pgm,Bth32Bit32Bit##name)
2054	#define PGM_BTH_NAME_PAE_REAL(name) PGM_CTX(pgm,BthPAEReal##name)
2055	#define PGM_BTH_NAME_PAE_PROT(name) PGM_CTX(pgm,BthPAEProt##name)
2056	#define PGM_BTH_NAME_PAE_32BIT(name) PGM_CTX(pgm,BthPAE32Bit##name)
2057	#define PGM_BTH_NAME_PAE_PAE(name) PGM_CTX(pgm,BthPAEPAE##name)
2058	#define PGM_BTH_NAME_AMD64_PROT(name) PGM_CTX(pgm,BthAMD64Prot##name)
2059	#define PGM_BTH_NAME_AMD64_AMD64(name) PGM_CTX(pgm,BthAMD64AMD64##name)
2060	#define PGM_BTH_NAME_NESTED_REAL(name) PGM_CTX(pgm,BthNestedReal##name)
2061	#define PGM_BTH_NAME_NESTED_PROT(name) PGM_CTX(pgm,BthNestedProt##name)
2062	#define PGM_BTH_NAME_NESTED_32BIT(name) PGM_CTX(pgm,BthNested32Bit##name)
2063	#define PGM_BTH_NAME_NESTED_PAE(name) PGM_CTX(pgm,BthNestedPAE##name)
2064	#define PGM_BTH_NAME_NESTED_AMD64(name) PGM_CTX(pgm,BthNestedAMD64##name)
2065	#define PGM_BTH_NAME_EPT_REAL(name) PGM_CTX(pgm,BthEPTReal##name)
2066	#define PGM_BTH_NAME_EPT_PROT(name) PGM_CTX(pgm,BthEPTProt##name)
2067	#define PGM_BTH_NAME_EPT_32BIT(name) PGM_CTX(pgm,BthEPT32Bit##name)
2068	#define PGM_BTH_NAME_EPT_PAE(name) PGM_CTX(pgm,BthEPTPAE##name)
2069	#define PGM_BTH_NAME_EPT_AMD64(name) PGM_CTX(pgm,BthEPTAMD64##name)
2070
2071	#define PGM_BTH_NAME_RC_32BIT_REAL_STR(name) "pgmRCBth32BitReal" #name
2072	#define PGM_BTH_NAME_RC_32BIT_PROT_STR(name) "pgmRCBth32BitProt" #name
2073	#define PGM_BTH_NAME_RC_32BIT_32BIT_STR(name) "pgmRCBth32Bit32Bit" #name
2074	#define PGM_BTH_NAME_RC_PAE_REAL_STR(name) "pgmRCBthPAEReal" #name
2075	#define PGM_BTH_NAME_RC_PAE_PROT_STR(name) "pgmRCBthPAEProt" #name
2076	#define PGM_BTH_NAME_RC_PAE_32BIT_STR(name) "pgmRCBthPAE32Bit" #name
2077	#define PGM_BTH_NAME_RC_PAE_PAE_STR(name) "pgmRCBthPAEPAE" #name
2078	#define PGM_BTH_NAME_RC_AMD64_AMD64_STR(name) "pgmRCBthAMD64AMD64" #name
2079	#define PGM_BTH_NAME_RC_NESTED_REAL_STR(name) "pgmRCBthNestedReal" #name
2080	#define PGM_BTH_NAME_RC_NESTED_PROT_STR(name) "pgmRCBthNestedProt" #name
2081	#define PGM_BTH_NAME_RC_NESTED_32BIT_STR(name) "pgmRCBthNested32Bit" #name
2082	#define PGM_BTH_NAME_RC_NESTED_PAE_STR(name) "pgmRCBthNestedPAE" #name
2083	#define PGM_BTH_NAME_RC_NESTED_AMD64_STR(name) "pgmRCBthNestedAMD64" #name
2084	#define PGM_BTH_NAME_RC_EPT_REAL_STR(name) "pgmRCBthEPTReal" #name
2085	#define PGM_BTH_NAME_RC_EPT_PROT_STR(name) "pgmRCBthEPTProt" #name
2086	#define PGM_BTH_NAME_RC_EPT_32BIT_STR(name) "pgmRCBthEPT32Bit" #name
2087	#define PGM_BTH_NAME_RC_EPT_PAE_STR(name) "pgmRCBthEPTPAE" #name
2088	#define PGM_BTH_NAME_RC_EPT_AMD64_STR(name) "pgmRCBthEPTAMD64" #name
2089	#define PGM_BTH_NAME_R0_32BIT_REAL_STR(name) "pgmR0Bth32BitReal" #name
2090	#define PGM_BTH_NAME_R0_32BIT_PROT_STR(name) "pgmR0Bth32BitProt" #name
2091	#define PGM_BTH_NAME_R0_32BIT_32BIT_STR(name) "pgmR0Bth32Bit32Bit" #name
2092	#define PGM_BTH_NAME_R0_PAE_REAL_STR(name) "pgmR0BthPAEReal" #name
2093	#define PGM_BTH_NAME_R0_PAE_PROT_STR(name) "pgmR0BthPAEProt" #name
2094	#define PGM_BTH_NAME_R0_PAE_32BIT_STR(name) "pgmR0BthPAE32Bit" #name
2095	#define PGM_BTH_NAME_R0_PAE_PAE_STR(name) "pgmR0BthPAEPAE" #name
2096	#define PGM_BTH_NAME_R0_AMD64_PROT_STR(name) "pgmR0BthAMD64Prot" #name
2097	#define PGM_BTH_NAME_R0_AMD64_AMD64_STR(name) "pgmR0BthAMD64AMD64" #name
2098	#define PGM_BTH_NAME_R0_NESTED_REAL_STR(name) "pgmR0BthNestedReal" #name
2099	#define PGM_BTH_NAME_R0_NESTED_PROT_STR(name) "pgmR0BthNestedProt" #name
2100	#define PGM_BTH_NAME_R0_NESTED_32BIT_STR(name) "pgmR0BthNested32Bit" #name
2101	#define PGM_BTH_NAME_R0_NESTED_PAE_STR(name) "pgmR0BthNestedPAE" #name
2102	#define PGM_BTH_NAME_R0_NESTED_AMD64_STR(name) "pgmR0BthNestedAMD64" #name
2103	#define PGM_BTH_NAME_R0_EPT_REAL_STR(name) "pgmR0BthEPTReal" #name
2104	#define PGM_BTH_NAME_R0_EPT_PROT_STR(name) "pgmR0BthEPTProt" #name
2105	#define PGM_BTH_NAME_R0_EPT_32BIT_STR(name) "pgmR0BthEPT32Bit" #name
2106	#define PGM_BTH_NAME_R0_EPT_PAE_STR(name) "pgmR0BthEPTPAE" #name
2107	#define PGM_BTH_NAME_R0_EPT_AMD64_STR(name) "pgmR0BthEPTAMD64" #name
2108
2109	#define PGM_BTH_DECL(type, name) PGM_CTX_DECL(type) PGM_BTH_NAME(name)
2110	#define PGM_BTH_PFN(name, pVCpu) ((pVCpu)->pgm.s.PGM_CTX(pfn,Bth##name))
2111	/** @} */
2112
2113	/**
2114	* Data for each paging mode.
2115	*/
2116	typedef struct PGMMODEDATA
2117	{
2118	/** The guest mode type. */
2119	uint32_t uGstType;
2120	/** The shadow mode type. */
2121	uint32_t uShwType;
2122
2123	/** @name Function pointers for Shadow paging.
2124	* @{
2125	*/
2126	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2127	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2128	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2129	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2130
2131	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2132	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2133
2134	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2135	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2136	/** @} */
2137
2138	/** @name Function pointers for Guest paging.
2139	* @{
2140	*/
2141	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2142	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2143	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2144	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2145	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2146	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2147	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2148	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2149	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2150	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2151	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2152	/** @} */
2153
2154	/** @name Function pointers for Both Shadow and Guest paging.
2155	* @{
2156	*/
2157	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2158	/* no pfnR3BthTrap0eHandler */
2159	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2160	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2161	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2162	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2163	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2164	#ifdef VBOX_STRICT
2165	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2166	#endif
2167	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2168	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2169
2170	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2171	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2172	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2173	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2174	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2175	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2176	#ifdef VBOX_STRICT
2177	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2178	#endif
2179	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2180	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2181
2182	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2183	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2184	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2185	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2186	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2187	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2188	#ifdef VBOX_STRICT
2189	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2190	#endif
2191	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2192	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2193	/** @} */
2194	} PGMMODEDATA, *PPGMMODEDATA;
2195
2196
2197
2198	/**
2199	* Converts a PGM pointer into a VM pointer.
2200	* @returns Pointer to the VM structure the PGM is part of.
2201	* @param pPGM Pointer to PGM instance data.
2202	*/
2203	#define PGM2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2204
2205	/**
2206	* PGM Data (part of VM)
2207	*/
2208	typedef struct PGM
2209	{
2210	/** Offset to the VM structure. */
2211	RTINT offVM;
2212	/** Offset of the PGMCPU structure relative to VMCPU. */
2213	RTINT offVCpuPGM;
2214
2215	/** @cfgm{RamPreAlloc, boolean, false}
2216	* Indicates whether the base RAM should all be allocated before starting
2217	* the VM (default), or if it should be allocated when first written to.
2218	*/
2219	bool fRamPreAlloc;
2220	/** Alignment padding. */
2221	bool afAlignment0[7];
2222
2223	/** What needs syncing (PGM_SYNC_*).
2224	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2225	* PGMFlushTLB, and PGMR3Load. */
2226	RTUINT fGlobalSyncFlags;
2227
2228	/*
2229	* This will be redefined at least two more times before we're done, I'm sure.
2230	* The current code is only to get on with the coding.
2231	* - 2004-06-10: initial version, bird.
2232	* - 2004-07-02: 1st time, bird.
2233	* - 2004-10-18: 2nd time, bird.
2234	* - 2005-07-xx: 3rd time, bird.
2235	*/
2236
2237	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2238	RCPTRTYPE(PX86PTE) paDynPageMap32BitPTEsGC;
2239	/** Pointer to the page table entries for the dynamic page mapping area - GCPtr. */
2240	RCPTRTYPE(PX86PTEPAE) paDynPageMapPaePTEsGC;
2241
2242	/** The host paging mode. (This is what SUPLib reports.) */
2243	SUPPAGINGMODE enmHostMode;
2244
2245	/** 4 MB page mask; 32 or 36 bits depending on PSE-36 (identical for all VCPUs) */
2246	RTGCPHYS GCPhys4MBPSEMask;
2247
2248	/** Pointer to the list of RAM ranges (Phys GC -> Phys HC conversion) - for R3.
2249	* This is sorted by physical address and contains no overlapping ranges. */
2250	R3PTRTYPE(PPGMRAMRANGE) pRamRangesR3;
2251	/** R0 pointer corresponding to PGM::pRamRangesR3. */
2252	R0PTRTYPE(PPGMRAMRANGE) pRamRangesR0;
2253	/** RC pointer corresponding to PGM::pRamRangesR3. */
2254	RCPTRTYPE(PPGMRAMRANGE) pRamRangesRC;
2255	RTRCPTR alignment4; /*< structure alignment. /
2256
2257	/** Pointer to the list of ROM ranges - for R3.
2258	* This is sorted by physical address and contains no overlapping ranges. */
2259	R3PTRTYPE(PPGMROMRANGE) pRomRangesR3;
2260	/** R0 pointer corresponding to PGM::pRomRangesR3. */
2261	R0PTRTYPE(PPGMROMRANGE) pRomRangesR0;
2262	/** RC pointer corresponding to PGM::pRomRangesR3. */
2263	RCPTRTYPE(PPGMROMRANGE) pRomRangesRC;
2264	/** Alignment padding. */
2265	RTRCPTR GCPtrPadding2;
2266
2267	/** Pointer to the list of MMIO2 ranges - for R3.
2268	* Registration order. */
2269	R3PTRTYPE(PPGMMMIO2RANGE) pMmio2RangesR3;
2270
2271	/** PGM offset based trees - R3 Ptr. */
2272	R3PTRTYPE(PPGMTREES) pTreesR3;
2273	/** PGM offset based trees - R0 Ptr. */
2274	R0PTRTYPE(PPGMTREES) pTreesR0;
2275	/** PGM offset based trees - RC Ptr. */
2276	RCPTRTYPE(PPGMTREES) pTreesRC;
2277
2278	/** Linked list of GC mappings - for RC.
2279	* The list is sorted ascending on address.
2280	*/
2281	RCPTRTYPE(PPGMMAPPING) pMappingsRC;
2282	/** Linked list of GC mappings - for HC.
2283	* The list is sorted ascending on address.
2284	*/
2285	R3PTRTYPE(PPGMMAPPING) pMappingsR3;
2286	/** Linked list of GC mappings - for R0.
2287	* The list is sorted ascending on address.
2288	*/
2289	R0PTRTYPE(PPGMMAPPING) pMappingsR0;
2290
2291	/** Pointer to the 5 page CR3 content mapping.
2292	* The first page is always the CR3 (in some form) while the 4 other pages
2293	* are used of the PDs in PAE mode. */
2294	RTGCPTR GCPtrCR3Mapping;
2295	#if HC_ARCH_BITS == 64 && GC_ARCH_BITS == 32
2296	uint32_t u32Alignment;
2297	#endif
2298
2299	/** Indicates that PGMR3FinalizeMappings has been called and that further
2300	* PGMR3MapIntermediate calls will be rejected. */
2301	bool fFinalizedMappings;
2302	/** If set no conflict checks are required. (boolean) */
2303	bool fMappingsFixed;
2304	/** If set, then no mappings are put into the shadow page table. (boolean) */
2305	bool fDisableMappings;
2306	/** Size of fixed mapping */
2307	uint32_t cbMappingFixed;
2308	/** Base address (GC) of fixed mapping */
2309	RTGCPTR GCPtrMappingFixed;
2310	/** The address of the previous RAM range mapping. */
2311	RTGCPTR GCPtrPrevRamRangeMapping;
2312
2313	/** @name Intermediate Context
2314	* @{ */
2315	/** Pointer to the intermediate page directory - Normal. */
2316	R3PTRTYPE(PX86PD) pInterPD;
2317	/** Pointer to the intermedate page tables - Normal.
2318	* There are two page tables, one for the identity mapping and one for
2319	* the host context mapping (of the core code). */
2320	R3PTRTYPE(PX86PT) apInterPTs[2];
2321	/** Pointer to the intermedate page tables - PAE. */
2322	R3PTRTYPE(PX86PTPAE) apInterPaePTs[2];
2323	/** Pointer to the intermedate page directory - PAE. */
2324	R3PTRTYPE(PX86PDPAE) apInterPaePDs[4];
2325	/** Pointer to the intermedate page directory - PAE. */
2326	R3PTRTYPE(PX86PDPT) pInterPaePDPT;
2327	/** Pointer to the intermedate page-map level 4 - AMD64. */
2328	R3PTRTYPE(PX86PML4) pInterPaePML4;
2329	/** Pointer to the intermedate page directory - AMD64. */
2330	R3PTRTYPE(PX86PDPT) pInterPaePDPT64;
2331	/** The Physical Address (HC) of the intermediate Page Directory - Normal. */
2332	RTHCPHYS HCPhysInterPD;
2333	/** The Physical Address (HC) of the intermediate Page Directory Pointer Table - PAE. */
2334	RTHCPHYS HCPhysInterPaePDPT;
2335	/** The Physical Address (HC) of the intermediate Page Map Level 4 table - AMD64. */
2336	RTHCPHYS HCPhysInterPaePML4;
2337	/** @} */
2338
2339	/** Base address of the dynamic page mapping area.
2340	* The array is MM_HYPER_DYNAMIC_SIZE bytes big.
2341	*/
2342	RCPTRTYPE(uint8_t *) pbDynPageMapBaseGC;
2343	/** The index of the last entry used in the dynamic page mapping area. */
2344	RTUINT iDynPageMapLast;
2345	/** Cache containing the last entries in the dynamic page mapping area.
2346	* The cache size is covering half of the mapping area. */
2347	RTHCPHYS aHCPhysDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT + 1)];
2348	/** Keep a lock counter for the full (!) mapping area. */
2349	uint32_t aLockedDynPageMapCache[MM_HYPER_DYNAMIC_SIZE >> (PAGE_SHIFT)];
2350
2351	/** The address of the ring-0 mapping cache if we're making use of it. */
2352	RTR0PTR pvR0DynMapUsed;
2353
2354	/** PGM critical section.
2355	* This protects the physical & virtual access handlers, ram ranges,
2356	* and the page flag updating (some of it anyway).
2357	*/
2358	PDMCRITSECT CritSect;
2359
2360	/** Pointer to SHW+GST mode data (function pointers).
2361	* The index into this table is made up from */
2362	R3PTRTYPE(PPGMMODEDATA) paModeData;
2363
2364	/** Shadow Page Pool - R3 Ptr. */
2365	R3PTRTYPE(PPGMPOOL) pPoolR3;
2366	/** Shadow Page Pool - R0 Ptr. */
2367	R0PTRTYPE(PPGMPOOL) pPoolR0;
2368	/** Shadow Page Pool - RC Ptr. */
2369	RCPTRTYPE(PPGMPOOL) pPoolRC;
2370
2371	/** We're not in a state which permits writes to guest memory.
2372	* (Only used in strict builds.) */
2373	bool fNoMorePhysWrites;
2374
2375	/** Flush the cache on the next access. */
2376	bool fPhysCacheFlushPending;
2377	/** @todo r=bird: Fix member names!*/
2378	/** PGMPhysRead cache */
2379	PGMPHYSCACHE pgmphysreadcache;
2380	/** PGMPhysWrite cache */
2381	PGMPHYSCACHE pgmphyswritecache;
2382
2383	/**
2384	* Data associated with managing the ring-3 mappings of the allocation chunks.
2385	*/
2386	struct
2387	{
2388	/** The chunk tree, ordered by chunk id. */
2389	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2390	R3PTRTYPE(PAVLU32NODECORE) pTree;
2391	#else
2392	R3R0PTRTYPE(PAVLU32NODECORE) pTree;
2393	#endif
2394	/** The chunk mapping TLB. */
2395	PGMCHUNKR3MAPTLB Tlb;
2396	/** The number of mapped chunks. */
2397	uint32_t c;
2398	/** The maximum number of mapped chunks.
2399	* @cfgm PGM/MaxRing3Chunks */
2400	uint32_t cMax;
2401	/** The chunk age tree, ordered by ageing sequence number. */
2402	R3PTRTYPE(PAVLLU32NODECORE) pAgeTree;
2403	/** The current time. */
2404	uint32_t iNow;
2405	/** Number of pgmR3PhysChunkFindUnmapCandidate calls left to the next ageing. */
2406	uint32_t AgeingCountdown;
2407	} ChunkR3Map;
2408
2409	/**
2410	* The page mapping TLB for ring-3 and (for the time being) ring-0.
2411	*/
2412	PGMPAGER3MAPTLB PhysTlbHC;
2413
2414	/** @name The zero page.
2415	* @{ */
2416	/** The host physical address of the zero page. */
2417	RTHCPHYS HCPhysZeroPg;
2418	/** The ring-3 mapping of the zero page. */
2419	RTR3PTR pvZeroPgR3;
2420	/** The ring-0 mapping of the zero page. */
2421	RTR0PTR pvZeroPgR0;
2422	/** The GC mapping of the zero page. */
2423	RTGCPTR pvZeroPgRC;
2424	#if GC_ARCH_BITS != 32
2425	uint32_t u32ZeroAlignment; /*< Alignment padding. /
2426	#endif
2427	/** @}*/
2428
2429	/** The number of handy pages. */
2430	uint32_t cHandyPages;
2431	/**
2432	* Array of handy pages.
2433	*
2434	* This array is used in a two way communication between pgmPhysAllocPage
2435	* and GMMR0AllocateHandyPages, with PGMR3PhysAllocateHandyPages serving as
2436	* an intermediary.
2437	*
2438	* The size of this array is important, see pgmPhysEnsureHandyPage for details.
2439	* (The current size of 32 pages, means 128 KB of handy memory.)
2440	*/
2441	GMMPAGEDESC aHandyPages[PGM_HANDY_PAGES];
2442
2443	/** @name Error injection.
2444	* @{ */
2445	/** Inject handy page allocation errors pretending we're completely out of
2446	* memory. */
2447	bool volatile fErrInjHandyPages;
2448	/** Padding. */
2449	bool afReserved[7];
2450	/** @} */
2451
2452	/** @name Release Statistics
2453	* @{ */
2454	uint32_t cAllPages; /*< The total number of pages. (Should be Private + Shared + Zero.) /
2455	uint32_t cPrivatePages; /*< The number of private pages. /
2456	uint32_t cSharedPages; /*< The number of shared pages. /
2457	uint32_t cZeroPages; /*< The number of zero backed pages. /
2458
2459	/** The number of times we were forced to change the hypervisor region location. */
2460	STAMCOUNTER cRelocations;
2461	/** @} */
2462
2463	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2464	/* R3 only: */
2465	STAMCOUNTER StatR3DetectedConflicts; /*< R3: Number of times PGMR3MapHasConflicts() detected a conflict. /
2466	STAMPROFILE StatR3ResolveConflict; /*< R3: pgmR3SyncPTResolveConflict() profiling (includes the entire relocation). /
2467
2468	STAMCOUNTER StatRZChunkR3MapTlbHits; /*< RC/R0: Ring-3/0 chunk mapper TLB hits. /
2469	STAMCOUNTER StatRZChunkR3MapTlbMisses; /*< RC/R0: Ring-3/0 chunk mapper TLB misses. /
2470	STAMCOUNTER StatRZPageMapTlbHits; /*< RC/R0: Ring-3/0 page mapper TLB hits. /
2471	STAMCOUNTER StatRZPageMapTlbMisses; /*< RC/R0: Ring-3/0 page mapper TLB misses. /
2472	STAMCOUNTER StatR3ChunkR3MapTlbHits; /*< R3: Ring-3/0 chunk mapper TLB hits. /
2473	STAMCOUNTER StatR3ChunkR3MapTlbMisses; /*< R3: Ring-3/0 chunk mapper TLB misses. /
2474	STAMCOUNTER StatR3PageMapTlbHits; /*< R3: Ring-3/0 page mapper TLB hits. /
2475	STAMCOUNTER StatR3PageMapTlbMisses; /*< R3: Ring-3/0 page mapper TLB misses. /
2476	STAMPROFILE StatRZSyncCR3HandlerVirtualReset; /*< RC/R0: Profiling of the virtual handler resets. /
2477	STAMPROFILE StatRZSyncCR3HandlerVirtualUpdate; /*< RC/R0: Profiling of the virtual handler updates. /
2478	STAMPROFILE StatR3SyncCR3HandlerVirtualReset; /*< R3: Profiling of the virtual handler resets. /
2479	STAMPROFILE StatR3SyncCR3HandlerVirtualUpdate; /*< R3: Profiling of the virtual handler updates. /
2480	STAMCOUNTER StatR3PhysHandlerReset; /*< R3: The number of times PGMHandlerPhysicalReset is called. /
2481	STAMCOUNTER StatRZPhysHandlerReset; /*< RC/R0: The number of times PGMHandlerPhysicalReset is called. /
2482	STAMPROFILE StatRZVirtHandlerSearchByPhys; /*< RC/R0: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2483	STAMPROFILE StatR3VirtHandlerSearchByPhys; /*< R3: Profiling of pgmHandlerVirtualFindByPhysAddr. /
2484	STAMCOUNTER StatRZPageReplaceShared; /*< RC/R0: Times a shared page has been replaced by a private one. /
2485	STAMCOUNTER StatRZPageReplaceZero; /*< RC/R0: Times the zero page has been replaced by a private one. /
2486	/// @todo STAMCOUNTER StatRZPageHandyAllocs; /*< RC/R0: The number of times we've executed GMMR3AllocateHandyPages. /
2487	STAMCOUNTER StatR3PageReplaceShared; /*< R3: Times a shared page has been replaced by a private one. /
2488	STAMCOUNTER StatR3PageReplaceZero; /*< R3: Times the zero page has been replaced by a private one. /
2489	/// @todo STAMCOUNTER StatR3PageHandyAllocs; /*< R3: The number of times we've executed GMMR3AllocateHandyPages. /
2490
2491	/* RC only: */
2492	STAMCOUNTER StatRCDynMapCacheMisses; /*< RC: The number of dynamic page mapping cache misses /
2493	STAMCOUNTER StatRCDynMapCacheHits; /*< RC: The number of dynamic page mapping cache hits /
2494	STAMCOUNTER StatRCInvlPgConflict; /*< RC: Number of times PGMInvalidatePage() detected a mapping conflict. /
2495	STAMCOUNTER StatRCInvlPgSyncMonCR3; /*< RC: Number of times PGMInvalidatePage() ran into PGM_SYNC_MONITOR_CR3. /
2496
2497	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2498	STAMCOUNTER StatTrackVirgin; /*< The number of first time shadowings. /
2499	STAMCOUNTER StatTrackAliased; /*< The number of times switching to cRef2, i.e. the page is being shadowed by two PTs. /
2500	STAMCOUNTER StatTrackAliasedMany; /*< The number of times we're tracking using cRef2. /
2501	STAMCOUNTER StatTrackAliasedLots; /*< The number of times we're hitting pages which has overflowed cRef2. /
2502	STAMCOUNTER StatTrackOverflows; /*< The number of times the extent list grows to long. /
2503	STAMPROFILE StatTrackDeref; /*< Profiling of SyncPageWorkerTrackDeref (expensive). /
2504	# endif
2505	#endif
2506	} PGM;
2507	/** Pointer to the PGM instance data. */
2508	typedef PGM *PPGM;
2509
2510
2511	/**
2512	* Converts a PGMCPU pointer into a VM pointer.
2513	* @returns Pointer to the VM structure the PGM is part of.
2514	* @param pPGM Pointer to PGMCPU instance data.
2515	*/
2516	#define PGMCPU2VM(pPGM) ( (PVM)((char*)pPGM - pPGM->offVM) )
2517
2518	/**
2519	* Converts a PGMCPU pointer into a PGM pointer.
2520	* @returns Pointer to the VM structure the PGM is part of.
2521	* @param pPGM Pointer to PGMCPU instance data.
2522	*/
2523	#define PGMCPU2PGM(pPGMCpu) ( (PPGM)((char*)pPGMCpu - pPGMCpu->offPGM) )
2524
2525	/**
2526	* PGMCPU Data (part of VMCPU).
2527	*/
2528	typedef struct PGMCPU
2529	{
2530	/** Offset to the VM structure. */
2531	RTINT offVM;
2532	/** Offset to the VMCPU structure. */
2533	RTINT offVCpu;
2534	/** Offset of the PGM structure relative to VMCPU. */
2535	RTINT offPGM;
2536	RTINT uPadding0; /*< structure size alignment. /
2537
2538	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE
2539	/** Automatically tracked physical memory mapping set.
2540	* Ring-0 and strict raw-mode builds. */
2541	PGMMAPSET AutoSet;
2542	#endif
2543
2544	/** A20 gate mask.
2545	* Our current approach to A20 emulation is to let REM do it and don't bother
2546	* anywhere else. The interesting Guests will be operating with it enabled anyway.
2547	* But whould need arrise, we'll subject physical addresses to this mask. */
2548	RTGCPHYS GCPhysA20Mask;
2549	/** A20 gate state - boolean! */
2550	bool fA20Enabled;
2551
2552	/** What needs syncing (PGM_SYNC_*).
2553	* This is used to queue operations for PGMSyncCR3, PGMInvalidatePage,
2554	* PGMFlushTLB, and PGMR3Load. */
2555	RTUINT fSyncFlags;
2556
2557	/** The shadow paging mode. */
2558	PGMMODE enmShadowMode;
2559	/** The guest paging mode. */
2560	PGMMODE enmGuestMode;
2561
2562	/** The current physical address representing in the guest CR3 register. */
2563	RTGCPHYS GCPhysCR3;
2564
2565	/** @name 32-bit Guest Paging.
2566	* @{ */
2567	/** The guest's page directory, R3 pointer. */
2568	R3PTRTYPE(PX86PD) pGst32BitPdR3;
2569	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2570	/** The guest's page directory, R0 pointer. */
2571	R0PTRTYPE(PX86PD) pGst32BitPdR0;
2572	#endif
2573	/** The guest's page directory, static RC mapping. */
2574	RCPTRTYPE(PX86PD) pGst32BitPdRC;
2575	/** @} */
2576
2577	/** @name PAE Guest Paging.
2578	* @{ */
2579	/** The guest's page directory pointer table, static RC mapping. */
2580	RCPTRTYPE(PX86PDPT) pGstPaePdptRC;
2581	/** The guest's page directory pointer table, R3 pointer. */
2582	R3PTRTYPE(PX86PDPT) pGstPaePdptR3;
2583	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2584	/** The guest's page directory pointer table, R0 pointer. */
2585	R0PTRTYPE(PX86PDPT) pGstPaePdptR0;
2586	#endif
2587
2588	/** The guest's page directories, R3 pointers.
2589	* These are individual pointers and don't have to be adjecent.
2590	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2591	R3PTRTYPE(PX86PDPAE) apGstPaePDsR3[4];
2592	/** The guest's page directories, R0 pointers.
2593	* Same restrictions as apGstPaePDsR3. */
2594	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2595	R0PTRTYPE(PX86PDPAE) apGstPaePDsR0[4];
2596	#endif
2597	/** The guest's page directories, static GC mapping.
2598	* Unlike the R3/R0 array the first entry can be accessed as a 2048 entry PD.
2599	* These don't have to be up-to-date - use pgmGstGetPaePD() to access them. */
2600	RCPTRTYPE(PX86PDPAE) apGstPaePDsRC[4];
2601	/** The physical addresses of the guest page directories (PAE) pointed to by apGstPagePDsHC/GC. */
2602	RTGCPHYS aGCPhysGstPaePDs[4];
2603	/** The physical addresses of the monitored guest page directories (PAE). */
2604	RTGCPHYS aGCPhysGstPaePDsMonitored[4];
2605	/** @} */
2606
2607	/** @name AMD64 Guest Paging.
2608	* @{ */
2609	/** The guest's page directory pointer table, R3 pointer. */
2610	R3PTRTYPE(PX86PML4) pGstAmd64Pml4R3;
2611	#ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
2612	/** The guest's page directory pointer table, R0 pointer. */
2613	R0PTRTYPE(PX86PML4) pGstAmd64Pml4R0;
2614	#endif
2615	/** @} */
2616
2617	/** Pointer to the page of the current active CR3 - R3 Ptr. */
2618	R3PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R3;
2619	/** Pointer to the page of the current active CR3 - R0 Ptr. */
2620	R0PTRTYPE(PPGMPOOLPAGE) pShwPageCR3R0;
2621	/** Pointer to the page of the current active CR3 - RC Ptr. */
2622	RCPTRTYPE(PPGMPOOLPAGE) pShwPageCR3RC;
2623	/* The shadow page pool index of the user table as specified during allocation; useful for freeing root pages */
2624	uint32_t iShwUser;
2625	/* The index into the user table (shadowed) as specified during allocation; useful for freeing root pages. */
2626	uint32_t iShwUserTable;
2627	# if HC_ARCH_BITS == 64
2628	RTRCPTR alignment6; /*< structure size alignment. /
2629	# endif
2630	/** @} */
2631
2632	/** @name Function pointers for Shadow paging.
2633	* @{
2634	*/
2635	DECLR3CALLBACKMEMBER(int, pfnR3ShwRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2636	DECLR3CALLBACKMEMBER(int, pfnR3ShwExit,(PVMCPU pVCpu));
2637	DECLR3CALLBACKMEMBER(int, pfnR3ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2638	DECLR3CALLBACKMEMBER(int, pfnR3ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2639
2640	DECLRCCALLBACKMEMBER(int, pfnRCShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2641	DECLRCCALLBACKMEMBER(int, pfnRCShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2642
2643	DECLR0CALLBACKMEMBER(int, pfnR0ShwGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys));
2644	DECLR0CALLBACKMEMBER(int, pfnR0ShwModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2645
2646	/** @} */
2647
2648	/** @name Function pointers for Guest paging.
2649	* @{
2650	*/
2651	DECLR3CALLBACKMEMBER(int, pfnR3GstRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2652	DECLR3CALLBACKMEMBER(int, pfnR3GstExit,(PVMCPU pVCpu));
2653	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2654	DECLR3CALLBACKMEMBER(int, pfnR3GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2655	DECLR3CALLBACKMEMBER(int, pfnR3GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2656	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2657	DECLRCCALLBACKMEMBER(int, pfnRCGstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2658	DECLRCCALLBACKMEMBER(int, pfnRCGstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2659	#if HC_ARCH_BITS == 64
2660	RTRCPTR alignment3; /*< structure size alignment. /
2661	#endif
2662
2663	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPage,(PVMCPU pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys));
2664	DECLR0CALLBACKMEMBER(int, pfnR0GstModifyPage,(PVMCPU pVCpu, RTGCPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask));
2665	DECLR0CALLBACKMEMBER(int, pfnR0GstGetPDE,(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDEPAE pPde));
2666	/** @} */
2667
2668	/** @name Function pointers for Both Shadow and Guest paging.
2669	* @{
2670	*/
2671	DECLR3CALLBACKMEMBER(int, pfnR3BthRelocate,(PVMCPU pVCpu, RTGCPTR offDelta));
2672	/* no pfnR3BthTrap0eHandler */
2673	DECLR3CALLBACKMEMBER(int, pfnR3BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2674	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2675	DECLR3CALLBACKMEMBER(int, pfnR3BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2676	DECLR3CALLBACKMEMBER(int, pfnR3BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2677	DECLR3CALLBACKMEMBER(int, pfnR3BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2678	DECLR3CALLBACKMEMBER(unsigned, pfnR3BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2679	DECLR3CALLBACKMEMBER(int, pfnR3BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2680	DECLR3CALLBACKMEMBER(int, pfnR3BthUnmapCR3,(PVMCPU pVCpu));
2681
2682	DECLR0CALLBACKMEMBER(int, pfnR0BthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2683	DECLR0CALLBACKMEMBER(int, pfnR0BthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2684	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2685	DECLR0CALLBACKMEMBER(int, pfnR0BthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2686	DECLR0CALLBACKMEMBER(int, pfnR0BthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2687	DECLR0CALLBACKMEMBER(int, pfnR0BthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2688	DECLR0CALLBACKMEMBER(unsigned, pfnR0BthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2689	DECLR0CALLBACKMEMBER(int, pfnR0BthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2690	DECLR0CALLBACKMEMBER(int, pfnR0BthUnmapCR3,(PVMCPU pVCpu));
2691
2692	DECLRCCALLBACKMEMBER(int, pfnRCBthTrap0eHandler,(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault));
2693	DECLRCCALLBACKMEMBER(int, pfnRCBthInvalidatePage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2694	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncCR3,(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal));
2695	DECLRCCALLBACKMEMBER(int, pfnRCBthSyncPage,(PVMCPU pVCpu, X86PDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uError));
2696	DECLRCCALLBACKMEMBER(int, pfnRCBthPrefetchPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage));
2697	DECLRCCALLBACKMEMBER(int, pfnRCBthVerifyAccessSyncPage,(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fFlags, unsigned uError));
2698	DECLRCCALLBACKMEMBER(unsigned, pfnRCBthAssertCR3,(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb));
2699	DECLRCCALLBACKMEMBER(int, pfnRCBthMapCR3,(PVMCPU pVCpu, RTGCPHYS GCPhysCR3));
2700	DECLRCCALLBACKMEMBER(int, pfnRCBthUnmapCR3,(PVMCPU pVCpu));
2701	#if HC_ARCH_BITS == 64
2702	RTRCPTR alignment2; /*< structure size alignment. /
2703	#endif
2704	/** @} */
2705
2706	/** @name Release Statistics
2707	* @{ */
2708	/** The number of times the guest has switched mode since last reset or statistics reset. */
2709	STAMCOUNTER cGuestModeChanges;
2710	/** @} */
2711
2712	#ifdef VBOX_WITH_STATISTICS /** @todo move this chunk to the heap. */
2713	/** @name Statistics
2714	* @{ */
2715	/** RC: Which statistic this \#PF should be attributed to. */
2716	RCPTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionRC;
2717	RTRCPTR padding0;
2718	/** R0: Which statistic this \#PF should be attributed to. */
2719	R0PTRTYPE(PSTAMPROFILE) pStatTrap0eAttributionR0;
2720	RTR0PTR padding1;
2721
2722	/* Common */
2723	STAMCOUNTER StatSyncPtPD[X86_PG_ENTRIES]; /*< SyncPT - PD distribution. /
2724	STAMCOUNTER StatSyncPagePD[X86_PG_ENTRIES]; /*< SyncPage - PD distribution. /
2725
2726	/* R0 only: */
2727	STAMCOUNTER StatR0DynMapMigrateInvlPg; /*< R0: invlpg in PGMDynMapMigrateAutoSet. /
2728	STAMPROFILE StatR0DynMapGCPageInl; /*< R0: Calls to pgmR0DynMapGCPageInlined. /
2729	STAMCOUNTER StatR0DynMapGCPageInlHits; /*< R0: Hash table lookup hits. /
2730	STAMCOUNTER StatR0DynMapGCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2731	STAMCOUNTER StatR0DynMapGCPageInlRamHits; /*< R0: 1st ram range hits. /
2732	STAMCOUNTER StatR0DynMapGCPageInlRamMisses; /*< R0: 1st ram range misses, takes slow path. /
2733	STAMPROFILE StatR0DynMapHCPageInl; /*< R0: Calls to pgmR0DynMapHCPageInlined. /
2734	STAMCOUNTER StatR0DynMapHCPageInlHits; /*< R0: Hash table lookup hits. /
2735	STAMCOUNTER StatR0DynMapHCPageInlMisses; /*< R0: Misses that falls back to code common with PGMDynMapHCPage. /
2736	STAMPROFILE StatR0DynMapHCPage; /*< R0: Calls to PGMDynMapHCPage. /
2737	STAMCOUNTER StatR0DynMapSetOptimize; /*< R0: Calls to pgmDynMapOptimizeAutoSet. /
2738	STAMCOUNTER StatR0DynMapSetSearchFlushes; /*< R0: Set search restorting to subset flushes. /
2739	STAMCOUNTER StatR0DynMapSetSearchHits; /*< R0: Set search hits. /
2740	STAMCOUNTER StatR0DynMapSetSearchMisses; /*< R0: Set search misses. /
2741	STAMCOUNTER StatR0DynMapPage; /*< R0: Calls to pgmR0DynMapPage. /
2742	STAMCOUNTER StatR0DynMapPageHits0; /*< R0: Hits at iPage+0. /
2743	STAMCOUNTER StatR0DynMapPageHits1; /*< R0: Hits at iPage+1. /
2744	STAMCOUNTER StatR0DynMapPageHits2; /*< R0: Hits at iPage+2. /
2745	STAMCOUNTER StatR0DynMapPageInvlPg; /*< R0: invlpg. /
2746	STAMCOUNTER StatR0DynMapPageSlow; /*< R0: Calls to pgmR0DynMapPageSlow. /
2747	STAMCOUNTER StatR0DynMapPageSlowLoopHits; /*< R0: Hits in the pgmR0DynMapPageSlow search loop. /
2748	STAMCOUNTER StatR0DynMapPageSlowLoopMisses; /*< R0: Misses in the pgmR0DynMapPageSlow search loop. /
2749	//STAMCOUNTER StatR0DynMapPageSlowLostHits; /*< R0: Lost hits. /
2750	STAMCOUNTER StatR0DynMapSubsets; /*< R0: Times PGMDynMapPushAutoSubset was called. /
2751	STAMCOUNTER StatR0DynMapPopFlushes; /*< R0: Times PGMDynMapPopAutoSubset flushes the subset. /
2752	STAMCOUNTER aStatR0DynMapSetSize[11]; /*< R0: Set size distribution. /
2753
2754	/* RZ only: */
2755	STAMPROFILE StatRZTrap0e; /*< RC/R0: PGMTrap0eHandler() profiling. /
2756	STAMPROFILE StatRZTrap0eTimeCheckPageFault;
2757	STAMPROFILE StatRZTrap0eTimeSyncPT;
2758	STAMPROFILE StatRZTrap0eTimeMapping;
2759	STAMPROFILE StatRZTrap0eTimeOutOfSync;
2760	STAMPROFILE StatRZTrap0eTimeHandlers;
2761	STAMPROFILE StatRZTrap0eTime2CSAM; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is CSAM. /
2762	STAMPROFILE StatRZTrap0eTime2DirtyAndAccessed; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is dirty and/or accessed bit emulation. /
2763	STAMPROFILE StatRZTrap0eTime2GuestTrap; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a guest trap. /
2764	STAMPROFILE StatRZTrap0eTime2HndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a physical handler. /
2765	STAMPROFILE StatRZTrap0eTime2HndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is a virtual handler. /
2766	STAMPROFILE StatRZTrap0eTime2HndUnhandled; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is access outside the monitored areas of a monitored page. /
2767	STAMPROFILE StatRZTrap0eTime2Misc; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is not known. /
2768	STAMPROFILE StatRZTrap0eTime2OutOfSync; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync page. /
2769	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndPhys; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync physical handler page. /
2770	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndVirt; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an out-of-sync virtual handler page. /
2771	STAMPROFILE StatRZTrap0eTime2OutOfSyncHndObs; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is an obsolete handler page. /
2772	STAMPROFILE StatRZTrap0eTime2SyncPT; /*< RC/R0: Profiling of the Trap0eHandler body when the cause is lazy syncing of a PT. /
2773	STAMCOUNTER StatRZTrap0eConflicts; /*< RC/R0: The number of times \#PF was caused by an undetected conflict. /
2774	STAMCOUNTER StatRZTrap0eHandlersMapping; /*< RC/R0: Number of traps due to access handlers in mappings. /
2775	STAMCOUNTER StatRZTrap0eHandlersOutOfSync; /*< RC/R0: Number of out-of-sync handled pages. /
2776	STAMCOUNTER StatRZTrap0eHandlersPhysical; /*< RC/R0: Number of traps due to physical access handlers. /
2777	STAMCOUNTER StatRZTrap0eHandlersVirtual; /*< RC/R0: Number of traps due to virtual access handlers. /
2778	STAMCOUNTER StatRZTrap0eHandlersVirtualByPhys; /*< RC/R0: Number of traps due to virtual access handlers found by physical address. /
2779	STAMCOUNTER StatRZTrap0eHandlersVirtualUnmarked;/*< RC/R0: Number of traps due to virtual access handlers found by virtual address (without proper physical flags). /
2780	STAMCOUNTER StatRZTrap0eHandlersUnhandled; /*< RC/R0: Number of traps due to access outside range of monitored page(s). /
2781	STAMCOUNTER StatRZTrap0eHandlersInvalid; /*< RC/R0: Number of traps due to access to invalid physical memory. /
2782	STAMCOUNTER StatRZTrap0eUSNotPresentRead; /*< RC/R0: #PF err kind /
2783	STAMCOUNTER StatRZTrap0eUSNotPresentWrite; /*< RC/R0: #PF err kind /
2784	STAMCOUNTER StatRZTrap0eUSWrite; /*< RC/R0: #PF err kind /
2785	STAMCOUNTER StatRZTrap0eUSReserved; /*< RC/R0: #PF err kind /
2786	STAMCOUNTER StatRZTrap0eUSNXE; /*< RC/R0: #PF err kind /
2787	STAMCOUNTER StatRZTrap0eUSRead; /*< RC/R0: #PF err kind /
2788	STAMCOUNTER StatRZTrap0eSVNotPresentRead; /*< RC/R0: #PF err kind /
2789	STAMCOUNTER StatRZTrap0eSVNotPresentWrite; /*< RC/R0: #PF err kind /
2790	STAMCOUNTER StatRZTrap0eSVWrite; /*< RC/R0: #PF err kind /
2791	STAMCOUNTER StatRZTrap0eSVReserved; /*< RC/R0: #PF err kind /
2792	STAMCOUNTER StatRZTrap0eSNXE; /*< RC/R0: #PF err kind /
2793	STAMCOUNTER StatRZTrap0eGuestPF; /*< RC/R0: Real guest #PFs. /
2794	STAMCOUNTER StatRZTrap0eGuestPFUnh; /*< RC/R0: Real guest #PF ending up at the end of the #PF code. /
2795	STAMCOUNTER StatRZTrap0eGuestPFMapping; /*< RC/R0: Real guest #PF to HMA or other mapping. /
2796	STAMCOUNTER StatRZTrap0eWPEmulInRZ; /*< RC/R0: WP=0 virtualization trap, handled. /
2797	STAMCOUNTER StatRZTrap0eWPEmulToR3; /*< RC/R0: WP=0 virtualization trap, chickened out. /
2798	STAMCOUNTER StatRZTrap0ePD[X86_PG_ENTRIES]; /*< RC/R0: PD distribution of the #PFs. /
2799	STAMCOUNTER StatRZGuestCR3WriteHandled; /*< RC/R0: The number of times WriteHandlerCR3() was successfully called. /
2800	STAMCOUNTER StatRZGuestCR3WriteUnhandled; /*< RC/R0: The number of times WriteHandlerCR3() was called and we had to fall back to the recompiler. /
2801	STAMCOUNTER StatRZGuestCR3WriteConflict; /*< RC/R0: The number of times WriteHandlerCR3() was called and a conflict was detected. /
2802	STAMCOUNTER StatRZGuestROMWriteHandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was successfully called. /
2803	STAMCOUNTER StatRZGuestROMWriteUnhandled; /*< RC/R0: The number of times pgmPhysRomWriteHandler() was called and we had to fall back to the recompiler /
2804
2805	/* HC - R3 and (maybe) R0: */
2806
2807	/* RZ & R3: */
2808	STAMPROFILE StatRZSyncCR3; /*< RC/R0: PGMSyncCR3() profiling. /
2809	STAMPROFILE StatRZSyncCR3Handlers; /*< RC/R0: Profiling of the PGMSyncCR3() update handler section. /
2810	STAMCOUNTER StatRZSyncCR3Global; /*< RC/R0: The number of global CR3 syncs. /
2811	STAMCOUNTER StatRZSyncCR3NotGlobal; /*< RC/R0: The number of non-global CR3 syncs. /
2812	STAMCOUNTER StatRZSyncCR3DstCacheHit; /*< RC/R0: The number of times we got some kind of cache hit on a page table. /
2813	STAMCOUNTER StatRZSyncCR3DstFreed; /*< RC/R0: The number of times we've had to free a shadow entry. /
2814	STAMCOUNTER StatRZSyncCR3DstFreedSrcNP; /*< RC/R0: The number of times we've had to free a shadow entry for which the source entry was not present. /
2815	STAMCOUNTER StatRZSyncCR3DstNotPresent; /*< RC/R0: The number of times we've encountered a not present shadow entry for a present guest entry. /
2816	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPD; /*< RC/R0: The number of times a global page directory wasn't flushed. /
2817	STAMCOUNTER StatRZSyncCR3DstSkippedGlobalPT; /*< RC/R0: The number of times a page table with only global entries wasn't flushed. /
2818	STAMPROFILE StatRZSyncPT; /*< RC/R0: PGMSyncPT() profiling. /
2819	STAMCOUNTER StatRZSyncPTFailed; /*< RC/R0: The number of times PGMSyncPT() failed. /
2820	STAMCOUNTER StatRZSyncPT4K; /*< RC/R0: Number of 4KB syncs. /
2821	STAMCOUNTER StatRZSyncPT4M; /*< RC/R0: Number of 4MB syncs. /
2822	STAMCOUNTER StatRZSyncPagePDNAs; /*< RC/R0: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2823	STAMCOUNTER StatRZSyncPagePDOutOfSync; /*< RC/R0: The number of time we've encountered an out-of-sync PD in SyncPage. /
2824	STAMCOUNTER StatRZAccessedPage; /*< RC/R0: The number of pages marked not present for accessed bit emulation. /
2825	STAMPROFILE StatRZDirtyBitTracking; /*< RC/R0: Profiling the dirty bit tracking in CheckPageFault().. /
2826	STAMCOUNTER StatRZDirtyPage; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2827	STAMCOUNTER StatRZDirtyPageBig; /*< RC/R0: The number of pages marked read-only for dirty bit tracking. /
2828	STAMCOUNTER StatRZDirtyPageSkipped; /*< RC/R0: The number of pages already dirty or readonly. /
2829	STAMCOUNTER StatRZDirtyPageTrap; /*< RC/R0: The number of traps generated for dirty bit tracking. /
2830	STAMCOUNTER StatRZDirtyPageStale; /*< RC/R0: The number of traps generated for dirty bit tracking. (stale tlb entries) /
2831	STAMCOUNTER StatRZDirtyTrackRealPF; /*< RC/R0: The number of real pages faults during dirty bit tracking. /
2832	STAMCOUNTER StatRZDirtiedPage; /*< RC/R0: The number of pages marked dirty because of write accesses. /
2833	STAMCOUNTER StatRZPageAlreadyDirty; /*< RC/R0: The number of pages already marked dirty because of write accesses. /
2834	STAMPROFILE StatRZInvalidatePage; /*< RC/R0: PGMInvalidatePage() profiling. /
2835	STAMCOUNTER StatRZInvalidatePage4KBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4KB page. /
2836	STAMCOUNTER StatRZInvalidatePage4MBPages; /*< RC/R0: The number of times PGMInvalidatePage() was called for a 4MB page. /
2837	STAMCOUNTER StatRZInvalidatePage4MBPagesSkip; /*< RC/R0: The number of times PGMInvalidatePage() skipped a 4MB page. /
2838	STAMCOUNTER StatRZInvalidatePagePDMappings; /*< RC/R0: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2839	STAMCOUNTER StatRZInvalidatePagePDNAs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2840	STAMCOUNTER StatRZInvalidatePagePDNPs; /*< RC/R0: The number of times PGMInvalidatePage() was called for a not present page directory. /
2841	STAMCOUNTER StatRZInvalidatePagePDOutOfSync; /*< RC/R0: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2842	STAMCOUNTER StatRZInvalidatePageSkipped; /*< RC/R0: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2843	STAMCOUNTER StatRZPageOutOfSyncUser; /*< RC/R0: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2844	STAMCOUNTER StatRZPageOutOfSyncSupervisor; /*< RC/R0: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2845	STAMPROFILE StatRZPrefetch; /*< RC/R0: PGMPrefetchPage. /
2846	STAMPROFILE StatRZFlushTLB; /*< RC/R0: Profiling of the PGMFlushTLB() body. /
2847	STAMCOUNTER StatRZFlushTLBNewCR3; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2848	STAMCOUNTER StatRZFlushTLBNewCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2849	STAMCOUNTER StatRZFlushTLBSameCR3; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2850	STAMCOUNTER StatRZFlushTLBSameCR3Global; /*< RC/R0: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2851	STAMPROFILE StatRZGstModifyPage; /*< RC/R0: Profiling of the PGMGstModifyPage() body /
2852
2853	STAMPROFILE StatR3SyncCR3; /*< R3: PGMSyncCR3() profiling. /
2854	STAMPROFILE StatR3SyncCR3Handlers; /*< R3: Profiling of the PGMSyncCR3() update handler section. /
2855	STAMCOUNTER StatR3SyncCR3Global; /*< R3: The number of global CR3 syncs. /
2856	STAMCOUNTER StatR3SyncCR3NotGlobal; /*< R3: The number of non-global CR3 syncs. /
2857	STAMCOUNTER StatR3SyncCR3DstFreed; /*< R3: The number of times we've had to free a shadow entry. /
2858	STAMCOUNTER StatR3SyncCR3DstFreedSrcNP; /*< R3: The number of times we've had to free a shadow entry for which the source entry was not present. /
2859	STAMCOUNTER StatR3SyncCR3DstNotPresent; /*< R3: The number of times we've encountered a not present shadow entry for a present guest entry. /
2860	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPD; /*< R3: The number of times a global page directory wasn't flushed. /
2861	STAMCOUNTER StatR3SyncCR3DstSkippedGlobalPT; /*< R3: The number of times a page table with only global entries wasn't flushed. /
2862	STAMCOUNTER StatR3SyncCR3DstCacheHit; /*< R3: The number of times we got some kind of cache hit on a page table. /
2863	STAMPROFILE StatR3SyncPT; /*< R3: PGMSyncPT() profiling. /
2864	STAMCOUNTER StatR3SyncPTFailed; /*< R3: The number of times PGMSyncPT() failed. /
2865	STAMCOUNTER StatR3SyncPT4K; /*< R3: Number of 4KB syncs. /
2866	STAMCOUNTER StatR3SyncPT4M; /*< R3: Number of 4MB syncs. /
2867	STAMCOUNTER StatR3SyncPagePDNAs; /*< R3: The number of time we've marked a PD not present from SyncPage to virtualize the accessed bit. /
2868	STAMCOUNTER StatR3SyncPagePDOutOfSync; /*< R3: The number of time we've encountered an out-of-sync PD in SyncPage. /
2869	STAMCOUNTER StatR3AccessedPage; /*< R3: The number of pages marked not present for accessed bit emulation. /
2870	STAMPROFILE StatR3DirtyBitTracking; /*< R3: Profiling the dirty bit tracking in CheckPageFault(). /
2871	STAMCOUNTER StatR3DirtyPage; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2872	STAMCOUNTER StatR3DirtyPageBig; /*< R3: The number of pages marked read-only for dirty bit tracking. /
2873	STAMCOUNTER StatR3DirtyPageSkipped; /*< R3: The number of pages already dirty or readonly. /
2874	STAMCOUNTER StatR3DirtyPageTrap; /*< R3: The number of traps generated for dirty bit tracking. /
2875	STAMCOUNTER StatR3DirtyTrackRealPF; /*< R3: The number of real pages faults during dirty bit tracking. /
2876	STAMCOUNTER StatR3DirtiedPage; /*< R3: The number of pages marked dirty because of write accesses. /
2877	STAMCOUNTER StatR3PageAlreadyDirty; /*< R3: The number of pages already marked dirty because of write accesses. /
2878	STAMPROFILE StatR3InvalidatePage; /*< R3: PGMInvalidatePage() profiling. /
2879	STAMCOUNTER StatR3InvalidatePage4KBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4KB page. /
2880	STAMCOUNTER StatR3InvalidatePage4MBPages; /*< R3: The number of times PGMInvalidatePage() was called for a 4MB page. /
2881	STAMCOUNTER StatR3InvalidatePage4MBPagesSkip; /*< R3: The number of times PGMInvalidatePage() skipped a 4MB page. /
2882	STAMCOUNTER StatR3InvalidatePagePDNAs; /*< R3: The number of times PGMInvalidatePage() was called for a not accessed page directory. /
2883	STAMCOUNTER StatR3InvalidatePagePDNPs; /*< R3: The number of times PGMInvalidatePage() was called for a not present page directory. /
2884	STAMCOUNTER StatR3InvalidatePagePDMappings; /*< R3: The number of times PGMInvalidatePage() was called for a page directory containing mappings (no conflict). /
2885	STAMCOUNTER StatR3InvalidatePagePDOutOfSync; /*< R3: The number of times PGMInvalidatePage() was called for an out of sync page directory. /
2886	STAMCOUNTER StatR3InvalidatePageSkipped; /*< R3: The number of times PGMInvalidatePage() was skipped due to not present shw or pending pending SyncCR3. /
2887	STAMCOUNTER StatR3PageOutOfSyncUser; /*< R3: The number of times user page is out of sync was detected in #PF or VerifyAccessSyncPage. /
2888	STAMCOUNTER StatR3PageOutOfSyncSupervisor; /*< R3: The number of times supervisor page is out of sync was detected in in #PF or VerifyAccessSyncPage. /
2889	STAMPROFILE StatR3Prefetch; /*< R3: PGMPrefetchPage. /
2890	STAMPROFILE StatR3FlushTLB; /*< R3: Profiling of the PGMFlushTLB() body. /
2891	STAMCOUNTER StatR3FlushTLBNewCR3; /*< R3: The number of times PGMFlushTLB was called with a new CR3, non-global. (switch) /
2892	STAMCOUNTER StatR3FlushTLBNewCR3Global; /*< R3: The number of times PGMFlushTLB was called with a new CR3, global. (switch) /
2893	STAMCOUNTER StatR3FlushTLBSameCR3; /*< R3: The number of times PGMFlushTLB was called with the same CR3, non-global. (flush) /
2894	STAMCOUNTER StatR3FlushTLBSameCR3Global; /*< R3: The number of times PGMFlushTLB was called with the same CR3, global. (flush) /
2895	STAMPROFILE StatR3GstModifyPage; /*< R3: Profiling of the PGMGstModifyPage() body /
2896	/** @} */
2897	#endif /* VBOX_WITH_STATISTICS */
2898	} PGMCPU;
2899	/** Pointer to the per-cpu PGM data. */
2900	typedef PGMCPU *PPGMCPU;
2901
2902
2903	/** @name PGM::fSyncFlags Flags
2904	* @{
2905	*/
2906	/** Updates the virtual access handler state bit in PGMPAGE. */
2907	#define PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL RT_BIT(0)
2908	/** Always sync CR3. */
2909	#define PGM_SYNC_ALWAYS RT_BIT(1)
2910	/** Check monitoring on next CR3 (re)load and invalidate page.
2911	* @todo This is obsolete now. Remove after 2.2.0 is branched off. */
2912	#define PGM_SYNC_MONITOR_CR3 RT_BIT(2)
2913	/** Check guest mapping in SyncCR3. */
2914	#define PGM_SYNC_MAP_CR3 RT_BIT(3)
2915	/** Clear the page pool (a light weight flush). */
2916	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT 8
2917	#define PGM_GLOBAL_SYNC_CLEAR_PGM_POOL RT_BIT(PGM_GLOBAL_SYNC_CLEAR_PGM_POOL_BIT)
2918	/** @} */
2919
2920
2921	__BEGIN_DECLS
2922
2923	int pgmLock(PVM pVM);
2924	void pgmUnlock(PVM pVM);
2925
2926	int pgmR3SyncPTResolveConflict(PVM pVM, PPGMMAPPING pMapping, PX86PD pPDSrc, RTGCPTR GCPtrOldMapping);
2927	int pgmR3SyncPTResolveConflictPAE(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping);
2928	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr);
2929	void pgmR3MapRelocate(PVM pVM, PPGMMAPPING pMapping, RTGCPTR GCPtrOldMapping, RTGCPTR GCPtrNewMapping);
2930	DECLCALLBACK(void) pgmR3MapInfo(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2931
2932	void pgmR3HandlerPhysicalUpdateAll(PVM pVM);
2933	bool pgmHandlerPhysicalIsAll(PVM pVM, RTGCPHYS GCPhys);
2934	void pgmHandlerPhysicalResetAliasedPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage);
2935	int pgmHandlerVirtualFindByPhysAddr(PVM pVM, RTGCPHYS GCPhys, PPGMVIRTHANDLER ppVirt, unsigned piPage);
2936	DECLCALLBACK(int) pgmHandlerVirtualResetOne(PAVLROGCPTRNODECORE pNode, void *pvUser);
2937	#if defined(VBOX_STRICT) \|\| defined(LOG_ENABLED)
2938	void pgmHandlerVirtualDumpPhysPages(PVM pVM);
2939	#else
2940	# define pgmHandlerVirtualDumpPhysPages(a) do { } while (0)
2941	#endif
2942	DECLCALLBACK(void) pgmR3InfoHandlers(PVM pVM, PCDBGFINFOHLP pHlp, const char *pszArgs);
2943
2944
2945	int pgmPhysAllocPage(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2946	int pgmPhysPageLoadIntoTlb(PPGM pPGM, RTGCPHYS GCPhys);
2947	int pgmPhysPageLoadIntoTlbWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2948	int pgmPhysPageMakeWritable(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2949	int pgmPhysPageMakeWritableUnlocked(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys);
2950	int pgmPhysPageMap(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv);
2951	int pgmPhysPageMapByPageID(PVM pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv);
2952	int pgmPhysGCPhys2CCPtrInternal(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv);
2953	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVM pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv);
2954	VMMDECL(int) pgmPhysRomWriteHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
2955	#ifdef IN_RING3
2956	void pgmR3PhysRelinkRamRanges(PVM pVM);
2957	int pgmR3PhysRamPreAllocate(PVM pVM);
2958	int pgmR3PhysRamReset(PVM pVM);
2959	int pgmR3PhysRomReset(PVM pVM);
2960	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk);
2961
2962	int pgmR3PoolInit(PVM pVM);
2963	void pgmR3PoolRelocate(PVM pVM);
2964	void pgmR3PoolReset(PVM pVM);
2965
2966	#endif /* IN_RING3 */
2967	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2968	int pgmR0DynMapHCPageCommon(PVM pVM, PPGMMAPSET pSet, RTHCPHYS HCPhys, void **ppv);
2969	#endif
2970	int pgmPoolAlloc(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage);
2971	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable);
2972	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
2973	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2974	void pgmPoolClearAll(PVM pVM);
2975	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys);
2976	int pgmPoolSyncCR3(PVM pVM);
2977	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs);
2978	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT);
2979	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage);
2980	#ifdef PGMPOOL_WITH_MONITORING
2981	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pCpu);
2982	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2983	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
2984	#endif
2985
2986	int pgmR3ExitShadowModeBeforePoolFlush(PVM pVM, PVMCPU pVCpu);
2987	int pgmR3ReEnterShadowModeAfterPoolFlush(PVM pVM, PVMCPU pVCpu);
2988
2989	void pgmMapSetShadowPDEs(PVM pVM, PPGMMAPPING pMap, unsigned iNewPDE);
2990	void pgmMapClearShadowPDEs(PVM pVM, PPGMPOOLPAGE pShwPageCR3, PPGMMAPPING pMap, unsigned iOldPDE, bool fDeactivateCR3);
2991	int pgmMapActivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2992	int pgmMapDeactivateCR3(PVM pVM, PPGMPOOLPAGE pShwPageCR3);
2993
2994	int pgmShwSyncPaePDPtr(PVMCPU pVCpu, RTGCPTR GCPtr, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2995	#ifndef IN_RC
2996	int pgmShwSyncLongModePDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PX86PML4E pGstPml4e, PX86PDPE pGstPdpe, PX86PDPAE *ppPD);
2997	#endif
2998	int pgmShwGetEPTPDPtr(PVMCPU pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
2999
3000	PX86PD pgmGstLazyMap32BitPD(PPGMCPU pPGM);
3001	PX86PDPT pgmGstLazyMapPaePDPT(PPGMCPU pPGM);
3002	PX86PDPAE pgmGstLazyMapPaePD(PPGMCPU pPGM, uint32_t iPdpt);
3003	PX86PML4 pgmGstLazyMapPml4(PPGMCPU pPGM);
3004
3005	__END_DECLS
3006
3007
3008	/**
3009	* Gets the PGMRAMRANGE structure for a guest page.
3010	*
3011	* @returns Pointer to the RAM range on success.
3012	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3013	*
3014	* @param pPGM PGM handle.
3015	* @param GCPhys The GC physical address.
3016	*/
3017	DECLINLINE(PPGMRAMRANGE) pgmPhysGetRange(PPGM pPGM, RTGCPHYS GCPhys)
3018	{
3019	/*
3020	* Optimize for the first range.
3021	*/
3022	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3023	RTGCPHYS off = GCPhys - pRam->GCPhys;
3024	if (RT_UNLIKELY(off >= pRam->cb))
3025	{
3026	do
3027	{
3028	pRam = pRam->CTX_SUFF(pNext);
3029	if (RT_UNLIKELY(!pRam))
3030	break;
3031	off = GCPhys - pRam->GCPhys;
3032	} while (off >= pRam->cb);
3033	}
3034	return pRam;
3035	}
3036
3037
3038	/**
3039	* Gets the PGMPAGE structure for a guest page.
3040	*
3041	* @returns Pointer to the page on success.
3042	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3043	*
3044	* @param pPGM PGM handle.
3045	* @param GCPhys The GC physical address.
3046	*/
3047	DECLINLINE(PPGMPAGE) pgmPhysGetPage(PPGM pPGM, RTGCPHYS GCPhys)
3048	{
3049	/*
3050	* Optimize for the first range.
3051	*/
3052	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3053	RTGCPHYS off = GCPhys - pRam->GCPhys;
3054	if (RT_UNLIKELY(off >= pRam->cb))
3055	{
3056	do
3057	{
3058	pRam = pRam->CTX_SUFF(pNext);
3059	if (RT_UNLIKELY(!pRam))
3060	return NULL;
3061	off = GCPhys - pRam->GCPhys;
3062	} while (off >= pRam->cb);
3063	}
3064	return &pRam->aPages[off >> PAGE_SHIFT];
3065	}
3066
3067
3068	/**
3069	* Gets the PGMPAGE structure for a guest page.
3070	*
3071	* Old Phys code: Will make sure the page is present.
3072	*
3073	* @returns VBox status code.
3074	* @retval VINF_SUCCESS and a valid *ppPage on success.
3075	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3076	*
3077	* @param pPGM PGM handle.
3078	* @param GCPhys The GC physical address.
3079	* @param ppPage Where to store the page poitner on success.
3080	*/
3081	DECLINLINE(int) pgmPhysGetPageEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
3082	{
3083	/*
3084	* Optimize for the first range.
3085	*/
3086	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3087	RTGCPHYS off = GCPhys - pRam->GCPhys;
3088	if (RT_UNLIKELY(off >= pRam->cb))
3089	{
3090	do
3091	{
3092	pRam = pRam->CTX_SUFF(pNext);
3093	if (RT_UNLIKELY(!pRam))
3094	{
3095	ppPage = NULL; / avoid incorrect and very annoying GCC warnings */
3096	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3097	}
3098	off = GCPhys - pRam->GCPhys;
3099	} while (off >= pRam->cb);
3100	}
3101	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3102	return VINF_SUCCESS;
3103	}
3104
3105
3106
3107
3108	/**
3109	* Gets the PGMPAGE structure for a guest page.
3110	*
3111	* Old Phys code: Will make sure the page is present.
3112	*
3113	* @returns VBox status code.
3114	* @retval VINF_SUCCESS and a valid *ppPage on success.
3115	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if the address isn't valid.
3116	*
3117	* @param pPGM PGM handle.
3118	* @param GCPhys The GC physical address.
3119	* @param ppPage Where to store the page poitner on success.
3120	* @param ppRamHint Where to read and store the ram list hint.
3121	* The caller initializes this to NULL before the call.
3122	*/
3123	DECLINLINE(int) pgmPhysGetPageWithHintEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRamHint)
3124	{
3125	RTGCPHYS off;
3126	PPGMRAMRANGE pRam = *ppRamHint;
3127	if ( !pRam
3128	\|\| RT_UNLIKELY((off = GCPhys - pRam->GCPhys) >= pRam->cb))
3129	{
3130	pRam = pPGM->CTX_SUFF(pRamRanges);
3131	off = GCPhys - pRam->GCPhys;
3132	if (RT_UNLIKELY(off >= pRam->cb))
3133	{
3134	do
3135	{
3136	pRam = pRam->CTX_SUFF(pNext);
3137	if (RT_UNLIKELY(!pRam))
3138	{
3139	ppPage = NULL; / Kill the incorrect and extremely annoying GCC warnings. */
3140	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3141	}
3142	off = GCPhys - pRam->GCPhys;
3143	} while (off >= pRam->cb);
3144	}
3145	*ppRamHint = pRam;
3146	}
3147	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3148	return VINF_SUCCESS;
3149	}
3150
3151
3152	/**
3153	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3154	*
3155	* @returns Pointer to the page on success.
3156	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3157	*
3158	* @param pPGM PGM handle.
3159	* @param GCPhys The GC physical address.
3160	* @param ppRam Where to store the pointer to the PGMRAMRANGE.
3161	*/
3162	DECLINLINE(PPGMPAGE) pgmPhysGetPageAndRange(PPGM pPGM, RTGCPHYS GCPhys, PPGMRAMRANGE *ppRam)
3163	{
3164	/*
3165	* Optimize for the first range.
3166	*/
3167	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3168	RTGCPHYS off = GCPhys - pRam->GCPhys;
3169	if (RT_UNLIKELY(off >= pRam->cb))
3170	{
3171	do
3172	{
3173	pRam = pRam->CTX_SUFF(pNext);
3174	if (RT_UNLIKELY(!pRam))
3175	return NULL;
3176	off = GCPhys - pRam->GCPhys;
3177	} while (off >= pRam->cb);
3178	}
3179	*ppRam = pRam;
3180	return &pRam->aPages[off >> PAGE_SHIFT];
3181	}
3182
3183
3184	/**
3185	* Gets the PGMPAGE structure for a guest page together with the PGMRAMRANGE.
3186	*
3187	* @returns Pointer to the page on success.
3188	* @returns NULL on a VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS condition.
3189	*
3190	* @param pPGM PGM handle.
3191	* @param GCPhys The GC physical address.
3192	* @param ppPage Where to store the pointer to the PGMPAGE structure.
3193	* @param ppRam Where to store the pointer to the PGMRAMRANGE structure.
3194	*/
3195	DECLINLINE(int) pgmPhysGetPageAndRangeEx(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
3196	{
3197	/*
3198	* Optimize for the first range.
3199	*/
3200	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3201	RTGCPHYS off = GCPhys - pRam->GCPhys;
3202	if (RT_UNLIKELY(off >= pRam->cb))
3203	{
3204	do
3205	{
3206	pRam = pRam->CTX_SUFF(pNext);
3207	if (RT_UNLIKELY(!pRam))
3208	{
3209	ppRam = NULL; / Shut up silly GCC warnings. */
3210	ppPage = NULL; / ditto */
3211	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
3212	}
3213	off = GCPhys - pRam->GCPhys;
3214	} while (off >= pRam->cb);
3215	}
3216	*ppRam = pRam;
3217	*ppPage = &pRam->aPages[off >> PAGE_SHIFT];
3218	return VINF_SUCCESS;
3219	}
3220
3221
3222	/**
3223	* Convert GC Phys to HC Phys.
3224	*
3225	* @returns VBox status.
3226	* @param pPGM PGM handle.
3227	* @param GCPhys The GC physical address.
3228	* @param pHCPhys Where to store the corresponding HC physical address.
3229	*
3230	* @deprecated Doesn't deal with zero, shared or write monitored pages.
3231	* Avoid when writing new code!
3232	*/
3233	DECLINLINE(int) pgmRamGCPhys2HCPhys(PPGM pPGM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
3234	{
3235	PPGMPAGE pPage;
3236	int rc = pgmPhysGetPageEx(pPGM, GCPhys, &pPage);
3237	if (RT_FAILURE(rc))
3238	return rc;
3239	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & PAGE_OFFSET_MASK);
3240	return VINF_SUCCESS;
3241	}
3242
3243	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3244
3245	/**
3246	* Inlined version of the ring-0 version of PGMDynMapHCPage that
3247	* optimizes access to pages already in the set.
3248	*
3249	* @returns VINF_SUCCESS. Will bail out to ring-3 on failure.
3250	* @param pPGM Pointer to the PVM instance data.
3251	* @param HCPhys The physical address of the page.
3252	* @param ppv Where to store the mapping address.
3253	*/
3254	DECLINLINE(int) pgmR0DynMapHCPageInlined(PPGM pPGM, RTHCPHYS HCPhys, void **ppv)
3255	{
3256	PVM pVM = PGM2VM(pPGM);
3257	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3258	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3259
3260	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapHCPageInl, a);
3261	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3262	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3263
3264	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3265	unsigned iEntry = pSet->aiHashTable[iHash];
3266	if ( iEntry < pSet->cEntries
3267	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3268	{
3269	*ppv = pSet->aEntries[iEntry].pvPage;
3270	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlHits);
3271	}
3272	else
3273	{
3274	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapHCPageInlMisses);
3275	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3276	}
3277
3278	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapHCPageInl, a);
3279	return VINF_SUCCESS;
3280	}
3281
3282
3283	/**
3284	* Inlined version of the ring-0 version of PGMDynMapGCPage that optimizes
3285	* access to pages already in the set.
3286	*
3287	* @returns See PGMDynMapGCPage.
3288	* @param pPGM Pointer to the PVM instance data.
3289	* @param HCPhys The physical address of the page.
3290	* @param ppv Where to store the mapping address.
3291	*/
3292	DECLINLINE(int) pgmR0DynMapGCPageInlined(PPGM pPGM, RTGCPHYS GCPhys, void **ppv)
3293	{
3294	PVM pVM = PGM2VM(pPGM);
3295	PPGMCPU pPGMCPU = (PPGMCPU)((uint8_t )VMMGetCpu(pVM) + pPGM->offVCpuPGM); / very pretty ;-) */
3296
3297	STAM_PROFILE_START(&pPGMCPU->StatR0DynMapGCPageInl, a);
3298	Assert(!(GCPhys & PAGE_OFFSET_MASK));
3299
3300	/*
3301	* Get the ram range.
3302	*/
3303	PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRanges);
3304	RTGCPHYS off = GCPhys - pRam->GCPhys;
3305	if (RT_UNLIKELY(off >= pRam->cb
3306	/** @todo \|\| page state stuff */))
3307	{
3308	/* This case is not counted into StatR0DynMapGCPageInl. */
3309	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamMisses);
3310	return PGMDynMapGCPage(pVM, GCPhys, ppv);
3311	}
3312
3313	RTHCPHYS HCPhys = PGM_PAGE_GET_HCPHYS(&pRam->aPages[off >> PAGE_SHIFT]);
3314	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlRamHits);
3315
3316	/*
3317	* pgmR0DynMapHCPageInlined with out stats.
3318	*/
3319	PPGMMAPSET pSet = &pPGMCPU->AutoSet;
3320	Assert(!(HCPhys & PAGE_OFFSET_MASK));
3321	Assert(pSet->cEntries <= RT_ELEMENTS(pSet->aEntries));
3322
3323	unsigned iHash = PGMMAPSET_HASH(HCPhys);
3324	unsigned iEntry = pSet->aiHashTable[iHash];
3325	if ( iEntry < pSet->cEntries
3326	&& pSet->aEntries[iEntry].HCPhys == HCPhys)
3327	{
3328	*ppv = pSet->aEntries[iEntry].pvPage;
3329	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlHits);
3330	}
3331	else
3332	{
3333	STAM_COUNTER_INC(&pPGMCPU->StatR0DynMapGCPageInlMisses);
3334	pgmR0DynMapHCPageCommon(pVM, pSet, HCPhys, ppv);
3335	}
3336
3337	STAM_PROFILE_STOP(&pPGMCPU->StatR0DynMapGCPageInl, a);
3338	return VINF_SUCCESS;
3339	}
3340
3341	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
3342	#if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3343
3344	/**
3345	* Maps the page into current context (RC and maybe R0).
3346	*
3347	* @returns pointer to the mapping.
3348	* @param pVM Pointer to the PGM instance data.
3349	* @param pPage The page.
3350	*/
3351	DECLINLINE(void *) pgmPoolMapPageInlined(PPGM pPGM, PPGMPOOLPAGE pPage)
3352	{
3353	if (pPage->idx >= PGMPOOL_IDX_FIRST)
3354	{
3355	Assert(pPage->idx < pPGM->CTX_SUFF(pPool)->cCurPages);
3356	void *pv;
3357	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3358	pgmR0DynMapHCPageInlined(pPGM, pPage->Core.Key, &pv);
3359	# else
3360	PGMDynMapHCPage(PGM2VM(pPGM), pPage->Core.Key, &pv);
3361	# endif
3362	return pv;
3363	}
3364	AssertFatalMsgFailed(("pgmPoolMapPageInlined invalid page index %x\n", pPage->idx));
3365	}
3366
3367	/**
3368	* Temporarily maps one host page specified by HC physical address, returning
3369	* pointer within the page.
3370	*
3371	* Be WARNED that the dynamic page mapping area is small, 8 pages, thus the space is
3372	* reused after 8 mappings (or perhaps a few more if you score with the cache).
3373	*
3374	* @returns The address corresponding to HCPhys.
3375	* @param pPGM Pointer to the PVM instance data.
3376	* @param HCPhys HC Physical address of the page.
3377	*/
3378	DECLINLINE(void *) pgmDynMapHCPageOff(PPGM pPGM, RTHCPHYS HCPhys)
3379	{
3380	void *pv;
3381	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3382	pgmR0DynMapHCPageInlined(pPGM, HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3383	# else
3384	PGMDynMapHCPage(PGM2VM(pPGM), HCPhys & ~(RTHCPHYS)PAGE_OFFSET_MASK, &pv);
3385	# endif
3386	pv = (void *)((uintptr_t)pv \| (HCPhys & PAGE_OFFSET_MASK));
3387	return pv;
3388	}
3389
3390	#endif /* VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 \|\| IN_RC */
3391	#ifndef IN_RC
3392
3393	/**
3394	* Queries the Physical TLB entry for a physical guest page,
3395	* attempting to load the TLB entry if necessary.
3396	*
3397	* @returns VBox status code.
3398	* @retval VINF_SUCCESS on success
3399	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3400	*
3401	* @param pPGM The PGM instance handle.
3402	* @param GCPhys The address of the guest page.
3403	* @param ppTlbe Where to store the pointer to the TLB entry.
3404	*/
3405	DECLINLINE(int) pgmPhysPageQueryTlbe(PPGM pPGM, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3406	{
3407	int rc;
3408	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3409	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3410	{
3411	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3412	rc = VINF_SUCCESS;
3413	}
3414	else
3415	rc = pgmPhysPageLoadIntoTlb(pPGM, GCPhys);
3416	*ppTlbe = pTlbe;
3417	return rc;
3418	}
3419
3420
3421	/**
3422	* Queries the Physical TLB entry for a physical guest page,
3423	* attempting to load the TLB entry if necessary.
3424	*
3425	* @returns VBox status code.
3426	* @retval VINF_SUCCESS on success
3427	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3428	*
3429	* @param pPGM The PGM instance handle.
3430	* @param pPage Pointer to the PGMPAGE structure corresponding to
3431	* GCPhys.
3432	* @param GCPhys The address of the guest page.
3433	* @param ppTlbe Where to store the pointer to the TLB entry.
3434	*/
3435	DECLINLINE(int) pgmPhysPageQueryTlbeWithPage(PPGM pPGM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAPTLBE ppTlbe)
3436	{
3437	int rc;
3438	PPGMPAGEMAPTLBE pTlbe = &pPGM->CTXSUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3439	if (pTlbe->GCPhys == (GCPhys & X86_PTE_PAE_PG_MASK))
3440	{
3441	STAM_COUNTER_INC(&pPGM->CTX_MID_Z(Stat,PageMapTlbHits));
3442	rc = VINF_SUCCESS;
3443	}
3444	else
3445	rc = pgmPhysPageLoadIntoTlbWithPage(pPGM, pPage, GCPhys);
3446	*ppTlbe = pTlbe;
3447	return rc;
3448	}
3449
3450	#endif /* !IN_RC */
3451
3452	/**
3453	* Calculated the guest physical address of the large (4 MB) page in 32 bits paging mode.
3454	* Takes PSE-36 into account.
3455	*
3456	* @returns guest physical address
3457	* @param pPGM Pointer to the PGM instance data.
3458	* @param Pde Guest Pde
3459	*/
3460	DECLINLINE(RTGCPHYS) pgmGstGet4MBPhysPage(PPGM pPGM, X86PDE Pde)
3461	{
3462	RTGCPHYS GCPhys = Pde.u & X86_PDE4M_PG_MASK;
3463	GCPhys \|= (RTGCPHYS)Pde.b.u8PageNoHigh << 32;
3464
3465	return GCPhys & pPGM->GCPhys4MBPSEMask;
3466	}
3467
3468
3469	/**
3470	* Gets the page directory entry for the specified address (32-bit paging).
3471	*
3472	* @returns The page directory entry in question.
3473	* @param pPGM Pointer to the PGM instance data.
3474	* @param GCPtr The address.
3475	*/
3476	DECLINLINE(X86PDE) pgmGstGet32bitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3477	{
3478	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3479	PCX86PD pGuestPD = NULL;
3480	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3481	if (RT_FAILURE(rc))
3482	{
3483	X86PDE ZeroPde = {0};
3484	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPde);
3485	}
3486	#else
3487	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3488	# ifdef IN_RING3
3489	if (!pGuestPD)
3490	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3491	# endif
3492	#endif
3493	return pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3494	}
3495
3496
3497	/**
3498	* Gets the address of a specific page directory entry (32-bit paging).
3499	*
3500	* @returns Pointer the page directory entry in question.
3501	* @param pPGM Pointer to the PGM instance data.
3502	* @param GCPtr The address.
3503	*/
3504	DECLINLINE(PX86PDE) pgmGstGet32bitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3505	{
3506	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3507	PX86PD pGuestPD = NULL;
3508	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3509	AssertRCReturn(rc, NULL);
3510	#else
3511	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3512	# ifdef IN_RING3
3513	if (!pGuestPD)
3514	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3515	# endif
3516	#endif
3517	return &pGuestPD->a[GCPtr >> X86_PD_SHIFT];
3518	}
3519
3520
3521	/**
3522	* Gets the address the guest page directory (32-bit paging).
3523	*
3524	* @returns Pointer the page directory entry in question.
3525	* @param pPGM Pointer to the PGM instance data.
3526	*/
3527	DECLINLINE(PX86PD) pgmGstGet32bitPDPtr(PPGMCPU pPGM)
3528	{
3529	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3530	PX86PD pGuestPD = NULL;
3531	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPD);
3532	AssertRCReturn(rc, NULL);
3533	#else
3534	PX86PD pGuestPD = pPGM->CTX_SUFF(pGst32BitPd);
3535	# ifdef IN_RING3
3536	if (!pGuestPD)
3537	pGuestPD = pgmGstLazyMap32BitPD(pPGM);
3538	# endif
3539	#endif
3540	return pGuestPD;
3541	}
3542
3543
3544	/**
3545	* Gets the guest page directory pointer table.
3546	*
3547	* @returns Pointer to the page directory in question.
3548	* @returns NULL if the page directory is not present or on an invalid page.
3549	* @param pPGM Pointer to the PGM instance data.
3550	*/
3551	DECLINLINE(PX86PDPT) pgmGstGetPaePDPTPtr(PPGMCPU pPGM)
3552	{
3553	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3554	PX86PDPT pGuestPDPT = NULL;
3555	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3556	AssertRCReturn(rc, NULL);
3557	#else
3558	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3559	# ifdef IN_RING3
3560	if (!pGuestPDPT)
3561	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3562	# endif
3563	#endif
3564	return pGuestPDPT;
3565	}
3566
3567
3568	/**
3569	* Gets the guest page directory pointer table entry for the specified address.
3570	*
3571	* @returns Pointer to the page directory in question.
3572	* @returns NULL if the page directory is not present or on an invalid page.
3573	* @param pPGM Pointer to the PGM instance data.
3574	* @param GCPtr The address.
3575	*/
3576	DECLINLINE(PX86PDPE) pgmGstGetPaePDPEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3577	{
3578	AssertGCPtr32(GCPtr);
3579
3580	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3581	PX86PDPT pGuestPDPT = 0;
3582	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPDPT);
3583	AssertRCReturn(rc, 0);
3584	#else
3585	PX86PDPT pGuestPDPT = pPGM->CTX_SUFF(pGstPaePdpt);
3586	# ifdef IN_RING3
3587	if (!pGuestPDPT)
3588	pGuestPDPT = pgmGstLazyMapPaePDPT(pPGM);
3589	# endif
3590	#endif
3591	return &pGuestPDPT->a[(GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE];
3592	}
3593
3594
3595	/**
3596	* Gets the page directory for the specified address.
3597	*
3598	* @returns Pointer to the page directory in question.
3599	* @returns NULL if the page directory is not present or on an invalid page.
3600	* @param pPGM Pointer to the PGM instance data.
3601	* @param GCPtr The address.
3602	*/
3603	DECLINLINE(PX86PDPAE) pgmGstGetPaePD(PPGMCPU pPGM, RTGCPTR GCPtr)
3604	{
3605	AssertGCPtr32(GCPtr);
3606
3607	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3608	AssertReturn(pGuestPDPT, NULL);
3609	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3610	if (pGuestPDPT->a[iPdpt].n.u1Present)
3611	{
3612	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3613	PX86PDPAE pGuestPD = NULL;
3614	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3615	AssertRCReturn(rc, NULL);
3616	#else
3617	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3618	if ( !pGuestPD
3619	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3620	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3621	#endif
3622	return pGuestPD;
3623	/* returning NULL is ok if we assume it's just an invalid page of some kind emulated as all 0s. (not quite true) */
3624	}
3625	return NULL;
3626	}
3627
3628
3629	/**
3630	* Gets the page directory entry for the specified address.
3631	*
3632	* @returns Pointer to the page directory entry in question.
3633	* @returns NULL if the page directory is not present or on an invalid page.
3634	* @param pPGM Pointer to the PGM instance data.
3635	* @param GCPtr The address.
3636	*/
3637	DECLINLINE(PX86PDEPAE) pgmGstGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
3638	{
3639	AssertGCPtr32(GCPtr);
3640
3641	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3642	AssertReturn(pGuestPDPT, NULL);
3643	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3644	if (pGuestPDPT->a[iPdpt].n.u1Present)
3645	{
3646	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3647	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3648	PX86PDPAE pGuestPD = NULL;
3649	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3650	AssertRCReturn(rc, NULL);
3651	#else
3652	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3653	if ( !pGuestPD
3654	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3655	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3656	#endif
3657	return &pGuestPD->a[iPD];
3658	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page or something which we'll emulate as all 0s. (not quite true) */
3659	}
3660	return NULL;
3661	}
3662
3663
3664	/**
3665	* Gets the page directory entry for the specified address.
3666	*
3667	* @returns The page directory entry in question.
3668	* @returns A non-present entry if the page directory is not present or on an invalid page.
3669	* @param pPGM Pointer to the PGM instance data.
3670	* @param GCPtr The address.
3671	*/
3672	DECLINLINE(X86PDEPAE) pgmGstGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
3673	{
3674	AssertGCPtr32(GCPtr);
3675	X86PDEPAE ZeroPde = {0};
3676	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3677	if (RT_LIKELY(pGuestPDPT))
3678	{
3679	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3680	if (pGuestPDPT->a[iPdpt].n.u1Present)
3681	{
3682	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3683	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3684	PX86PDPAE pGuestPD = NULL;
3685	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3686	AssertRCReturn(rc, ZeroPde);
3687	#else
3688	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3689	if ( !pGuestPD
3690	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3691	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3692	#endif
3693	return pGuestPD->a[iPD];
3694	}
3695	}
3696	return ZeroPde;
3697	}
3698
3699
3700	/**
3701	* Gets the page directory pointer table entry for the specified address
3702	* and returns the index into the page directory
3703	*
3704	* @returns Pointer to the page directory in question.
3705	* @returns NULL if the page directory is not present or on an invalid page.
3706	* @param pPGM Pointer to the PGM instance data.
3707	* @param GCPtr The address.
3708	* @param piPD Receives the index into the returned page directory
3709	* @param pPdpe Receives the page directory pointer entry. Optional.
3710	*/
3711	DECLINLINE(PX86PDPAE) pgmGstGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr, unsigned *piPD, PX86PDPE pPdpe)
3712	{
3713	AssertGCPtr32(GCPtr);
3714
3715	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pPGM);
3716	AssertReturn(pGuestPDPT, NULL);
3717	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
3718	if (pPdpe)
3719	*pPdpe = pGuestPDPT->a[iPdpt];
3720	if (pGuestPDPT->a[iPdpt].n.u1Present)
3721	{
3722	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3723	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3724	PX86PDPAE pGuestPD = NULL;
3725	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK, (void **)&pGuestPD);
3726	AssertRCReturn(rc, NULL);
3727	#else
3728	PX86PDPAE pGuestPD = pPGM->CTX_SUFF(apGstPaePDs)[iPdpt];
3729	if ( !pGuestPD
3730	\|\| (pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK) != pPGM->aGCPhysGstPaePDs[iPdpt])
3731	pGuestPD = pgmGstLazyMapPaePD(pPGM, iPdpt);
3732	#endif
3733	*piPD = iPD;
3734	return pGuestPD;
3735	/* returning NIL_RTGCPHYS is ok if we assume it's just an invalid page of some kind emulated as all 0s. */
3736	}
3737	return NULL;
3738	}
3739
3740	#ifndef IN_RC
3741
3742	/**
3743	* Gets the page map level-4 pointer for the guest.
3744	*
3745	* @returns Pointer to the PML4 page.
3746	* @param pPGM Pointer to the PGM instance data.
3747	*/
3748	DECLINLINE(PX86PML4) pgmGstGetLongModePML4Ptr(PPGMCPU pPGM)
3749	{
3750	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3751	PX86PML4 pGuestPml4;
3752	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3753	AssertRCReturn(rc, NULL);
3754	#else
3755	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3756	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3757	if (!pGuestPml4)
3758	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3759	# endif
3760	Assert(pGuestPml4);
3761	#endif
3762	return pGuestPml4;
3763	}
3764
3765
3766	/**
3767	* Gets the pointer to a page map level-4 entry.
3768	*
3769	* @returns Pointer to the PML4 entry.
3770	* @param pPGM Pointer to the PGM instance data.
3771	* @param iPml4 The index.
3772	*/
3773	DECLINLINE(PX86PML4E) pgmGstGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
3774	{
3775	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3776	PX86PML4 pGuestPml4;
3777	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3778	AssertRCReturn(rc, NULL);
3779	#else
3780	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3781	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3782	if (!pGuestPml4)
3783	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3784	# endif
3785	Assert(pGuestPml4);
3786	#endif
3787	return &pGuestPml4->a[iPml4];
3788	}
3789
3790
3791	/**
3792	* Gets a page map level-4 entry.
3793	*
3794	* @returns The PML4 entry.
3795	* @param pPGM Pointer to the PGM instance data.
3796	* @param iPml4 The index.
3797	*/
3798	DECLINLINE(X86PML4E) pgmGstGetLongModePML4E(PPGMCPU pPGM, unsigned int iPml4)
3799	{
3800	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3801	PX86PML4 pGuestPml4;
3802	int rc = pgmR0DynMapGCPageInlined(PGMCPU2PGM(pPGM), pPGM->GCPhysCR3, (void **)&pGuestPml4);
3803	if (RT_FAILURE(rc))
3804	{
3805	X86PML4E ZeroPml4e = {0};
3806	AssertMsgFailedReturn(("%Rrc\n", rc), ZeroPml4e);
3807	}
3808	#else
3809	PX86PML4 pGuestPml4 = pPGM->CTX_SUFF(pGstAmd64Pml4);
3810	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R3
3811	if (!pGuestPml4)
3812	pGuestPml4 = pgmGstLazyMapPml4(pPGM);
3813	# endif
3814	Assert(pGuestPml4);
3815	#endif
3816	return pGuestPml4->a[iPml4];
3817	}
3818
3819
3820	/**
3821	* Gets the page directory pointer entry for the specified address.
3822	*
3823	* @returns Pointer to the page directory pointer entry in question.
3824	* @returns NULL if the page directory is not present or on an invalid page.
3825	* @param pPGM Pointer to the PGM instance data.
3826	* @param GCPtr The address.
3827	* @param ppPml4e Page Map Level-4 Entry (out)
3828	*/
3829	DECLINLINE(PX86PDPE) pgmGstGetLongModePDPTPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e)
3830	{
3831	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3832	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3833	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3834	if (pPml4e->n.u1Present)
3835	{
3836	PX86PDPT pPdpt;
3837	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdpt);
3838	AssertRCReturn(rc, NULL);
3839
3840	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3841	return &pPdpt->a[iPdpt];
3842	}
3843	return NULL;
3844	}
3845
3846
3847	/**
3848	* Gets the page directory entry for the specified address.
3849	*
3850	* @returns The page directory entry in question.
3851	* @returns A non-present entry if the page directory is not present or on an invalid page.
3852	* @param pPGM Pointer to the PGM instance data.
3853	* @param GCPtr The address.
3854	* @param ppPml4e Page Map Level-4 Entry (out)
3855	* @param pPdpe Page directory pointer table entry (out)
3856	*/
3857	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDEEx(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPE pPdpe)
3858	{
3859	X86PDEPAE ZeroPde = {0};
3860	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3861	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3862	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3863	if (pPml4e->n.u1Present)
3864	{
3865	PCX86PDPT pPdptTemp;
3866	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3867	AssertRCReturn(rc, ZeroPde);
3868
3869	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3870	*pPdpe = pPdptTemp->a[iPdpt];
3871	if (pPdptTemp->a[iPdpt].n.u1Present)
3872	{
3873	PCX86PDPAE pPD;
3874	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3875	AssertRCReturn(rc, ZeroPde);
3876
3877	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3878	return pPD->a[iPD];
3879	}
3880	}
3881
3882	return ZeroPde;
3883	}
3884
3885
3886	/**
3887	* Gets the page directory entry for the specified address.
3888	*
3889	* @returns The page directory entry in question.
3890	* @returns A non-present entry if the page directory is not present or on an invalid page.
3891	* @param pPGM Pointer to the PGM instance data.
3892	* @param GCPtr The address.
3893	*/
3894	DECLINLINE(X86PDEPAE) pgmGstGetLongModePDE(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3895	{
3896	X86PDEPAE ZeroPde = {0};
3897	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3898	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3899	if (pGuestPml4->a[iPml4].n.u1Present)
3900	{
3901	PCX86PDPT pPdptTemp;
3902	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3903	AssertRCReturn(rc, ZeroPde);
3904
3905	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3906	if (pPdptTemp->a[iPdpt].n.u1Present)
3907	{
3908	PCX86PDPAE pPD;
3909	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3910	AssertRCReturn(rc, ZeroPde);
3911
3912	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3913	return pPD->a[iPD];
3914	}
3915	}
3916	return ZeroPde;
3917	}
3918
3919
3920	/**
3921	* Gets the page directory entry for the specified address.
3922	*
3923	* @returns Pointer to the page directory entry in question.
3924	* @returns NULL if the page directory is not present or on an invalid page.
3925	* @param pPGM Pointer to the PGM instance data.
3926	* @param GCPtr The address.
3927	*/
3928	DECLINLINE(PX86PDEPAE) pgmGstGetLongModePDEPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr)
3929	{
3930	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3931	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3932	if (pGuestPml4->a[iPml4].n.u1Present)
3933	{
3934	PCX86PDPT pPdptTemp;
3935	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
3936	AssertRCReturn(rc, NULL);
3937
3938	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3939	if (pPdptTemp->a[iPdpt].n.u1Present)
3940	{
3941	PX86PDPAE pPD;
3942	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3943	AssertRCReturn(rc, NULL);
3944
3945	const unsigned iPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3946	return &pPD->a[iPD];
3947	}
3948	}
3949	return NULL;
3950	}
3951
3952
3953	/**
3954	* Gets the GUEST page directory pointer for the specified address.
3955	*
3956	* @returns The page directory in question.
3957	* @returns NULL if the page directory is not present or on an invalid page.
3958	* @param pPGM Pointer to the PGM instance data.
3959	* @param GCPtr The address.
3960	* @param ppPml4e Page Map Level-4 Entry (out)
3961	* @param pPdpe Page directory pointer table entry (out)
3962	* @param piPD Receives the index into the returned page directory
3963	*/
3964	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPE pPdpe, unsigned piPD)
3965	{
3966	PX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
3967	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3968	PCX86PML4E pPml4e = *ppPml4e = &pGuestPml4->a[iPml4];
3969	if (pPml4e->n.u1Present)
3970	{
3971	PCX86PDPT pPdptTemp;
3972	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPml4e->u & X86_PML4E_PG_MASK, &pPdptTemp);
3973	AssertRCReturn(rc, NULL);
3974
3975	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3976	*pPdpe = pPdptTemp->a[iPdpt];
3977	if (pPdptTemp->a[iPdpt].n.u1Present)
3978	{
3979	PX86PDPAE pPD;
3980	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
3981	AssertRCReturn(rc, NULL);
3982
3983	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
3984	return pPD;
3985	}
3986	}
3987	return 0;
3988	}
3989
3990	#endif /* !IN_RC */
3991
3992	/**
3993	* Gets the shadow page directory, 32-bit.
3994	*
3995	* @returns Pointer to the shadow 32-bit PD.
3996	* @param pPGM Pointer to the PGM instance data.
3997	*/
3998	DECLINLINE(PX86PD) pgmShwGet32BitPDPtr(PPGMCPU pPGM)
3999	{
4000	return (PX86PD)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4001	}
4002
4003
4004	/**
4005	* Gets the shadow page directory entry for the specified address, 32-bit.
4006	*
4007	* @returns Shadow 32-bit PDE.
4008	* @param pPGM Pointer to the PGM instance data.
4009	* @param GCPtr The address.
4010	*/
4011	DECLINLINE(X86PDE) pgmShwGet32BitPDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4012	{
4013	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4014
4015	PX86PD pShwPde = pgmShwGet32BitPDPtr(pPGM);
4016	if (!pShwPde)
4017	{
4018	X86PDE ZeroPde = {0};
4019	return ZeroPde;
4020	}
4021	return pShwPde->a[iPd];
4022	}
4023
4024
4025	/**
4026	* Gets the pointer to the shadow page directory entry for the specified
4027	* address, 32-bit.
4028	*
4029	* @returns Pointer to the shadow 32-bit PDE.
4030	* @param pPGM Pointer to the PGM instance data.
4031	* @param GCPtr The address.
4032	*/
4033	DECLINLINE(PX86PDE) pgmShwGet32BitPDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4034	{
4035	const unsigned iPd = (GCPtr >> X86_PD_SHIFT) & X86_PD_MASK;
4036
4037	PX86PD pPde = pgmShwGet32BitPDPtr(pPGM);
4038	AssertReturn(pPde, NULL);
4039	return &pPde->a[iPd];
4040	}
4041
4042
4043	/**
4044	* Gets the shadow page pointer table, PAE.
4045	*
4046	* @returns Pointer to the shadow PAE PDPT.
4047	* @param pPGM Pointer to the PGM instance data.
4048	*/
4049	DECLINLINE(PX86PDPT) pgmShwGetPaePDPTPtr(PPGMCPU pPGM)
4050	{
4051	return (PX86PDPT)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4052	}
4053
4054
4055	/**
4056	* Gets the shadow page directory for the specified address, PAE.
4057	*
4058	* @returns Pointer to the shadow PD.
4059	* @param pPGM Pointer to the PGM instance data.
4060	* @param GCPtr The address.
4061	*/
4062	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4063	{
4064	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4065	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pPGM);
4066
4067	if (!pPdpt->a[iPdpt].n.u1Present)
4068	return NULL;
4069
4070	/* Fetch the pgm pool shadow descriptor. */
4071	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4072	AssertReturn(pShwPde, NULL);
4073
4074	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4075	}
4076
4077
4078	/**
4079	* Gets the shadow page directory for the specified address, PAE.
4080	*
4081	* @returns Pointer to the shadow PD.
4082	* @param pPGM Pointer to the PGM instance data.
4083	* @param GCPtr The address.
4084	*/
4085	DECLINLINE(PX86PDPAE) pgmShwGetPaePDPtr(PPGMCPU pPGM, PX86PDPT pPdpt, RTGCPTR GCPtr)
4086	{
4087	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
4088
4089	if (!pPdpt->a[iPdpt].n.u1Present)
4090	return NULL;
4091
4092	/* Fetch the pgm pool shadow descriptor. */
4093	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(PGMCPU2PGM(pPGM)->CTX_SUFF(pPool), pPdpt->a[iPdpt].u & X86_PDPE_PG_MASK);
4094	AssertReturn(pShwPde, NULL);
4095
4096	return (PX86PDPAE)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pShwPde);
4097	}
4098
4099
4100	/**
4101	* Gets the shadow page directory entry, PAE.
4102	*
4103	* @returns PDE.
4104	* @param pPGM Pointer to the PGM instance data.
4105	* @param GCPtr The address.
4106	*/
4107	DECLINLINE(X86PDEPAE) pgmShwGetPaePDE(PPGMCPU pPGM, RTGCPTR GCPtr)
4108	{
4109	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4110
4111	PX86PDPAE pShwPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4112	if (!pShwPde)
4113	{
4114	X86PDEPAE ZeroPde = {0};
4115	return ZeroPde;
4116	}
4117	return pShwPde->a[iPd];
4118	}
4119
4120
4121	/**
4122	* Gets the pointer to the shadow page directory entry for an address, PAE.
4123	*
4124	* @returns Pointer to the PDE.
4125	* @param pPGM Pointer to the PGM instance data.
4126	* @param GCPtr The address.
4127	*/
4128	DECLINLINE(PX86PDEPAE) pgmShwGetPaePDEPtr(PPGMCPU pPGM, RTGCPTR GCPtr)
4129	{
4130	const unsigned iPd = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4131
4132	PX86PDPAE pPde = pgmShwGetPaePDPtr(pPGM, GCPtr);
4133	AssertReturn(pPde, NULL);
4134	return &pPde->a[iPd];
4135	}
4136
4137	#ifndef IN_RC
4138
4139	/**
4140	* Gets the shadow page map level-4 pointer.
4141	*
4142	* @returns Pointer to the shadow PML4.
4143	* @param pPGM Pointer to the PGM instance data.
4144	*/
4145	DECLINLINE(PX86PML4) pgmShwGetLongModePML4Ptr(PPGMCPU pPGM)
4146	{
4147	return (PX86PML4)PGMPOOL_PAGE_2_PTR_BY_PGMCPU(pPGM, pPGM->CTX_SUFF(pShwPageCR3));
4148	}
4149
4150
4151	/**
4152	* Gets the shadow page map level-4 entry for the specified address.
4153	*
4154	* @returns The entry.
4155	* @param pPGM Pointer to the PGM instance data.
4156	* @param GCPtr The address.
4157	*/
4158	DECLINLINE(X86PML4E) pgmShwGetLongModePML4E(PPGMCPU pPGM, RTGCPTR GCPtr)
4159	{
4160	const unsigned iPml4 = ((RTGCUINTPTR64)GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4161	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4162
4163	if (!pShwPml4)
4164	{
4165	X86PML4E ZeroPml4e = {0};
4166	return ZeroPml4e;
4167	}
4168	return pShwPml4->a[iPml4];
4169	}
4170
4171
4172	/**
4173	* Gets the pointer to the specified shadow page map level-4 entry.
4174	*
4175	* @returns The entry.
4176	* @param pPGM Pointer to the PGM instance data.
4177	* @param iPml4 The PML4 index.
4178	*/
4179	DECLINLINE(PX86PML4E) pgmShwGetLongModePML4EPtr(PPGMCPU pPGM, unsigned int iPml4)
4180	{
4181	PX86PML4 pShwPml4 = pgmShwGetLongModePML4Ptr(pPGM);
4182	if (!pShwPml4)
4183	return NULL;
4184	return &pShwPml4->a[iPml4];
4185	}
4186
4187
4188	/**
4189	* Gets the GUEST page directory pointer for the specified address.
4190	*
4191	* @returns The page directory in question.
4192	* @returns NULL if the page directory is not present or on an invalid page.
4193	* @param pPGM Pointer to the PGM instance data.
4194	* @param GCPtr The address.
4195	* @param piPD Receives the index into the returned page directory
4196	*/
4197	DECLINLINE(PX86PDPAE) pgmGstGetLongModePDPtr(PPGMCPU pPGM, RTGCPTR64 GCPtr, unsigned *piPD)
4198	{
4199	PCX86PML4 pGuestPml4 = pgmGstGetLongModePML4Ptr(pPGM);
4200	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
4201	if (pGuestPml4->a[iPml4].n.u1Present)
4202	{
4203	PCX86PDPT pPdptTemp;
4204	int rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pGuestPml4->a[iPml4].u & X86_PML4E_PG_MASK, &pPdptTemp);
4205	AssertRCReturn(rc, NULL);
4206
4207	const unsigned iPdpt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
4208	if (pPdptTemp->a[iPdpt].n.u1Present)
4209	{
4210	PX86PDPAE pPD;
4211	rc = PGM_GCPHYS_2_PTR_BY_PGMCPU(pPGM, pPdptTemp->a[iPdpt].u & X86_PDPE_PG_MASK, &pPD);
4212	AssertRCReturn(rc, NULL);
4213
4214	*piPD = (GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK;
4215	return pPD;
4216	}
4217	}
4218	return NULL;
4219	}
4220
4221	#endif /* !IN_RC */
4222
4223	/**
4224	* Gets the page state for a physical handler.
4225	*
4226	* @returns The physical handler page state.
4227	* @param pCur The physical handler in question.
4228	*/
4229	DECLINLINE(unsigned) pgmHandlerPhysicalCalcState(PPGMPHYSHANDLER pCur)
4230	{
4231	switch (pCur->enmType)
4232	{
4233	case PGMPHYSHANDLERTYPE_PHYSICAL_WRITE:
4234	return PGM_PAGE_HNDL_PHYS_STATE_WRITE;
4235
4236	case PGMPHYSHANDLERTYPE_MMIO:
4237	case PGMPHYSHANDLERTYPE_PHYSICAL_ALL:
4238	return PGM_PAGE_HNDL_PHYS_STATE_ALL;
4239
4240	default:
4241	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4242	}
4243	}
4244
4245
4246	/**
4247	* Gets the page state for a virtual handler.
4248	*
4249	* @returns The virtual handler page state.
4250	* @param pCur The virtual handler in question.
4251	* @remarks This should never be used on a hypervisor access handler.
4252	*/
4253	DECLINLINE(unsigned) pgmHandlerVirtualCalcState(PPGMVIRTHANDLER pCur)
4254	{
4255	switch (pCur->enmType)
4256	{
4257	case PGMVIRTHANDLERTYPE_WRITE:
4258	return PGM_PAGE_HNDL_VIRT_STATE_WRITE;
4259	case PGMVIRTHANDLERTYPE_ALL:
4260	return PGM_PAGE_HNDL_VIRT_STATE_ALL;
4261	default:
4262	AssertFatalMsgFailed(("Invalid type %d\n", pCur->enmType));
4263	}
4264	}
4265
4266
4267	/**
4268	* Clears one physical page of a virtual handler
4269	*
4270	* @param pPGM Pointer to the PGM instance.
4271	* @param pCur Virtual handler structure
4272	* @param iPage Physical page index
4273	*
4274	* @remark Only used when PGM_SYNC_UPDATE_PAGE_BIT_VIRTUAL is being set, so no
4275	* need to care about other handlers in the same page.
4276	*/
4277	DECLINLINE(void) pgmHandlerVirtualClearPage(PPGM pPGM, PPGMVIRTHANDLER pCur, unsigned iPage)
4278	{
4279	const PPGMPHYS2VIRTHANDLER pPhys2Virt = &pCur->aPhysToVirt[iPage];
4280
4281	/*
4282	* Remove the node from the tree (it's supposed to be in the tree if we get here!).
4283	*/
4284	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4285	AssertReleaseMsg(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4286	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4287	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4288	#endif
4289	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_IS_HEAD)
4290	{
4291	/* We're the head of the alias chain. */
4292	PPGMPHYS2VIRTHANDLER pRemove = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysRemove(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key); NOREF(pRemove);
4293	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4294	AssertReleaseMsg(pRemove != NULL,
4295	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4296	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias));
4297	AssertReleaseMsg(pRemove == pPhys2Virt,
4298	("wanted: pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n"
4299	" got: pRemove=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4300	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias,
4301	pRemove, pRemove->Core.Key, pRemove->Core.KeyLast, pRemove->offVirtHandler, pRemove->offNextAlias));
4302	#endif
4303	if (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK)
4304	{
4305	/* Insert the next list in the alias chain into the tree. */
4306	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4307	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4308	AssertReleaseMsg(pNext->offNextAlias & PGMPHYS2VIRTHANDLER_IN_TREE,
4309	("pNext=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32}\n",
4310	pNext, pNext->Core.Key, pNext->Core.KeyLast, pNext->offVirtHandler, pNext->offNextAlias));
4311	#endif
4312	pNext->offNextAlias \|= PGMPHYS2VIRTHANDLER_IS_HEAD;
4313	bool fRc = RTAvlroGCPhysInsert(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, &pNext->Core);
4314	AssertRelease(fRc);
4315	}
4316	}
4317	else
4318	{
4319	/* Locate the previous node in the alias chain. */
4320	PPGMPHYS2VIRTHANDLER pPrev = (PPGMPHYS2VIRTHANDLER)RTAvlroGCPhysGet(&pPGM->CTX_SUFF(pTrees)->PhysToVirtHandlers, pPhys2Virt->Core.Key);
4321	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4322	AssertReleaseMsg(pPrev != pPhys2Virt,
4323	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4324	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4325	#endif
4326	for (;;)
4327	{
4328	PPGMPHYS2VIRTHANDLER pNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPrev + (pPrev->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4329	if (pNext == pPhys2Virt)
4330	{
4331	/* unlink. */
4332	LogFlow(("pgmHandlerVirtualClearPage: removed %p:{.offNextAlias=%#RX32} from alias chain. prev %p:{.offNextAlias=%#RX32} [%RGp-%RGp]\n",
4333	pPhys2Virt, pPhys2Virt->offNextAlias, pPrev, pPrev->offNextAlias, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast));
4334	if (!(pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK))
4335	pPrev->offNextAlias &= ~PGMPHYS2VIRTHANDLER_OFF_MASK;
4336	else
4337	{
4338	PPGMPHYS2VIRTHANDLER pNewNext = (PPGMPHYS2VIRTHANDLER)((intptr_t)pPhys2Virt + (pPhys2Virt->offNextAlias & PGMPHYS2VIRTHANDLER_OFF_MASK));
4339	pPrev->offNextAlias = ((intptr_t)pNewNext - (intptr_t)pPrev)
4340	\| (pPrev->offNextAlias & ~PGMPHYS2VIRTHANDLER_OFF_MASK);
4341	}
4342	break;
4343	}
4344
4345	/* next */
4346	if (pNext == pPrev)
4347	{
4348	#ifdef VBOX_STRICT_PGM_HANDLER_VIRTUAL
4349	AssertReleaseMsg(pNext != pPrev,
4350	("pPhys2Virt=%p:{.Core.Key=%RGp, .Core.KeyLast=%RGp, .offVirtHandler=%#RX32, .offNextAlias=%#RX32} pPrev=%p\n",
4351	pPhys2Virt, pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offVirtHandler, pPhys2Virt->offNextAlias, pPrev));
4352	#endif
4353	break;
4354	}
4355	pPrev = pNext;
4356	}
4357	}
4358	Log2(("PHYS2VIRT: Removing %RGp-%RGp %#RX32 %s\n",
4359	pPhys2Virt->Core.Key, pPhys2Virt->Core.KeyLast, pPhys2Virt->offNextAlias, R3STRING(pCur->pszDesc)));
4360	pPhys2Virt->offNextAlias = 0;
4361	pPhys2Virt->Core.KeyLast = NIL_RTGCPHYS; /* require reinsert */
4362
4363	/*
4364	* Clear the ram flags for this page.
4365	*/
4366	PPGMPAGE pPage = pgmPhysGetPage(pPGM, pPhys2Virt->Core.Key);
4367	AssertReturnVoid(pPage);
4368	PGM_PAGE_SET_HNDL_VIRT_STATE(pPage, PGM_PAGE_HNDL_VIRT_STATE_NONE);
4369	}
4370
4371
4372	/**
4373	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4374	*
4375	* @returns Pointer to the shadow page structure.
4376	* @param pPool The pool.
4377	* @param idx The pool page index.
4378	*/
4379	DECLINLINE(PPGMPOOLPAGE) pgmPoolGetPageByIdx(PPGMPOOL pPool, unsigned idx)
4380	{
4381	AssertFatalMsg(idx >= PGMPOOL_IDX_FIRST && idx < pPool->cCurPages, ("idx=%d\n", idx));
4382	return &pPool->aPages[idx];
4383	}
4384
4385
4386	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4387	/**
4388	* Clear references to guest physical memory.
4389	*
4390	* @param pPool The pool.
4391	* @param pPoolPage The pool page.
4392	* @param pPhysPage The physical guest page tracking structure.
4393	*/
4394	DECLINLINE(void) pgmTrackDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPoolPage, PPGMPAGE pPhysPage)
4395	{
4396	/*
4397	* Just deal with the simple case here.
4398	*/
4399	# ifdef LOG_ENABLED
4400	const unsigned uOrg = PGM_PAGE_GET_TRACKING(pPhysPage);
4401	# endif
4402	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
4403	if (cRefs == 1)
4404	{
4405	Assert(pPoolPage->idx == PGM_PAGE_GET_TD_IDX(pPhysPage));
4406	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
4407	}
4408	else
4409	pgmPoolTrackPhysExtDerefGCPhys(pPool, pPoolPage, pPhysPage);
4410	Log2(("pgmTrackDerefGCPhys: %x -> %x pPhysPage=%R[pgmpage]\n", uOrg, PGM_PAGE_GET_TRACKING(pPhysPage), pPhysPage ));
4411	}
4412	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4413
4414
4415	#ifdef PGMPOOL_WITH_CACHE
4416	/**
4417	* Moves the page to the head of the age list.
4418	*
4419	* This is done when the cached page is used in one way or another.
4420	*
4421	* @param pPool The pool.
4422	* @param pPage The cached page.
4423	*/
4424	DECLINLINE(void) pgmPoolCacheUsed(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4425	{
4426	PVM pVM = pPool->CTX_SUFF(pVM);
4427	pgmLock(pVM);
4428
4429	/*
4430	* Move to the head of the age list.
4431	*/
4432	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
4433	{
4434	/* unlink */
4435	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
4436	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
4437	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
4438	else
4439	pPool->iAgeTail = pPage->iAgePrev;
4440
4441	/* insert at head */
4442	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4443	pPage->iAgeNext = pPool->iAgeHead;
4444	Assert(pPage->iAgeNext != NIL_PGMPOOL_IDX); /* we would've already been head then */
4445	pPool->iAgeHead = pPage->idx;
4446	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->idx;
4447	}
4448	pgmUnlock(pVM);
4449	}
4450	#endif /* PGMPOOL_WITH_CACHE */
4451
4452	/**
4453	* Locks a page to prevent flushing (important for cr3 root pages or shadow pae pd pages).
4454	*
4455	* @param pVM VM Handle.
4456	* @param pPage PGM pool page
4457	*/
4458	DECLINLINE(void) pgmPoolLockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4459	{
4460	ASMAtomicIncU32(&pPage->cLocked);
4461	}
4462
4463
4464	/**
4465	* Unlocks a page to allow flushing again
4466	*
4467	* @param pVM VM Handle.
4468	* @param pPage PGM pool page
4469	*/
4470	DECLINLINE(void) pgmPoolUnlockPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4471	{
4472	Assert(pPage->cLocked);
4473	ASMAtomicDecU32(&pPage->cLocked);
4474	}
4475
4476
4477	/**
4478	* Checks if the page is locked (e.g. the active CR3 or one of the four PDs of a PAE PDPT)
4479	*
4480	* @returns VBox status code.
4481	* @param pPage PGM pool page
4482	*/
4483	DECLINLINE(bool) pgmPoolIsPageLocked(PPGM pPGM, PPGMPOOLPAGE pPage)
4484	{
4485	if (pPage->cLocked)
4486	{
4487	LogFlow(("pgmPoolIsPageLocked found root page %d\n", pPage->enmKind));
4488	if (pPage->cModifications)
4489	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
4490	return true;
4491	}
4492	return false;
4493	}
4494
4495	/**
4496	* Tells if mappings are to be put into the shadow page table or not
4497	*
4498	* @returns boolean result
4499	* @param pVM VM handle.
4500	*/
4501	DECLINLINE(bool) pgmMapAreMappingsEnabled(PPGM pPGM)
4502	{
4503	#ifdef IN_RING0
4504	/* There are no mappings in VT-x and AMD-V mode. */
4505	Assert(pPGM->fDisableMappings);
4506	return false;
4507	#else
4508	return !pPGM->fDisableMappings;
4509	#endif
4510	}
4511
4512	/** @} */
4513
4514	#endif
4515
4516

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/PGMInternal.h@ 20125

以其他格式下載: