PGMAllBth.h@ 73246

最後變更在這個檔案從73246是 73246,由 vboxsync 提交於 7 年前
PGM: Working on eliminating PGMMODEDATA and the corresponding PGMCPU section so we can do mode switching in ring-0. This second part deals with shadow paging pointers and expands PGM_TYPE_NESTED & PGMMODE_NESTED into 32BIT, PAE and AMD64 variants to better map to reality at the expense of a little bit of more code. bugref:9044
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Id Revision`
檔案大小: 213.3 KB

行
1	/* $Id: PGMAllBth.h 73246 2018-07-19 15:51:20Z vboxsync $ */
2	/** @file
3	* VBox - Page Manager, Shadow+Guest Paging Template - All context code.
4	*
5	* @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
6	* PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
7	* bird: WTF does this mean these days? Looking at PGMAll.cpp it's
8	*
9	* @remarks This file is one big \#ifdef-orgy!
10	*
11	*/
12
13	/*
14	* Copyright (C) 2006-2017 Oracle Corporation
15	*
16	* This file is part of VirtualBox Open Source Edition (OSE), as
17	* available from http://www.alldomusa.eu.org. This file is free software;
18	* you can redistribute it and/or modify it under the terms of the GNU
19	* General Public License (GPL) as published by the Free Software
20	* Foundation, in version 2 as it comes in the "COPYING" file of the
21	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
22	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
23	*/
24
25	#ifdef _MSC_VER
26	/** @todo we're generating unnecessary code in nested/ept shadow mode and for
27	* real/prot-guest+RC mode. */
28	# pragma warning(disable: 4505)
29	#endif
30
31	/*******************************************************************************
32	* Internal Functions *
33	*******************************************************************************/
34	RT_C_DECLS_BEGIN
35	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken);
36	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
37	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
38	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
39	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
40	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
41	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
42	# else
43	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
44	#endif
45	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
46	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage);
47	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
48	#ifdef VBOX_STRICT
49	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
50	#endif
51	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3);
52	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu);
53	RT_C_DECLS_END
54
55
56	/*
57	* Filter out some illegal combinations of guest and shadow paging, so we can
58	* remove redundant checks inside functions.
59	*/
60	#if PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
61	# error "Invalid combination; PAE guest implies PAE shadow"
62	#endif
63
64	#if (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
65	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64 \|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE))
66	# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
67	#endif
68
69	#if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) \
70	&& !(PGM_SHW_TYPE == PGM_TYPE_32BIT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE))
71	# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
72	#endif
73
74	#if (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)) \
75	\|\| (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
76	# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
77	#endif
78
79	#ifndef IN_RING3
80
81	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
82	/**
83	* Deal with a guest page fault.
84	*
85	* @returns Strict VBox status code.
86	* @retval VINF_EM_RAW_GUEST_TRAP
87	* @retval VINF_EM_RAW_EMULATE_INSTR
88	*
89	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
90	* @param pGstWalk The guest page table walk result.
91	* @param uErr The error code.
92	*/
93	PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPU pVCpu, PGSTPTWALK pGstWalk, RTGCUINT uErr)
94	{
95	# if !defined(PGM_WITHOUT_MAPPINGS) && (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE)
96	/*
97	* Check for write conflicts with our hypervisor mapping.
98	*
99	* If the guest happens to access a non-present page, where our hypervisor
100	* is currently mapped, then we'll create a #PF storm in the guest.
101	*/
102	if ( (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW)
103	&& pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM))
104	&& MMHyperIsInsideArea(pVCpu->CTX_SUFF(pVM), pGstWalk->Core.GCPtr))
105	{
106	/* Force a CR3 sync to check for conflicts and emulate the instruction. */
107	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
108	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
109	return VINF_EM_RAW_EMULATE_INSTR;
110	}
111	# endif
112
113	/*
114	* Calc the error code for the guest trap.
115	*/
116	uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
117	? uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID)
118	: uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
119	if ( pGstWalk->Core.fRsvdError
120	\|\| pGstWalk->Core.fBadPhysAddr)
121	{
122	uNewErr \|= X86_TRAP_PF_RSVD \| X86_TRAP_PF_P;
123	Assert(!pGstWalk->Core.fNotPresent);
124	}
125	else if (!pGstWalk->Core.fNotPresent)
126	uNewErr \|= X86_TRAP_PF_P;
127	TRPMSetErrorCode(pVCpu, uNewErr);
128
129	LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pGstWalk->Core.GCPtr, uErr, pGstWalk->Core.uLevel));
130	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
131	return VINF_EM_RAW_GUEST_TRAP;
132	}
133	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
134
135
136	#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE)
137	/**
138	* Deal with a guest page fault.
139	*
140	* The caller has taken the PGM lock.
141	*
142	* @returns Strict VBox status code.
143	*
144	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
145	* @param uErr The error code.
146	* @param pRegFrame The register frame.
147	* @param pvFault The fault address.
148	* @param pPage The guest page at @a pvFault.
149	* @param pGstWalk The guest page table walk result.
150	* @param pfLockTaken PGM lock taken here or not (out). This is true
151	* when we're called.
152	*/
153	static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame,
154	RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
155	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
156	, PGSTPTWALK pGstWalk
157	# endif
158	)
159	{
160	# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
161	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
162	# endif
163	PVM pVM = pVCpu->CTX_SUFF(pVM);
164	VBOXSTRICTRC rcStrict;
165
166	if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
167	{
168	/*
169	* Physical page access handler.
170	*/
171	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
172	const RTGCPHYS GCPhysFault = pGstWalk->Core.GCPhys;
173	# else
174	const RTGCPHYS GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
175	# endif
176	PPGMPHYSHANDLER pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
177	if (pCur)
178	{
179	PPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
180
181	# ifdef PGM_SYNC_N_PAGES
182	/*
183	* If the region is write protected and we got a page not present fault, then sync
184	* the pages. If the fault was caused by a read, then restart the instruction.
185	* In case of write access continue to the GC write handler.
186	*
187	* ASSUMES that there is only one handler per page or that they have similar write properties.
188	*/
189	if ( !(uErr & X86_TRAP_PF_P)
190	&& pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
191	{
192	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
193	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
194	# else
195	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
196	# endif
197	if ( RT_FAILURE(rcStrict)
198	\|\| !(uErr & X86_TRAP_PF_RW)
199	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
200	{
201	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
202	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
203	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
204	return rcStrict;
205	}
206	}
207	# endif
208	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
209	/*
210	* If the access was not thru a #PF(RSVD\|...) resync the page.
211	*/
212	if ( !(uErr & X86_TRAP_PF_RSVD)
213	&& pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
214	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
215	&& pGstWalk->Core.fEffectiveRW
216	&& !pGstWalk->Core.fEffectiveUS /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
217	# endif
218	)
219	{
220	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
221	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
222	# else
223	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
224	# endif
225	if ( RT_FAILURE(rcStrict)
226	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
227	{
228	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
229	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
230	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
231	return rcStrict;
232	}
233	}
234	# endif
235
236	AssertMsg( pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
237	\|\| (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
238	("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
239	pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
240	if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
241	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysWrite);
242	else
243	{
244	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAll);
245	if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersPhysAllOpt);
246	}
247
248	if (pCurType->CTX_SUFF(pfnPfHandler))
249	{
250	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
251	void *pvUser = pCur->CTX_SUFF(pvUser);
252
253	STAM_PROFILE_START(&pCur->Stat, h);
254	if (pCur->hType != pPool->hAccessHandlerType)
255	{
256	pgmUnlock(pVM);
257	*pfLockTaken = false;
258	}
259
260	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPhysFault, pvUser);
261
262	# ifdef VBOX_WITH_STATISTICS
263	pgmLock(pVM);
264	pCur = pgmHandlerPhysicalLookup(pVM, GCPhysFault);
265	if (pCur)
266	STAM_PROFILE_STOP(&pCur->Stat, h);
267	pgmUnlock(pVM);
268	# endif
269	}
270	else
271	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
272
273	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndPhys; });
274	return rcStrict;
275	}
276	}
277	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
278	else
279	{
280	# ifdef PGM_SYNC_N_PAGES
281	/*
282	* If the region is write protected and we got a page not present fault, then sync
283	* the pages. If the fault was caused by a read, then restart the instruction.
284	* In case of write access continue to the GC write handler.
285	*/
286	if ( PGM_PAGE_GET_HNDL_VIRT_STATE(pPage) < PGM_PAGE_HNDL_PHYS_STATE_ALL
287	&& !(uErr & X86_TRAP_PF_P))
288	{
289	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
290	if ( RT_FAILURE(rcStrict)
291	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
292	\|\| !(uErr & X86_TRAP_PF_RW))
293	{
294	AssertRC(rcStrict);
295	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
296	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndVirt; });
297	return rcStrict;
298	}
299	}
300	# endif
301	/*
302	* Ok, it's an virtual page access handler.
303	*
304	* Since it's faster to search by address, we'll do that first
305	* and then retry by GCPhys if that fails.
306	*/
307	/** @todo r=bird: perhaps we should consider looking up by physical address directly now?
308	* r=svl: true, but lookup on virtual address should remain as a fallback as phys & virt trees might be
309	* out of sync, because the page was changed without us noticing it (not-present -> present
310	* without invlpg or mov cr3, xxx).
311	*/
312	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
313	if (pCur)
314	{
315	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
316	AssertMsg(!(pvFault - pCur->Core.Key < pCur->cb)
317	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
318	\|\| !(uErr & X86_TRAP_PF_P)
319	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
320	("Unexpected trap for virtual handler: %RGv (phys=%RGp) pPage=%R[pgmpage] uErr=%X, enumKind=%d\n",
321	pvFault, pGstWalk->Core.GCPhys, pPage, uErr, pCurType->enmKind));
322
323	if ( pvFault - pCur->Core.Key < pCur->cb
324	&& ( uErr & X86_TRAP_PF_RW
325	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
326	{
327	# ifdef IN_RC
328	STAM_PROFILE_START(&pCur->Stat, h);
329	RTGCPTR GCPtrStart = pCur->Core.Key;
330	void *pvUser = pCur->CTX_SUFF(pvUser);
331	pgmUnlock(pVM);
332	*pfLockTaken = false;
333
334	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart,
335	pvFault - GCPtrStart, pvUser);
336
337	# ifdef VBOX_WITH_STATISTICS
338	pgmLock(pVM);
339	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
340	if (pCur)
341	STAM_PROFILE_STOP(&pCur->Stat, h);
342	pgmUnlock(pVM);
343	# endif
344	# else
345	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
346	# endif
347	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtual);
348	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
349	return rcStrict;
350	}
351	/* Unhandled part of a monitored page */
352	Log(("Unhandled part of monitored page %RGv\n", pvFault));
353	}
354	else
355	{
356	/* Check by physical address. */
357	unsigned iPage;
358	pCur = pgmHandlerVirtualFindByPhysAddr(pVM, pGstWalk->Core.GCPhys, &iPage);
359	if (pCur)
360	{
361	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
362	if ( uErr & X86_TRAP_PF_RW
363	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE )
364	{
365	Assert( (pCur->aPhysToVirt[iPage].Core.Key & X86_PTE_PAE_PG_MASK)
366	== (pGstWalk->Core.GCPhys & X86_PTE_PAE_PG_MASK));
367	# ifdef IN_RC
368	STAM_PROFILE_START(&pCur->Stat, h);
369	RTGCPTR GCPtrStart = pCur->Core.Key;
370	void *pvUser = pCur->CTX_SUFF(pvUser);
371	pgmUnlock(pVM);
372	*pfLockTaken = false;
373
374	RTGCPTR off = (iPage << PAGE_SHIFT)
375	+ (pvFault & PAGE_OFFSET_MASK)
376	- (GCPtrStart & PAGE_OFFSET_MASK);
377	Assert(off < pCur->cb);
378	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, GCPtrStart, off, pvUser);
379
380	# ifdef VBOX_WITH_STATISTICS
381	pgmLock(pVM);
382	pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, GCPtrStart);
383	if (pCur)
384	STAM_PROFILE_STOP(&pCur->Stat, h);
385	pgmUnlock(pVM);
386	# endif
387	# else
388	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
389	# endif
390	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersVirtualByPhys);
391	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
392	return rcStrict;
393	}
394	}
395	}
396	}
397	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
398
399	/*
400	* There is a handled area of the page, but this fault doesn't belong to it.
401	* We must emulate the instruction.
402	*
403	* To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
404	* we first check if this was a page-not-present fault for a page with only
405	* write access handlers. Restart the instruction if it wasn't a write access.
406	*/
407	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersUnhandled);
408
409	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
410	&& !(uErr & X86_TRAP_PF_P))
411	{
412	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
413	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
414	# else
415	rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
416	# endif
417	if ( RT_FAILURE(rcStrict)
418	\|\| rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
419	\|\| !(uErr & X86_TRAP_PF_RW))
420	{
421	AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
422	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersOutOfSync);
423	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndPhys; });
424	return rcStrict;
425	}
426	}
427
428	/** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
429	* It's writing to an unhandled part of the LDT page several million times.
430	*/
431	rcStrict = PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault);
432	LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
433	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndUnhandled; });
434	return rcStrict;
435	} /* if any kind of handler */
436	# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) */
437
438
439	/**
440	* \#PF Handler for raw-mode guest execution.
441	*
442	* @returns VBox status code (appropriate for trap handling and GC return).
443	*
444	* @param pVCpu The cross context virtual CPU structure.
445	* @param uErr The trap error code.
446	* @param pRegFrame Trap register frame.
447	* @param pvFault The fault address.
448	* @param pfLockTaken PGM lock taken here or not (out)
449	*/
450	PGM_BTH_DECL(int, Trap0eHandler)(PVMCPU pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, bool *pfLockTaken)
451	{
452	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
453
454	*pfLockTaken = false;
455
456	# if ( PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT \
457	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
458	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
459	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT)
460	int rc;
461
462	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
463	/*
464	* Walk the guest page translation tables and check if it's a guest fault.
465	*/
466	GSTPTWALK GstWalk;
467	rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &GstWalk);
468	if (RT_FAILURE_NP(rc))
469	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
470
471	/* assert some GstWalk sanity. */
472	# if PGM_GST_TYPE == PGM_TYPE_AMD64
473	/AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u)); - not always true with SMP guests. /
474	# endif
475	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
476	/AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto /
477	# endif
478	/AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto /
479	/AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto /
480	Assert(GstWalk.Core.fSucceeded);
481
482	if (uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID))
483	{
484	if ( ( (uErr & X86_TRAP_PF_RW)
485	&& !GstWalk.Core.fEffectiveRW
486	&& ( (uErr & X86_TRAP_PF_US)
487	\|\| CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
488	\|\| ((uErr & X86_TRAP_PF_US) && !GstWalk.Core.fEffectiveUS)
489	\|\| ((uErr & X86_TRAP_PF_ID) && GstWalk.Core.fEffectiveNX)
490	)
491	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &GstWalk, uErr));
492	}
493
494	/*
495	* Set the accessed and dirty flags.
496	*/
497	# if PGM_GST_TYPE == PGM_TYPE_AMD64
498	GstWalk.Pml4e.u \|= X86_PML4E_A;
499	GstWalk.pPml4e->u \|= X86_PML4E_A;
500	GstWalk.Pdpe.u \|= X86_PDPE_A;
501	GstWalk.pPdpe->u \|= X86_PDPE_A;
502	# endif
503	if (GstWalk.Core.fBigPage)
504	{
505	Assert(GstWalk.Pde.b.u1Size);
506	if (uErr & X86_TRAP_PF_RW)
507	{
508	GstWalk.Pde.u \|= X86_PDE4M_A \| X86_PDE4M_D;
509	GstWalk.pPde->u \|= X86_PDE4M_A \| X86_PDE4M_D;
510	}
511	else
512	{
513	GstWalk.Pde.u \|= X86_PDE4M_A;
514	GstWalk.pPde->u \|= X86_PDE4M_A;
515	}
516	}
517	else
518	{
519	Assert(!GstWalk.Pde.b.u1Size);
520	GstWalk.Pde.u \|= X86_PDE_A;
521	GstWalk.pPde->u \|= X86_PDE_A;
522	if (uErr & X86_TRAP_PF_RW)
523	{
524	# ifdef VBOX_WITH_STATISTICS
525	if (!GstWalk.Pte.n.u1Dirty)
526	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtiedPage));
527	else
528	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageAlreadyDirty));
529	# endif
530	GstWalk.Pte.u \|= X86_PTE_A \| X86_PTE_D;
531	GstWalk.pPte->u \|= X86_PTE_A \| X86_PTE_D;
532	}
533	else
534	{
535	GstWalk.Pte.u \|= X86_PTE_A;
536	GstWalk.pPte->u \|= X86_PTE_A;
537	}
538	Assert(GstWalk.Pte.u == GstWalk.pPte->u);
539	}
540	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u,
541	("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
542	# else /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
543	GSTPDE const PdeSrcDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A}; /** @todo eliminate this */
544	# endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
545
546	/* Take the big lock now. */
547	*pfLockTaken = true;
548	pgmLock(pVM);
549
550	# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
551	/*
552	* If it is a reserved bit fault we know that it is an MMIO (access
553	* handler) related fault and can skip some 200 lines of code.
554	*/
555	if (uErr & X86_TRAP_PF_RSVD)
556	{
557	Assert(uErr & X86_TRAP_PF_P);
558	PPGMPAGE pPage;
559	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
560	rc = pgmPhysGetPageEx(pVM, GstWalk.Core.GCPhys, &pPage);
561	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
562	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
563	pfLockTaken, &GstWalk));
564	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
565	# else
566	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
567	if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
568	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage,
569	pfLockTaken));
570	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
571	# endif
572	AssertRC(rc);
573	PGM_INVL_PG(pVCpu, pvFault);
574	return rc; /* Restart with the corrected entry. */
575	}
576	# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
577
578	/*
579	* Fetch the guest PDE, PDPE and PML4E.
580	*/
581	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
582	const unsigned iPDDst = pvFault >> SHW_PD_SHIFT;
583	PX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
584
585	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
586	const unsigned iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK; /* pPDDst index, not used with the pool. */
587	PX86PDPAE pPDDst;
588	# if PGM_GST_TYPE == PGM_TYPE_PAE
589	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
590	# else
591	rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst); /* RW, US and A are reserved in PAE mode. */
592	# endif
593	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
594
595	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
596	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
597	PX86PDPAE pPDDst;
598	# if PGM_GST_TYPE == PGM_TYPE_PROT /* (AMD-V nested paging) */
599	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A,
600	X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A, &pPDDst);
601	# else
602	rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
603	# endif
604	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
605
606	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
607	const unsigned iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
608	PEPTPD pPDDst;
609	rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
610	AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
611	# endif
612	Assert(pPDDst);
613
614	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
615	/*
616	* Dirty page handling.
617	*
618	* If we successfully correct the write protection fault due to dirty bit
619	* tracking, then return immediately.
620	*/
621	if (uErr & X86_TRAP_PF_RW) /* write fault? */
622	{
623	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
624	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
625	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyBitTracking), a);
626	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
627	{
628	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution)
629	= rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
630	? &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2DirtyAndAccessed
631	: &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2GuestTrap; });
632	Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
633	return VINF_SUCCESS;
634	}
635	#ifdef DEBUG_bird
636	AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u \|\| GstWalk.pPte->u == GstWalk.pPde->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
637	AssertMsg(GstWalk.Core.fBigPage \|\| GstWalk.Pte.u == GstWalk.pPte->u \|\| pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
638	#endif
639	}
640
641	# if 0 /* rarely useful; leave for debugging. */
642	STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
643	# endif
644	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
645
646	/*
647	* A common case is the not-present error caused by lazy page table syncing.
648	*
649	* It is IMPORTANT that we weed out any access to non-present shadow PDEs
650	* here so we can safely assume that the shadow PT is present when calling
651	* SyncPage later.
652	*
653	* On failure, we ASSUME that SyncPT is out of memory or detected some kind
654	* of mapping conflict and defer to SyncCR3 in R3.
655	* (Again, we do NOT support access handlers for non-present guest pages.)
656	*
657	*/
658	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
659	Assert(GstWalk.Pde.n.u1Present);
660	# endif
661	if ( !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
662	&& !pPDDst->a[iPDDst].n.u1Present)
663	{
664	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2SyncPT; });
665	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
666	LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
667	rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
668	# else
669	LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
670	rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
671	# endif
672	if (RT_SUCCESS(rc))
673	return rc;
674	Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
675	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
676	return VINF_PGM_SYNC_CR3;
677	}
678
679	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(PGM_WITHOUT_MAPPINGS)
680	/*
681	* Check if this address is within any of our mappings.
682	*
683	* This is very fast and it's gonna save us a bit of effort below and prevent
684	* us from screwing ourself with MMIO2 pages which have a GC Mapping (VRam).
685	* (BTW, it's impossible to have physical access handlers in a mapping.)
686	*/
687	if (pgmMapAreMappingsEnabled(pVM))
688	{
689	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
690	for ( ; pMapping; pMapping = pMapping->CTX_SUFF(pNext))
691	{
692	if (pvFault < pMapping->GCPtr)
693	break;
694	if (pvFault - pMapping->GCPtr < pMapping->cb)
695	{
696	/*
697	* The first thing we check is if we've got an undetected conflict.
698	*/
699	if (pgmMapAreMappingsFloating(pVM))
700	{
701	unsigned iPT = pMapping->cb >> GST_PD_SHIFT;
702	while (iPT-- > 0)
703	if (GstWalk.pPde[iPT].n.u1Present)
704	{
705	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eConflicts);
706	Log(("Trap0e: Detected Conflict %RGv-%RGv\n", pMapping->GCPtr, pMapping->GCPtrLast));
707	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync,right? */
708	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
709	return VINF_PGM_SYNC_CR3;
710	}
711	}
712
713	/*
714	* Check if the fault address is in a virtual page access handler range.
715	*/
716	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->HyperVirtHandlers,
717	pvFault);
718	if ( pCur
719	&& pvFault - pCur->Core.Key < pCur->cb
720	&& uErr & X86_TRAP_PF_RW)
721	{
722	VBOXSTRICTRC rcStrict;
723	# ifdef IN_RC
724	STAM_PROFILE_START(&pCur->Stat, h);
725	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
726	void *pvUser = pCur->CTX_SUFF(pvUser);
727	pgmUnlock(pVM);
728	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
729	pvFault - pCur->Core.Key, pvUser);
730	pgmLock(pVM);
731	STAM_PROFILE_STOP(&pCur->Stat, h);
732	# else
733	AssertFailed();
734	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /* can't happen with VMX */
735	# endif
736	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersMapping);
737	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
738	return VBOXSTRICTRC_TODO(rcStrict);
739	}
740
741	/*
742	* Pretend we're not here and let the guest handle the trap.
743	*/
744	TRPMSetErrorCode(pVCpu, uErr & ~X86_TRAP_PF_P);
745	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eGuestPFMapping);
746	LogFlow(("PGM: Mapping access -> route trap to recompiler!\n"));
747	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Mapping; });
748	return VINF_EM_RAW_GUEST_TRAP;
749	}
750	}
751	} /* pgmAreMappingsEnabled(&pVM->pgm.s) */
752	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
753
754	/*
755	* Check if this fault address is flagged for special treatment,
756	* which means we'll have to figure out the physical address and
757	* check flags associated with it.
758	*
759	* ASSUME that we can limit any special access handling to pages
760	* in page tables which the guest believes to be present.
761	*/
762	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
763	RTGCPHYS GCPhys = GstWalk.Core.GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
764	# else
765	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)PAGE_OFFSET_MASK);
766	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
767	PPGMPAGE pPage;
768	rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
769	if (RT_FAILURE(rc))
770	{
771	/*
772	* When the guest accesses invalid physical memory (e.g. probing
773	* of RAM or accessing a remapped MMIO range), then we'll fall
774	* back to the recompiler to emulate the instruction.
775	*/
776	LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
777	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eHandlersInvalid);
778	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2InvalidPhys; });
779	return VINF_EM_RAW_EMULATE_INSTR;
780	}
781
782	/*
783	* Any handlers for this page?
784	*/
785	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
786	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
787	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken,
788	&GstWalk));
789	# else
790	return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pRegFrame, pvFault, pPage, pfLockTaken));
791	# endif
792
793	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
794	if (uErr & X86_TRAP_PF_P)
795	{
796	/*
797	* The page isn't marked, but it might still be monitored by a virtual page access handler.
798	* (ASSUMES no temporary disabling of virtual handlers.)
799	*/
800	/** @todo r=bird: Since the purpose is to catch out of sync pages with virtual handler(s) here,
801	* we should correct both the shadow page table and physical memory flags, and not only check for
802	* accesses within the handler region but for access to pages with virtual handlers. */
803	PPGMVIRTHANDLER pCur = (PPGMVIRTHANDLER)RTAvlroGCPtrRangeGet(&pVM->pgm.s.CTX_SUFF(pTrees)->VirtHandlers, pvFault);
804	if (pCur)
805	{
806	PPGMVIRTHANDLERTYPEINT pCurType = PGMVIRTANDLER_GET_TYPE(pVM, pCur);
807	AssertMsg( !(pvFault - pCur->Core.Key < pCur->cb)
808	\|\| ( pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE
809	\|\| !(uErr & X86_TRAP_PF_P)
810	\|\| (pCurType->enmKind == PGMVIRTHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW))),
811	("Unexpected trap for virtual handler: %08X (phys=%08x) %R[pgmpage] uErr=%X, enumKind=%d\n",
812	pvFault, GCPhys, pPage, uErr, pCurType->enmKind));
813
814	if ( pvFault - pCur->Core.Key < pCur->cb
815	&& ( uErr & X86_TRAP_PF_RW
816	\|\| pCurType->enmKind != PGMVIRTHANDLERKIND_WRITE ) )
817	{
818	VBOXSTRICTRC rcStrict;
819	# ifdef IN_RC
820	STAM_PROFILE_START(&pCur->Stat, h);
821	void *pvUser = pCur->CTX_SUFF(pvUser);
822	pgmUnlock(pVM);
823	rcStrict = pCurType->CTX_SUFF(pfnPfHandler)(pVM, pVCpu, uErr, pRegFrame, pvFault, pCur->Core.Key,
824	pvFault - pCur->Core.Key, pvUser);
825	pgmLock(pVM);
826	STAM_PROFILE_STOP(&pCur->Stat, h);
827	# else
828	rcStrict = VINF_EM_RAW_EMULATE_INSTR; /** @todo for VMX */
829	# endif
830	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2HndVirt; });
831	return VBOXSTRICTRC_TODO(rcStrict);
832	}
833	}
834	}
835	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
836
837	/*
838	* We are here only if page is present in Guest page tables and
839	* trap is not handled by our handlers.
840	*
841	* Check it for page out-of-sync situation.
842	*/
843	if (!(uErr & X86_TRAP_PF_P))
844	{
845	/*
846	* Page is not present in our page tables. Try to sync it!
847	*/
848	if (uErr & X86_TRAP_PF_US)
849	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
850	else /* supervisor */
851	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
852
853	if (PGM_PAGE_IS_BALLOONED(pPage))
854	{
855	/* Emulate reads from ballooned pages as they are not present in
856	our shadow page tables. (Required for e.g. Solaris guests; soft
857	ecc, random nr generator.) */
858	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
859	LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
860	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncBallloon));
861	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Ballooned; });
862	return rc;
863	}
864
865	# if defined(LOG_ENABLED) && !defined(IN_RING0)
866	RTGCPHYS GCPhys2;
867	uint64_t fPageGst2;
868	PGMGstGetPage(pVCpu, pvFault, &fPageGst2, &GCPhys2);
869	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
870	Log(("Page out of sync: %RGv eip=%08x PdeSrc.US=%d fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
871	pvFault, pRegFrame->eip, GstWalk.Pde.n.u1User, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
872	# else
873	Log(("Page out of sync: %RGv eip=%08x fPageGst2=%08llx GCPhys2=%RGp scan=%d\n",
874	pvFault, pRegFrame->eip, fPageGst2, GCPhys2, CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)));
875	# endif
876	# endif /* LOG_ENABLED */
877
878	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0)
879	if ( !GstWalk.Core.fEffectiveUS
880	&& CSAMIsEnabled(pVM)
881	&& CPUMGetGuestCPL(pVCpu) == 0)
882	{
883	/* Note: Can't check for X86_TRAP_ID bit, because that requires execute disable support on the CPU. */
884	if ( pvFault == (RTGCPTR)pRegFrame->eip
885	\|\| pvFault - pRegFrame->eip < 8 /* instruction crossing a page boundary */
886	# ifdef CSAM_DETECT_NEW_CODE_PAGES
887	\|\| ( !PATMIsPatchGCAddr(pVM, pRegFrame->eip)
888	&& CSAMDoesPageNeedScanning(pVM, pRegFrame->eip)) /* any new code we encounter here */
889	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
890	)
891	{
892	LogFlow(("CSAMExecFault %RX32\n", pRegFrame->eip));
893	rc = CSAMExecFault(pVM, (RTRCPTR)pRegFrame->eip);
894	if (rc != VINF_SUCCESS)
895	{
896	/*
897	* CSAM needs to perform a job in ring 3.
898	*
899	* Sync the page before going to the host context; otherwise we'll end up in a loop if
900	* CSAM fails (e.g. instruction crosses a page boundary and the next page is not present)
901	*/
902	LogFlow(("CSAM ring 3 job\n"));
903	int rc2 = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
904	AssertRC(rc2);
905
906	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2CSAM; });
907	return rc;
908	}
909	}
910	# ifdef CSAM_DETECT_NEW_CODE_PAGES
911	else if ( uErr == X86_TRAP_PF_RW
912	&& pRegFrame->ecx >= 0x100 /* early check for movswd count */
913	&& pRegFrame->ecx < 0x10000)
914	{
915	/* In case of a write to a non-present supervisor shadow page, we'll take special precautions
916	* to detect loading of new code pages.
917	*/
918
919	/*
920	* Decode the instruction.
921	*/
922	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
923	uint32_t cbOp;
924	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
925
926	/* For now we'll restrict this to rep movsw/d instructions */
927	if ( rc == VINF_SUCCESS
928	&& pDis->pCurInstr->opcode == OP_MOVSWD
929	&& (pDis->prefix & DISPREFIX_REP))
930	{
931	CSAMMarkPossibleCodePage(pVM, pvFault);
932	}
933	}
934	# endif /* CSAM_DETECT_NEW_CODE_PAGES */
935
936	/*
937	* Mark this page as safe.
938	*/
939	/** @todo not correct for pages that contain both code and data!! */
940	Log2(("CSAMMarkPage %RGv; scanned=%d\n", pvFault, true));
941	CSAMMarkPage(pVM, pvFault, true);
942	}
943	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && !defined(IN_RING0) */
944	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
945	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
946	# else
947	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
948	# endif
949	if (RT_SUCCESS(rc))
950	{
951	/* The page was successfully synced, return to the guest. */
952	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSync; });
953	return VINF_SUCCESS;
954	}
955	}
956	else /* uErr & X86_TRAP_PF_P: */
957	{
958	/*
959	* Write protected pages are made writable when the guest makes the
960	* first write to it. This happens for pages that are shared, write
961	* monitored or not yet allocated.
962	*
963	* We may also end up here when CR0.WP=0 in the guest.
964	*
965	* Also, a side effect of not flushing global PDEs are out of sync
966	* pages due to physical monitored regions, that are no longer valid.
967	* Assume for now it only applies to the read/write flag.
968	*/
969	if (uErr & X86_TRAP_PF_RW)
970	{
971	/*
972	* Check if it is a read-only page.
973	*/
974	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
975	{
976	Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
977	Assert(!PGM_PAGE_IS_ZERO(pPage));
978	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
979	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2MakeWritable; });
980
981	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
982	if (rc != VINF_SUCCESS)
983	{
984	AssertMsg(rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("%Rrc\n", rc));
985	return rc;
986	}
987	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
988	return VINF_EM_NO_MEMORY;
989	}
990
991	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
992	/*
993	* Check to see if we need to emulate the instruction if CR0.WP=0.
994	*/
995	if ( !GstWalk.Core.fEffectiveRW
996	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
997	&& CPUMGetGuestCPL(pVCpu) < 3)
998	{
999	Assert((uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RW \| X86_TRAP_PF_P));
1000
1001	/*
1002	* The Netware WP0+RO+US hack.
1003	*
1004	* Netware sometimes(/always?) runs with WP0. It has been observed doing
1005	* excessive write accesses to pages which are mapped with US=1 and RW=0
1006	* while WP=0. This causes a lot of exits and extremely slow execution.
1007	* To avoid trapping and emulating every write here, we change the shadow
1008	* page table entry to map it as US=0 and RW=1 until user mode tries to
1009	* access it again (see further below). We count these shadow page table
1010	* changes so we can avoid having to clear the page pool every time the WP
1011	* bit changes to 1 (see PGMCr0WpEnabled()).
1012	*/
1013	# if (PGM_GST_TYPE == PGM_TYPE_32BIT \|\| PGM_GST_TYPE == PGM_TYPE_PAE) && 1
1014	if ( GstWalk.Core.fEffectiveUS
1015	&& !GstWalk.Core.fEffectiveRW
1016	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1017	&& pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
1018	{
1019	Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, GstWalk.Core.fBigPage));
1020	rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, GstWalk.Core.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
1021	if (rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3)
1022	{
1023	PGM_INVL_PG(pVCpu, pvFault);
1024	pVCpu->pgm.s.cNetwareWp0Hacks++;
1025	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsHack; });
1026	return rc;
1027	}
1028	AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
1029	Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
1030	}
1031	# endif
1032
1033	/* Interpret the access. */
1034	rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVM, pVCpu, pRegFrame, pvFault));
1035	Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), GstWalk.Core.fBigPage, GstWalk.Core.fEffectiveUS));
1036	if (RT_SUCCESS(rc))
1037	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulInRZ);
1038	else
1039	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eWPEmulToR3);
1040	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2WPEmulation; });
1041	return rc;
1042	}
1043	# endif
1044	/// @todo count the above case; else
1045	if (uErr & X86_TRAP_PF_US)
1046	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
1047	else /* supervisor */
1048	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
1049
1050	/*
1051	* Sync the page.
1052	*
1053	* Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
1054	* page is not present, which is not true in this case.
1055	*/
1056	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1057	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1058	# else
1059	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
1060	# endif
1061	if (RT_SUCCESS(rc))
1062	{
1063	/*
1064	* Page was successfully synced, return to guest but invalidate
1065	* the TLB first as the page is very likely to be in it.
1066	*/
1067	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1068	HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
1069	# else
1070	PGM_INVL_PG(pVCpu, pvFault);
1071	# endif
1072	# ifdef VBOX_STRICT
1073	RTGCPHYS GCPhys2 = RTGCPHYS_MAX;
1074	uint64_t fPageGst = UINT64_MAX;
1075	if (!pVM->pgm.s.fNestedPaging)
1076	{
1077	rc = PGMGstGetPage(pVCpu, pvFault, &fPageGst, &GCPhys2);
1078	AssertMsg(RT_SUCCESS(rc) && ((fPageGst & X86_PTE_RW) \|\| ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, fPageGst));
1079	LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GCPhys2, (uint64_t)fPageGst));
1080	}
1081	# if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
1082	uint64_t fPageShw = 0;
1083	rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1084	AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) \|\| pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
1085	("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GCPhys2, fPageGst, pvFault));
1086	# endif
1087	# endif /* VBOX_STRICT */
1088	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2OutOfSyncHndObs; });
1089	return VINF_SUCCESS;
1090	}
1091	}
1092	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1093	/*
1094	* Check for Netware WP0+RO+US hack from above and undo it when user
1095	* mode accesses the page again.
1096	*/
1097	else if ( GstWalk.Core.fEffectiveUS
1098	&& !GstWalk.Core.fEffectiveRW
1099	&& (GstWalk.Core.fBigPage \|\| GstWalk.Pde.n.u1Write)
1100	&& pVCpu->pgm.s.cNetwareWp0Hacks > 0
1101	&& (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP \| X86_CR0_PG)) == X86_CR0_PG
1102	&& CPUMGetGuestCPL(pVCpu) == 3
1103	&& pVM->cCpus == 1
1104	)
1105	{
1106	Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
1107	rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
1108	if (RT_SUCCESS(rc))
1109	{
1110	PGM_INVL_PG(pVCpu, pvFault);
1111	pVCpu->pgm.s.cNetwareWp0Hacks--;
1112	STAM_STATS({ pVCpu->pgm.s.CTX_SUFF(pStatTrap0eAttribution) = &pVCpu->pgm.s.CTX_SUFF(pStats)->StatRZTrap0eTime2Wp0RoUsUnhack; });
1113	return VINF_SUCCESS;
1114	}
1115	}
1116	# endif /* PGM_WITH_PAGING */
1117
1118	/** @todo else: why are we here? */
1119
1120	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
1121	/*
1122	* Check for VMM page flags vs. Guest page flags consistency.
1123	* Currently only for debug purposes.
1124	*/
1125	if (RT_SUCCESS(rc))
1126	{
1127	/* Get guest page flags. */
1128	uint64_t fPageGst;
1129	int rc2 = PGMGstGetPage(pVCpu, pvFault, &fPageGst, NULL);
1130	if (RT_SUCCESS(rc2))
1131	{
1132	uint64_t fPageShw = 0;
1133	rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
1134
1135	#if 0
1136	/*
1137	* Compare page flags.
1138	* Note: we have AVL, A, D bits desynced.
1139	*/
1140	AssertMsg( (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1141	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK))
1142	\|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0
1143	&& (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1144	== (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US))
1145	&& (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW
1146	&& (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US),
1147	("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
1148	pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
1149	01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
1150	01:01:15.623511 00:08:43.266064 Location : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
1151
1152	01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK)) \|\| ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) == (fPageGst & ~(X86_PTE_A \| X86_PTE_D \| X86_PTE_AVL_MASK \| X86_PTE_RW \| X86_PTE_US)) && (fPageShw & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_US)
1153	01:01:15.625516 00:08:43.268051 Location :
1154	e:\vbox\svn\trunk\srcPage flags mismatch!
1155	pvFault=fffff801b0d7b000
1156	uErr=11 X86_TRAP_PF_ID \| X86_TRAP_PF_P
1157	GCPhys=0000000019b52000
1158	fPageShw=0
1159	fPageGst=77b0000000000121
1160	rc=0
1161	#endif
1162
1163	}
1164	else
1165	AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
1166	}
1167	else
1168	AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
1169	# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
1170	}
1171
1172
1173	/*
1174	* If we get here it is because something failed above, i.e. most like guru
1175	* meditiation time.
1176	*/
1177	LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
1178	__PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pRegFrame->cs.Sel, pRegFrame->rip));
1179	return rc;
1180
1181	# else /* Nested paging, EPT except PGM_GST_TYPE = PROT */
1182	NOREF(uErr); NOREF(pRegFrame); NOREF(pvFault);
1183	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
1184	return VERR_PGM_NOT_USED_IN_MODE;
1185	# endif
1186	}
1187	#endif /* !IN_RING3 */
1188
1189
1190	/**
1191	* Emulation of the invlpg instruction.
1192	*
1193	*
1194	* @returns VBox status code.
1195	*
1196	* @param pVCpu The cross context virtual CPU structure.
1197	* @param GCPtrPage Page to invalidate.
1198	*
1199	* @remark ASSUMES that the guest is updating before invalidating. This order
1200	* isn't required by the CPU, so this is speculative and could cause
1201	* trouble.
1202	* @remark No TLB shootdown is done on any other VCPU as we assume that
1203	* invlpg emulation is the only reason for calling this function.
1204	* (The guest has to shoot down TLB entries on other CPUs itself)
1205	* Currently true, but keep in mind!
1206	*
1207	* @todo Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
1208	* Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
1209	*/
1210	PGM_BTH_DECL(int, InvalidatePage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
1211	{
1212	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1213	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
1214	int rc;
1215	PVM pVM = pVCpu->CTX_SUFF(pVM);
1216	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1217
1218	PGM_LOCK_ASSERT_OWNER(pVM);
1219
1220	LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
1221
1222	/*
1223	* Get the shadow PD entry and skip out if this PD isn't present.
1224	* (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
1225	*/
1226	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1227	const unsigned iPDDst = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
1228	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1229
1230	/* Fetch the pgm pool shadow descriptor. */
1231	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1232	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1233	if (!pShwPde)
1234	{
1235	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1236	return VINF_SUCCESS;
1237	}
1238	# else
1239	Assert(pShwPde);
1240	# endif
1241
1242	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1243	const unsigned iPdpt = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
1244	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
1245
1246	/* If the shadow PDPE isn't present, then skip the invalidate. */
1247	# ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
1248	if (!pPdptDst \|\| !pPdptDst->a[iPdpt].n.u1Present)
1249	# else
1250	if (!pPdptDst->a[iPdpt].n.u1Present)
1251	# endif
1252	{
1253	Assert(!pPdptDst \|\| !(pPdptDst->a[iPdpt].u & PGM_PLXFLAGS_MAPPING));
1254	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1255	PGM_INVL_PG(pVCpu, GCPtrPage);
1256	return VINF_SUCCESS;
1257	}
1258
1259	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1260	PPGMPOOLPAGE pShwPde = NULL;
1261	PX86PDPAE pPDDst;
1262
1263	/* Fetch the pgm pool shadow descriptor. */
1264	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1265	AssertRCSuccessReturn(rc, rc);
1266	Assert(pShwPde);
1267
1268	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1269	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1270
1271	# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1272	/* PML4 */
1273	/const unsigned iPml4 = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;/
1274	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1275	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1276	PX86PDPAE pPDDst;
1277	PX86PDPT pPdptDst;
1278	PX86PML4E pPml4eDst;
1279	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
1280	if (rc != VINF_SUCCESS)
1281	{
1282	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT \|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
1283	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1284	PGM_INVL_PG(pVCpu, GCPtrPage);
1285	return VINF_SUCCESS;
1286	}
1287	Assert(pPDDst);
1288
1289	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1290	PX86PDPE pPdpeDst = &pPdptDst->a[iPdpt];
1291
1292	if (!pPdpeDst->n.u1Present)
1293	{
1294	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1295	PGM_INVL_PG(pVCpu, GCPtrPage);
1296	return VINF_SUCCESS;
1297	}
1298
1299	/* Fetch the pgm pool shadow descriptor. */
1300	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
1301	Assert(pShwPde);
1302
1303	# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
1304
1305	const SHWPDE PdeDst = *pPdeDst;
1306	if (!PdeDst.n.u1Present)
1307	{
1308	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1309	PGM_INVL_PG(pVCpu, GCPtrPage);
1310	return VINF_SUCCESS;
1311	}
1312
1313	/*
1314	* Get the guest PD entry and calc big page.
1315	*/
1316	# if PGM_GST_TYPE == PGM_TYPE_32BIT
1317	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
1318	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
1319	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
1320	# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1321	unsigned iPDSrc = 0;
1322	# if PGM_GST_TYPE == PGM_TYPE_PAE
1323	X86PDPE PdpeSrcIgn;
1324	PX86PDPAE pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
1325	# else /* AMD64 */
1326	PX86PML4E pPml4eSrcIgn;
1327	X86PDPE PdpeSrcIgn;
1328	PX86PDPAE pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
1329	# endif
1330	GSTPDE PdeSrc;
1331
1332	if (pPDSrc)
1333	PdeSrc = pPDSrc->a[iPDSrc];
1334	else
1335	PdeSrc.u = 0;
1336	# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
1337	const bool fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
1338	const bool fIsBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
1339	if (fWasBigPage != fIsBigPage)
1340	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1341
1342	# ifdef IN_RING3
1343	/*
1344	* If a CR3 Sync is pending we may ignore the invalidate page operation
1345	* depending on the kind of sync and if it's a global page or not.
1346	* This doesn't make sense in GC/R0 so we'll skip it entirely there.
1347	*/
1348	# ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
1349	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
1350	\|\| ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
1351	&& fIsBigPage
1352	&& PdeSrc.b.u1Global
1353	)
1354	)
1355	# else
1356	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
1357	# endif
1358	{
1359	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePageSkipped));
1360	return VINF_SUCCESS;
1361	}
1362	# endif /* IN_RING3 */
1363
1364	/*
1365	* Deal with the Guest PDE.
1366	*/
1367	rc = VINF_SUCCESS;
1368	if (PdeSrc.n.u1Present)
1369	{
1370	Assert( PdeSrc.n.u1User == PdeDst.n.u1User
1371	&& (PdeSrc.n.u1Write \|\| !PdeDst.n.u1Write \|\| pVCpu->pgm.s.cNetwareWp0Hacks > 0));
1372	# ifndef PGM_WITHOUT_MAPPING
1373	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
1374	{
1375	/*
1376	* Conflict - Let SyncPT deal with it to avoid duplicate code.
1377	*/
1378	Assert(pgmMapAreMappingsEnabled(pVM));
1379	Assert(PGMGetGuestMode(pVCpu) <= PGMMODE_PAE);
1380	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
1381	}
1382	else
1383	# endif /* !PGM_WITHOUT_MAPPING */
1384	if (!fIsBigPage)
1385	{
1386	/*
1387	* 4KB - page.
1388	*/
1389	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1390	RTGCPHYS GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
1391
1392	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1393	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
1394	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
1395	# endif
1396	if (pShwPage->GCPhys == GCPhys)
1397	{
1398	/* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
1399	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1400
1401	PGSTPT pPTSrc;
1402	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
1403	if (RT_SUCCESS(rc))
1404	{
1405	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
1406	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
1407	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
1408	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
1409	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
1410	GCPtrPage, PteSrc.n.u1Present,
1411	PteSrc.n.u1Write & PdeSrc.n.u1Write,
1412	PteSrc.n.u1User & PdeSrc.n.u1User,
1413	(uint64_t)PteSrc.u,
1414	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
1415	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
1416	}
1417	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4KBPages));
1418	PGM_INVL_PG(pVCpu, GCPtrPage);
1419	}
1420	else
1421	{
1422	/*
1423	* The page table address changed.
1424	*/
1425	LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
1426	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
1427	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1428	ASMAtomicWriteSize(pPdeDst, 0);
1429	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
1430	PGM_INVL_VCPU_TLBS(pVCpu);
1431	}
1432	}
1433	else
1434	{
1435	/*
1436	* 2/4MB - page.
1437	*/
1438	/* Before freeing the page, check if anything really changed. */
1439	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
1440	RTGCPHYS GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
1441	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
1442	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
1443	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
1444	# endif
1445	if ( pShwPage->GCPhys == GCPhys
1446	&& pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
1447	{
1448	/* ASSUMES a the given bits are identical for 4M and normal PDEs */
1449	/** @todo This test is wrong as it cannot check the G bit!
1450	* FIXME */
1451	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1452	== (PdeDst.u & (X86_PDE_P \| X86_PDE_RW \| X86_PDE_US))
1453	&& ( PdeSrc.b.u1Dirty /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
1454	\|\| (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
1455	{
1456	LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
1457	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
1458	return VINF_SUCCESS;
1459	}
1460	}
1461
1462	/*
1463	* Ok, the page table is present and it's been changed in the guest.
1464	* If we're in host context, we'll just mark it as not present taking the lazy approach.
1465	* We could do this for some flushes in GC too, but we need an algorithm for
1466	* deciding which 4MB pages containing code likely to be executed very soon.
1467	*/
1468	LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
1469	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
1470	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1471	ASMAtomicWriteSize(pPdeDst, 0);
1472	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePage4MBPages));
1473	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
1474	}
1475	}
1476	else
1477	{
1478	/*
1479	* Page directory is not present, mark shadow PDE not present.
1480	*/
1481	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
1482	{
1483	pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
1484	ASMAtomicWriteSize(pPdeDst, 0);
1485	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDNPs));
1486	PGM_INVL_PG(pVCpu, GCPtrPage);
1487	}
1488	else
1489	{
1490	Assert(pgmMapAreMappingsEnabled(pVM));
1491	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,InvalidatePagePDMappings));
1492	}
1493	}
1494	return rc;
1495
1496	#else /* guest real and protected mode */
1497	/* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
1498	NOREF(pVCpu); NOREF(GCPtrPage);
1499	return VINF_SUCCESS;
1500	#endif
1501	}
1502
1503
1504	/**
1505	* Update the tracking of shadowed pages.
1506	*
1507	* @param pVCpu The cross context virtual CPU structure.
1508	* @param pShwPage The shadow page.
1509	* @param HCPhys The physical page we is being dereferenced.
1510	* @param iPte Shadow PTE index
1511	* @param GCPhysPage Guest physical address (only valid if pShwPage->fDirty is set)
1512	*/
1513	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
1514	RTGCPHYS GCPhysPage)
1515	{
1516	PVM pVM = pVCpu->CTX_SUFF(pVM);
1517
1518	# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1519	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1520	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1521
1522	/* Use the hint we retrieved from the cached guest PT. */
1523	if (pShwPage->fDirty)
1524	{
1525	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1526
1527	Assert(pShwPage->cPresent);
1528	Assert(pPool->cPresent);
1529	pShwPage->cPresent--;
1530	pPool->cPresent--;
1531
1532	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
1533	AssertRelease(pPhysPage);
1534	pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
1535	return;
1536	}
1537	# else
1538	NOREF(GCPhysPage);
1539	# endif
1540
1541	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1542	LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
1543
1544	/** @todo If this turns out to be a bottle neck (very likely) two things can be done:
1545	* 1. have a medium sized HCPhys -> GCPhys TLB (hash?)
1546	* 2. write protect all shadowed pages. I.e. implement caching.
1547	*/
1548	/** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
1549
1550	/*
1551	* Find the guest address.
1552	*/
1553	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
1554	pRam;
1555	pRam = pRam->CTX_SUFF(pNext))
1556	{
1557	unsigned iPage = pRam->cb >> PAGE_SHIFT;
1558	while (iPage-- > 0)
1559	{
1560	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
1561	{
1562	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1563
1564	Assert(pShwPage->cPresent);
1565	Assert(pPool->cPresent);
1566	pShwPage->cPresent--;
1567	pPool->cPresent--;
1568
1569	pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
1570	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackDeref, a);
1571	return;
1572	}
1573	}
1574	}
1575
1576	for (;;)
1577	AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
1578	}
1579
1580
1581	/**
1582	* Update the tracking of shadowed pages.
1583	*
1584	* @param pVCpu The cross context virtual CPU structure.
1585	* @param pShwPage The shadow page.
1586	* @param u16 The top 16-bit of the pPage->HCPhys.
1587	* @param pPage Pointer to the guest page. this will be modified.
1588	* @param iPTDst The index into the shadow table.
1589	*/
1590	DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPU pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16, PPGMPAGE pPage, const unsigned iPTDst)
1591	{
1592	PVM pVM = pVCpu->CTX_SUFF(pVM);
1593
1594	/*
1595	* Just deal with the simple first time here.
1596	*/
1597	if (!u16)
1598	{
1599	STAM_COUNTER_INC(&pVM->pgm.s.CTX_SUFF(pStats)->StatTrackVirgin);
1600	u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
1601	/* Save the page table index. */
1602	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
1603	}
1604	else
1605	u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
1606
1607	/* write back */
1608	Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
1609	PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
1610
1611	/* update statistics. */
1612	pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
1613	pShwPage->cPresent++;
1614	if (pShwPage->iFirstPresent > iPTDst)
1615	pShwPage->iFirstPresent = iPTDst;
1616	}
1617
1618
1619	/**
1620	* Modifies a shadow PTE to account for access handlers.
1621	*
1622	* @param pVM The cross context VM structure.
1623	* @param pPage The page in question.
1624	* @param fPteSrc The shadowed flags of the source PTE. Must include the
1625	* A (accessed) bit so it can be emulated correctly.
1626	* @param pPteDst The shadow PTE (output). This is temporary storage and
1627	* does not need to be set atomically.
1628	*/
1629	DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVM pVM, PCPGMPAGE pPage, uint64_t fPteSrc, PSHWPTE pPteDst)
1630	{
1631	NOREF(pVM); RT_NOREF_PV(fPteSrc);
1632
1633	/** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
1634	* Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
1635	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
1636	{
1637	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
1638	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1639	pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage);
1640	pPteDst->n.u1Present = 1;
1641	pPteDst->n.u1Execute = 1;
1642	pPteDst->n.u1IgnorePAT = 1;
1643	pPteDst->n.u3EMT = VMX_EPT_MEMTYPE_WB;
1644	/* PteDst.n.u1Write = 0 && PteDst.n.u1Size = 0 */
1645	#else
1646	if (fPteSrc & X86_PTE_A)
1647	{
1648	SHW_PTE_SET(*pPteDst, fPteSrc \| PGM_PAGE_GET_HCPHYS(pPage));
1649	SHW_PTE_SET_RO(*pPteDst);
1650	}
1651	else
1652	SHW_PTE_SET(*pPteDst, 0);
1653	#endif
1654	}
1655	#ifdef PGM_WITH_MMIO_OPTIMIZATIONS
1656	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
1657	else if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
1658	&& ( BTH_IS_NP_ACTIVE(pVM)
1659	\|\| (fPteSrc & (X86_PTE_RW \| X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
1660	# if PGM_SHW_TYPE == PGM_TYPE_AMD64
1661	&& pVM->pgm.s.fLessThan52PhysicalAddressBits
1662	# endif
1663	)
1664	{
1665	LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
1666	# if PGM_SHW_TYPE == PGM_TYPE_EPT
1667	/* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
1668	pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg;
1669	/* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
1670	pPteDst->n.u1Present = 0;
1671	pPteDst->n.u1Write = 1;
1672	pPteDst->n.u1Execute = 0;
1673	/* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
1674	pPteDst->n.u3EMT = 7;
1675	# else
1676	/* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64). */
1677	SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg \| X86_PTE_PAE_MBZ_MASK_NO_NX \| X86_PTE_P);
1678	# endif
1679	}
1680	# endif
1681	#endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
1682	else
1683	{
1684	LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
1685	SHW_PTE_SET(*pPteDst, 0);
1686	}
1687	/** @todo count these kinds of entries. */
1688	}
1689
1690
1691	/**
1692	* Creates a 4K shadow page for a guest page.
1693	*
1694	* For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
1695	* physical address. The PdeSrc argument only the flags are used. No page
1696	* structured will be mapped in this function.
1697	*
1698	* @param pVCpu The cross context virtual CPU structure.
1699	* @param pPteDst Destination page table entry.
1700	* @param PdeSrc Source page directory entry (i.e. Guest OS page directory entry).
1701	* Can safely assume that only the flags are being used.
1702	* @param PteSrc Source page table entry (i.e. Guest OS page table entry).
1703	* @param pShwPage Pointer to the shadow page.
1704	* @param iPTDst The index into the shadow table.
1705	*
1706	* @remark Not used for 2/4MB pages!
1707	*/
1708	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \|\| defined(DOXYGEN_RUNNING)
1709	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
1710	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1711	#else
1712	static void PGM_BTH_NAME(SyncPageWorker)(PVMCPU pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
1713	PPGMPOOLPAGE pShwPage, unsigned iPTDst)
1714	#endif
1715	{
1716	PVM pVM = pVCpu->CTX_SUFF(pVM);
1717	RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
1718
1719	#if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
1720	&& PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
1721	&& (PGM_GST_TYPE == PGM_TYPE_PAE \|\| PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
1722
1723	if (pShwPage->fDirty)
1724	{
1725	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1726	PGSTPT pGstPT;
1727
1728	/* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
1729	pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
1730	GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
1731	pGstPT->a[iPTDst].u = PteSrc.u;
1732	}
1733	#else
1734	Assert(!pShwPage->fDirty);
1735	#endif
1736
1737	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1738	if ( PteSrc.n.u1Present
1739	&& GST_IS_PTE_VALID(pVCpu, PteSrc))
1740	#endif
1741	{
1742	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1743	RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
1744	# endif
1745	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
1746
1747	/*
1748	* Find the ram range.
1749	*/
1750	PPGMPAGE pPage;
1751	int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
1752	if (RT_SUCCESS(rc))
1753	{
1754	/* Ignore ballooned pages.
1755	Don't return errors or use a fatal assert here as part of a
1756	shadow sync range might included ballooned pages. */
1757	if (PGM_PAGE_IS_BALLOONED(pPage))
1758	{
1759	Assert(!SHW_PTE_IS_P(pPteDst)); /* @todo user tracking needs updating if this triggers. */
1760	return;
1761	}
1762
1763	#ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
1764	/* Make the page writable if necessary. */
1765	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
1766	&& ( PGM_PAGE_IS_ZERO(pPage)
1767	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1768	\|\| ( PteSrc.n.u1Write
1769	# else
1770	\|\| ( 1
1771	# endif
1772	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
1773	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
1774	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
1775	# endif
1776	# ifdef VBOX_WITH_PAGE_SHARING
1777	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
1778	# endif
1779	)
1780	)
1781	)
1782	{
1783	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
1784	AssertRC(rc);
1785	}
1786	#endif
1787
1788	/*
1789	* Make page table entry.
1790	*/
1791	SHWPTE PteDst;
1792	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1793	uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
1794	# else
1795	uint64_t fGstShwPteFlags = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D;
1796	# endif
1797	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
1798	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, fGstShwPteFlags, &PteDst);
1799	else
1800	{
1801	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1802	/*
1803	* If the page or page directory entry is not marked accessed,
1804	* we mark the page not present.
1805	*/
1806	if (!PteSrc.n.u1Accessed \|\| !PdeSrc.n.u1Accessed)
1807	{
1808	LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
1809	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,AccessedPage));
1810	SHW_PTE_SET(PteDst, 0);
1811	}
1812	/*
1813	* If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
1814	* when the page is modified.
1815	*/
1816	else if (!PteSrc.n.u1Dirty && (PdeSrc.n.u1Write & PteSrc.n.u1Write))
1817	{
1818	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPage));
1819	SHW_PTE_SET(PteDst,
1820	fGstShwPteFlags
1821	\| PGM_PAGE_GET_HCPHYS(pPage)
1822	\| PGM_PTFLAGS_TRACK_DIRTY);
1823	SHW_PTE_SET_RO(PteDst);
1824	}
1825	else
1826	#endif
1827	{
1828	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageSkipped));
1829	#if PGM_SHW_TYPE == PGM_TYPE_EPT
1830	PteDst.u = PGM_PAGE_GET_HCPHYS(pPage);
1831	PteDst.n.u1Present = 1;
1832	PteDst.n.u1Write = 1;
1833	PteDst.n.u1Execute = 1;
1834	PteDst.n.u1IgnorePAT = 1;
1835	PteDst.n.u3EMT = VMX_EPT_MEMTYPE_WB;
1836	/* PteDst.n.u1Size = 0 */
1837	#else
1838	SHW_PTE_SET(PteDst, fGstShwPteFlags \| PGM_PAGE_GET_HCPHYS(pPage));
1839	#endif
1840	}
1841
1842	/*
1843	* Make sure only allocated pages are mapped writable.
1844	*/
1845	if ( SHW_PTE_IS_P_RW(PteDst)
1846	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
1847	{
1848	/* Still applies to shared pages. */
1849	Assert(!PGM_PAGE_IS_ZERO(pPage));
1850	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet. Why, isn't it? */
1851	Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
1852	}
1853	}
1854
1855	/*
1856	* Keep user track up to date.
1857	*/
1858	if (SHW_PTE_IS_P(PteDst))
1859	{
1860	if (!SHW_PTE_IS_P(*pPteDst))
1861	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1862	else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
1863	{
1864	Log2(("SyncPageWorker: deref! pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(pPteDst), SHW_PTE_LOG64(PteDst)));
1865	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1866	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
1867	}
1868	}
1869	else if (SHW_PTE_IS_P(*pPteDst))
1870	{
1871	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1872	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1873	}
1874
1875	/*
1876	* Update statistics and commit the entry.
1877	*/
1878	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1879	if (!PteSrc.n.u1Global)
1880	pShwPage->fSeenNonGlobal = true;
1881	#endif
1882	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
1883	return;
1884	}
1885
1886	/** @todo count these three different kinds. */
1887	Log2(("SyncPageWorker: invalid address in Pte\n"));
1888	}
1889	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
1890	else if (!PteSrc.n.u1Present)
1891	Log2(("SyncPageWorker: page not present in Pte\n"));
1892	else
1893	Log2(("SyncPageWorker: invalid Pte\n"));
1894	#endif
1895
1896	/*
1897	* The page is not present or the PTE is bad. Replace the shadow PTE by
1898	* an empty entry, making sure to keep the user tracking up to date.
1899	*/
1900	if (SHW_PTE_IS_P(*pPteDst))
1901	{
1902	Log2(("SyncPageWorker: deref! pPteDst=%RX64\n", SHW_PTE_LOG64(pPteDst)));
1903	PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
1904	}
1905	SHW_PTE_ATOMIC_SET(*pPteDst, 0);
1906	}
1907
1908
1909	/**
1910	* Syncs a guest OS page.
1911	*
1912	* There are no conflicts at this point, neither is there any need for
1913	* page table allocations.
1914	*
1915	* When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
1916	* When called in AMD64 guest mode, the guest PML4E shall be valid.
1917	*
1918	* @returns VBox status code.
1919	* @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
1920	* @param pVCpu The cross context virtual CPU structure.
1921	* @param PdeSrc Page directory entry of the guest.
1922	* @param GCPtrPage Guest context page address.
1923	* @param cPages Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
1924	* @param uErr Fault error (X86_TRAP_PF_*).
1925	*/
1926	static int PGM_BTH_NAME(SyncPage)(PVMCPU pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
1927	{
1928	PVM pVM = pVCpu->CTX_SUFF(pVM);
1929	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
1930	LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
1931	RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
1932
1933	PGM_LOCK_ASSERT_OWNER(pVM);
1934
1935	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
1936	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
1937	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
1938	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
1939
1940	/*
1941	* Assert preconditions.
1942	*/
1943	Assert(PdeSrc.n.u1Present);
1944	Assert(cPages);
1945	# if 0 /* rarely useful; leave for debugging. */
1946	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
1947	# endif
1948
1949	/*
1950	* Get the shadow PDE, find the shadow page table in the pool.
1951	*/
1952	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
1953	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1954	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
1955
1956	/* Fetch the pgm pool shadow descriptor. */
1957	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
1958	Assert(pShwPde);
1959
1960	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
1961	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1962	PPGMPOOLPAGE pShwPde = NULL;
1963	PX86PDPAE pPDDst;
1964
1965	/* Fetch the pgm pool shadow descriptor. */
1966	int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
1967	AssertRCSuccessReturn(rc2, rc2);
1968	Assert(pShwPde);
1969
1970	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
1971	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1972
1973	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
1974	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
1975	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1976	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
1977	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
1978
1979	int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
1980	AssertRCSuccessReturn(rc2, rc2);
1981	Assert(pPDDst && pPdptDst);
1982	PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
1983	# endif
1984	SHWPDE PdeDst = *pPdeDst;
1985
1986	/*
1987	* - In the guest SMP case we could have blocked while another VCPU reused
1988	* this page table.
1989	* - With W7-64 we may also take this path when the A bit is cleared on
1990	* higher level tables (PDPE/PML4E). The guest does not invalidate the
1991	* relevant TLB entries. If we're write monitoring any page mapped by
1992	* the modified entry, we may end up here with a "stale" TLB entry.
1993	*/
1994	if (!PdeDst.n.u1Present)
1995	{
1996	Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1997	AssertMsg(pVM->cCpus > 1 \|\| (uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RW)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RW),
1998	("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
1999	if (uErr & X86_TRAP_PF_P)
2000	PGM_INVL_PG(pVCpu, GCPtrPage);
2001	return VINF_SUCCESS; /* force the instruction to be executed again. */
2002	}
2003
2004	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2005	Assert(pShwPage);
2006
2007	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2008	/* Fetch the pgm pool shadow descriptor. */
2009	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2010	Assert(pShwPde);
2011	# endif
2012
2013	/*
2014	* Check that the page is present and that the shadow PDE isn't out of sync.
2015	*/
2016	const bool fBigPage = PdeSrc.b.u1Size && GST_IS_PSE_ACTIVE(pVCpu);
2017	const bool fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
2018	RTGCPHYS GCPhys;
2019	if (!fBigPage)
2020	{
2021	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2022	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2023	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2024	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2025	# endif
2026	}
2027	else
2028	{
2029	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2030	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2031	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2032	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2033	# endif
2034	}
2035	/** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
2036	if ( fPdeValid
2037	&& pShwPage->GCPhys == GCPhys
2038	&& PdeSrc.n.u1Present
2039	&& PdeSrc.n.u1User == PdeDst.n.u1User
2040	&& (PdeSrc.n.u1Write == PdeDst.n.u1Write \|\| !PdeDst.n.u1Write)
2041	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2042	&& (PdeSrc.n.u1NoExecute == PdeDst.n.u1NoExecute \|\| !GST_IS_NX_ACTIVE(pVCpu))
2043	# endif
2044	)
2045	{
2046	/*
2047	* Check that the PDE is marked accessed already.
2048	* Since we set the accessed bit before getting here on a #PF, this
2049	* check is only meant for dealing with non-#PF'ing paths.
2050	*/
2051	if (PdeSrc.n.u1Accessed)
2052	{
2053	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2054	if (!fBigPage)
2055	{
2056	/*
2057	* 4KB Page - Map the guest page table.
2058	*/
2059	PGSTPT pPTSrc;
2060	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2061	if (RT_SUCCESS(rc))
2062	{
2063	# ifdef PGM_SYNC_N_PAGES
2064	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2065	if ( cPages > 1
2066	&& !(uErr & X86_TRAP_PF_P)
2067	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2068	{
2069	/*
2070	* This code path is currently only taken when the caller is PGMTrap0eHandler
2071	* for non-present pages!
2072	*
2073	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2074	* deal with locality.
2075	*/
2076	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2077	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2078	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2079	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2080	# else
2081	const unsigned offPTSrc = 0;
2082	# endif
2083	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2084	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2085	iPTDst = 0;
2086	else
2087	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2088
2089	for (; iPTDst < iPTDstEnd; iPTDst++)
2090	{
2091	const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
2092
2093	if ( pPteSrc->n.u1Present
2094	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2095	{
2096	RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT)) \| ((offPTSrc + iPTDst) << PAGE_SHIFT);
2097	NOREF(GCPtrCurPage);
2098	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2099	/*
2100	* Assuming kernel code will be marked as supervisor - and not as user level
2101	* and executed using a conforming code selector - And marked as readonly.
2102	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2103	*/
2104	PPGMPAGE pPage;
2105	if ( ((PdeSrc.u & pPteSrc->u) & (X86_PTE_RW \| X86_PTE_US))
2106	\|\| iPTDst == ((GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK) /* always sync GCPtrPage */
2107	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCurPage)
2108	\|\| ( (pPage = pgmPhysGetPage(pVM, pPteSrc->u & GST_PTE_PG_MASK))
2109	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2110	)
2111	# endif /* else: CSAM not active */
2112	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
2113	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
2114	GCPtrCurPage, pPteSrc->n.u1Present,
2115	pPteSrc->n.u1Write & PdeSrc.n.u1Write,
2116	pPteSrc->n.u1User & PdeSrc.n.u1User,
2117	(uint64_t)pPteSrc->u,
2118	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2119	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2120	}
2121	}
2122	}
2123	else
2124	# endif /* PGM_SYNC_N_PAGES */
2125	{
2126	const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
2127	GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2128	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2129	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2130	Log2(("SyncPage: 4K %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
2131	GCPtrPage, PteSrc.n.u1Present,
2132	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2133	PteSrc.n.u1User & PdeSrc.n.u1User,
2134	(uint64_t)PteSrc.u,
2135	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2136	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2137	}
2138	}
2139	else /* MMIO or invalid page: emulated in #PF handler. */
2140	{
2141	LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
2142	Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
2143	}
2144	}
2145	else
2146	{
2147	/*
2148	* 4/2MB page - lazy syncing shadow 4K pages.
2149	* (There are many causes of getting here, it's no longer only CSAM.)
2150	*/
2151	/* Calculate the GC physical address of this 4KB shadow page. */
2152	GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) \| (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
2153	/* Find ram range. */
2154	PPGMPAGE pPage;
2155	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2156	if (RT_SUCCESS(rc))
2157	{
2158	AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
2159
2160	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
2161	/* Try to make the page writable if necessary. */
2162	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2163	&& ( PGM_PAGE_IS_ZERO(pPage)
2164	\|\| ( PdeSrc.n.u1Write
2165	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2166	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
2167	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
2168	# endif
2169	# ifdef VBOX_WITH_PAGE_SHARING
2170	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
2171	# endif
2172	)
2173	)
2174	)
2175	{
2176	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2177	AssertRC(rc);
2178	}
2179	# endif
2180
2181	/*
2182	* Make shadow PTE entry.
2183	*/
2184	SHWPTE PteDst;
2185	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2186	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
2187	else
2188	SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) \| PGM_PAGE_GET_HCPHYS(pPage));
2189
2190	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2191	if ( SHW_PTE_IS_P(PteDst)
2192	&& !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2193	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
2194
2195	/* Make sure only allocated pages are mapped writable. */
2196	if ( SHW_PTE_IS_P_RW(PteDst)
2197	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
2198	{
2199	/* Still applies to shared pages. */
2200	Assert(!PGM_PAGE_IS_ZERO(pPage));
2201	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
2202	Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
2203	}
2204
2205	SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
2206
2207	/*
2208	* If the page is not flagged as dirty and is writable, then make it read-only
2209	* at PD level, so we can set the dirty bit when the page is modified.
2210	*
2211	* ASSUMES that page access handlers are implemented on page table entry level.
2212	* Thus we will first catch the dirty access and set PDE.D and restart. If
2213	* there is an access handler, we'll trap again and let it work on the problem.
2214	*/
2215	/** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
2216	* As for invlpg, it simply frees the whole shadow PT.
2217	* ...It's possibly because the guest clears it and the guest doesn't really tell us... */
2218	if ( !PdeSrc.b.u1Dirty
2219	&& PdeSrc.b.u1Write)
2220	{
2221	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2222	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2223	PdeDst.n.u1Write = 0;
2224	}
2225	else
2226	{
2227	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2228	PdeDst.n.u1Write = PdeSrc.n.u1Write;
2229	}
2230	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2231	Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
2232	GCPtrPage, PdeSrc.n.u1Present, PdeSrc.n.u1Write, PdeSrc.n.u1User, (uint64_t)PdeSrc.u, GCPhys,
2233	PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
2234	}
2235	else
2236	{
2237	LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
2238	/** @todo must wipe the shadow page table entry in this
2239	* case. */
2240	}
2241	}
2242	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2243	return VINF_SUCCESS;
2244	}
2245
2246	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDNAs));
2247	}
2248	else if (fPdeValid)
2249	{
2250	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPagePDOutOfSync));
2251	Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2252	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2253	}
2254	else
2255	{
2256	/// @todo STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
2257	Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
2258	GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
2259	}
2260
2261	/*
2262	* Mark the PDE not present. Restart the instruction and let #PF call SyncPT.
2263	* Yea, I'm lazy.
2264	*/
2265	pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
2266	ASMAtomicWriteSize(pPdeDst, 0);
2267
2268	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2269	PGM_INVL_VCPU_TLBS(pVCpu);
2270	return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
2271
2272
2273	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2274	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
2275	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
2276	&& !defined(IN_RC)
2277	NOREF(PdeSrc);
2278
2279	# ifdef PGM_SYNC_N_PAGES
2280	/*
2281	* Get the shadow PDE, find the shadow page table in the pool.
2282	*/
2283	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2284	X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
2285
2286	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2287	X86PDEPAE PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
2288
2289	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2290	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2291	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
2292	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2293	X86PDEPAE PdeDst;
2294	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2295
2296	int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2297	AssertRCSuccessReturn(rc, rc);
2298	Assert(pPDDst && pPdptDst);
2299	PdeDst = pPDDst->a[iPDDst];
2300	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
2301	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
2302	PEPTPD pPDDst;
2303	EPTPDE PdeDst;
2304
2305	int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
2306	if (rc != VINF_SUCCESS)
2307	{
2308	AssertRC(rc);
2309	return rc;
2310	}
2311	Assert(pPDDst);
2312	PdeDst = pPDDst->a[iPDDst];
2313	# endif
2314	/* In the guest SMP case we could have blocked while another VCPU reused this page table. */
2315	if (!PdeDst.n.u1Present)
2316	{
2317	AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
2318	Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
2319	return VINF_SUCCESS; /* force the instruction to be executed again. */
2320	}
2321
2322	/* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
2323	if (PdeDst.n.u1Size)
2324	{
2325	Assert(pVM->pgm.s.fNestedPaging);
2326	Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
2327	return VINF_SUCCESS;
2328	}
2329
2330	/* Mask away the page offset. */
2331	GCPtrPage &= ~((RTGCPTR)0xfff);
2332
2333	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
2334	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2335
2336	Assert(cPages == 1 \|\| !(uErr & X86_TRAP_PF_P));
2337	if ( cPages > 1
2338	&& !(uErr & X86_TRAP_PF_P)
2339	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
2340	{
2341	/*
2342	* This code path is currently only taken when the caller is PGMTrap0eHandler
2343	* for non-present pages!
2344	*
2345	* We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
2346	* deal with locality.
2347	*/
2348	unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2349	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2350	if (iPTDst < PGM_SYNC_NR_PAGES / 2)
2351	iPTDst = 0;
2352	else
2353	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2354	for (; iPTDst < iPTDstEnd; iPTDst++)
2355	{
2356	if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
2357	{
2358	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2359	\| (iPTDst << PAGE_SHIFT));
2360
2361	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2362	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
2363	GCPtrCurPage,
2364	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2365	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2366
2367	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
2368	break;
2369	}
2370	else
2371	Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n", (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) \| (iPTDst << PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
2372	}
2373	}
2374	else
2375	# endif /* PGM_SYNC_N_PAGES */
2376	{
2377	const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2378	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
2379	\| (iPTDst << PAGE_SHIFT));
2380
2381	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
2382
2383	Log2(("SyncPage: 4K %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
2384	GCPtrPage,
2385	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
2386	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
2387	}
2388	return VINF_SUCCESS;
2389
2390	#else
2391	NOREF(PdeSrc);
2392	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
2393	return VERR_PGM_NOT_USED_IN_MODE;
2394	#endif
2395	}
2396
2397
2398	#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
2399
2400	/**
2401	* CheckPageFault helper for returning a page fault indicating a non-present
2402	* (NP) entry in the page translation structures.
2403	*
2404	* @returns VINF_EM_RAW_GUEST_TRAP.
2405	* @param pVCpu The cross context virtual CPU structure.
2406	* @param uErr The error code of the shadow fault. Corrections to
2407	* TRPM's copy will be made if necessary.
2408	* @param GCPtrPage For logging.
2409	* @param uPageFaultLevel For logging.
2410	*/
2411	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnNP)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2412	{
2413	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2414	AssertMsg(!(uErr & X86_TRAP_PF_P), ("%#x\n", uErr));
2415	AssertMsg(!(uErr & X86_TRAP_PF_RSVD), ("%#x\n", uErr));
2416	if (uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2417	TRPMSetErrorCode(pVCpu, uErr & ~(X86_TRAP_PF_RSVD \| X86_TRAP_PF_P));
2418
2419	Log(("CheckPageFault: real page fault (notp) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2420	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2421	return VINF_EM_RAW_GUEST_TRAP;
2422	}
2423
2424
2425	/**
2426	* CheckPageFault helper for returning a page fault indicating a reserved bit
2427	* (RSVD) error in the page translation structures.
2428	*
2429	* @returns VINF_EM_RAW_GUEST_TRAP.
2430	* @param pVCpu The cross context virtual CPU structure.
2431	* @param uErr The error code of the shadow fault. Corrections to
2432	* TRPM's copy will be made if necessary.
2433	* @param GCPtrPage For logging.
2434	* @param uPageFaultLevel For logging.
2435	*/
2436	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnRSVD)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2437	{
2438	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2439	if ((uErr & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
2440	TRPMSetErrorCode(pVCpu, uErr \| X86_TRAP_PF_RSVD \| X86_TRAP_PF_P);
2441
2442	Log(("CheckPageFault: real page fault (rsvd) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2443	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2444	return VINF_EM_RAW_GUEST_TRAP;
2445	}
2446
2447
2448	/**
2449	* CheckPageFault helper for returning a page protection fault (P).
2450	*
2451	* @returns VINF_EM_RAW_GUEST_TRAP.
2452	* @param pVCpu The cross context virtual CPU structure.
2453	* @param uErr The error code of the shadow fault. Corrections to
2454	* TRPM's copy will be made if necessary.
2455	* @param GCPtrPage For logging.
2456	* @param uPageFaultLevel For logging.
2457	*/
2458	DECLINLINE(int) PGM_BTH_NAME(CheckPageFaultReturnProt)(PVMCPU pVCpu, uint32_t uErr, RTGCPTR GCPtrPage, unsigned uPageFaultLevel)
2459	{
2460	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyTrackRealPF));
2461	AssertMsg(uErr & (X86_TRAP_PF_RW \| X86_TRAP_PF_US \| X86_TRAP_PF_ID), ("%#x\n", uErr));
2462	if ((uErr & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) != X86_TRAP_PF_P)
2463	TRPMSetErrorCode(pVCpu, (uErr & ~X86_TRAP_PF_RSVD) \| X86_TRAP_PF_P);
2464
2465	Log(("CheckPageFault: real page fault (prot) at %RGv (%d)\n", GCPtrPage, uPageFaultLevel));
2466	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(uPageFaultLevel);
2467	return VINF_EM_RAW_GUEST_TRAP;
2468	}
2469
2470
2471	/**
2472	* Handle dirty bit tracking faults.
2473	*
2474	* @returns VBox status code.
2475	* @param pVCpu The cross context virtual CPU structure.
2476	* @param uErr Page fault error code.
2477	* @param pPdeSrc Guest page directory entry.
2478	* @param pPdeDst Shadow page directory entry.
2479	* @param GCPtrPage Guest context page address.
2480	*/
2481	static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPU pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
2482	RTGCPTR GCPtrPage)
2483	{
2484	PVM pVM = pVCpu->CTX_SUFF(pVM);
2485	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2486	NOREF(uErr);
2487
2488	PGM_LOCK_ASSERT_OWNER(pVM);
2489
2490	/*
2491	* Handle big page.
2492	*/
2493	if (pPdeSrc->b.u1Size && GST_IS_PSE_ACTIVE(pVCpu))
2494	{
2495	if ( pPdeDst->n.u1Present
2496	&& (pPdeDst->u & PGM_PDFLAGS_TRACK_DIRTY))
2497	{
2498	SHWPDE PdeDst = *pPdeDst;
2499
2500	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2501	Assert(pPdeSrc->b.u1Write);
2502
2503	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
2504	* fault again and take this path to only invalidate the entry (see below).
2505	*/
2506	PdeDst.n.u1Write = 1;
2507	PdeDst.n.u1Accessed = 1;
2508	PdeDst.au32[0] &= ~PGM_PDFLAGS_TRACK_DIRTY;
2509	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2510	PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
2511	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2512	}
2513
2514	# ifdef IN_RING0
2515	/* Check for stale TLB entry; only applies to the SMP guest case. */
2516	if ( pVM->cCpus > 1
2517	&& pPdeDst->n.u1Write
2518	&& pPdeDst->n.u1Accessed)
2519	{
2520	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2521	if (pShwPage)
2522	{
2523	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2524	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2525	if (SHW_PTE_IS_P_RW(*pPteDst))
2526	{
2527	/* Stale TLB entry. */
2528	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2529	PGM_INVL_PG(pVCpu, GCPtrPage);
2530	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2531	}
2532	}
2533	}
2534	# endif /* IN_RING0 */
2535	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2536	}
2537
2538	/*
2539	* Map the guest page table.
2540	*/
2541	PGSTPT pPTSrc;
2542	int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
2543	if (RT_FAILURE(rc))
2544	{
2545	AssertRC(rc);
2546	return rc;
2547	}
2548
2549	if (pPdeDst->n.u1Present)
2550	{
2551	GSTPTE const *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
2552	const GSTPTE PteSrc = *pPteSrc;
2553
2554	#ifdef VBOX_WITH_RAW_MODE_NOT_R0
2555	/* Bail out here as pgmPoolGetPage will return NULL and we'll crash below.
2556	* Our individual shadow handlers will provide more information and force a fatal exit.
2557	*/
2558	if ( VM_IS_RAW_MODE_ENABLED(pVM)
2559	&& MMHyperIsInsideArea(pVM, (RTGCPTR)GCPtrPage))
2560	{
2561	LogRel(("CheckPageFault: write to hypervisor region %RGv\n", GCPtrPage));
2562	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2563	}
2564	#endif
2565	/*
2566	* Map shadow page table.
2567	*/
2568	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
2569	if (pShwPage)
2570	{
2571	PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2572	PSHWPTE pPteDst = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
2573	if (SHW_PTE_IS_P(pPteDst)) /* @todo Optimize accessed bit emulation? */
2574	{
2575	if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
2576	{
2577	PPGMPAGE pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
2578	SHWPTE PteDst = *pPteDst;
2579
2580	LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
2581	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageTrap));
2582
2583	Assert(PteSrc.n.u1Write);
2584
2585	/* Note: No need to invalidate this entry on other VCPUs as a stale TLB
2586	* entry will not harm; write access will simply fault again and
2587	* take this path to only invalidate the entry.
2588	*/
2589	if (RT_LIKELY(pPage))
2590	{
2591	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2592	{
2593	//AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
2594	Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
2595	/* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
2596	SHW_PTE_SET_RO(PteDst);
2597	}
2598	else
2599	{
2600	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
2601	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2602	{
2603	rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
2604	AssertRC(rc);
2605	}
2606	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
2607	SHW_PTE_SET_RW(PteDst);
2608	else
2609	{
2610	/* Still applies to shared pages. */
2611	Assert(!PGM_PAGE_IS_ZERO(pPage));
2612	SHW_PTE_SET_RO(PteDst);
2613	}
2614	}
2615	}
2616	else
2617	SHW_PTE_SET_RW(PteDst); /** @todo r=bird: This doesn't make sense to me. */
2618
2619	SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) \| X86_PTE_D \| X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
2620	SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
2621	PGM_INVL_PG(pVCpu, GCPtrPage);
2622	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2623	}
2624
2625	# ifdef IN_RING0
2626	/* Check for stale TLB entry; only applies to the SMP guest case. */
2627	if ( pVM->cCpus > 1
2628	&& SHW_PTE_IS_RW(*pPteDst)
2629	&& SHW_PTE_IS_A(*pPteDst))
2630	{
2631	/* Stale TLB entry. */
2632	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageStale));
2633	PGM_INVL_PG(pVCpu, GCPtrPage);
2634	return VINF_PGM_HANDLED_DIRTY_BIT_FAULT; /* restarts the instruction. */
2635	}
2636	# endif
2637	}
2638	}
2639	else
2640	AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
2641	}
2642
2643	return VINF_PGM_NO_DIRTY_BIT_TRACKING;
2644	}
2645
2646	#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
2647
2648
2649	/**
2650	* Sync a shadow page table.
2651	*
2652	* The shadow page table is not present in the shadow PDE.
2653	*
2654	* Handles mapping conflicts.
2655	*
2656	* This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
2657	* conflict), and Trap0eHandler.
2658	*
2659	* A precondition for this method is that the shadow PDE is not present. The
2660	* caller must take the PGM lock before checking this and continue to hold it
2661	* when calling this method.
2662	*
2663	* @returns VBox status code.
2664	* @param pVCpu The cross context virtual CPU structure.
2665	* @param iPDSrc Page directory index.
2666	* @param pPDSrc Source page directory (i.e. Guest OS page directory).
2667	* Assume this is a temporary mapping.
2668	* @param GCPtrPage GC Pointer of the page that caused the fault
2669	*/
2670	static int PGM_BTH_NAME(SyncPT)(PVMCPU pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
2671	{
2672	PVM pVM = pVCpu->CTX_SUFF(pVM);
2673	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
2674
2675	#if 0 /* rarely useful; leave for debugging. */
2676	STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
2677	#endif
2678	LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
2679
2680	PGM_LOCK_ASSERT_OWNER(pVM);
2681
2682	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
2683	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
2684	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64) \
2685	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
2686	int rc = VINF_SUCCESS;
2687
2688	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2689
2690	/*
2691	* Some input validation first.
2692	*/
2693	AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
2694
2695	/*
2696	* Get the relevant shadow PDE entry.
2697	*/
2698	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
2699	const unsigned iPDDst = GCPtrPage >> SHW_PD_SHIFT;
2700	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
2701
2702	/* Fetch the pgm pool shadow descriptor. */
2703	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2704	Assert(pShwPde);
2705
2706	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
2707	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2708	PPGMPOOLPAGE pShwPde = NULL;
2709	PX86PDPAE pPDDst;
2710	PSHWPDE pPdeDst;
2711
2712	/* Fetch the pgm pool shadow descriptor. */
2713	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
2714	AssertRCSuccessReturn(rc, rc);
2715	Assert(pShwPde);
2716
2717	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
2718	pPdeDst = &pPDDst->a[iPDDst];
2719
2720	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
2721	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
2722	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
2723	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
2724	PX86PDPT pPdptDst = NULL; /* initialized to shut up gcc */
2725	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
2726	AssertRCSuccessReturn(rc, rc);
2727	Assert(pPDDst);
2728	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
2729	# endif
2730	SHWPDE PdeDst = *pPdeDst;
2731
2732	# if PGM_GST_TYPE == PGM_TYPE_AMD64
2733	/* Fetch the pgm pool shadow descriptor. */
2734	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
2735	Assert(pShwPde);
2736	# endif
2737
2738	# ifndef PGM_WITHOUT_MAPPINGS
2739	/*
2740	* Check for conflicts.
2741	* RC: In case of a conflict we'll go to Ring-3 and do a full SyncCR3.
2742	* R3: Simply resolve the conflict.
2743	*/
2744	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
2745	{
2746	Assert(pgmMapAreMappingsEnabled(pVM));
2747	# ifndef IN_RING3
2748	Log(("SyncPT: Conflict at %RGv\n", GCPtrPage));
2749	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2750	return VERR_ADDRESS_CONFLICT;
2751
2752	# else /* IN_RING3 */
2753	PPGMMAPPING pMapping = pgmGetMapping(pVM, (RTGCPTR)GCPtrPage);
2754	Assert(pMapping);
2755	# if PGM_GST_TYPE == PGM_TYPE_32BIT
2756	rc = pgmR3SyncPTResolveConflict(pVM, pMapping, pPDSrc, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2757	# elif PGM_GST_TYPE == PGM_TYPE_PAE
2758	rc = pgmR3SyncPTResolveConflictPAE(pVM, pMapping, GCPtrPage & (GST_PD_MASK << GST_PD_SHIFT));
2759	# else
2760	AssertFailed(); NOREF(pMapping); /* can't happen for amd64 */
2761	# endif
2762	if (RT_FAILURE(rc))
2763	{
2764	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
2765	return rc;
2766	}
2767	PdeDst = *pPdeDst;
2768	# endif /* IN_RING3 */
2769	}
2770	# endif /* !PGM_WITHOUT_MAPPINGS */
2771	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
2772
2773	/*
2774	* Sync the page directory entry.
2775	*/
2776	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
2777	const bool fPageTable = !PdeSrc.b.u1Size \|\| !GST_IS_PSE_ACTIVE(pVCpu);
2778	if ( PdeSrc.n.u1Present
2779	&& (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
2780	{
2781	/*
2782	* Allocate & map the page table.
2783	*/
2784	PSHWPT pPTDst;
2785	PPGMPOOLPAGE pShwPage;
2786	RTGCPHYS GCPhys;
2787	if (fPageTable)
2788	{
2789	GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
2790	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2791	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2792	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
2793	# endif
2794	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
2795	pShwPde->idx, iPDDst, false /fLockPage/,
2796	&pShwPage);
2797	}
2798	else
2799	{
2800	PGMPOOLACCESS enmAccess;
2801	# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
2802	const bool fNoExecute = PdeSrc.n.u1NoExecute && GST_IS_NX_ACTIVE(pVCpu);
2803	# else
2804	const bool fNoExecute = false;
2805	# endif
2806
2807	GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
2808	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2809	/* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
2810	GCPhys = PGM_A20_APPLY(pVCpu, GCPhys \| (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
2811	# endif
2812	/* Determine the right kind of large page to avoid incorrect cached entry reuse. */
2813	if (PdeSrc.n.u1User)
2814	{
2815	if (PdeSrc.n.u1Write)
2816	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
2817	else
2818	enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX : PGMPOOLACCESS_USER_R;
2819	}
2820	else
2821	{
2822	if (PdeSrc.n.u1Write)
2823	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
2824	else
2825	enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX : PGMPOOLACCESS_SUPERVISOR_R;
2826	}
2827	rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
2828	pShwPde->idx, iPDDst, false /fLockPage/,
2829	&pShwPage);
2830	}
2831	if (rc == VINF_SUCCESS)
2832	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
2833	else if (rc == VINF_PGM_CACHED_PAGE)
2834	{
2835	/*
2836	* The PT was cached, just hook it up.
2837	*/
2838	if (fPageTable)
2839	PdeDst.u = pShwPage->Core.Key \| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2840	else
2841	{
2842	PdeDst.u = pShwPage->Core.Key \| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2843	/* (see explanation and assumptions further down.) */
2844	if ( !PdeSrc.b.u1Dirty
2845	&& PdeSrc.b.u1Write)
2846	{
2847	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
2848	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
2849	PdeDst.b.u1Write = 0;
2850	}
2851	}
2852	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2853	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2854	return VINF_SUCCESS;
2855	}
2856	else if (rc == VERR_PGM_POOL_FLUSHED)
2857	{
2858	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2859	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2860	return VINF_PGM_SYNC_CR3;
2861	}
2862	else
2863	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
2864	/** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
2865	* Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
2866	* irrelevant at this point. */
2867	PdeDst.u &= X86_PDE_AVL_MASK;
2868	PdeDst.u \|= pShwPage->Core.Key;
2869
2870	/*
2871	* Page directory has been accessed (this is a fault situation, remember).
2872	*/
2873	/** @todo
2874	* Well, when the caller is PrefetchPage or InvalidatePage is isn't a
2875	* fault situation. What's more, the Trap0eHandler has already set the
2876	* accessed bit. So, it's actually just VerifyAccessSyncPage which
2877	* might need setting the accessed flag.
2878	*
2879	* The best idea is to leave this change to the caller and add an
2880	* assertion that it's set already. */
2881	pPDSrc->a[iPDSrc].n.u1Accessed = 1;
2882	if (fPageTable)
2883	{
2884	/*
2885	* Page table - 4KB.
2886	*
2887	* Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
2888	*/
2889	Log2(("SyncPT: 4K %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
2890	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u));
2891	PGSTPT pPTSrc;
2892	rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
2893	if (RT_SUCCESS(rc))
2894	{
2895	/*
2896	* Start by syncing the page directory entry so CSAM's TLB trick works.
2897	*/
2898	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| X86_PDE_AVL_MASK))
2899	\| GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2900	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
2901	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
2902
2903	/*
2904	* Directory/page user or supervisor privilege: (same goes for read/write)
2905	*
2906	* Directory Page Combined
2907	* U/S U/S U/S
2908	* 0 0 0
2909	* 0 1 0
2910	* 1 0 0
2911	* 1 1 1
2912	*
2913	* Simple AND operation. Table listed for completeness.
2914	*
2915	*/
2916	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4K));
2917	# ifdef PGM_SYNC_N_PAGES
2918	unsigned iPTBase = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
2919	unsigned iPTDst = iPTBase;
2920	const unsigned iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
2921	if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
2922	iPTDst = 0;
2923	else
2924	iPTDst -= PGM_SYNC_NR_PAGES / 2;
2925	# else /* !PGM_SYNC_N_PAGES */
2926	unsigned iPTDst = 0;
2927	const unsigned iPTDstEnd = RT_ELEMENTS(pPTDst->a);
2928	# endif /* !PGM_SYNC_N_PAGES */
2929	RTGCPTR GCPtrCur = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
2930	\| ((RTGCPTR)iPTDst << PAGE_SHIFT);
2931	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
2932	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
2933	const unsigned offPTSrc = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
2934	# else
2935	const unsigned offPTSrc = 0;
2936	# endif
2937	for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += PAGE_SIZE)
2938	{
2939	const unsigned iPTSrc = iPTDst + offPTSrc;
2940	const GSTPTE PteSrc = pPTSrc->a[iPTSrc];
2941
2942	if (PteSrc.n.u1Present)
2943	{
2944	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
2945	/*
2946	* Assuming kernel code will be marked as supervisor - and not as user level
2947	* and executed using a conforming code selector - And marked as readonly.
2948	* Also assume that if we're monitoring a page, it's of no interest to CSAM.
2949	*/
2950	PPGMPAGE pPage;
2951	if ( ((PdeSrc.u & pPTSrc->a[iPTSrc].u) & (X86_PTE_RW \| X86_PTE_US))
2952	\|\| !CSAMDoesPageNeedScanning(pVM, GCPtrCur)
2953	\|\| ( (pPage = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc)))
2954	&& PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
2955	)
2956	# endif
2957	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
2958	Log2(("SyncPT: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
2959	GCPtrCur,
2960	PteSrc.n.u1Present,
2961	PteSrc.n.u1Write & PdeSrc.n.u1Write,
2962	PteSrc.n.u1User & PdeSrc.n.u1User,
2963	(uint64_t)PteSrc.u,
2964	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
2965	(RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
2966	}
2967	/* else: the page table was cleared by the pool */
2968	} /* for PTEs */
2969	}
2970	}
2971	else
2972	{
2973	/*
2974	* Big page - 2/4MB.
2975	*
2976	* We'll walk the ram range list in parallel and optimize lookups.
2977	* We will only sync one shadow page table at a time.
2978	*/
2979	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT4M));
2980
2981	/**
2982	* @todo It might be more efficient to sync only a part of the 4MB
2983	* page (similar to what we do for 4KB PDs).
2984	*/
2985
2986	/*
2987	* Start by syncing the page directory entry.
2988	*/
2989	PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK \| (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
2990	\| GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
2991
2992	/*
2993	* If the page is not flagged as dirty and is writable, then make it read-only
2994	* at PD level, so we can set the dirty bit when the page is modified.
2995	*
2996	* ASSUMES that page access handlers are implemented on page table entry level.
2997	* Thus we will first catch the dirty access and set PDE.D and restart. If
2998	* there is an access handler, we'll trap again and let it work on the problem.
2999	*/
3000	/** @todo move the above stuff to a section in the PGM documentation. */
3001	Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
3002	if ( !PdeSrc.b.u1Dirty
3003	&& PdeSrc.b.u1Write)
3004	{
3005	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,DirtyPageBig));
3006	PdeDst.u \|= PGM_PDFLAGS_TRACK_DIRTY;
3007	PdeDst.b.u1Write = 0;
3008	}
3009	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3010	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3011
3012	/*
3013	* Fill the shadow page table.
3014	*/
3015	/* Get address and flags from the source PDE. */
3016	SHWPTE PteDstBase;
3017	SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
3018
3019	/* Loop thru the entries in the shadow PT. */
3020	const RTGCPTR GCPtr = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
3021	Log2(("SyncPT: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
3022	GCPtrPage, PdeSrc.b.u1Present, PdeSrc.b.u1Write, PdeSrc.b.u1User, (uint64_t)PdeSrc.u, GCPtr,
3023	GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
3024	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
3025	unsigned iPTDst = 0;
3026	while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3027	&& !VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
3028	{
3029	if (pRam && GCPhys >= pRam->GCPhys)
3030	{
3031	# ifndef PGM_WITH_A20
3032	unsigned iHCPage = (GCPhys - pRam->GCPhys) >> PAGE_SHIFT;
3033	# endif
3034	do
3035	{
3036	/* Make shadow PTE. */
3037	# ifdef PGM_WITH_A20
3038	PPGMPAGE pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> PAGE_SHIFT];
3039	# else
3040	PPGMPAGE pPage = &pRam->aPages[iHCPage];
3041	# endif
3042	SHWPTE PteDst;
3043
3044	# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3045	/* Try to make the page writable if necessary. */
3046	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
3047	&& ( PGM_PAGE_IS_ZERO(pPage)
3048	\|\| ( SHW_PTE_IS_RW(PteDstBase)
3049	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
3050	# ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
3051	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
3052	# endif
3053	# ifdef VBOX_WITH_PAGE_SHARING
3054	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
3055	# endif
3056	&& !PGM_PAGE_IS_BALLOONED(pPage))
3057	)
3058	)
3059	{
3060	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3061	AssertRCReturn(rc, rc);
3062	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY))
3063	break;
3064	}
3065	# endif
3066
3067	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
3068	PGM_BTH_NAME(SyncHandlerPte)(pVM, pPage, SHW_PTE_GET_U(PteDstBase), &PteDst);
3069	else if (PGM_PAGE_IS_BALLOONED(pPage))
3070	SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
3071	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3072	/*
3073	* Assuming kernel code will be marked as supervisor and not as user level and executed
3074	* using a conforming code selector. Don't check for readonly, as that implies the whole
3075	* 4MB can be code or readonly data. Linux enables write access for its large pages.
3076	*/
3077	else if ( !PdeSrc.n.u1User
3078	&& CSAMDoesPageNeedScanning(pVM, GCPtr \| (iPTDst << SHW_PT_SHIFT)))
3079	SHW_PTE_SET(PteDst, 0);
3080	# endif
3081	else
3082	SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) \| SHW_PTE_GET_U(PteDstBase));
3083
3084	/* Only map writable pages writable. */
3085	if ( SHW_PTE_IS_P_RW(PteDst)
3086	&& PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
3087	{
3088	/* Still applies to shared pages. */
3089	Assert(!PGM_PAGE_IS_ZERO(pPage));
3090	SHW_PTE_SET_RO(PteDst); /** @todo this isn't quite working yet... */
3091	Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT))));
3092	}
3093
3094	if (SHW_PTE_IS_P(PteDst))
3095	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
3096
3097	/* commit it (not atomic, new table) */
3098	pPTDst->a[iPTDst] = PteDst;
3099	Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
3100	(RTGCPTR)(GCPtr \| (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
3101	SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
3102
3103	/* advance */
3104	GCPhys += PAGE_SIZE;
3105	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
3106	# ifndef PGM_WITH_A20
3107	iHCPage++;
3108	# endif
3109	iPTDst++;
3110	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3111	&& GCPhys <= pRam->GCPhysLast);
3112
3113	/* Advance ram range list. */
3114	while (pRam && GCPhys > pRam->GCPhysLast)
3115	pRam = pRam->CTX_SUFF(pNext);
3116	}
3117	else if (pRam)
3118	{
3119	Log(("Invalid pages at %RGp\n", GCPhys));
3120	do
3121	{
3122	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3123	GCPhys += PAGE_SIZE;
3124	iPTDst++;
3125	} while ( iPTDst < RT_ELEMENTS(pPTDst->a)
3126	&& GCPhys < pRam->GCPhys);
3127	PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
3128	}
3129	else
3130	{
3131	Log(("Invalid pages at %RGp (2)\n", GCPhys));
3132	for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3133	SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
3134	}
3135	} /* while more PTEs */
3136	} /* 4KB / 4MB */
3137	}
3138	else
3139	AssertRelease(!PdeDst.n.u1Present);
3140
3141	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3142	if (RT_FAILURE(rc))
3143	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3144	return rc;
3145
3146	#elif (PGM_GST_TYPE == PGM_TYPE_REAL \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3147	&& !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
3148	&& (PGM_SHW_TYPE != PGM_TYPE_EPT \|\| PGM_GST_TYPE == PGM_TYPE_PROT) \
3149	&& !defined(IN_RC)
3150	NOREF(iPDSrc); NOREF(pPDSrc);
3151
3152	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3153
3154	/*
3155	* Validate input a little bit.
3156	*/
3157	int rc = VINF_SUCCESS;
3158	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3159	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3160	PSHWPDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3161
3162	/* Fetch the pgm pool shadow descriptor. */
3163	PPGMPOOLPAGE pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
3164	Assert(pShwPde);
3165
3166	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3167	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3168	PPGMPOOLPAGE pShwPde = NULL; /* initialized to shut up gcc */
3169	PX86PDPAE pPDDst;
3170	PSHWPDE pPdeDst;
3171
3172	/* Fetch the pgm pool shadow descriptor. */
3173	rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
3174	AssertRCSuccessReturn(rc, rc);
3175	Assert(pShwPde);
3176
3177	pPDDst = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
3178	pPdeDst = &pPDDst->a[iPDDst];
3179
3180	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3181	const unsigned iPdpt = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
3182	const unsigned iPDDst = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
3183	PX86PDPAE pPDDst = NULL; /* initialized to shut up gcc */
3184	PX86PDPT pPdptDst= NULL; /* initialized to shut up gcc */
3185	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
3186	AssertRCSuccessReturn(rc, rc);
3187	Assert(pPDDst);
3188	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3189
3190	/* Fetch the pgm pool shadow descriptor. */
3191	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
3192	Assert(pShwPde);
3193
3194	# elif PGM_SHW_TYPE == PGM_TYPE_EPT
3195	const unsigned iPdpt = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
3196	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3197	PEPTPD pPDDst;
3198	PEPTPDPT pPdptDst;
3199
3200	rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
3201	if (rc != VINF_SUCCESS)
3202	{
3203	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3204	AssertRC(rc);
3205	return rc;
3206	}
3207	Assert(pPDDst);
3208	PSHWPDE pPdeDst = &pPDDst->a[iPDDst];
3209
3210	/* Fetch the pgm pool shadow descriptor. */
3211	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
3212	Assert(pShwPde);
3213	# endif
3214	SHWPDE PdeDst = *pPdeDst;
3215
3216	Assert(!(PdeDst.u & PGM_PDFLAGS_MAPPING));
3217	Assert(!PdeDst.n.u1Present); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
3218
3219	# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
3220	if ( BTH_IS_NP_ACTIVE(pVM)
3221	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Large page support. */
3222	{
3223	/* Check if we allocated a big page before for this 2 MB range. */
3224	PPGMPAGE pPage;
3225	rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
3226	if (RT_SUCCESS(rc))
3227	{
3228	RTHCPHYS HCPhys = NIL_RTHCPHYS;
3229	if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
3230	{
3231	if (PGM_A20_IS_ENABLED(pVCpu))
3232	{
3233	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
3234	AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3235	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3236	}
3237	else
3238	{
3239	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
3240	pVM->pgm.s.cLargePagesDisabled++;
3241	}
3242	}
3243	else if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
3244	&& PGM_A20_IS_ENABLED(pVCpu))
3245	{
3246	/* Recheck the entire 2 MB range to see if we can use it again as a large page. */
3247	rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
3248	if (RT_SUCCESS(rc))
3249	{
3250	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3251	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3252	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3253	}
3254	}
3255	else if ( PGMIsUsingLargePages(pVM)
3256	&& PGM_A20_IS_ENABLED(pVCpu))
3257	{
3258	rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
3259	if (RT_SUCCESS(rc))
3260	{
3261	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
3262	Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
3263	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
3264	}
3265	else
3266	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
3267	}
3268
3269	if (HCPhys != NIL_RTHCPHYS)
3270	{
3271	PdeDst.u &= X86_PDE_AVL_MASK;
3272	PdeDst.u \|= HCPhys;
3273	PdeDst.n.u1Present = 1;
3274	PdeDst.n.u1Write = 1;
3275	PdeDst.b.u1Size = 1;
3276	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3277	PdeDst.n.u1Execute = 1;
3278	PdeDst.b.u1IgnorePAT = 1;
3279	PdeDst.b.u3EMT = VMX_EPT_MEMTYPE_WB;
3280	# else
3281	PdeDst.n.u1User = 1;
3282	# endif
3283	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3284
3285	Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
3286	/* Add a reference to the first page only. */
3287	PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
3288
3289	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3290	return VINF_SUCCESS;
3291	}
3292	}
3293	}
3294	# endif /* HC_ARCH_BITS == 64 */
3295
3296	/*
3297	* Allocate & map the page table.
3298	*/
3299	PSHWPT pPTDst;
3300	PPGMPOOLPAGE pShwPage;
3301	RTGCPHYS GCPhys;
3302
3303	/* Virtual address = physical address */
3304	GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
3305	rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
3306	PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /fLockPage/,
3307	&pShwPage);
3308	if ( rc == VINF_SUCCESS
3309	\|\| rc == VINF_PGM_CACHED_PAGE)
3310	pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
3311	else
3312	{
3313	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3314	AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
3315	}
3316
3317	if (rc == VINF_SUCCESS)
3318	{
3319	/* New page table; fully set it up. */
3320	Assert(pPTDst);
3321
3322	/* Mask away the page offset. */
3323	GCPtrPage &= ~(RTGCPTR)PAGE_OFFSET_MASK;
3324
3325	for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
3326	{
3327	RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
3328	\| (iPTDst << PAGE_SHIFT));
3329
3330	PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
3331	Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
3332	GCPtrCurPage,
3333	SHW_PTE_LOG64(pPTDst->a[iPTDst]),
3334	SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
3335
3336	if (RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)))
3337	break;
3338	}
3339	}
3340	else
3341	rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
3342
3343	/* Save the new PDE. */
3344	PdeDst.u &= X86_PDE_AVL_MASK;
3345	PdeDst.u \|= pShwPage->Core.Key;
3346	PdeDst.n.u1Present = 1;
3347	PdeDst.n.u1Write = 1;
3348	# if PGM_SHW_TYPE == PGM_TYPE_EPT
3349	PdeDst.n.u1Execute = 1;
3350	# else
3351	PdeDst.n.u1User = 1;
3352	PdeDst.n.u1Accessed = 1;
3353	# endif
3354	ASMAtomicWriteSize(pPdeDst, PdeDst.u);
3355
3356	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPT), a);
3357	if (RT_FAILURE(rc))
3358	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncPTFailed));
3359	return rc;
3360
3361	#else
3362	NOREF(iPDSrc); NOREF(pPDSrc);
3363	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
3364	return VERR_PGM_NOT_USED_IN_MODE;
3365	#endif
3366	}
3367
3368
3369
3370	/**
3371	* Prefetch a page/set of pages.
3372	*
3373	* Typically used to sync commonly used pages before entering raw mode
3374	* after a CR3 reload.
3375	*
3376	* @returns VBox status code.
3377	* @param pVCpu The cross context virtual CPU structure.
3378	* @param GCPtrPage Page to invalidate.
3379	*/
3380	PGM_BTH_DECL(int, PrefetchPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage)
3381	{
3382	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3383	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3384	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3385	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3386	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3387	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3388	/*
3389	* Check that all Guest levels thru the PDE are present, getting the
3390	* PD and PDE in the processes.
3391	*/
3392	int rc = VINF_SUCCESS;
3393	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3394	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3395	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3396	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3397	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3398	unsigned iPDSrc;
3399	X86PDPE PdpeSrc;
3400	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3401	if (!pPDSrc)
3402	return VINF_SUCCESS; /* not present */
3403	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3404	unsigned iPDSrc;
3405	PX86PML4E pPml4eSrc;
3406	X86PDPE PdpeSrc;
3407	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3408	if (!pPDSrc)
3409	return VINF_SUCCESS; /* not present */
3410	# endif
3411	const GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3412	# else
3413	PGSTPD pPDSrc = NULL;
3414	const unsigned iPDSrc = 0;
3415	GSTPDE PdeSrc;
3416
3417	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3418	PdeSrc.n.u1Present = 1;
3419	PdeSrc.n.u1Write = 1;
3420	PdeSrc.n.u1Accessed = 1;
3421	PdeSrc.n.u1User = 1;
3422	# endif
3423
3424	if (PdeSrc.n.u1Present && PdeSrc.n.u1Accessed)
3425	{
3426	PVM pVM = pVCpu->CTX_SUFF(pVM);
3427	pgmLock(pVM);
3428
3429	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3430	const X86PDE PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
3431	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3432	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3433	PX86PDPAE pPDDst;
3434	X86PDEPAE PdeDst;
3435	# if PGM_GST_TYPE != PGM_TYPE_PAE
3436	X86PDPE PdpeSrc;
3437
3438	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3439	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3440	# endif
3441	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3442	if (rc != VINF_SUCCESS)
3443	{
3444	pgmUnlock(pVM);
3445	AssertRC(rc);
3446	return rc;
3447	}
3448	Assert(pPDDst);
3449	PdeDst = pPDDst->a[iPDDst];
3450
3451	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3452	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3453	PX86PDPAE pPDDst;
3454	X86PDEPAE PdeDst;
3455
3456	# if PGM_GST_TYPE == PGM_TYPE_PROT
3457	/* AMD-V nested paging */
3458	X86PML4E Pml4eSrc;
3459	X86PDPE PdpeSrc;
3460	PX86PML4E pPml4eSrc = &Pml4eSrc;
3461
3462	/* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3463	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3464	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3465	# endif
3466
3467	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3468	if (rc != VINF_SUCCESS)
3469	{
3470	pgmUnlock(pVM);
3471	AssertRC(rc);
3472	return rc;
3473	}
3474	Assert(pPDDst);
3475	PdeDst = pPDDst->a[iPDDst];
3476	# endif
3477	if (!(PdeDst.u & PGM_PDFLAGS_MAPPING))
3478	{
3479	if (!PdeDst.n.u1Present)
3480	{
3481	/** @todo r=bird: This guy will set the A bit on the PDE,
3482	* probably harmless. */
3483	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3484	}
3485	else
3486	{
3487	/* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
3488	* R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
3489	* makes no sense to prefetch more than one page.
3490	*/
3491	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3492	if (RT_SUCCESS(rc))
3493	rc = VINF_SUCCESS;
3494	}
3495	}
3496	pgmUnlock(pVM);
3497	}
3498	return rc;
3499
3500	#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3501	NOREF(pVCpu); NOREF(GCPtrPage);
3502	return VINF_SUCCESS; /* ignore */
3503	#else
3504	AssertCompile(0);
3505	#endif
3506	}
3507
3508
3509
3510
3511	/**
3512	* Syncs a page during a PGMVerifyAccess() call.
3513	*
3514	* @returns VBox status code (informational included).
3515	* @param pVCpu The cross context virtual CPU structure.
3516	* @param GCPtrPage The address of the page to sync.
3517	* @param fPage The effective guest page flags.
3518	* @param uErr The trap error code.
3519	* @remarks This will normally never be called on invalid guest page
3520	* translation entries.
3521	*/
3522	PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPU pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
3523	{
3524	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3525
3526	LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
3527	RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
3528
3529	Assert(!pVM->pgm.s.fNestedPaging);
3530	#if ( PGM_GST_TYPE == PGM_TYPE_32BIT \
3531	\|\| PGM_GST_TYPE == PGM_TYPE_REAL \
3532	\|\| PGM_GST_TYPE == PGM_TYPE_PROT \
3533	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3534	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
3535	&& !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3536
3537	# ifdef VBOX_WITH_RAW_MODE_NOT_R0
3538	if (!(fPage & X86_PTE_US))
3539	{
3540	/*
3541	* Mark this page as safe.
3542	*/
3543	/** @todo not correct for pages that contain both code and data!! */
3544	Log(("CSAMMarkPage %RGv; scanned=%d\n", GCPtrPage, true));
3545	CSAMMarkPage(pVM, GCPtrPage, true);
3546	}
3547	# endif
3548
3549	/*
3550	* Get guest PD and index.
3551	*/
3552	/** @todo Performance: We've done all this a jiffy ago in the
3553	* PGMGstGetPage call. */
3554	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3555	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3556	const unsigned iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
3557	PGSTPD pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
3558
3559	# elif PGM_GST_TYPE == PGM_TYPE_PAE
3560	unsigned iPDSrc = 0;
3561	X86PDPE PdpeSrc;
3562	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
3563	if (RT_UNLIKELY(!pPDSrc))
3564	{
3565	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3566	return VINF_EM_RAW_GUEST_TRAP;
3567	}
3568
3569	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
3570	unsigned iPDSrc = 0; /* shut up gcc */
3571	PX86PML4E pPml4eSrc = NULL; /* ditto */
3572	X86PDPE PdpeSrc;
3573	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
3574	if (RT_UNLIKELY(!pPDSrc))
3575	{
3576	Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
3577	return VINF_EM_RAW_GUEST_TRAP;
3578	}
3579	# endif
3580
3581	# else /* !PGM_WITH_PAGING */
3582	PGSTPD pPDSrc = NULL;
3583	const unsigned iPDSrc = 0;
3584	# endif /* !PGM_WITH_PAGING */
3585	int rc = VINF_SUCCESS;
3586
3587	pgmLock(pVM);
3588
3589	/*
3590	* First check if the shadow pd is present.
3591	*/
3592	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3593	PX86PDE pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
3594
3595	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3596	PX86PDEPAE pPdeDst;
3597	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3598	PX86PDPAE pPDDst;
3599	# if PGM_GST_TYPE != PGM_TYPE_PAE
3600	/* Fake PDPT entry; access control handled on the page table level, so allow everything. */
3601	X86PDPE PdpeSrc;
3602	PdpeSrc.u = X86_PDPE_P; /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
3603	# endif
3604	rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
3605	if (rc != VINF_SUCCESS)
3606	{
3607	pgmUnlock(pVM);
3608	AssertRC(rc);
3609	return rc;
3610	}
3611	Assert(pPDDst);
3612	pPdeDst = &pPDDst->a[iPDDst];
3613
3614	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3615	const unsigned iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
3616	PX86PDPAE pPDDst;
3617	PX86PDEPAE pPdeDst;
3618
3619	# if PGM_GST_TYPE == PGM_TYPE_PROT
3620	/* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
3621	X86PML4E Pml4eSrc;
3622	X86PDPE PdpeSrc;
3623	PX86PML4E pPml4eSrc = &Pml4eSrc;
3624	Pml4eSrc.u = X86_PML4E_P \| X86_PML4E_RW \| X86_PML4E_US \| X86_PML4E_A;
3625	PdpeSrc.u = X86_PDPE_P \| X86_PDPE_RW \| X86_PDPE_US \| X86_PDPE_A;
3626	# endif
3627
3628	rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
3629	if (rc != VINF_SUCCESS)
3630	{
3631	pgmUnlock(pVM);
3632	AssertRC(rc);
3633	return rc;
3634	}
3635	Assert(pPDDst);
3636	pPdeDst = &pPDDst->a[iPDDst];
3637	# endif
3638
3639	if (!pPdeDst->n.u1Present)
3640	{
3641	rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
3642	if (rc != VINF_SUCCESS)
3643	{
3644	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3645	pgmUnlock(pVM);
3646	AssertRC(rc);
3647	return rc;
3648	}
3649	}
3650
3651	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3652	/* Check for dirty bit fault */
3653	rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
3654	if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
3655	Log(("PGMVerifyAccess: success (dirty)\n"));
3656	else
3657	# endif
3658	{
3659	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
3660	GSTPDE PdeSrc = pPDSrc->a[iPDSrc];
3661	# else
3662	GSTPDE PdeSrc;
3663	PdeSrc.u = 0; /* faked so we don't have to #ifdef everything */
3664	PdeSrc.n.u1Present = 1;
3665	PdeSrc.n.u1Write = 1;
3666	PdeSrc.n.u1Accessed = 1;
3667	PdeSrc.n.u1User = 1;
3668	# endif
3669
3670	Assert(rc != VINF_EM_RAW_GUEST_TRAP);
3671	if (uErr & X86_TRAP_PF_US)
3672	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncUser));
3673	else /* supervisor */
3674	STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
3675
3676	rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
3677	if (RT_SUCCESS(rc))
3678	{
3679	/* Page was successfully synced */
3680	Log2(("PGMVerifyAccess: success (sync)\n"));
3681	rc = VINF_SUCCESS;
3682	}
3683	else
3684	{
3685	Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
3686	rc = VINF_EM_RAW_GUEST_TRAP;
3687	}
3688	}
3689	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
3690	pgmUnlock(pVM);
3691	return rc;
3692
3693	#else /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3694
3695	AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
3696	return VERR_PGM_NOT_USED_IN_MODE;
3697	#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
3698	}
3699
3700
3701	/**
3702	* Syncs the paging hierarchy starting at CR3.
3703	*
3704	* @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
3705	* informational status codes.
3706	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
3707	* the VMM into guest context.
3708	* @param pVCpu The cross context virtual CPU structure.
3709	* @param cr0 Guest context CR0 register.
3710	* @param cr3 Guest context CR3 register. Not subjected to the A20
3711	* mask.
3712	* @param cr4 Guest context CR4 register.
3713	* @param fGlobal Including global page directories or not
3714	*/
3715	PGM_BTH_DECL(int, SyncCR3)(PVMCPU pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
3716	{
3717	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
3718	NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
3719
3720	LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
3721
3722	#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3723
3724	pgmLock(pVM);
3725
3726	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
3727	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3728	if (pPool->cDirtyPages)
3729	pgmPoolResetDirtyPages(pVM);
3730	# endif
3731
3732	/*
3733	* Update page access handlers.
3734	* The virtual are always flushed, while the physical are only on demand.
3735	* WARNING: We are incorrectly not doing global flushing on Virtual Handler updates. We'll
3736	* have to look into that later because it will have a bad influence on the performance.
3737	* @note SvL: There's no need for that. Just invalidate the virtual range(s).
3738	* bird: Yes, but that won't work for aliases.
3739	*/
3740	/** @todo this MUST go away. See @bugref{1557}. */
3741	STAM_PROFILE_START(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3742	PGM_GST_NAME(HandlerVirtualUpdate)(pVM, cr4);
3743	STAM_PROFILE_STOP(&pVCpu->pgm.s.CTX_SUFF(pStats)->CTX_MID_Z(Stat,SyncCR3Handlers), h);
3744	pgmUnlock(pVM);
3745	#endif /* !NESTED && !EPT */
3746
3747	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3748	/*
3749	* Nested / EPT - almost no work.
3750	*/
3751	Assert(!pgmMapAreMappingsEnabled(pVM));
3752	return VINF_SUCCESS;
3753
3754	#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3755	/*
3756	* AMD64 (Shw & Gst) - No need to check all paging levels; we zero
3757	* out the shadow parts when the guest modifies its tables.
3758	*/
3759	Assert(!pgmMapAreMappingsEnabled(pVM));
3760	return VINF_SUCCESS;
3761
3762	#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3763
3764	# ifndef PGM_WITHOUT_MAPPINGS
3765	/*
3766	* Check for and resolve conflicts with our guest mappings if they
3767	* are enabled and not fixed.
3768	*/
3769	if (pgmMapAreMappingsFloating(pVM))
3770	{
3771	int rc = pgmMapResolveConflicts(pVM);
3772	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
3773	if (rc == VINF_SUCCESS)
3774	{ /* likely */ }
3775	else if (rc == VINF_PGM_SYNC_CR3)
3776	{
3777	LogFlow(("SyncCR3: detected conflict -> VINF_PGM_SYNC_CR3\n"));
3778	return VINF_PGM_SYNC_CR3;
3779	}
3780	else if (RT_FAILURE(rc))
3781	return rc;
3782	else
3783	AssertMsgFailed(("%Rrc\n", rc));
3784	}
3785	# else
3786	Assert(!pgmMapAreMappingsEnabled(pVM));
3787	# endif
3788	return VINF_SUCCESS;
3789	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
3790	}
3791
3792
3793
3794
3795	#ifdef VBOX_STRICT
3796	# ifdef IN_RC
3797	# undef AssertMsgFailed
3798	# define AssertMsgFailed Log
3799	# endif
3800
3801	/**
3802	* Checks that the shadow page table is in sync with the guest one.
3803	*
3804	* @returns The number of errors.
3805	* @param pVCpu The cross context virtual CPU structure.
3806	* @param cr3 Guest context CR3 register.
3807	* @param cr4 Guest context CR4 register.
3808	* @param GCPtr Where to start. Defaults to 0.
3809	* @param cb How much to check. Defaults to everything.
3810	*/
3811	PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPU pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
3812	{
3813	NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
3814	#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
3815	return 0;
3816	#else
3817	unsigned cErrors = 0;
3818	PVM pVM = pVCpu->CTX_SUFF(pVM);
3819	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3820
3821	# if PGM_GST_TYPE == PGM_TYPE_PAE
3822	/** @todo currently broken; crashes below somewhere */
3823	AssertFailed();
3824	# endif
3825
3826	# if PGM_GST_TYPE == PGM_TYPE_32BIT \
3827	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
3828	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
3829
3830	bool fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
3831	PPGMCPU pPGM = &pVCpu->pgm.s;
3832	RTGCPHYS GCPhysGst; /* page address derived from the guest page tables. */
3833	RTHCPHYS HCPhysShw; /* page address derived from the shadow page tables. */
3834	# ifndef IN_RING0
3835	RTHCPHYS HCPhys; /* general usage. */
3836	# endif
3837	int rc;
3838
3839	/*
3840	* Check that the Guest CR3 and all its mappings are correct.
3841	*/
3842	AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
3843	("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
3844	false);
3845	# if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
3846	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3847	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
3848	# else
3849	rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
3850	# endif
3851	AssertRCReturn(rc, 1);
3852	HCPhys = NIL_RTHCPHYS;
3853	rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
3854	AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
3855	# if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
3856	pgmGstGet32bitPDPtr(pVCpu);
3857	RTGCPHYS GCPhys;
3858	rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
3859	AssertRCReturn(rc, 1);
3860	AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
3861	# endif
3862	# endif /* !IN_RING0 */
3863
3864	/*
3865	* Get and check the Shadow CR3.
3866	*/
3867	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
3868	unsigned cPDEs = X86_PG_ENTRIES;
3869	unsigned cIncrement = X86_PG_ENTRIES * PAGE_SIZE;
3870	# elif PGM_SHW_TYPE == PGM_TYPE_PAE
3871	# if PGM_GST_TYPE == PGM_TYPE_32BIT
3872	unsigned cPDEs = X86_PG_PAE_ENTRIES * 4; /* treat it as a 2048 entry table. */
3873	# else
3874	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3875	# endif
3876	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3877	# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
3878	unsigned cPDEs = X86_PG_PAE_ENTRIES;
3879	unsigned cIncrement = X86_PG_PAE_ENTRIES * PAGE_SIZE;
3880	# endif
3881	if (cb != ~(RTGCPTR)0)
3882	cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
3883
3884	/** @todo call the other two PGMAssert() functions. /
3885
3886	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3887	unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
3888
3889	for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
3890	{
3891	PPGMPOOLPAGE pShwPdpt = NULL;
3892	PX86PML4E pPml4eSrc;
3893	PX86PML4E pPml4eDst;
3894	RTGCPHYS GCPhysPdptSrc;
3895
3896	pPml4eSrc = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
3897	pPml4eDst = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
3898
3899	/* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
3900	if (!pPml4eDst->n.u1Present)
3901	{
3902	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3903	continue;
3904	}
3905
3906	pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
3907	GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
3908
3909	if (pPml4eSrc->n.u1Present != pPml4eDst->n.u1Present)
3910	{
3911	AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3912	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3913	cErrors++;
3914	continue;
3915	}
3916
3917	if (GCPhysPdptSrc != pShwPdpt->GCPhys)
3918	{
3919	AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
3920	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3921	cErrors++;
3922	continue;
3923	}
3924
3925	if ( pPml4eDst->n.u1User != pPml4eSrc->n.u1User
3926	\|\| pPml4eDst->n.u1Write != pPml4eSrc->n.u1Write
3927	\|\| pPml4eDst->n.u1NoExecute != pPml4eSrc->n.u1NoExecute)
3928	{
3929	AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
3930	GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
3931	cErrors++;
3932	continue;
3933	}
3934	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3935	{
3936	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
3937
3938	# if PGM_GST_TYPE == PGM_TYPE_AMD64 \|\| PGM_GST_TYPE == PGM_TYPE_PAE
3939	/*
3940	* Check the PDPTEs too.
3941	*/
3942	unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
3943
3944	for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
3945	{
3946	unsigned iPDSrc = 0; /* initialized to shut up gcc */
3947	PPGMPOOLPAGE pShwPde = NULL;
3948	PX86PDPE pPdpeDst;
3949	RTGCPHYS GCPhysPdeSrc;
3950	X86PDPE PdpeSrc;
3951	PdpeSrc.u = 0; /* initialized to shut up gcc 4.5 */
3952	# if PGM_GST_TYPE == PGM_TYPE_PAE
3953	PGSTPD pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
3954	PX86PDPT pPdptDst = pgmShwGetPaePDPTPtr(pVCpu);
3955	# else
3956	PX86PML4E pPml4eSrcIgn;
3957	PX86PDPT pPdptDst;
3958	PX86PDPAE pPDDst;
3959	PGSTPD pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
3960
3961	rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
3962	if (rc != VINF_SUCCESS)
3963	{
3964	AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
3965	GCPtr += 512 * _2M;
3966	continue; /* next PDPTE */
3967	}
3968	Assert(pPDDst);
3969	# endif
3970	Assert(iPDSrc == 0);
3971
3972	pPdpeDst = &pPdptDst->a[iPdpt];
3973
3974	if (!pPdpeDst->n.u1Present)
3975	{
3976	GCPtr += 512 * _2M;
3977	continue; /* next PDPTE */
3978	}
3979
3980	pShwPde = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
3981	GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
3982
3983	if (pPdpeDst->n.u1Present != PdpeSrc.n.u1Present)
3984	{
3985	AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
3986	GCPtr += 512 * _2M;
3987	cErrors++;
3988	continue;
3989	}
3990
3991	if (GCPhysPdeSrc != pShwPde->GCPhys)
3992	{
3993	# if PGM_GST_TYPE == PGM_TYPE_AMD64
3994	AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3995	# else
3996	AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
3997	# endif
3998	GCPtr += 512 * _2M;
3999	cErrors++;
4000	continue;
4001	}
4002
4003	# if PGM_GST_TYPE == PGM_TYPE_AMD64
4004	if ( pPdpeDst->lm.u1User != PdpeSrc.lm.u1User
4005	\|\| pPdpeDst->lm.u1Write != PdpeSrc.lm.u1Write
4006	\|\| pPdpeDst->lm.u1NoExecute != PdpeSrc.lm.u1NoExecute)
4007	{
4008	AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
4009	GCPtr += 512 * _2M;
4010	cErrors++;
4011	continue;
4012	}
4013	# endif
4014
4015	# else /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4016	{
4017	# endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
4018	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4019	GSTPD const *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
4020	# if PGM_SHW_TYPE == PGM_TYPE_32BIT
4021	PCX86PD pPDDst = pgmShwGet32BitPDPtr(pVCpu);
4022	# endif
4023	# endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
4024	/*
4025	* Iterate the shadow page directory.
4026	*/
4027	GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
4028	unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
4029
4030	for (;
4031	iPDDst < cPDEs;
4032	iPDDst++, GCPtr += cIncrement)
4033	{
4034	# if PGM_SHW_TYPE == PGM_TYPE_PAE
4035	const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
4036	# else
4037	const SHWPDE PdeDst = pPDDst->a[iPDDst];
4038	# endif
4039	if (PdeDst.u & PGM_PDFLAGS_MAPPING)
4040	{
4041	Assert(pgmMapAreMappingsEnabled(pVM));
4042	if ((PdeDst.u & X86_PDE_AVL_MASK) != PGM_PDFLAGS_MAPPING)
4043	{
4044	AssertMsgFailed(("Mapping shall only have PGM_PDFLAGS_MAPPING set! PdeDst.u=%#RX64\n", (uint64_t)PdeDst.u));
4045	cErrors++;
4046	continue;
4047	}
4048	}
4049	else if ( (PdeDst.u & X86_PDE_P)
4050	\|\| ((PdeDst.u & (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P \| PGM_PDFLAGS_TRACK_DIRTY))
4051	)
4052	{
4053	HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
4054	PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
4055	if (!pPoolPage)
4056	{
4057	AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
4058	HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
4059	cErrors++;
4060	continue;
4061	}
4062	const SHWPT pPTDst = (const SHWPT )PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
4063
4064	if (PdeDst.u & (X86_PDE4M_PWT \| X86_PDE4M_PCD))
4065	{
4066	AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
4067	GCPtr, (uint64_t)PdeDst.u));
4068	cErrors++;
4069	}
4070
4071	if (PdeDst.u & (X86_PDE4M_G \| X86_PDE4M_D))
4072	{
4073	AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
4074	GCPtr, (uint64_t)PdeDst.u));
4075	cErrors++;
4076	}
4077
4078	const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
4079	if (!PdeSrc.n.u1Present)
4080	{
4081	AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
4082	GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
4083	cErrors++;
4084	continue;
4085	}
4086
4087	if ( !PdeSrc.b.u1Size
4088	\|\| !fBigPagesSupported)
4089	{
4090	GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
4091	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4092	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| ((iPDDst & 1) * (PAGE_SIZE / 2)));
4093	# endif
4094	}
4095	else
4096	{
4097	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4098	if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
4099	{
4100	AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
4101	GCPtr, (uint64_t)PdeSrc.u));
4102	cErrors++;
4103	continue;
4104	}
4105	# endif
4106	GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
4107	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4108	GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst \| (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
4109	# endif
4110	}
4111
4112	if ( pPoolPage->enmKind
4113	!= (!PdeSrc.b.u1Size \|\| !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
4114	{
4115	AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
4116	pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
4117	cErrors++;
4118	}
4119
4120	PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4121	if (!pPhysPage)
4122	{
4123	AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4124	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4125	cErrors++;
4126	continue;
4127	}
4128
4129	if (GCPhysGst != pPoolPage->GCPhys)
4130	{
4131	AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
4132	GCPhysGst, pPoolPage->GCPhys, GCPtr));
4133	cErrors++;
4134	continue;
4135	}
4136
4137	if ( !PdeSrc.b.u1Size
4138	\|\| !fBigPagesSupported)
4139	{
4140	/*
4141	* Page Table.
4142	*/
4143	const GSTPT *pPTSrc;
4144	rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(PAGE_SIZE - 1)),
4145	&pPTSrc);
4146	if (RT_FAILURE(rc))
4147	{
4148	AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
4149	GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
4150	cErrors++;
4151	continue;
4152	}
4153	if ( (PdeSrc.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/))
4154	!= (PdeDst.u & (X86_PDE_P \| X86_PDE_US \| X86_PDE_RW/* \| X86_PDE_A*/)))
4155	{
4156	/// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
4157	// (This problem will go away when/if we shadow multiple CR3s.)
4158	AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4159	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4160	cErrors++;
4161	continue;
4162	}
4163	if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4164	{
4165	AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
4166	GCPtr, (uint64_t)PdeDst.u));
4167	cErrors++;
4168	continue;
4169	}
4170
4171	/* iterate the page table. */
4172	# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
4173	/* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
4174	const unsigned offPTSrc = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
4175	# else
4176	const unsigned offPTSrc = 0;
4177	# endif
4178	for (unsigned iPT = 0, off = 0;
4179	iPT < RT_ELEMENTS(pPTDst->a);
4180	iPT++, off += PAGE_SIZE)
4181	{
4182	const SHWPTE PteDst = pPTDst->a[iPT];
4183
4184	/* skip not-present and dirty tracked entries. */
4185	if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P \| PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
4186	continue;
4187	Assert(SHW_PTE_IS_P(PteDst));
4188
4189	const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
4190	if (!PteSrc.n.u1Present)
4191	{
4192	# ifdef IN_RING3
4193	PGMAssertHandlerAndFlagsInSync(pVM);
4194	DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 \| DBGFPGDMP_FLAGS_CURRENT_MODE
4195	\| DBGFPGDMP_FLAGS_GUEST \| DBGFPGDMP_FLAGS_HEADER \| DBGFPGDMP_FLAGS_PRINT_CR3,
4196	0, 0, UINT64_MAX, 99, NULL);
4197	# endif
4198	AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
4199	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
4200	(uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
4201	cErrors++;
4202	continue;
4203	}
4204
4205	uint64_t fIgnoreFlags = GST_PTE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_G \| X86_PTE_D \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT;
4206	# if 1 /** @todo sync accessed bit properly... */
4207	fIgnoreFlags \|= X86_PTE_A;
4208	# endif
4209
4210	/* match the physical addresses */
4211	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4212	GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
4213
4214	# ifdef IN_RING3
4215	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4216	if (RT_FAILURE(rc))
4217	{
4218	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4219	{
4220	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4221	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4222	cErrors++;
4223	continue;
4224	}
4225	}
4226	else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
4227	{
4228	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4229	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4230	cErrors++;
4231	continue;
4232	}
4233	# endif
4234
4235	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4236	if (!pPhysPage)
4237	{
4238	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4239	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4240	{
4241	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
4242	GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4243	cErrors++;
4244	continue;
4245	}
4246	# endif
4247	if (SHW_PTE_IS_RW(PteDst))
4248	{
4249	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4250	GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4251	cErrors++;
4252	}
4253	fIgnoreFlags \|= X86_PTE_RW;
4254	}
4255	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4256	{
4257	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
4258	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4259	cErrors++;
4260	continue;
4261	}
4262
4263	/* flags */
4264	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4265	{
4266	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4267	{
4268	if (SHW_PTE_IS_RW(PteDst))
4269	{
4270	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4271	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4272	cErrors++;
4273	continue;
4274	}
4275	fIgnoreFlags \|= X86_PTE_RW;
4276	}
4277	else
4278	{
4279	if ( SHW_PTE_IS_P(PteDst)
4280	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4281	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4282	# endif
4283	)
4284	{
4285	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
4286	GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4287	cErrors++;
4288	continue;
4289	}
4290	fIgnoreFlags \|= X86_PTE_P;
4291	}
4292	}
4293	else
4294	{
4295	if (!PteSrc.n.u1Dirty && PteSrc.n.u1Write)
4296	{
4297	if (SHW_PTE_IS_RW(PteDst))
4298	{
4299	AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
4300	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4301	cErrors++;
4302	continue;
4303	}
4304	if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
4305	{
4306	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4307	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4308	cErrors++;
4309	continue;
4310	}
4311	if (SHW_PTE_IS_D(PteDst))
4312	{
4313	AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4314	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4315	cErrors++;
4316	}
4317	# if 0 /** @todo sync access bit properly... */
4318	if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
4319	{
4320	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4321	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4322	cErrors++;
4323	}
4324	fIgnoreFlags \|= X86_PTE_RW;
4325	# else
4326	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4327	# endif
4328	}
4329	else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4330	{
4331	/* access bit emulation (not implemented). */
4332	if (PteSrc.n.u1Accessed \|\| SHW_PTE_IS_P(PteDst))
4333	{
4334	AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
4335	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4336	cErrors++;
4337	continue;
4338	}
4339	if (!SHW_PTE_IS_A(PteDst))
4340	{
4341	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
4342	GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4343	cErrors++;
4344	}
4345	fIgnoreFlags \|= X86_PTE_P;
4346	}
4347	# ifdef DEBUG_sandervl
4348	fIgnoreFlags \|= X86_PTE_D \| X86_PTE_A;
4349	# endif
4350	}
4351
4352	if ( (PteSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4353	&& (PteSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4354	)
4355	{
4356	AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
4357	GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4358	fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
4359	cErrors++;
4360	continue;
4361	}
4362	} /* foreach PTE */
4363	}
4364	else
4365	{
4366	/*
4367	* Big Page.
4368	*/
4369	uint64_t fIgnoreFlags = X86_PDE_AVL_MASK \| GST_PDE_PG_MASK \| X86_PDE4M_G \| X86_PDE4M_D \| X86_PDE4M_PS \| X86_PDE4M_PWT \| X86_PDE4M_PCD;
4370	if (!PdeSrc.b.u1Dirty && PdeSrc.b.u1Write)
4371	{
4372	if (PdeDst.n.u1Write)
4373	{
4374	AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4375	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4376	cErrors++;
4377	continue;
4378	}
4379	if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
4380	{
4381	AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
4382	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4383	cErrors++;
4384	continue;
4385	}
4386	# if 0 /** @todo sync access bit properly... */
4387	if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
4388	{
4389	AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
4390	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4391	cErrors++;
4392	}
4393	fIgnoreFlags \|= X86_PTE_RW;
4394	# else
4395	fIgnoreFlags \|= X86_PTE_RW \| X86_PTE_A;
4396	# endif
4397	}
4398	else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
4399	{
4400	/* access bit emulation (not implemented). */
4401	if (PdeSrc.b.u1Accessed \|\| PdeDst.n.u1Present)
4402	{
4403	AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4404	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4405	cErrors++;
4406	continue;
4407	}
4408	if (!PdeDst.n.u1Accessed)
4409	{
4410	AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
4411	GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4412	cErrors++;
4413	}
4414	fIgnoreFlags \|= X86_PTE_P;
4415	}
4416
4417	if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
4418	{
4419	AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
4420	GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
4421	fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
4422	cErrors++;
4423	}
4424
4425	/* iterate the page table. */
4426	for (unsigned iPT = 0, off = 0;
4427	iPT < RT_ELEMENTS(pPTDst->a);
4428	iPT++, off += PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + PAGE_SIZE))
4429	{
4430	const SHWPTE PteDst = pPTDst->a[iPT];
4431
4432	if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
4433	{
4434	AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
4435	GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4436	cErrors++;
4437	}
4438
4439	/* skip not-present entries. */
4440	if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
4441	continue;
4442
4443	fIgnoreFlags = X86_PTE_PAE_PG_MASK \| X86_PTE_AVL_MASK \| X86_PTE_PWT \| X86_PTE_PCD \| X86_PTE_PAT \| X86_PTE_D \| X86_PTE_A \| X86_PTE_G \| X86_PTE_PAE_NX;
4444
4445	/* match the physical addresses */
4446	HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
4447
4448	# ifdef IN_RING3
4449	rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
4450	if (RT_FAILURE(rc))
4451	{
4452	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4453	{
4454	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4455	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4456	cErrors++;
4457	}
4458	}
4459	else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
4460	{
4461	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4462	GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4463	cErrors++;
4464	continue;
4465	}
4466	# endif
4467	pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
4468	if (!pPhysPage)
4469	{
4470	# ifdef IN_RING3 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
4471	if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
4472	{
4473	AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
4474	GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4475	cErrors++;
4476	continue;
4477	}
4478	# endif
4479	if (SHW_PTE_IS_RW(PteDst))
4480	{
4481	AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4482	GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4483	cErrors++;
4484	}
4485	fIgnoreFlags \|= X86_PTE_RW;
4486	}
4487	else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
4488	{
4489	AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
4490	GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4491	cErrors++;
4492	continue;
4493	}
4494
4495	/* flags */
4496	if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
4497	{
4498	if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
4499	{
4500	if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
4501	{
4502	if (SHW_PTE_IS_RW(PteDst))
4503	{
4504	AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4505	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4506	cErrors++;
4507	continue;
4508	}
4509	fIgnoreFlags \|= X86_PTE_RW;
4510	}
4511	}
4512	else
4513	{
4514	if ( SHW_PTE_IS_P(PteDst)
4515	# if PGM_SHW_TYPE == PGM_TYPE_EPT \|\| PGM_SHW_TYPE == PGM_TYPE_PAE \|\| PGM_SHW_TYPE == PGM_TYPE_AMD64
4516	&& !PGM_PAGE_IS_MMIO(pPhysPage)
4517	# endif
4518	)
4519	{
4520	AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
4521	GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4522	cErrors++;
4523	continue;
4524	}
4525	fIgnoreFlags \|= X86_PTE_P;
4526	}
4527	}
4528
4529	if ( (PdeSrc.u & ~fIgnoreFlags) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
4530	&& (PdeSrc.u & ~(fIgnoreFlags \| X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
4531	)
4532	{
4533	AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
4534	GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
4535	fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
4536	cErrors++;
4537	continue;
4538	}
4539	} /* for each PTE */
4540	}
4541	}
4542	/* not present */
4543
4544	} /* for each PDE */
4545
4546	} /* for each PDPTE */
4547
4548	} /* for each PML4E */
4549
4550	# ifdef DEBUG
4551	if (cErrors)
4552	LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
4553	# endif
4554	# endif /* GST is in {32BIT, PAE, AMD64} */
4555	return cErrors;
4556	#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
4557	}
4558	#endif /* VBOX_STRICT */
4559
4560
4561	/**
4562	* Sets up the CR3 for shadow paging
4563	*
4564	* @returns Strict VBox status code.
4565	* @retval VINF_SUCCESS.
4566	*
4567	* @param pVCpu The cross context virtual CPU structure.
4568	* @param GCPhysCR3 The physical address in the CR3 register. (A20
4569	* mask already applied.)
4570	*/
4571	PGM_BTH_DECL(int, MapCR3)(PVMCPU pVCpu, RTGCPHYS GCPhysCR3)
4572	{
4573	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4574
4575	/* Update guest paging info. */
4576	#if PGM_GST_TYPE == PGM_TYPE_32BIT \
4577	\|\| PGM_GST_TYPE == PGM_TYPE_PAE \
4578	\|\| PGM_GST_TYPE == PGM_TYPE_AMD64
4579
4580	LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
4581	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4582
4583	/*
4584	* Map the page CR3 points at.
4585	*/
4586	RTHCPTR HCPtrGuestCR3;
4587	RTHCPHYS HCPhysGuestCR3;
4588	pgmLock(pVM);
4589	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
4590	AssertReturn(pPageCR3, VERR_PGM_INVALID_CR3_ADDR);
4591	HCPhysGuestCR3 = PGM_PAGE_GET_HCPHYS(pPageCR3);
4592	/** @todo this needs some reworking wrt. locking? */
4593	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4594	HCPtrGuestCR3 = NIL_RTHCPTR;
4595	int rc = VINF_SUCCESS;
4596	# else
4597	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3 & GST_CR3_PAGE_MASK, (void )&HCPtrGuestCR3); / @todo r=bird: This GCPhysCR3 masking isn't necessary. */
4598	# endif
4599	pgmUnlock(pVM);
4600	if (RT_SUCCESS(rc))
4601	{
4602	rc = PGMMap(pVM, (RTGCPTR)pVM->pgm.s.GCPtrCR3Mapping, HCPhysGuestCR3, PAGE_SIZE, 0);
4603	if (RT_SUCCESS(rc))
4604	{
4605	# ifdef IN_RC
4606	PGM_INVL_PG(pVCpu, pVM->pgm.s.GCPtrCR3Mapping);
4607	# endif
4608	# if PGM_GST_TYPE == PGM_TYPE_32BIT
4609	pVCpu->pgm.s.pGst32BitPdR3 = (R3PTRTYPE(PX86PD))HCPtrGuestCR3;
4610	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4611	pVCpu->pgm.s.pGst32BitPdR0 = (R0PTRTYPE(PX86PD))HCPtrGuestCR3;
4612	# endif
4613	pVCpu->pgm.s.pGst32BitPdRC = (RCPTRTYPE(PX86PD))(RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping;
4614
4615	# elif PGM_GST_TYPE == PGM_TYPE_PAE
4616	unsigned off = GCPhysCR3 & GST_CR3_PAGE_MASK & PAGE_OFFSET_MASK;
4617	pVCpu->pgm.s.pGstPaePdptR3 = (R3PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4618	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4619	pVCpu->pgm.s.pGstPaePdptR0 = (R0PTRTYPE(PX86PDPT))HCPtrGuestCR3;
4620	# endif
4621	pVCpu->pgm.s.pGstPaePdptRC = (RCPTRTYPE(PX86PDPT))((RTRCUINTPTR)pVM->pgm.s.GCPtrCR3Mapping + off);
4622	LogFlow(("Cached mapping %RRv\n", pVCpu->pgm.s.pGstPaePdptRC));
4623
4624	/*
4625	* Map the 4 PDs too.
4626	*/
4627	PX86PDPT pGuestPDPT = pgmGstGetPaePDPTPtr(pVCpu);
4628	RTGCPTR GCPtr = pVM->pgm.s.GCPtrCR3Mapping + PAGE_SIZE;
4629	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++, GCPtr += PAGE_SIZE)
4630	{
4631	pVCpu->pgm.s.aGstPaePdpeRegs[i].u = pGuestPDPT->a[i].u;
4632	if (pGuestPDPT->a[i].n.u1Present)
4633	{
4634	RTHCPTR HCPtr;
4635	RTHCPHYS HCPhys;
4636	RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, pGuestPDPT->a[i].u & X86_PDPE_PG_MASK);
4637	pgmLock(pVM);
4638	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
4639	AssertReturn(pPage, VERR_PGM_INVALID_PDPE_ADDR);
4640	HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
4641	# if defined(IN_RC) \|\| defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4642	HCPtr = NIL_RTHCPTR;
4643	int rc2 = VINF_SUCCESS;
4644	# else
4645	int rc2 = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
4646	# endif
4647	pgmUnlock(pVM);
4648	if (RT_SUCCESS(rc2))
4649	{
4650	rc = PGMMap(pVM, GCPtr, HCPhys, PAGE_SIZE, 0);
4651	AssertRCReturn(rc, rc);
4652
4653	pVCpu->pgm.s.apGstPaePDsR3[i] = (R3PTRTYPE(PX86PDPAE))HCPtr;
4654	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4655	pVCpu->pgm.s.apGstPaePDsR0[i] = (R0PTRTYPE(PX86PDPAE))HCPtr;
4656	# endif
4657	pVCpu->pgm.s.apGstPaePDsRC[i] = (RCPTRTYPE(PX86PDPAE))(RTRCUINTPTR)GCPtr;
4658	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
4659	# ifdef IN_RC
4660	PGM_INVL_PG(pVCpu, GCPtr);
4661	# endif
4662	continue;
4663	}
4664	AssertMsgFailed(("pgmR3Gst32BitMapCR3: rc2=%d GCPhys=%RGp i=%d\n", rc2, GCPhys, i));
4665	}
4666
4667	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4668	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4669	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4670	# endif
4671	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4672	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4673	# ifdef IN_RC
4674	PGM_INVL_PG(pVCpu, GCPtr); /** @todo this shouldn't be necessary? */
4675	# endif
4676	}
4677
4678	# elif PGM_GST_TYPE == PGM_TYPE_AMD64
4679	pVCpu->pgm.s.pGstAmd64Pml4R3 = (R3PTRTYPE(PX86PML4))HCPtrGuestCR3;
4680	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4681	pVCpu->pgm.s.pGstAmd64Pml4R0 = (R0PTRTYPE(PX86PML4))HCPtrGuestCR3;
4682	# endif
4683	# endif
4684	}
4685	else
4686	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4687	}
4688	else
4689	AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
4690
4691	#else /* prot/real stub */
4692	int rc = VINF_SUCCESS;
4693	#endif
4694
4695	/* Update shadow paging info for guest modes with paging (32, pae, 64). */
4696	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4697	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4698	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64) \
4699	&& ( PGM_GST_TYPE != PGM_TYPE_REAL \
4700	&& PGM_GST_TYPE != PGM_TYPE_PROT))
4701
4702	Assert(!pVM->pgm.s.fNestedPaging);
4703	PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
4704
4705	/*
4706	* Update the shadow root page as well since that's not fixed.
4707	*/
4708	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4709	PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
4710	PPGMPOOLPAGE pNewShwPageCR3;
4711
4712	pgmLock(pVM);
4713
4714	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4715	if (pPool->cDirtyPages)
4716	pgmPoolResetDirtyPages(pVM);
4717	# endif
4718
4719	Assert(!(GCPhysCR3 >> (PAGE_SHIFT + 32)));
4720	rc = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
4721	NIL_PGMPOOL_IDX, UINT32_MAX, true /fLockPage/,
4722	&pNewShwPageCR3);
4723	AssertFatalRC(rc);
4724	rc = VINF_SUCCESS;
4725
4726	# ifdef IN_RC
4727	/*
4728	* WARNING! We can't deal with jumps to ring 3 in the code below as the
4729	* state will be inconsistent! Flush important things now while
4730	* we still can and then make sure there are no ring-3 calls.
4731	*/
4732	# ifdef VBOX_WITH_REM
4733	REMNotifyHandlerPhysicalFlushIfAlmostFull(pVM, pVCpu);
4734	# endif
4735	VMMRZCallRing3Disable(pVCpu);
4736	# endif
4737
4738	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3) = pNewShwPageCR3;
4739	# ifdef IN_RING0
4740	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4741	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4742	# elif defined(IN_RC)
4743	pVCpu->pgm.s.pShwPageCR3R3 = MMHyperCCToR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4744	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4745	# else
4746	pVCpu->pgm.s.pShwPageCR3R0 = MMHyperCCToR0(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4747	pVCpu->pgm.s.pShwPageCR3RC = MMHyperCCToRC(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4748	# endif
4749
4750	# ifndef PGM_WITHOUT_MAPPINGS
4751	/*
4752	* Apply all hypervisor mappings to the new CR3.
4753	* Note that SyncCR3 will be executed in case CR3 is changed in a guest paging mode; this will
4754	* make sure we check for conflicts in the new CR3 root.
4755	*/
4756	# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
4757	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL) \|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4758	# endif
4759	rc = pgmMapActivateCR3(pVM, pNewShwPageCR3);
4760	AssertRCReturn(rc, rc);
4761	# endif
4762
4763	/* Set the current hypervisor CR3. */
4764	CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
4765	SELMShadowCR3Changed(pVM, pVCpu);
4766
4767	# ifdef IN_RC
4768	/* NOTE: The state is consistent again. */
4769	VMMRZCallRing3Enable(pVCpu);
4770	# endif
4771
4772	/* Clean up the old CR3 root. */
4773	if ( pOldShwPageCR3
4774	&& pOldShwPageCR3 != pNewShwPageCR3 /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
4775	{
4776	Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
4777	# ifndef PGM_WITHOUT_MAPPINGS
4778	/* Remove the hypervisor mappings from the shadow page table. */
4779	pgmMapDeactivateCR3(pVM, pOldShwPageCR3);
4780	# endif
4781	/* Mark the page as unlocked; allow flushing again. */
4782	pgmPoolUnlockPage(pPool, pOldShwPageCR3);
4783
4784	pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
4785	}
4786	pgmUnlock(pVM);
4787	# else
4788	NOREF(GCPhysCR3);
4789	# endif
4790
4791	return rc;
4792	}
4793
4794	/**
4795	* Unmaps the shadow CR3.
4796	*
4797	* @returns VBox status, no specials.
4798	* @param pVCpu The cross context virtual CPU structure.
4799	*/
4800	PGM_BTH_DECL(int, UnmapCR3)(PVMCPU pVCpu)
4801	{
4802	LogFlow(("UnmapCR3\n"));
4803
4804	int rc = VINF_SUCCESS;
4805	PVM pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
4806
4807	/*
4808	* Update guest paging info.
4809	*/
4810	#if PGM_GST_TYPE == PGM_TYPE_32BIT
4811	pVCpu->pgm.s.pGst32BitPdR3 = 0;
4812	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4813	pVCpu->pgm.s.pGst32BitPdR0 = 0;
4814	# endif
4815	pVCpu->pgm.s.pGst32BitPdRC = 0;
4816
4817	#elif PGM_GST_TYPE == PGM_TYPE_PAE
4818	pVCpu->pgm.s.pGstPaePdptR3 = 0;
4819	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4820	pVCpu->pgm.s.pGstPaePdptR0 = 0;
4821	# endif
4822	pVCpu->pgm.s.pGstPaePdptRC = 0;
4823	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4824	{
4825	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
4826	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4827	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
4828	# endif
4829	pVCpu->pgm.s.apGstPaePDsRC[i] = 0;
4830	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
4831	}
4832
4833	#elif PGM_GST_TYPE == PGM_TYPE_AMD64
4834	pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
4835	# ifndef VBOX_WITH_2X_4GB_ADDR_SPACE
4836	pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
4837	# endif
4838
4839	#else /* prot/real mode stub */
4840	/* nothing to do */
4841	#endif
4842
4843	#if !defined(IN_RC) /* In RC we rely on MapCR3 to do the shadow part for us at a safe time */
4844	/*
4845	* Update shadow paging info.
4846	*/
4847	# if ( ( PGM_SHW_TYPE == PGM_TYPE_32BIT \
4848	\|\| PGM_SHW_TYPE == PGM_TYPE_PAE \
4849	\|\| PGM_SHW_TYPE == PGM_TYPE_AMD64))
4850
4851	# if PGM_GST_TYPE != PGM_TYPE_REAL
4852	Assert(!pVM->pgm.s.fNestedPaging);
4853	# endif
4854
4855	pgmLock(pVM);
4856
4857	# ifndef PGM_WITHOUT_MAPPINGS
4858	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4859	/* Remove the hypervisor mappings from the shadow page table. */
4860	pgmMapDeactivateCR3(pVM, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4861	# endif
4862
4863	if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
4864	{
4865	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4866
4867	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4868	if (pPool->cDirtyPages)
4869	pgmPoolResetDirtyPages(pVM);
4870	# endif
4871
4872	/* Mark the page as unlocked; allow flushing again. */
4873	pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
4874
4875	pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
4876	pVCpu->pgm.s.pShwPageCR3R3 = 0;
4877	pVCpu->pgm.s.pShwPageCR3R0 = 0;
4878	pVCpu->pgm.s.pShwPageCR3RC = 0;
4879	}
4880	pgmUnlock(pVM);
4881	# endif
4882	#endif /* !IN_RC*/
4883
4884	return rc;
4885	}
4886

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllBth.h@ 73246

以其他格式下載: