PGMAllPool.cpp@ 22788

最後變更在這個檔案從22788是 22788,由 vboxsync 提交於 16 年前
Minor pgm pool optimizations.
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Id`
檔案大小: 188.9 KB

行
1	/* $Id: PGMAllPool.cpp 22788 2009-09-04 15:37:07Z vboxsync $ */
2	/** @file
3	* PGM Shadow Page Pool.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.alldomusa.eu.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_PGM_POOL
27	#include <VBox/pgm.h>
28	#include <VBox/mm.h>
29	#include <VBox/em.h>
30	#include <VBox/cpum.h>
31	#ifdef IN_RC
32	# include <VBox/patm.h>
33	#endif
34	#include "PGMInternal.h"
35	#include <VBox/vm.h>
36	#include <VBox/disopcode.h>
37	#include <VBox/hwacc_vmx.h>
38
39	#include <VBox/log.h>
40	#include <VBox/err.h>
41	#include <iprt/asm.h>
42	#include <iprt/string.h>
43
44
45	/*******************************************************************************
46	* Internal Functions *
47	*******************************************************************************/
48	RT_C_DECLS_BEGIN
49	static void pgmPoolFlushAllInt(PPGMPOOL pPool);
50	#ifdef PGMPOOL_WITH_USER_TRACKING
51	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
52	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
53	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
54	#endif
55	#ifdef PGMPOOL_WITH_CACHE
56	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
57	#endif
58	#ifdef PGMPOOL_WITH_MONITORING
59	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
60	#endif
61	#ifndef IN_RING3
62	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser);
63	#endif
64	#ifdef LOG_ENABLED
65	static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
66	#endif
67
68	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs);
69	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt);
70	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage);
71	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt);
72	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt);
73	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt);
74
75	RT_C_DECLS_END
76
77
78	/**
79	* Checks if the specified page pool kind is for a 4MB or 2MB guest page.
80	*
81	* @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
82	* @param enmKind The page kind.
83	*/
84	DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
85	{
86	switch (enmKind)
87	{
88	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
89	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
90	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
91	return true;
92	default:
93	return false;
94	}
95	}
96
97	/** @def PGMPOOL_PAGE_2_LOCKED_PTR
98	* Maps a pool page pool into the current context and lock it (RC only).
99	*
100	* @returns VBox status code.
101	* @param pVM The VM handle.
102	* @param pPage The pool page.
103	*
104	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
105	* small page window employeed by that function. Be careful.
106	* @remark There is no need to assert on the result.
107	*/
108	#if defined(IN_RC)
109	DECLINLINE(void *) PGMPOOL_PAGE_2_LOCKED_PTR(PVM pVM, PPGMPOOLPAGE pPage)
110	{
111	void *pv = pgmPoolMapPageInlined(&pVM->pgm.s, pPage);
112
113	/* Make sure the dynamic mapping will not be reused. */
114	if (pv)
115	PGMDynLockHCPage(pVM, (uint8_t *)pv);
116
117	return pv;
118	}
119	#else
120	# define PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage) PGMPOOL_PAGE_2_PTR(pVM, pPage)
121	#endif
122
123	/** @def PGMPOOL_UNLOCK_PTR
124	* Unlock a previously locked dynamic caching (RC only).
125	*
126	* @returns VBox status code.
127	* @param pVM The VM handle.
128	* @param pPage The pool page.
129	*
130	* @remark In RC this uses PGMGCDynMapHCPage(), so it will consume of the
131	* small page window employeed by that function. Be careful.
132	* @remark There is no need to assert on the result.
133	*/
134	#if defined(IN_RC)
135	DECLINLINE(void) PGMPOOL_UNLOCK_PTR(PVM pVM, void *pvPage)
136	{
137	if (pvPage)
138	PGMDynUnlockHCPage(pVM, (uint8_t *)pvPage);
139	}
140	#else
141	# define PGMPOOL_UNLOCK_PTR(pVM, pPage) do {} while (0)
142	#endif
143
144
145	#ifdef PGMPOOL_WITH_MONITORING
146	/**
147	* Determin the size of a write instruction.
148	* @returns number of bytes written.
149	* @param pDis The disassembler state.
150	*/
151	static unsigned pgmPoolDisasWriteSize(PDISCPUSTATE pDis)
152	{
153	/*
154	* This is very crude and possibly wrong for some opcodes,
155	* but since it's not really supposed to be called we can
156	* probably live with that.
157	*/
158	return DISGetParamSize(pDis, &pDis->param1);
159	}
160
161
162	/**
163	* Flushes a chain of pages sharing the same access monitor.
164	*
165	* @returns VBox status code suitable for scheduling.
166	* @param pPool The pool.
167	* @param pPage A page in the chain.
168	*/
169	int pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
170	{
171	LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
172
173	/*
174	* Find the list head.
175	*/
176	uint16_t idx = pPage->idx;
177	if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
178	{
179	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
180	{
181	idx = pPage->iMonitoredPrev;
182	Assert(idx != pPage->idx);
183	pPage = &pPool->aPages[idx];
184	}
185	}
186
187	/*
188	* Iterate the list flushing each shadow page.
189	*/
190	int rc = VINF_SUCCESS;
191	for (;;)
192	{
193	idx = pPage->iMonitoredNext;
194	Assert(idx != pPage->idx);
195	if (pPage->idx >= PGMPOOL_IDX_FIRST)
196	{
197	int rc2 = pgmPoolFlushPage(pPool, pPage);
198	AssertRC(rc2);
199	}
200	/* next */
201	if (idx == NIL_PGMPOOL_IDX)
202	break;
203	pPage = &pPool->aPages[idx];
204	}
205	return rc;
206	}
207
208
209	/**
210	* Wrapper for getting the current context pointer to the entry being modified.
211	*
212	* @returns VBox status code suitable for scheduling.
213	* @param pVM VM Handle.
214	* @param pvDst Destination address
215	* @param pvSrc Source guest virtual address.
216	* @param GCPhysSrc The source guest physical address.
217	* @param cb Size of data to read
218	*/
219	DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVM pVM, void *pvDst, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
220	{
221	#if defined(IN_RING3)
222	memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
223	return VINF_SUCCESS;
224	#else
225	/* @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
226	return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
227	#endif
228	}
229
230	/**
231	* Process shadow entries before they are changed by the guest.
232	*
233	* For PT entries we will clear them. For PD entries, we'll simply check
234	* for mapping conflicts and set the SyncCR3 FF if found.
235	*
236	* @param pVCpu VMCPU handle
237	* @param pPool The pool.
238	* @param pPage The head page.
239	* @param GCPhysFault The guest physical fault address.
240	* @param uAddress In R0 and GC this is the guest context fault address (flat).
241	* In R3 this is the host context 'fault' address.
242	* @param pDis The disassembler state for figuring out the write size.
243	* This need not be specified if the caller knows we won't do cross entry accesses.
244	*/
245	void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault, CTXTYPE(RTGCPTR, RTHCPTR, RTGCPTR) pvAddress, PDISCPUSTATE pDis)
246	{
247	AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%#x (idx=%#x)\n", pPage->iMonitoredPrev, pPage->idx));
248	const unsigned off = GCPhysFault & PAGE_OFFSET_MASK;
249	const unsigned cbWrite = pDis ? pgmPoolDisasWriteSize(pDis) : 0;
250	PVM pVM = pPool->CTX_SUFF(pVM);
251
252	LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)pvAddress, GCPhysFault, cbWrite));
253
254	for (;;)
255	{
256	union
257	{
258	void *pv;
259	PX86PT pPT;
260	PX86PTPAE pPTPae;
261	PX86PD pPD;
262	PX86PDPAE pPDPae;
263	PX86PDPT pPDPT;
264	PX86PML4 pPML4;
265	} uShw;
266
267	LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s\n", pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
268
269	uShw.pv = NULL;
270	switch (pPage->enmKind)
271	{
272	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
273	{
274	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
275	const unsigned iShw = off / sizeof(X86PTE);
276	LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
277	if (uShw.pPT->a[iShw].n.u1Present)
278	{
279	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
280	X86PTE GstPte;
281
282	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
283	AssertRC(rc);
284	Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
285	pgmPoolTracDerefGCPhysHint(pPool, pPage,
286	uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK,
287	GstPte.u & X86_PTE_PG_MASK);
288	# endif
289	ASMAtomicWriteSize(&uShw.pPT->a[iShw], 0);
290	}
291	break;
292	}
293
294	/* page/2 sized */
295	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
296	{
297	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
298	if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
299	{
300	const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
301	LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
302	if (uShw.pPTPae->a[iShw].n.u1Present)
303	{
304	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
305	X86PTE GstPte;
306	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
307	AssertRC(rc);
308
309	Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
310	pgmPoolTracDerefGCPhysHint(pPool, pPage,
311	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
312	GstPte.u & X86_PTE_PG_MASK);
313	# endif
314	ASMAtomicWriteSize(&uShw.pPTPae->a[iShw], 0);
315	}
316	}
317	break;
318	}
319
320	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
321	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
322	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
323	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
324	{
325	unsigned iGst = off / sizeof(X86PDE);
326	unsigned iShwPdpt = iGst / 256;
327	unsigned iShw = (iGst % 256) * 2;
328	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
329
330	LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
331	if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
332	{
333	for (unsigned i = 0; i < 2; i++)
334	{
335	# ifndef IN_RING0
336	if ((uShw.pPDPae->a[iShw + i].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
337	{
338	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
339	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
340	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw=%#x!\n", iShwPdpt, iShw+i));
341	break;
342	}
343	else
344	# endif /* !IN_RING0 */
345	if (uShw.pPDPae->a[iShw+i].n.u1Present)
346	{
347	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw+i, uShw.pPDPae->a[iShw+i].u));
348	pgmPoolFree(pVM,
349	uShw.pPDPae->a[iShw+i].u & X86_PDE_PAE_PG_MASK,
350	pPage->idx,
351	iShw + i);
352	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw+i], 0);
353	}
354
355	/* paranoia / a bit assumptive. */
356	if ( pDis
357	&& (off & 3)
358	&& (off & 3) + cbWrite > 4)
359	{
360	const unsigned iShw2 = iShw + 2 + i;
361	if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
362	{
363	# ifndef IN_RING0
364	if ((uShw.pPDPae->a[iShw2].u & (PGM_PDFLAGS_MAPPING \| X86_PDE_P)) == (PGM_PDFLAGS_MAPPING \| X86_PDE_P))
365	{
366	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
367	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
368	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShwPdpt=%#x iShw2=%#x!\n", iShwPdpt, iShw2));
369	break;
370	}
371	else
372	# endif /* !IN_RING0 */
373	if (uShw.pPDPae->a[iShw2].n.u1Present)
374	{
375	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
376	pgmPoolFree(pVM,
377	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
378	pPage->idx,
379	iShw2);
380	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
381	}
382	}
383	}
384	}
385	}
386	break;
387	}
388
389	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
390	{
391	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
392	const unsigned iShw = off / sizeof(X86PTEPAE);
393	if (uShw.pPTPae->a[iShw].n.u1Present)
394	{
395	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
396	X86PTEPAE GstPte;
397	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
398	AssertRC(rc);
399
400	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
401	pgmPoolTracDerefGCPhysHint(pPool, pPage,
402	uShw.pPTPae->a[iShw].u & X86_PTE_PAE_PG_MASK,
403	GstPte.u & X86_PTE_PAE_PG_MASK);
404	# endif
405	ASMAtomicWriteSize(&uShw.pPTPae->a[iShw].u, 0);
406	}
407
408	/* paranoia / a bit assumptive. */
409	if ( pDis
410	&& (off & 7)
411	&& (off & 7) + cbWrite > sizeof(X86PTEPAE))
412	{
413	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
414	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
415
416	if (uShw.pPTPae->a[iShw2].n.u1Present)
417	{
418	# ifdef PGMPOOL_WITH_GCPHYS_TRACKING
419	X86PTEPAE GstPte;
420	# ifdef IN_RING3
421	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, (RTHCPTR)((RTHCUINTPTR)pvAddress + sizeof(GstPte)), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
422	# else
423	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress + sizeof(GstPte), GCPhysFault + sizeof(GstPte), sizeof(GstPte));
424	# endif
425	AssertRC(rc);
426	Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PAE_PG_MASK));
427	pgmPoolTracDerefGCPhysHint(pPool, pPage,
428	uShw.pPTPae->a[iShw2].u & X86_PTE_PAE_PG_MASK,
429	GstPte.u & X86_PTE_PAE_PG_MASK);
430	# endif
431	ASMAtomicWriteSize(&uShw.pPTPae->a[iShw2].u ,0);
432	}
433	}
434	break;
435	}
436
437	case PGMPOOLKIND_32BIT_PD:
438	{
439	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
440	const unsigned iShw = off / sizeof(X86PTE); // ASSUMING 32-bit guest paging!
441
442	LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
443	# ifndef IN_RING0
444	if (uShw.pPD->a[iShw].u & PGM_PDFLAGS_MAPPING)
445	{
446	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
447	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
448	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
449	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
450	break;
451	}
452	# endif /* !IN_RING0 */
453	# ifndef IN_RING0
454	else
455	# endif /* !IN_RING0 */
456	{
457	if (uShw.pPD->a[iShw].n.u1Present)
458	{
459	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
460	pgmPoolFree(pVM,
461	uShw.pPD->a[iShw].u & X86_PDE_PAE_PG_MASK,
462	pPage->idx,
463	iShw);
464	ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
465	}
466	}
467	/* paranoia / a bit assumptive. */
468	if ( pDis
469	&& (off & 3)
470	&& (off & 3) + cbWrite > sizeof(X86PTE))
471	{
472	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
473	if ( iShw2 != iShw
474	&& iShw2 < RT_ELEMENTS(uShw.pPD->a))
475	{
476	# ifndef IN_RING0
477	if (uShw.pPD->a[iShw2].u & PGM_PDFLAGS_MAPPING)
478	{
479	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
480	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
481	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
482	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
483	break;
484	}
485	# endif /* !IN_RING0 */
486	# ifndef IN_RING0
487	else
488	# endif /* !IN_RING0 */
489	{
490	if (uShw.pPD->a[iShw2].n.u1Present)
491	{
492	LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPD->a[iShw2].u));
493	pgmPoolFree(pVM,
494	uShw.pPD->a[iShw2].u & X86_PDE_PAE_PG_MASK,
495	pPage->idx,
496	iShw2);
497	ASMAtomicWriteSize(&uShw.pPD->a[iShw2].u, 0);
498	}
499	}
500	}
501	}
502	#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). */
503	if ( uShw.pPD->a[iShw].n.u1Present
504	&& !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
505	{
506	LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
507	# ifdef IN_RC /* TLB load - we're pushing things a bit... */
508	ASMProbeReadByte(pvAddress);
509	# endif
510	pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
511	ASMAtomicWriteSize(&uShw.pPD->a[iShw].u, 0);
512	}
513	#endif
514	break;
515	}
516
517	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
518	{
519	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
520	const unsigned iShw = off / sizeof(X86PDEPAE);
521	#ifndef IN_RING0
522	if (uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING)
523	{
524	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
525	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
526	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
527	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw=%#x!\n", iShw));
528	break;
529	}
530	#endif /* !IN_RING0 */
531	/*
532	* Causes trouble when the guest uses a PDE to refer to the whole page table level
533	* structure. (Invalidate here; faults later on when it tries to change the page
534	* table entries -> recheck; probably only applies to the RC case.)
535	*/
536	# ifndef IN_RING0
537	else
538	# endif /* !IN_RING0 */
539	{
540	if (uShw.pPDPae->a[iShw].n.u1Present)
541	{
542	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
543	pgmPoolFree(pVM,
544	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
545	pPage->idx,
546	iShw);
547	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
548	}
549	}
550	/* paranoia / a bit assumptive. */
551	if ( pDis
552	&& (off & 7)
553	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
554	{
555	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
556	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
557
558	#ifndef IN_RING0
559	if ( iShw2 != iShw
560	&& uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING)
561	{
562	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
563	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
564	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
565	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
566	break;
567	}
568	#endif /* !IN_RING0 */
569	# ifndef IN_RING0
570	else
571	# endif /* !IN_RING0 */
572	if (uShw.pPDPae->a[iShw2].n.u1Present)
573	{
574	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
575	pgmPoolFree(pVM,
576	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
577	pPage->idx,
578	iShw2);
579	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
580	}
581	}
582	break;
583	}
584
585	case PGMPOOLKIND_PAE_PDPT:
586	{
587	/*
588	* Hopefully this doesn't happen very often:
589	* - touching unused parts of the page
590	* - messing with the bits of pd pointers without changing the physical address
591	*/
592	/* PDPT roots are not page aligned; 32 byte only! */
593	const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
594
595	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
596	const unsigned iShw = offPdpt / sizeof(X86PDPE);
597	if (iShw < X86_PG_PAE_PDPE_ENTRIES) /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
598	{
599	# ifndef IN_RING0
600	if (uShw.pPDPT->a[iShw].u & PGM_PLXFLAGS_MAPPING)
601	{
602	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
603	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
604	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
605	LogFlow(("pgmPoolMonitorChainChanging: Detected pdpt conflict at iShw=%#x!\n", iShw));
606	break;
607	}
608	# endif /* !IN_RING0 */
609	# ifndef IN_RING0
610	else
611	# endif /* !IN_RING0 */
612	if (uShw.pPDPT->a[iShw].n.u1Present)
613	{
614	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
615	pgmPoolFree(pVM,
616	uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK,
617	pPage->idx,
618	iShw);
619	ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
620	}
621
622	/* paranoia / a bit assumptive. */
623	if ( pDis
624	&& (offPdpt & 7)
625	&& (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
626	{
627	const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
628	if ( iShw2 != iShw
629	&& iShw2 < X86_PG_PAE_PDPE_ENTRIES)
630	{
631	# ifndef IN_RING0
632	if (uShw.pPDPT->a[iShw2].u & PGM_PLXFLAGS_MAPPING)
633	{
634	Assert(pgmMapAreMappingsEnabled(&pVM->pgm.s));
635	STAM_COUNTER_INC(&(pVCpu->pgm.s.StatRZGuestCR3WriteConflict));
636	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
637	LogFlow(("pgmPoolMonitorChainChanging: Detected conflict at iShw2=%#x!\n", iShw2));
638	break;
639	}
640	# endif /* !IN_RING0 */
641	# ifndef IN_RING0
642	else
643	# endif /* !IN_RING0 */
644	if (uShw.pPDPT->a[iShw2].n.u1Present)
645	{
646	LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
647	pgmPoolFree(pVM,
648	uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK,
649	pPage->idx,
650	iShw2);
651	ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
652	}
653	}
654	}
655	}
656	break;
657	}
658
659	#ifndef IN_RC
660	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
661	{
662	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
663	const unsigned iShw = off / sizeof(X86PDEPAE);
664	Assert(!(uShw.pPDPae->a[iShw].u & PGM_PDFLAGS_MAPPING));
665	if (uShw.pPDPae->a[iShw].n.u1Present)
666	{
667	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPae->a[iShw].u));
668	pgmPoolFree(pVM,
669	uShw.pPDPae->a[iShw].u & X86_PDE_PAE_PG_MASK,
670	pPage->idx,
671	iShw);
672	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw].u, 0);
673	}
674	/* paranoia / a bit assumptive. */
675	if ( pDis
676	&& (off & 7)
677	&& (off & 7) + cbWrite > sizeof(X86PDEPAE))
678	{
679	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
680	AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
681
682	Assert(!(uShw.pPDPae->a[iShw2].u & PGM_PDFLAGS_MAPPING));
683	if (uShw.pPDPae->a[iShw2].n.u1Present)
684	{
685	LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPae->a[iShw2].u));
686	pgmPoolFree(pVM,
687	uShw.pPDPae->a[iShw2].u & X86_PDE_PAE_PG_MASK,
688	pPage->idx,
689	iShw2);
690	ASMAtomicWriteSize(&uShw.pPDPae->a[iShw2].u, 0);
691	}
692	}
693	break;
694	}
695
696	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
697	{
698	/*
699	* Hopefully this doesn't happen very often:
700	* - messing with the bits of pd pointers without changing the physical address
701	*/
702	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
703	{
704	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
705	const unsigned iShw = off / sizeof(X86PDPE);
706	if (uShw.pPDPT->a[iShw].n.u1Present)
707	{
708	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
709	pgmPoolFree(pVM, uShw.pPDPT->a[iShw].u & X86_PDPE_PG_MASK, pPage->idx, iShw);
710	ASMAtomicWriteSize(&uShw.pPDPT->a[iShw].u, 0);
711	}
712	/* paranoia / a bit assumptive. */
713	if ( pDis
714	&& (off & 7)
715	&& (off & 7) + cbWrite > sizeof(X86PDPE))
716	{
717	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
718	if (uShw.pPDPT->a[iShw2].n.u1Present)
719	{
720	LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
721	pgmPoolFree(pVM, uShw.pPDPT->a[iShw2].u & X86_PDPE_PG_MASK, pPage->idx, iShw2);
722	ASMAtomicWriteSize(&uShw.pPDPT->a[iShw2].u, 0);
723	}
724	}
725	}
726	break;
727	}
728
729	case PGMPOOLKIND_64BIT_PML4:
730	{
731	/*
732	* Hopefully this doesn't happen very often:
733	* - messing with the bits of pd pointers without changing the physical address
734	*/
735	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
736	{
737	uShw.pv = PGMPOOL_PAGE_2_LOCKED_PTR(pVM, pPage);
738	const unsigned iShw = off / sizeof(X86PDPE);
739	if (uShw.pPML4->a[iShw].n.u1Present)
740	{
741	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPML4->a[iShw].u));
742	pgmPoolFree(pVM, uShw.pPML4->a[iShw].u & X86_PML4E_PG_MASK, pPage->idx, iShw);
743	ASMAtomicWriteSize(&uShw.pPML4->a[iShw].u, 0);
744	}
745	/* paranoia / a bit assumptive. */
746	if ( pDis
747	&& (off & 7)
748	&& (off & 7) + cbWrite > sizeof(X86PDPE))
749	{
750	const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
751	if (uShw.pPML4->a[iShw2].n.u1Present)
752	{
753	LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPML4->a[iShw2].u));
754	pgmPoolFree(pVM, uShw.pPML4->a[iShw2].u & X86_PML4E_PG_MASK, pPage->idx, iShw2);
755	ASMAtomicWriteSize(&uShw.pPML4->a[iShw2].u, 0);
756	}
757	}
758	}
759	break;
760	}
761	#endif /* IN_RING0 */
762
763	default:
764	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
765	}
766	PGMPOOL_UNLOCK_PTR(pVM, uShw.pv);
767
768	/* next */
769	if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
770	return;
771	pPage = &pPool->aPages[pPage->iMonitoredNext];
772	}
773	}
774
775	# ifndef IN_RING3
776	/**
777	* Checks if a access could be a fork operation in progress.
778	*
779	* Meaning, that the guest is setting up the parent process for Copy-On-Write.
780	*
781	* @returns true if it's likly that we're forking, otherwise false.
782	* @param pPool The pool.
783	* @param pDis The disassembled instruction.
784	* @param offFault The access offset.
785	*/
786	DECLINLINE(bool) pgmPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
787	{
788	/*
789	* i386 linux is using btr to clear X86_PTE_RW.
790	* The functions involved are (2.6.16 source inspection):
791	* clear_bit
792	* ptep_set_wrprotect
793	* copy_one_pte
794	* copy_pte_range
795	* copy_pmd_range
796	* copy_pud_range
797	* copy_page_range
798	* dup_mmap
799	* dup_mm
800	* copy_mm
801	* copy_process
802	* do_fork
803	*/
804	if ( pDis->pCurInstr->opcode == OP_BTR
805	&& !(offFault & 4)
806	/** @todo Validate that the bit index is X86_PTE_RW. */
807	)
808	{
809	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,Fork));
810	return true;
811	}
812	return false;
813	}
814
815
816	/**
817	* Determine whether the page is likely to have been reused.
818	*
819	* @returns true if we consider the page as being reused for a different purpose.
820	* @returns false if we consider it to still be a paging page.
821	* @param pVM VM Handle.
822	* @param pVCpu VMCPU Handle.
823	* @param pRegFrame Trap register frame.
824	* @param pDis The disassembly info for the faulting instruction.
825	* @param pvFault The fault address.
826	*
827	* @remark The REP prefix check is left to the caller because of STOSD/W.
828	*/
829	DECLINLINE(bool) pgmPoolMonitorIsReused(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, PDISCPUSTATE pDis, RTGCPTR pvFault)
830	{
831	#ifndef IN_RC
832	/** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
833	if ( HWACCMHasPendingIrq(pVM)
834	&& (pRegFrame->rsp - pvFault) < 32)
835	{
836	/* Fault caused by stack writes while trying to inject an interrupt event. */
837	Log(("pgmPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pRegFrame->rsp));
838	return true;
839	}
840	#else
841	NOREF(pVM); NOREF(pvFault);
842	#endif
843
844	LogFlow(("Reused instr %RGv %d at %RGv param1.flags=%x param1.reg=%d\n", pRegFrame->rip, pDis->pCurInstr->opcode, pvFault, pDis->param1.flags, pDis->param1.base.reg_gen));
845
846	/* Non-supervisor mode write means it's used for something else. */
847	if (CPUMGetGuestCPL(pVCpu, pRegFrame) != 0)
848	return true;
849
850	switch (pDis->pCurInstr->opcode)
851	{
852	/* call implies the actual push of the return address faulted */
853	case OP_CALL:
854	Log4(("pgmPoolMonitorIsReused: CALL\n"));
855	return true;
856	case OP_PUSH:
857	Log4(("pgmPoolMonitorIsReused: PUSH\n"));
858	return true;
859	case OP_PUSHF:
860	Log4(("pgmPoolMonitorIsReused: PUSHF\n"));
861	return true;
862	case OP_PUSHA:
863	Log4(("pgmPoolMonitorIsReused: PUSHA\n"));
864	return true;
865	case OP_FXSAVE:
866	Log4(("pgmPoolMonitorIsReused: FXSAVE\n"));
867	return true;
868	case OP_MOVNTI: /* solaris - block_zero_no_xmm */
869	Log4(("pgmPoolMonitorIsReused: MOVNTI\n"));
870	return true;
871	case OP_MOVNTDQ: /* solaris - hwblkclr & hwblkpagecopy */
872	Log4(("pgmPoolMonitorIsReused: MOVNTDQ\n"));
873	return true;
874	case OP_MOVSWD:
875	case OP_STOSWD:
876	if ( pDis->prefix == (PREFIX_REP\|PREFIX_REX)
877	&& pRegFrame->rcx >= 0x40
878	)
879	{
880	Assert(pDis->mode == CPUMODE_64BIT);
881
882	Log(("pgmPoolMonitorIsReused: OP_STOSQ\n"));
883	return true;
884	}
885	return false;
886	}
887	if ( ( (pDis->param1.flags & USE_REG_GEN32)
888	\|\| (pDis->param1.flags & USE_REG_GEN64))
889	&& (pDis->param1.base.reg_gen == USE_REG_ESP))
890	{
891	Log4(("pgmPoolMonitorIsReused: ESP\n"));
892	return true;
893	}
894
895	return false;
896	}
897
898	/**
899	* Flushes the page being accessed.
900	*
901	* @returns VBox status code suitable for scheduling.
902	* @param pVM The VM handle.
903	* @param pVCpu The VMCPU handle.
904	* @param pPool The pool.
905	* @param pPage The pool page (head).
906	* @param pDis The disassembly of the write instruction.
907	* @param pRegFrame The trap register frame.
908	* @param GCPhysFault The fault address as guest physical address.
909	* @param pvFault The fault address.
910	*/
911	static int pgmPoolAccessHandlerFlush(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
912	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
913	{
914	/*
915	* First, do the flushing.
916	*/
917	int rc = pgmPoolMonitorChainFlush(pPool, pPage);
918
919	/*
920	* Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection). Must do this in raw mode (!); XP boot will fail otherwise
921	*/
922	uint32_t cbWritten;
923	int rc2 = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cbWritten);
924	if (RT_SUCCESS(rc2))
925	pRegFrame->rip += pDis->opsize;
926	else if (rc2 == VERR_EM_INTERPRETER)
927	{
928	#ifdef IN_RC
929	if (PATMIsPatchGCAddr(pVM, (RTRCPTR)pRegFrame->eip))
930	{
931	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for patch code %04x:%RGv, ignoring.\n",
932	pRegFrame->cs, (RTGCPTR)pRegFrame->eip));
933	rc = VINF_SUCCESS;
934	STAM_COUNTER_INC(&pPool->StatMonitorRZIntrFailPatch2);
935	}
936	else
937	#endif
938	{
939	rc = VINF_EM_RAW_EMULATE_INSTR;
940	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
941	}
942	}
943	else
944	rc = rc2;
945
946	/* See use in pgmPoolAccessHandlerSimple(). */
947	PGM_INVL_VCPU_TLBS(pVCpu);
948	LogFlow(("pgmPoolAccessHandlerPT: returns %Rrc (flushed)\n", rc));
949	return rc;
950	}
951
952	/**
953	* Handles the STOSD write accesses.
954	*
955	* @returns VBox status code suitable for scheduling.
956	* @param pVM The VM handle.
957	* @param pPool The pool.
958	* @param pPage The pool page (head).
959	* @param pDis The disassembly of the write instruction.
960	* @param pRegFrame The trap register frame.
961	* @param GCPhysFault The fault address as guest physical address.
962	* @param pvFault The fault address.
963	*/
964	DECLINLINE(int) pgmPoolAccessHandlerSTOSD(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
965	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
966	{
967	unsigned uIncrement = pDis->param1.size;
968
969	Assert(pDis->mode == CPUMODE_32BIT \|\| pDis->mode == CPUMODE_64BIT);
970	Assert(pRegFrame->rcx <= 0x20);
971
972	#ifdef VBOX_STRICT
973	if (pDis->opmode == CPUMODE_32BIT)
974	Assert(uIncrement == 4);
975	else
976	Assert(uIncrement == 8);
977	#endif
978
979	Log3(("pgmPoolAccessHandlerSTOSD\n"));
980
981	/*
982	* Increment the modification counter and insert it into the list
983	* of modified pages the first time.
984	*/
985	if (!pPage->cModifications++)
986	pgmPoolMonitorModifiedInsert(pPool, pPage);
987
988	/*
989	* Execute REP STOSD.
990	*
991	* This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
992	* write situation, meaning that it's safe to write here.
993	*/
994	PVMCPU pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
995	RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
996	while (pRegFrame->rcx)
997	{
998	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
999	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1000	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1001	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1002	#else
1003	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, (RTGCPTR)pu32, NULL);
1004	#endif
1005	#ifdef IN_RC
1006	(uint32_t )pu32 = pRegFrame->eax;
1007	#else
1008	PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pRegFrame->rax, uIncrement);
1009	#endif
1010	pu32 += uIncrement;
1011	GCPhysFault += uIncrement;
1012	pRegFrame->rdi += uIncrement;
1013	pRegFrame->rcx--;
1014	}
1015	pRegFrame->rip += pDis->opsize;
1016
1017	#ifdef IN_RC
1018	/* See use in pgmPoolAccessHandlerSimple(). */
1019	PGM_INVL_VCPU_TLBS(pVCpu);
1020	#endif
1021
1022	LogFlow(("pgmPoolAccessHandlerSTOSD: returns\n"));
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Handles the simple write accesses.
1029	*
1030	* @returns VBox status code suitable for scheduling.
1031	* @param pVM The VM handle.
1032	* @param pVCpu The VMCPU handle.
1033	* @param pPool The pool.
1034	* @param pPage The pool page (head).
1035	* @param pDis The disassembly of the write instruction.
1036	* @param pRegFrame The trap register frame.
1037	* @param GCPhysFault The fault address as guest physical address.
1038	* @param pvFault The fault address.
1039	* @param pfReused Reused state (out)
1040	*/
1041	DECLINLINE(int) pgmPoolAccessHandlerSimple(PVM pVM, PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
1042	PCPUMCTXCORE pRegFrame, RTGCPHYS GCPhysFault, RTGCPTR pvFault, bool *pfReused)
1043	{
1044	Log3(("pgmPoolAccessHandlerSimple\n"));
1045	/*
1046	* Increment the modification counter and insert it into the list
1047	* of modified pages the first time.
1048	*/
1049	if (!pPage->cModifications++)
1050	pgmPoolMonitorModifiedInsert(pPool, pPage);
1051
1052	/*
1053	* Clear all the pages. ASSUMES that pvFault is readable.
1054	*/
1055	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
1056	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
1057	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1058	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
1059	#else
1060	pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, pvFault, pDis);
1061	#endif
1062
1063	/*
1064	* Interpret the instruction.
1065	*/
1066	uint32_t cb;
1067	int rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, pRegFrame, pvFault, &cb);
1068	if (RT_SUCCESS(rc))
1069	pRegFrame->rip += pDis->opsize;
1070	else if (rc == VERR_EM_INTERPRETER)
1071	{
1072	LogFlow(("pgmPoolAccessHandlerPTWorker: Interpretation failed for %04x:%RGv - opcode=%d\n",
1073	pRegFrame->cs, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode));
1074	rc = VINF_EM_RAW_EMULATE_INSTR;
1075	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,EmulateInstr));
1076	}
1077
1078	#if 0 /* experimental code */
1079	if (rc == VINF_SUCCESS)
1080	{
1081	switch (pPage->enmKind)
1082	{
1083	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1084	{
1085	X86PTEPAE GstPte;
1086	int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
1087	AssertRC(rc);
1088
1089	/* Check the new value written by the guest. If present and with a bogus physical address, then
1090	* it's fairly safe to assume the guest is reusing the PT.
1091	*/
1092	if (GstPte.n.u1Present)
1093	{
1094	RTHCPHYS HCPhys = -1;
1095	int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
1096	if (rc != VINF_SUCCESS)
1097	{
1098	*pfReused = true;
1099	STAM_COUNTER_INC(&pPool->StatForceFlushReused);
1100	}
1101	}
1102	break;
1103	}
1104	}
1105	}
1106	#endif
1107
1108	#ifdef IN_RC
1109	/*
1110	* Quick hack, with logging enabled we're getting stale
1111	* code TLBs but no data TLB for EIP and crash in EMInterpretDisasOne.
1112	* Flushing here is BAD and expensive, I think EMInterpretDisasOne will
1113	* have to be fixed to support this. But that'll have to wait till next week.
1114	*
1115	* An alternative is to keep track of the changed PTEs together with the
1116	* GCPhys from the guest PT. This may proove expensive though.
1117	*
1118	* At the moment, it's VITAL that it's done AFTER the instruction interpreting
1119	* because we need the stale TLBs in some cases (XP boot). This MUST be fixed properly!
1120	*/
1121	PGM_INVL_VCPU_TLBS(pVCpu);
1122	#endif
1123
1124	LogFlow(("pgmPoolAccessHandlerSimple: returns %Rrc cb=%d\n", rc, cb));
1125	return rc;
1126	}
1127
1128	/**
1129	* \#PF Handler callback for PT write accesses.
1130	*
1131	* @returns VBox status code (appropriate for GC return).
1132	* @param pVM VM Handle.
1133	* @param uErrorCode CPU Error code.
1134	* @param pRegFrame Trap register frame.
1135	* NULL on DMA and other non CPU access.
1136	* @param pvFault The fault address (cr2).
1137	* @param GCPhysFault The GC physical address corresponding to pvFault.
1138	* @param pvUser User argument.
1139	*/
1140	DECLEXPORT(int) pgmPoolAccessHandler(PVM pVM, RTGCUINT uErrorCode, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault, RTGCPHYS GCPhysFault, void *pvUser)
1141	{
1142	STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1143	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1144	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)pvUser;
1145	PVMCPU pVCpu = VMMGetCpu(pVM);
1146	unsigned cMaxModifications;
1147	bool fForcedFlush = false;
1148
1149	LogFlow(("pgmPoolAccessHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
1150
1151	pgmLock(pVM);
1152	if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
1153	{
1154	/* Pool page changed while we were waiting for the lock; ignore. */
1155	Log(("CPU%d: pgmPoolAccessHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
1156	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1157	pgmUnlock(pVM);
1158	return VINF_SUCCESS;
1159	}
1160
1161	/*
1162	* Disassemble the faulting instruction.
1163	*/
1164	PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
1165	int rc = EMInterpretDisasOne(pVM, pVCpu, pRegFrame, pDis, NULL);
1166	AssertReturnStmt(rc == VINF_SUCCESS, pgmUnlock(pVM), rc);
1167
1168	Assert(pPage->enmKind != PGMPOOLKIND_FREE);
1169
1170	/*
1171	* We should ALWAYS have the list head as user parameter. This
1172	* is because we use that page to record the changes.
1173	*/
1174	Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1175	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1176	Assert(!pPage->fDirty);
1177	#endif
1178
1179	/* Maximum nr of modifications depends on the guest mode. */
1180	if (pDis->mode == CPUMODE_32BIT)
1181	cMaxModifications = 48;
1182	else
1183	cMaxModifications = 24;
1184
1185	/*
1186	* Incremental page table updates should weight more than random ones.
1187	* (Only applies when started from offset 0)
1188	*/
1189	pVCpu->pgm.s.cPoolAccessHandler++;
1190	if ( pPage->pvLastAccessHandlerRip >= pRegFrame->rip - 0x40 /* observed loops in Windows 7 x64 */
1191	&& pPage->pvLastAccessHandlerRip < pRegFrame->rip + 0x40
1192	&& pvFault == (pPage->pvLastAccessHandlerFault + pDis->param1.size)
1193	&& pVCpu->pgm.s.cPoolAccessHandler == (pPage->cLastAccessHandlerCount + 1))
1194	{
1195	Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1196	pPage->cModifications = pPage->cModifications * 2;
1197	pPage->pvLastAccessHandlerFault = pvFault;
1198	pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1199	if (pPage->cModifications >= cMaxModifications)
1200	{
1201	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushReinit));
1202	fForcedFlush = true;
1203	}
1204	}
1205
1206	if (pPage->cModifications >= cMaxModifications)
1207	Log(("Mod overflow %VGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(&pVM->pgm.s, pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
1208
1209	/*
1210	* Check if it's worth dealing with.
1211	*/
1212	bool fReused = false;
1213	bool fNotReusedNotForking = false;
1214	if ( ( pPage->cModifications < cMaxModifications /** @todo #define / /* @todo need to check that it's not mapping EIP. / /* @todo adjust this! */
1215	\|\| pgmPoolIsPageLocked(&pVM->pgm.s, pPage)
1216	)
1217	&& !(fReused = pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault))
1218	&& !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1219	{
1220	/*
1221	* Simple instructions, no REP prefix.
1222	*/
1223	if (!(pDis->prefix & (PREFIX_REP \| PREFIX_REPNE)))
1224	{
1225	rc = pgmPoolAccessHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault, &fReused);
1226	if (fReused)
1227	goto flushPage;
1228
1229	/* A mov instruction to change the first page table entry will be remembered so we can detect
1230	* full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
1231	*/
1232	if ( rc == VINF_SUCCESS
1233	&& pDis->pCurInstr->opcode == OP_MOV
1234	&& (pvFault & PAGE_OFFSET_MASK) == 0)
1235	{
1236	pPage->pvLastAccessHandlerFault = pvFault;
1237	pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1238	pPage->pvLastAccessHandlerRip = pRegFrame->rip;
1239	/* Make sure we don't kick out a page too quickly. */
1240	if (pPage->cModifications > 8)
1241	pPage->cModifications = 2;
1242	}
1243	else
1244	if (pPage->pvLastAccessHandlerFault == pvFault)
1245	{
1246	/* ignore the 2nd write to this page table entry. */
1247	pPage->cLastAccessHandlerCount = pVCpu->pgm.s.cPoolAccessHandler;
1248	}
1249	else
1250	{
1251	pPage->pvLastAccessHandlerFault = 0;
1252	pPage->pvLastAccessHandlerRip = 0;
1253	}
1254
1255	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,Handled), a);
1256	pgmUnlock(pVM);
1257	return rc;
1258	}
1259
1260	/*
1261	* Windows is frequently doing small memset() operations (netio test 4k+).
1262	* We have to deal with these or we'll kill the cache and performance.
1263	*/
1264	if ( pDis->pCurInstr->opcode == OP_STOSWD
1265	&& !pRegFrame->eflags.Bits.u1DF
1266	&& pDis->opmode == pDis->mode
1267	&& pDis->addrmode == pDis->mode)
1268	{
1269	bool fValidStosd = false;
1270
1271	if ( pDis->mode == CPUMODE_32BIT
1272	&& pDis->prefix == PREFIX_REP
1273	&& pRegFrame->ecx <= 0x20
1274	&& pRegFrame->ecx * 4 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1275	&& !((uintptr_t)pvFault & 3)
1276	&& (pRegFrame->eax == 0 \|\| pRegFrame->eax == 0x80) /* the two values observed. */
1277	)
1278	{
1279	fValidStosd = true;
1280	pRegFrame->rcx &= 0xffffffff; /* paranoia */
1281	}
1282	else
1283	if ( pDis->mode == CPUMODE_64BIT
1284	&& pDis->prefix == (PREFIX_REP \| PREFIX_REX)
1285	&& pRegFrame->rcx <= 0x20
1286	&& pRegFrame->rcx * 8 <= PAGE_SIZE - ((uintptr_t)pvFault & PAGE_OFFSET_MASK)
1287	&& !((uintptr_t)pvFault & 7)
1288	&& (pRegFrame->rax == 0 \|\| pRegFrame->rax == 0x80) /* the two values observed. */
1289	)
1290	{
1291	fValidStosd = true;
1292	}
1293
1294	if (fValidStosd)
1295	{
1296	rc = pgmPoolAccessHandlerSTOSD(pVM, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1297	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,RepStosd), a);
1298	pgmUnlock(pVM);
1299	return rc;
1300	}
1301	}
1302
1303	/* REP prefix, don't bother. */
1304	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,RepPrefix));
1305	Log4(("pgmPoolAccessHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
1306	pRegFrame->eax, pRegFrame->ecx, pRegFrame->edi, pRegFrame->esi, (RTGCPTR)pRegFrame->rip, pDis->pCurInstr->opcode, pDis->prefix));
1307	fNotReusedNotForking = true;
1308	}
1309
1310	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1311	/* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
1312	* leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
1313	*/
1314	if ( pPage->cModifications >= cMaxModifications
1315	&& !fForcedFlush
1316	&& pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
1317	&& ( fNotReusedNotForking
1318	\|\| ( !pgmPoolMonitorIsReused(pVM, pVCpu, pRegFrame, pDis, pvFault)
1319	&& !pgmPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
1320	)
1321	)
1322	{
1323	Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
1324	Assert(pPage->fDirty == false);
1325
1326	/* Flush any monitored duplicates as we will disable write protection. */
1327	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
1328	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
1329	{
1330	PPGMPOOLPAGE pPageHead = pPage;
1331
1332	/* Find the monitor head. */
1333	while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
1334	pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
1335
1336	while (pPageHead)
1337	{
1338	unsigned idxNext = pPageHead->iMonitoredNext;
1339
1340	if (pPageHead != pPage)
1341	{
1342	STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
1343	Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
1344	int rc2 = pgmPoolFlushPage(pPool, pPageHead);
1345	AssertRC(rc2);
1346	}
1347
1348	if (idxNext == NIL_PGMPOOL_IDX)
1349	break;
1350
1351	pPageHead = &pPool->aPages[idxNext];
1352	}
1353	}
1354
1355	/* The flushing above might fail for locked pages, so double check. */
1356	if ( pPage->iMonitoredNext == NIL_PGMPOOL_IDX
1357	&& pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
1358	{
1359	/* Temporarily allow write access to the page table again. */
1360	rc = PGMHandlerPhysicalPageTempOff(pVM, pPage->GCPhys, pPage->GCPhys);
1361	if (rc == VINF_SUCCESS)
1362	{
1363	rc = PGMShwModifyPage(pVCpu, pvFault, 1, X86_PTE_RW, ~(uint64_t)X86_PTE_RW);
1364	AssertMsg(rc == VINF_SUCCESS
1365	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1366	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1367	\|\| rc == VERR_PAGE_NOT_PRESENT,
1368	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
1369
1370	pgmPoolAddDirtyPage(pVM, pPool, pPage);
1371	pPage->pvDirtyFault = pvFault;
1372
1373	STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), a);
1374	pgmUnlock(pVM);
1375	return rc;
1376	}
1377	}
1378	}
1379	#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1380
1381	STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FlushModOverflow));
1382	flushPage:
1383	/*
1384	* Not worth it, so flush it.
1385	*
1386	* If we considered it to be reused, don't go back to ring-3
1387	* to emulate failed instructions since we usually cannot
1388	* interpret then. This may be a bit risky, in which case
1389	* the reuse detection must be fixed.
1390	*/
1391	rc = pgmPoolAccessHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pRegFrame, GCPhysFault, pvFault);
1392	if (rc == VINF_EM_RAW_EMULATE_INSTR && fReused)
1393	rc = VINF_SUCCESS;
1394	STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
1395	pgmUnlock(pVM);
1396	return rc;
1397	}
1398
1399	# endif /* !IN_RING3 */
1400
1401	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
1402
1403	# ifdef VBOX_STRICT
1404	/**
1405	* Check references to guest physical memory in a PAE / PAE page table.
1406	*
1407	* @param pPool The pool.
1408	* @param pPage The page.
1409	* @param pShwPT The shadow page table (mapping of the page).
1410	* @param pGstPT The guest page table.
1411	*/
1412	DECLINLINE(void) pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
1413	{
1414	unsigned cErrors = 0;
1415	#ifdef VBOX_STRICT
1416	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1417	AssertMsg(!pShwPT->a[i].n.u1Present, ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, pShwPT->a[i].u, pPage->iFirstPresent));
1418	#endif
1419	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1420	{
1421	if (pShwPT->a[i].n.u1Present)
1422	{
1423	RTHCPHYS HCPhys = -1;
1424	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1425	if ( rc != VINF_SUCCESS
1426	\|\| (pShwPT->a[i].u & X86_PTE_PAE_PG_MASK) != HCPhys)
1427	{
1428	RTHCPHYS HCPhysPT = -1;
1429	Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, pShwPT->a[i].u, HCPhys));
1430	cErrors++;
1431
1432	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pPage->GCPhys, &HCPhysPT);
1433	AssertRC(rc);
1434
1435	for (unsigned i = 0; i < pPool->cCurPages; i++)
1436	{
1437	PPGMPOOLPAGE pTempPage = &pPool->aPages[i];
1438
1439	if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
1440	{
1441	PX86PTPAE pShwPT2 = (PX86PTPAE)PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pTempPage);
1442
1443	for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
1444	{
1445	if ( pShwPT2->a[j].n.u1Present
1446	&& pShwPT2->a[j].n.u1Write
1447	&& ((pShwPT2->a[j].u & X86_PTE_PAE_PG_MASK) == HCPhysPT))
1448	{
1449	Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, pShwPT->a[j].u, pShwPT2->a[j].u));
1450	}
1451	}
1452	}
1453	}
1454	}
1455	}
1456	}
1457	Assert(!cErrors);
1458	}
1459	# endif /* VBOX_STRICT */
1460
1461	/**
1462	* Clear references to guest physical memory in a PAE / PAE page table.
1463	*
1464	* @returns nr of changed PTEs
1465	* @param pPool The pool.
1466	* @param pPage The page.
1467	* @param pShwPT The shadow page table (mapping of the page).
1468	* @param pGstPT The guest page table.
1469	* @param pOldGstPT The old cached guest page table.
1470	*/
1471	DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT, PCX86PTPAE pOldGstPT)
1472	{
1473	unsigned cChanged = 0;
1474
1475	#ifdef VBOX_STRICT
1476	for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
1477	AssertMsg(!pShwPT->a[i].n.u1Present, ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, pShwPT->a[i].u, pPage->iFirstPresent));
1478	#endif
1479	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
1480	{
1481	if (pShwPT->a[i].n.u1Present)
1482	{
1483	/* The the old cached PTE is identical, then there's no need to flush the shadow copy. */
1484	if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
1485	{
1486	#ifdef VBOX_STRICT
1487	RTHCPHYS HCPhys = -1;
1488	int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
1489	AssertMsg(rc == VINF_SUCCESS && (pShwPT->a[i].u & X86_PTE_PAE_PG_MASK) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, pShwPT->a[i].u, HCPhys));
1490	#endif
1491	uint64_t uHostAttr = pShwPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1492	bool fHostRW = !!(pShwPT->a[i].u & X86_PTE_RW);
1493	uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P \| X86_PTE_US \| X86_PTE_A \| X86_PTE_D \| X86_PTE_G \| X86_PTE_PAE_NX);
1494	bool fGuestRW = !!(pGstPT->a[i].u & X86_PTE_RW);
1495
1496	if ( uHostAttr == uGuestAttr
1497	&& fHostRW <= fGuestRW)
1498	continue;
1499	}
1500	cChanged++;
1501	/* Something was changed, so flush it. */
1502	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
1503	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
1504	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
1505	ASMAtomicWriteSize(&pShwPT->a[i].u, 0);
1506	}
1507	}
1508	return cChanged;
1509	}
1510
1511
1512	/**
1513	* Flush a dirty page
1514	*
1515	* @param pVM VM Handle.
1516	* @param pPool The pool.
1517	* @param idxSlot Dirty array slot index
1518	* @param fForceRemoval Force removal from the dirty page list
1519	*/
1520	static void pgmPoolFlushDirtyPage(PVM pVM, PPGMPOOL pPool, unsigned idxSlot, bool fForceRemoval = false)
1521	{
1522	PPGMPOOLPAGE pPage;
1523	unsigned idxPage;
1524
1525	Assert(idxSlot < RT_ELEMENTS(pPool->aIdxDirtyPages));
1526	if (pPool->aIdxDirtyPages[idxSlot] == NIL_PGMPOOL_IDX)
1527	return;
1528
1529	idxPage = pPool->aIdxDirtyPages[idxSlot];
1530	AssertRelease(idxPage != NIL_PGMPOOL_IDX);
1531	pPage = &pPool->aPages[idxPage];
1532	Assert(pPage->idx == idxPage);
1533	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1534
1535	AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
1536	Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1537
1538	/* Flush those PTEs that have changed. */
1539	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
1540	void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
1541	void *pvGst;
1542	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1543	unsigned cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst, (PCX86PTPAE)&pPool->aDirtyPages[idxSlot][0]);
1544	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
1545
1546	/** Note: we might want to consider keeping the dirty page active in case there were many changes. */
1547
1548	/* Write protect the page again to catch all write accesses. */
1549	rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys);
1550	Assert(rc == VINF_SUCCESS);
1551	pPage->fDirty = false;
1552
1553	#ifdef VBOX_STRICT
1554	uint64_t fFlags = 0;
1555	RTHCPHYS HCPhys;
1556	rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->pvDirtyFault, &fFlags, &HCPhys);
1557	AssertMsg( ( rc == VINF_SUCCESS
1558	&& (!(fFlags & X86_PTE_RW) \|\| HCPhys != pPage->Core.Key))
1559	/* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
1560	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1561	\|\| rc == VERR_PAGE_NOT_PRESENT,
1562	("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->pvDirtyFault, rc, fFlags));
1563	#endif
1564
1565	/* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
1566	Assert(pPage->cModifications);
1567	if (cChanges < 4)
1568	pPage->cModifications = 1; /* must use > 0 here */
1569	else
1570	pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
1571
1572	STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
1573	if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages))
1574	pPool->idxFreeDirtyPage = idxSlot;
1575
1576	pPool->cDirtyPages--;
1577	pPool->aIdxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
1578	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1579	Log(("Removed dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
1580	}
1581
1582	# ifndef IN_RING3
1583	/**
1584	* Add a new dirty page
1585	*
1586	* @param pVM VM Handle.
1587	* @param pPool The pool.
1588	* @param pPage The page.
1589	*/
1590	void pgmPoolAddDirtyPage(PVM pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1591	{
1592	unsigned idxFree;
1593
1594	Assert(PGMIsLocked(pVM));
1595	AssertCompile(RT_ELEMENTS(pPool->aIdxDirtyPages) == 8 \|\| RT_ELEMENTS(pPool->aIdxDirtyPages) == 16);
1596	Assert(!pPage->fDirty);
1597
1598	idxFree = pPool->idxFreeDirtyPage;
1599	Assert(idxFree < RT_ELEMENTS(pPool->aIdxDirtyPages));
1600	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
1601
1602	if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aIdxDirtyPages))
1603	{
1604	STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
1605	pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* force removal */);
1606	}
1607	Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages));
1608	AssertMsg(pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
1609
1610	Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
1611
1612	/* Make a copy of the guest page table as we require valid GCPhys addresses when removing
1613	* references to physical pages. (the HCPhys linear lookup is extremely expensive!)
1614	*/
1615	void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
1616	void *pvGst;
1617	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
1618	memcpy(&pPool->aDirtyPages[idxFree][0], pvGst, PAGE_SIZE);
1619	#ifdef VBOX_STRICT
1620	pgmPoolTrackCheckPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
1621	#endif
1622
1623	STAM_COUNTER_INC(&pPool->StatDirtyPage);
1624	pPage->fDirty = true;
1625	pPage->idxDirty = idxFree;
1626	pPool->aIdxDirtyPages[idxFree] = pPage->idx;
1627	pPool->cDirtyPages++;
1628
1629	pPool->idxFreeDirtyPage = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1630	if ( pPool->cDirtyPages < RT_ELEMENTS(pPool->aIdxDirtyPages)
1631	&& pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1632	{
1633	unsigned i;
1634	for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1635	{
1636	idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aIdxDirtyPages) - 1);
1637	if (pPool->aIdxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
1638	{
1639	pPool->idxFreeDirtyPage = idxFree;
1640	break;
1641	}
1642	}
1643	Assert(i != RT_ELEMENTS(pPool->aIdxDirtyPages));
1644	}
1645
1646	Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages) \|\| pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
1647	return;
1648	}
1649	# endif /* !IN_RING3 */
1650
1651	/**
1652	* Check if the specified page is dirty (not write monitored)
1653	*
1654	* @return dirty or not
1655	* @param pVM VM Handle.
1656	* @param GCPhys Guest physical address
1657	*/
1658	bool pgmPoolIsDirtyPage(PVM pVM, RTGCPHYS GCPhys)
1659	{
1660	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1661	Assert(PGMIsLocked(pVM));
1662	if (!pPool->cDirtyPages)
1663	return false;
1664
1665	GCPhys = GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
1666
1667	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1668	{
1669	if (pPool->aIdxDirtyPages[i] != NIL_PGMPOOL_IDX)
1670	{
1671	PPGMPOOLPAGE pPage;
1672	unsigned idxPage = pPool->aIdxDirtyPages[i];
1673
1674	pPage = &pPool->aPages[idxPage];
1675	if (pPage->GCPhys == GCPhys)
1676	return true;
1677	}
1678	}
1679	return false;
1680	}
1681
1682	/**
1683	* Reset all dirty pages by reinstating page monitoring.
1684	*
1685	* @param pVM VM Handle.
1686	* @param fForceRemoval Force removal of all dirty pages
1687	*/
1688	void pgmPoolResetDirtyPages(PVM pVM, bool fForceRemoval)
1689	{
1690	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1691	Assert(PGMIsLocked(pVM));
1692	Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aIdxDirtyPages));
1693
1694	if (!pPool->cDirtyPages)
1695	return;
1696
1697	Log(("pgmPoolResetDirtyPages\n"));
1698	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1699	pgmPoolFlushDirtyPage(pVM, pPool, i, fForceRemoval);
1700
1701	pPool->idxFreeDirtyPage = 0;
1702	if ( pPool->cDirtyPages != RT_ELEMENTS(pPool->aIdxDirtyPages)
1703	&& pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
1704	{
1705	unsigned i;
1706	for (i = 1; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
1707	{
1708	if (pPool->aIdxDirtyPages[i] == NIL_PGMPOOL_IDX)
1709	{
1710	pPool->idxFreeDirtyPage = i;
1711	break;
1712	}
1713	}
1714	AssertMsg(i != RT_ELEMENTS(pPool->aIdxDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
1715	}
1716
1717	Assert(pPool->aIdxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX \|\| pPool->cDirtyPages == RT_ELEMENTS(pPool->aIdxDirtyPages));
1718	return;
1719	}
1720	# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
1721	#endif /* PGMPOOL_WITH_MONITORING */
1722
1723	#ifdef PGMPOOL_WITH_CACHE
1724
1725	/**
1726	* Inserts a page into the GCPhys hash table.
1727	*
1728	* @param pPool The pool.
1729	* @param pPage The page.
1730	*/
1731	DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1732	{
1733	Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
1734	Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
1735	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1736	pPage->iNext = pPool->aiHash[iHash];
1737	pPool->aiHash[iHash] = pPage->idx;
1738	}
1739
1740
1741	/**
1742	* Removes a page from the GCPhys hash table.
1743	*
1744	* @param pPool The pool.
1745	* @param pPage The page.
1746	*/
1747	DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
1748	{
1749	Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
1750	uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
1751	if (pPool->aiHash[iHash] == pPage->idx)
1752	pPool->aiHash[iHash] = pPage->iNext;
1753	else
1754	{
1755	uint16_t iPrev = pPool->aiHash[iHash];
1756	for (;;)
1757	{
1758	const int16_t i = pPool->aPages[iPrev].iNext;
1759	if (i == pPage->idx)
1760	{
1761	pPool->aPages[iPrev].iNext = pPage->iNext;
1762	break;
1763	}
1764	if (i == NIL_PGMPOOL_IDX)
1765	{
1766	AssertReleaseMsgFailed(("GCPhys=%RGp idx=%#x\n", pPage->GCPhys, pPage->idx));
1767	break;
1768	}
1769	iPrev = i;
1770	}
1771	}
1772	pPage->iNext = NIL_PGMPOOL_IDX;
1773	}
1774
1775
1776	/**
1777	* Frees up one cache page.
1778	*
1779	* @returns VBox status code.
1780	* @retval VINF_SUCCESS on success.
1781	* @param pPool The pool.
1782	* @param iUser The user index.
1783	*/
1784	static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
1785	{
1786	#ifndef IN_RC
1787	const PVM pVM = pPool->CTX_SUFF(pVM);
1788	#endif
1789	Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
1790	STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
1791
1792	/*
1793	* Select one page from the tail of the age list.
1794	*/
1795	PPGMPOOLPAGE pPage;
1796	for (unsigned iLoop = 0; ; iLoop++)
1797	{
1798	uint16_t iToFree = pPool->iAgeTail;
1799	if (iToFree == iUser)
1800	iToFree = pPool->aPages[iToFree].iAgePrev;
1801	/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
1802	if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
1803	{
1804	uint16_t i = pPool->aPages[iToFree].iAgePrev;
1805	for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
1806	{
1807	if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
1808	continue;
1809	iToFree = i;
1810	break;
1811	}
1812	}
1813	*/
1814	Assert(iToFree != iUser);
1815	AssertRelease(iToFree != NIL_PGMPOOL_IDX);
1816	pPage = &pPool->aPages[iToFree];
1817
1818	/*
1819	* Reject any attempts at flushing the currently active shadow CR3 mapping.
1820	* Call pgmPoolCacheUsed to move the page to the head of the age list.
1821	*/
1822	if (!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage))
1823	break;
1824	LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
1825	pgmPoolCacheUsed(pPool, pPage);
1826	AssertLogRelReturn(iLoop < 8192, VERR_INTERNAL_ERROR);
1827	}
1828
1829	/*
1830	* Found a usable page, flush it and return.
1831	*/
1832	return pgmPoolFlushPage(pPool, pPage);
1833	}
1834
1835
1836	/**
1837	* Checks if a kind mismatch is really a page being reused
1838	* or if it's just normal remappings.
1839	*
1840	* @returns true if reused and the cached page (enmKind1) should be flushed
1841	* @returns false if not reused.
1842	* @param enmKind1 The kind of the cached page.
1843	* @param enmKind2 The kind of the requested page.
1844	*/
1845	static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
1846	{
1847	switch (enmKind1)
1848	{
1849	/*
1850	* Never reuse them. There is no remapping in non-paging mode.
1851	*/
1852	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1853	case PGMPOOLKIND_32BIT_PD_PHYS:
1854	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1855	case PGMPOOLKIND_PAE_PD_PHYS:
1856	case PGMPOOLKIND_PAE_PDPT_PHYS:
1857	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1858	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1859	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1860	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1861	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1862	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
1863	return false;
1864
1865	/*
1866	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1867	*/
1868	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1869	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1870	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1871	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1872	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1873	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1874	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1875	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1876	case PGMPOOLKIND_32BIT_PD:
1877	case PGMPOOLKIND_PAE_PDPT:
1878	switch (enmKind2)
1879	{
1880	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1881	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1882	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1883	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1884	case PGMPOOLKIND_64BIT_PML4:
1885	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1886	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1887	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1888	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1889	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1890	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1891	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1892	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1893	return true;
1894	default:
1895	return false;
1896	}
1897
1898	/*
1899	* It's perfectly fine to reuse these, except for PAE and non-paging stuff.
1900	*/
1901	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
1902	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
1903	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
1904	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
1905	case PGMPOOLKIND_64BIT_PML4:
1906	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
1907	switch (enmKind2)
1908	{
1909	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
1910	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
1911	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
1912	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
1913	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
1914	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
1915	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
1916	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
1917	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
1918	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
1919	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
1920	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
1921	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
1922	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
1923	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
1924	return true;
1925	default:
1926	return false;
1927	}
1928
1929	/*
1930	* These cannot be flushed, and it's common to reuse the PDs as PTs.
1931	*/
1932	case PGMPOOLKIND_ROOT_NESTED:
1933	return false;
1934
1935	default:
1936	AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
1937	}
1938	}
1939
1940
1941	/**
1942	* Attempts to satisfy a pgmPoolAlloc request from the cache.
1943	*
1944	* @returns VBox status code.
1945	* @retval VINF_PGM_CACHED_PAGE on success.
1946	* @retval VERR_FILE_NOT_FOUND if not found.
1947	* @param pPool The pool.
1948	* @param GCPhys The GC physical address of the page we're gonna shadow.
1949	* @param enmKind The kind of mapping.
1950	* @param enmAccess Access type for the mapping (only relevant for big pages)
1951	* @param iUser The shadow page pool index of the user table.
1952	* @param iUserTable The index into the user table (shadowed).
1953	* @param ppPage Where to store the pointer to the page.
1954	*/
1955	static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
1956	{
1957	#ifndef IN_RC
1958	const PVM pVM = pPool->CTX_SUFF(pVM);
1959	#endif
1960	/*
1961	* Look up the GCPhys in the hash.
1962	*/
1963	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
1964	Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%x iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
1965	if (i != NIL_PGMPOOL_IDX)
1966	{
1967	do
1968	{
1969	PPGMPOOLPAGE pPage = &pPool->aPages[i];
1970	Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
1971	if (pPage->GCPhys == GCPhys)
1972	{
1973	if ( (PGMPOOLKIND)pPage->enmKind == enmKind
1974	&& (PGMPOOLACCESS)pPage->enmAccess == enmAccess)
1975	{
1976	/* Put it at the start of the use list to make sure pgmPoolTrackAddUser
1977	* doesn't flush it in case there are no more free use records.
1978	*/
1979	pgmPoolCacheUsed(pPool, pPage);
1980
1981	int rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
1982	if (RT_SUCCESS(rc))
1983	{
1984	Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
1985	*ppPage = pPage;
1986	if (pPage->cModifications)
1987	pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
1988	STAM_COUNTER_INC(&pPool->StatCacheHits);
1989	return VINF_PGM_CACHED_PAGE;
1990	}
1991	return rc;
1992	}
1993
1994	if ((PGMPOOLKIND)pPage->enmKind != enmKind)
1995	{
1996	/*
1997	* The kind is different. In some cases we should now flush the page
1998	* as it has been reused, but in most cases this is normal remapping
1999	* of PDs as PT or big pages using the GCPhys field in a slightly
2000	* different way than the other kinds.
2001	*/
2002	if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
2003	{
2004	STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
2005	pgmPoolFlushPage(pPool, pPage);
2006	break;
2007	}
2008	}
2009	}
2010
2011	/* next */
2012	i = pPage->iNext;
2013	} while (i != NIL_PGMPOOL_IDX);
2014	}
2015
2016	Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
2017	STAM_COUNTER_INC(&pPool->StatCacheMisses);
2018	return VERR_FILE_NOT_FOUND;
2019	}
2020
2021
2022	/**
2023	* Inserts a page into the cache.
2024	*
2025	* @param pPool The pool.
2026	* @param pPage The cached page.
2027	* @param fCanBeCached Set if the page is fit for caching from the caller's point of view.
2028	*/
2029	static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
2030	{
2031	/*
2032	* Insert into the GCPhys hash if the page is fit for that.
2033	*/
2034	Assert(!pPage->fCached);
2035	if (fCanBeCached)
2036	{
2037	pPage->fCached = true;
2038	pgmPoolHashInsert(pPool, pPage);
2039	Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2040	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2041	STAM_COUNTER_INC(&pPool->StatCacheCacheable);
2042	}
2043	else
2044	{
2045	Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
2046	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
2047	STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
2048	}
2049
2050	/*
2051	* Insert at the head of the age list.
2052	*/
2053	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2054	pPage->iAgeNext = pPool->iAgeHead;
2055	if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
2056	pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
2057	else
2058	pPool->iAgeTail = pPage->idx;
2059	pPool->iAgeHead = pPage->idx;
2060	}
2061
2062
2063	/**
2064	* Flushes a cached page.
2065	*
2066	* @param pPool The pool.
2067	* @param pPage The cached page.
2068	*/
2069	static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2070	{
2071	Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
2072
2073	/*
2074	* Remove the page from the hash.
2075	*/
2076	if (pPage->fCached)
2077	{
2078	pPage->fCached = false;
2079	pgmPoolHashRemove(pPool, pPage);
2080	}
2081	else
2082	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
2083
2084	/*
2085	* Remove it from the age list.
2086	*/
2087	if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
2088	pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
2089	else
2090	pPool->iAgeTail = pPage->iAgePrev;
2091	if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
2092	pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
2093	else
2094	pPool->iAgeHead = pPage->iAgeNext;
2095	pPage->iAgeNext = NIL_PGMPOOL_IDX;
2096	pPage->iAgePrev = NIL_PGMPOOL_IDX;
2097	}
2098
2099	#endif /* PGMPOOL_WITH_CACHE */
2100	#ifdef PGMPOOL_WITH_MONITORING
2101
2102	/**
2103	* Looks for pages sharing the monitor.
2104	*
2105	* @returns Pointer to the head page.
2106	* @returns NULL if not found.
2107	* @param pPool The Pool
2108	* @param pNewPage The page which is going to be monitored.
2109	*/
2110	static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
2111	{
2112	#ifdef PGMPOOL_WITH_CACHE
2113	/*
2114	* Look up the GCPhys in the hash.
2115	*/
2116	RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
2117	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
2118	if (i == NIL_PGMPOOL_IDX)
2119	return NULL;
2120	do
2121	{
2122	PPGMPOOLPAGE pPage = &pPool->aPages[i];
2123	if ( pPage->GCPhys - GCPhys < PAGE_SIZE
2124	&& pPage != pNewPage)
2125	{
2126	switch (pPage->enmKind)
2127	{
2128	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2129	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2130	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2131	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2132	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2133	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2134	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2135	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2136	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2137	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2138	case PGMPOOLKIND_64BIT_PML4:
2139	case PGMPOOLKIND_32BIT_PD:
2140	case PGMPOOLKIND_PAE_PDPT:
2141	{
2142	/* find the head */
2143	while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2144	{
2145	Assert(pPage->iMonitoredPrev != pPage->idx);
2146	pPage = &pPool->aPages[pPage->iMonitoredPrev];
2147	}
2148	return pPage;
2149	}
2150
2151	/* ignore, no monitoring. */
2152	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2153	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2154	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2155	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2156	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2157	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2158	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2159	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2160	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2161	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2162	case PGMPOOLKIND_ROOT_NESTED:
2163	case PGMPOOLKIND_PAE_PD_PHYS:
2164	case PGMPOOLKIND_PAE_PDPT_PHYS:
2165	case PGMPOOLKIND_32BIT_PD_PHYS:
2166	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2167	break;
2168	default:
2169	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
2170	}
2171	}
2172
2173	/* next */
2174	i = pPage->iNext;
2175	} while (i != NIL_PGMPOOL_IDX);
2176	#endif
2177	return NULL;
2178	}
2179
2180
2181	/**
2182	* Enabled write monitoring of a guest page.
2183	*
2184	* @returns VBox status code.
2185	* @retval VINF_SUCCESS on success.
2186	* @param pPool The pool.
2187	* @param pPage The cached page.
2188	*/
2189	static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2190	{
2191	LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1)));
2192
2193	/*
2194	* Filter out the relevant kinds.
2195	*/
2196	switch (pPage->enmKind)
2197	{
2198	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2199	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2200	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2201	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2202	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2203	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2204	case PGMPOOLKIND_64BIT_PML4:
2205	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2206	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2207	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2208	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2209	case PGMPOOLKIND_32BIT_PD:
2210	case PGMPOOLKIND_PAE_PDPT:
2211	break;
2212
2213	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2214	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2215	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2216	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2217	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2218	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2219	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2220	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2221	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2222	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2223	case PGMPOOLKIND_ROOT_NESTED:
2224	/* Nothing to monitor here. */
2225	return VINF_SUCCESS;
2226
2227	case PGMPOOLKIND_32BIT_PD_PHYS:
2228	case PGMPOOLKIND_PAE_PDPT_PHYS:
2229	case PGMPOOLKIND_PAE_PD_PHYS:
2230	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
2231	/* Nothing to monitor here. */
2232	return VINF_SUCCESS;
2233	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2234	break;
2235	#else
2236	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2237	#endif
2238	default:
2239	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2240	}
2241
2242	/*
2243	* Install handler.
2244	*/
2245	int rc;
2246	PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
2247	if (pPageHead)
2248	{
2249	Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
2250	Assert(pPageHead->iMonitoredPrev != pPage->idx);
2251
2252	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2253	if (pPageHead->fDirty)
2254	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirty, true /* force removal */);
2255	#endif
2256
2257	pPage->iMonitoredPrev = pPageHead->idx;
2258	pPage->iMonitoredNext = pPageHead->iMonitoredNext;
2259	if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
2260	pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
2261	pPageHead->iMonitoredNext = pPage->idx;
2262	rc = VINF_SUCCESS;
2263	}
2264	else
2265	{
2266	Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
2267	PVM pVM = pPool->CTX_SUFF(pVM);
2268	const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
2269	rc = PGMHandlerPhysicalRegisterEx(pVM, PGMPHYSHANDLERTYPE_PHYSICAL_WRITE,
2270	GCPhysPage, GCPhysPage + (PAGE_SIZE - 1),
2271	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
2272	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
2273	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
2274	pPool->pszAccessHandler);
2275	/** @todo we should probably deal with out-of-memory conditions here, but for now increasing
2276	* the heap size should suffice. */
2277	AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
2278	Assert(!(VMMGetCpu(pVM)->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_ISSET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3));
2279	}
2280	pPage->fMonitored = true;
2281	return rc;
2282	}
2283
2284
2285	/**
2286	* Disables write monitoring of a guest page.
2287	*
2288	* @returns VBox status code.
2289	* @retval VINF_SUCCESS on success.
2290	* @param pPool The pool.
2291	* @param pPage The cached page.
2292	*/
2293	static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2294	{
2295	/*
2296	* Filter out the relevant kinds.
2297	*/
2298	switch (pPage->enmKind)
2299	{
2300	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2301	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2302	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2303	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2304	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2305	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2306	case PGMPOOLKIND_64BIT_PML4:
2307	case PGMPOOLKIND_32BIT_PD:
2308	case PGMPOOLKIND_PAE_PDPT:
2309	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2310	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2311	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2312	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2313	break;
2314
2315	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2316	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2317	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2318	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2319	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2320	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
2321	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
2322	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
2323	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
2324	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
2325	case PGMPOOLKIND_ROOT_NESTED:
2326	case PGMPOOLKIND_PAE_PD_PHYS:
2327	case PGMPOOLKIND_PAE_PDPT_PHYS:
2328	case PGMPOOLKIND_32BIT_PD_PHYS:
2329	/* Nothing to monitor here. */
2330	return VINF_SUCCESS;
2331
2332	#ifdef PGMPOOL_WITH_MIXED_PT_CR3
2333	break;
2334	#endif
2335	default:
2336	AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
2337	}
2338
2339	/*
2340	* Remove the page from the monitored list or uninstall it if last.
2341	*/
2342	const PVM pVM = pPool->CTX_SUFF(pVM);
2343	int rc;
2344	if ( pPage->iMonitoredNext != NIL_PGMPOOL_IDX
2345	\|\| pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
2346	{
2347	if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
2348	{
2349	PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
2350	pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
2351	rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
2352	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pNewHead),
2353	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pNewHead),
2354	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pNewHead),
2355	pPool->pszAccessHandler);
2356	AssertFatalRCSuccess(rc);
2357	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2358	}
2359	else
2360	{
2361	pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
2362	if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
2363	{
2364	pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
2365	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
2366	}
2367	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
2368	rc = VINF_SUCCESS;
2369	}
2370	}
2371	else
2372	{
2373	rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1));
2374	AssertFatalRC(rc);
2375	#ifdef VBOX_STRICT
2376	PVMCPU pVCpu = VMMGetCpu(pVM);
2377	#endif
2378	AssertMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) \|\| VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
2379	("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
2380	}
2381	pPage->fMonitored = false;
2382
2383	/*
2384	* Remove it from the list of modified pages (if in it).
2385	*/
2386	pgmPoolMonitorModifiedRemove(pPool, pPage);
2387
2388	return rc;
2389	}
2390
2391
2392	/**
2393	* Inserts the page into the list of modified pages.
2394	*
2395	* @param pPool The pool.
2396	* @param pPage The page.
2397	*/
2398	void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2399	{
2400	Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
2401	AssertMsg( pPage->iModifiedNext == NIL_PGMPOOL_IDX
2402	&& pPage->iModifiedPrev == NIL_PGMPOOL_IDX
2403	&& pPool->iModifiedHead != pPage->idx,
2404	("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
2405	pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
2406	pPool->iModifiedHead, pPool->cModifiedPages));
2407
2408	pPage->iModifiedNext = pPool->iModifiedHead;
2409	if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
2410	pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
2411	pPool->iModifiedHead = pPage->idx;
2412	pPool->cModifiedPages++;
2413	#ifdef VBOX_WITH_STATISTICS
2414	if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
2415	pPool->cModifiedPagesHigh = pPool->cModifiedPages;
2416	#endif
2417	}
2418
2419
2420	/**
2421	* Removes the page from the list of modified pages and resets the
2422	* moficiation counter.
2423	*
2424	* @param pPool The pool.
2425	* @param pPage The page which is believed to be in the list of modified pages.
2426	*/
2427	static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
2428	{
2429	Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
2430	if (pPool->iModifiedHead == pPage->idx)
2431	{
2432	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2433	pPool->iModifiedHead = pPage->iModifiedNext;
2434	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2435	{
2436	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
2437	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2438	}
2439	pPool->cModifiedPages--;
2440	}
2441	else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
2442	{
2443	pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
2444	if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
2445	{
2446	pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
2447	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2448	}
2449	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2450	pPool->cModifiedPages--;
2451	}
2452	else
2453	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
2454	pPage->cModifications = 0;
2455	}
2456
2457
2458	/**
2459	* Zaps the list of modified pages, resetting their modification counters in the process.
2460	*
2461	* @param pVM The VM handle.
2462	*/
2463	static void pgmPoolMonitorModifiedClearAll(PVM pVM)
2464	{
2465	pgmLock(pVM);
2466	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2467	LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
2468
2469	unsigned cPages = 0; NOREF(cPages);
2470
2471	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2472	pgmPoolResetDirtyPages(pVM, true /* force removal. */);
2473	#endif
2474
2475	uint16_t idx = pPool->iModifiedHead;
2476	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2477	while (idx != NIL_PGMPOOL_IDX)
2478	{
2479	PPGMPOOLPAGE pPage = &pPool->aPages[idx];
2480	idx = pPage->iModifiedNext;
2481	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2482	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2483	pPage->cModifications = 0;
2484	Assert(++cPages);
2485	}
2486	AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
2487	pPool->cModifiedPages = 0;
2488	pgmUnlock(pVM);
2489	}
2490
2491
2492	#ifdef IN_RING3
2493	/**
2494	* Callback to clear all shadow pages and clear all modification counters.
2495	*
2496	* @returns VBox status code.
2497	* @param pVM The VM handle.
2498	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
2499	* @param pvUser Unused parameter.
2500	*
2501	* @remark Should only be used when monitoring is available, thus placed in
2502	* the PGMPOOL_WITH_MONITORING \#ifdef.
2503	*/
2504	DECLCALLBACK(int) pgmPoolClearAll(PVM pVM, PVMCPU pVCpu, void *pvUser)
2505	{
2506	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2507	STAM_PROFILE_START(&pPool->StatClearAll, c);
2508	LogFlow(("pgmPoolClearAll: cUsedPages=%d\n", pPool->cUsedPages));
2509	NOREF(pvUser); NOREF(pVCpu);
2510
2511	pgmLock(pVM);
2512
2513	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2514	pgmPoolResetDirtyPages(pVM, true /* force removal. */);
2515	#endif
2516
2517	/*
2518	* Iterate all the pages until we've encountered all that in use.
2519	* This is simple but not quite optimal solution.
2520	*/
2521	unsigned cModifiedPages = 0; NOREF(cModifiedPages);
2522	unsigned cLeft = pPool->cUsedPages;
2523	unsigned iPage = pPool->cCurPages;
2524	while (--iPage >= PGMPOOL_IDX_FIRST)
2525	{
2526	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2527	if (pPage->GCPhys != NIL_RTGCPHYS)
2528	{
2529	switch (pPage->enmKind)
2530	{
2531	/*
2532	* We only care about shadow page tables.
2533	*/
2534	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2535	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2536	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2537	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2538	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2539	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2540	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2541	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2542	{
2543	#ifdef PGMPOOL_WITH_USER_TRACKING
2544	if (pPage->cPresent)
2545	#endif
2546	{
2547	void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
2548	STAM_PROFILE_START(&pPool->StatZeroPage, z);
2549	ASMMemZeroPage(pvShw);
2550	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
2551	#ifdef PGMPOOL_WITH_USER_TRACKING
2552	pPage->cPresent = 0;
2553	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
2554	#endif
2555	}
2556	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2557	else
2558	Assert(!pPage->fDirty);
2559	#endif
2560	}
2561	/* fall thru */
2562
2563	default:
2564	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2565	Assert(!pPage->fDirty);
2566	#endif
2567	Assert(!pPage->cModifications \|\| ++cModifiedPages);
2568	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2569	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2570	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2571	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2572	pPage->cModifications = 0;
2573	break;
2574
2575	}
2576	if (!--cLeft)
2577	break;
2578	}
2579	}
2580
2581	/* swipe the special pages too. */
2582	for (iPage = PGMPOOL_IDX_FIRST_SPECIAL; iPage < PGMPOOL_IDX_FIRST; iPage++)
2583	{
2584	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
2585	if (pPage->GCPhys != NIL_RTGCPHYS)
2586	{
2587	Assert(!pPage->cModifications \|\| ++cModifiedPages);
2588	Assert(pPage->iModifiedNext == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2589	Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX \|\| pPage->cModifications);
2590	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
2591	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
2592	pPage->cModifications = 0;
2593	}
2594	}
2595
2596	#ifndef DEBUG_michael
2597	AssertMsg(cModifiedPages == pPool->cModifiedPages, ("%d != %d\n", cModifiedPages, pPool->cModifiedPages));
2598	#endif
2599	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
2600	pPool->cModifiedPages = 0;
2601
2602	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
2603	/*
2604	* Clear all the GCPhys links and rebuild the phys ext free list.
2605	*/
2606	for (PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
2607	pRam;
2608	pRam = pRam->CTX_SUFF(pNext))
2609	{
2610	unsigned iPage = pRam->cb >> PAGE_SHIFT;
2611	while (iPage-- > 0)
2612	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
2613	}
2614
2615	pPool->iPhysExtFreeHead = 0;
2616	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
2617	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
2618	for (unsigned i = 0; i < cMaxPhysExts; i++)
2619	{
2620	paPhysExts[i].iNext = i + 1;
2621	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
2622	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
2623	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
2624	}
2625	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
2626	#endif
2627
2628	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2629	/* Clear all dirty pages. */
2630	pPool->idxFreeDirtyPage = 0;
2631	pPool->cDirtyPages = 0;
2632	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
2633	pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
2634	#endif
2635
2636	/* Clear the PGM_SYNC_CLEAR_PGM_POOL flag on all VCPUs to prevent redundant flushes. */
2637	for (unsigned idCpu = 0; idCpu < pVM->cCPUs; idCpu++)
2638	{
2639	PVMCPU pVCpu = &pVM->aCpus[idCpu];
2640
2641	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_CLEAR_PGM_POOL;
2642	}
2643
2644	pPool->cPresent = 0;
2645	pgmUnlock(pVM);
2646	PGM_INVL_ALL_VCPU_TLBS(pVM);
2647	STAM_PROFILE_STOP(&pPool->StatClearAll, c);
2648	return VINF_SUCCESS;
2649	}
2650	#endif /* IN_RING3 */
2651
2652
2653	/**
2654	* Handle SyncCR3 pool tasks
2655	*
2656	* @returns VBox status code.
2657	* @retval VINF_SUCCESS if successfully added.
2658	* @retval VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
2659	* @param pVCpu The VMCPU handle.
2660	* @remark Should only be used when monitoring is available, thus placed in
2661	* the PGMPOOL_WITH_MONITORING #ifdef.
2662	*/
2663	int pgmPoolSyncCR3(PVMCPU pVCpu)
2664	{
2665	PVM pVM = pVCpu->CTX_SUFF(pVM);
2666	LogFlow(("pgmPoolSyncCR3\n"));
2667
2668	/*
2669	* When monitoring shadowed pages, we reset the modification counters on CR3 sync.
2670	* Occasionally we will have to clear all the shadow page tables because we wanted
2671	* to monitor a page which was mapped by too many shadowed page tables. This operation
2672	* sometimes refered to as a 'lightweight flush'.
2673	*/
2674	# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
2675	if (ASMBitTestAndClear(&pVCpu->pgm.s.fSyncFlags, PGM_SYNC_CLEAR_PGM_POOL_BIT))
2676	{
2677	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmPoolClearAll, NULL);
2678	AssertRC(rc);
2679	}
2680	# else /* !IN_RING3 */
2681	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2682	{
2683	LogFlow(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
2684	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
2685	return VINF_PGM_SYNC_CR3;
2686	}
2687	# endif /* !IN_RING3 */
2688	else
2689	pgmPoolMonitorModifiedClearAll(pVM);
2690
2691	return VINF_SUCCESS;
2692	}
2693
2694	#endif /* PGMPOOL_WITH_MONITORING */
2695	#ifdef PGMPOOL_WITH_USER_TRACKING
2696
2697	/**
2698	* Frees up at least one user entry.
2699	*
2700	* @returns VBox status code.
2701	* @retval VINF_SUCCESS if successfully added.
2702	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2703	* @param pPool The pool.
2704	* @param iUser The user index.
2705	*/
2706	static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
2707	{
2708	STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
2709	#ifdef PGMPOOL_WITH_CACHE
2710	/*
2711	* Just free cached pages in a braindead fashion.
2712	*/
2713	/** @todo walk the age list backwards and free the first with usage. */
2714	int rc = VINF_SUCCESS;
2715	do
2716	{
2717	int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
2718	if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
2719	rc = rc2;
2720	} while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
2721	return rc;
2722	#else
2723	/*
2724	* Lazy approach.
2725	*/
2726	/* @todo This path no longer works (CR3 root pages will be flushed)!! */
2727	AssertCompileFailed();
2728	Assert(!CPUMIsGuestInLongMode(pVM));
2729	pgmPoolFlushAllInt(pPool);
2730	return VERR_PGM_POOL_FLUSHED;
2731	#endif
2732	}
2733
2734
2735	/**
2736	* Inserts a page into the cache.
2737	*
2738	* This will create user node for the page, insert it into the GCPhys
2739	* hash, and insert it into the age list.
2740	*
2741	* @returns VBox status code.
2742	* @retval VINF_SUCCESS if successfully added.
2743	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2744	* @param pPool The pool.
2745	* @param pPage The cached page.
2746	* @param GCPhys The GC physical address of the page we're gonna shadow.
2747	* @param iUser The user index.
2748	* @param iUserTable The user table index.
2749	*/
2750	DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
2751	{
2752	int rc = VINF_SUCCESS;
2753	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2754
2755	LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser %x iUserTable %x\n", GCPhys, iUser, iUserTable));
2756
2757	#ifdef VBOX_STRICT
2758	/*
2759	* Check that the entry doesn't already exists.
2760	*/
2761	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2762	{
2763	uint16_t i = pPage->iUserHead;
2764	do
2765	{
2766	Assert(i < pPool->cMaxUsers);
2767	AssertMsg(paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2768	i = paUsers[i].iNext;
2769	} while (i != NIL_PGMPOOL_USER_INDEX);
2770	}
2771	#endif
2772
2773	/*
2774	* Find free a user node.
2775	*/
2776	uint16_t i = pPool->iUserFreeHead;
2777	if (i == NIL_PGMPOOL_USER_INDEX)
2778	{
2779	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2780	if (RT_FAILURE(rc))
2781	return rc;
2782	i = pPool->iUserFreeHead;
2783	}
2784
2785	/*
2786	* Unlink the user node from the free list,
2787	* initialize and insert it into the user list.
2788	*/
2789	pPool->iUserFreeHead = paUsers[i].iNext;
2790	paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
2791	paUsers[i].iUser = iUser;
2792	paUsers[i].iUserTable = iUserTable;
2793	pPage->iUserHead = i;
2794
2795	/*
2796	* Insert into cache and enable monitoring of the guest page if enabled.
2797	*
2798	* Until we implement caching of all levels, including the CR3 one, we'll
2799	* have to make sure we don't try monitor & cache any recursive reuse of
2800	* a monitored CR3 page. Because all windows versions are doing this we'll
2801	* have to be able to do combined access monitoring, CR3 + PT and
2802	* PD + PT (guest PAE).
2803	*
2804	* Update:
2805	* We're now cooperating with the CR3 monitor if an uncachable page is found.
2806	*/
2807	#if defined(PGMPOOL_WITH_MONITORING) \|\| defined(PGMPOOL_WITH_CACHE)
2808	# ifdef PGMPOOL_WITH_MIXED_PT_CR3
2809	const bool fCanBeMonitored = true;
2810	# else
2811	bool fCanBeMonitored = pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored == NIL_RTGCPHYS
2812	\|\| (GCPhys & X86_PTE_PAE_PG_MASK) != (pPool->CTX_SUFF(pVM)->pgm.s.GCPhysGstCR3Monitored & X86_PTE_PAE_PG_MASK)
2813	\|\| pgmPoolIsBigPage((PGMPOOLKIND)pPage->enmKind);
2814	# endif
2815	# ifdef PGMPOOL_WITH_CACHE
2816	pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
2817	# endif
2818	if (fCanBeMonitored)
2819	{
2820	# ifdef PGMPOOL_WITH_MONITORING
2821	rc = pgmPoolMonitorInsert(pPool, pPage);
2822	AssertRC(rc);
2823	}
2824	# endif
2825	#endif /* PGMPOOL_WITH_MONITORING */
2826	return rc;
2827	}
2828
2829
2830	# ifdef PGMPOOL_WITH_CACHE /* (only used when the cache is enabled.) */
2831	/**
2832	* Adds a user reference to a page.
2833	*
2834	* This will move the page to the head of the
2835	*
2836	* @returns VBox status code.
2837	* @retval VINF_SUCCESS if successfully added.
2838	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
2839	* @param pPool The pool.
2840	* @param pPage The cached page.
2841	* @param iUser The user index.
2842	* @param iUserTable The user table.
2843	*/
2844	static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2845	{
2846	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2847
2848	Log3(("pgmPoolTrackAddUser GCPhys = %RGp iUser %x iUserTable %x\n", pPage->GCPhys, iUser, iUserTable));
2849
2850	# ifdef VBOX_STRICT
2851	/*
2852	* Check that the entry doesn't already exists. We only allow multiple users of top-level paging structures (SHW_POOL_ROOT_IDX).
2853	*/
2854	if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
2855	{
2856	uint16_t i = pPage->iUserHead;
2857	do
2858	{
2859	Assert(i < pPool->cMaxUsers);
2860	AssertMsg(iUser != PGMPOOL_IDX_PD \|\| iUser != PGMPOOL_IDX_PDPT \|\| iUser != PGMPOOL_IDX_NESTED_ROOT \|\| iUser != PGMPOOL_IDX_AMD64_CR3 \|\|
2861	paUsers[i].iUser != iUser \|\| paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
2862	i = paUsers[i].iNext;
2863	} while (i != NIL_PGMPOOL_USER_INDEX);
2864	}
2865	# endif
2866
2867	/*
2868	* Allocate a user node.
2869	*/
2870	uint16_t i = pPool->iUserFreeHead;
2871	if (i == NIL_PGMPOOL_USER_INDEX)
2872	{
2873	int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
2874	if (RT_FAILURE(rc))
2875	return rc;
2876	i = pPool->iUserFreeHead;
2877	}
2878	pPool->iUserFreeHead = paUsers[i].iNext;
2879
2880	/*
2881	* Initialize the user node and insert it.
2882	*/
2883	paUsers[i].iNext = pPage->iUserHead;
2884	paUsers[i].iUser = iUser;
2885	paUsers[i].iUserTable = iUserTable;
2886	pPage->iUserHead = i;
2887
2888	# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2889	if (pPage->fDirty)
2890	pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirty, true /* force removal */);
2891	# endif
2892
2893	# ifdef PGMPOOL_WITH_CACHE
2894	/*
2895	* Tell the cache to update its replacement stats for this page.
2896	*/
2897	pgmPoolCacheUsed(pPool, pPage);
2898	# endif
2899	return VINF_SUCCESS;
2900	}
2901	# endif /* PGMPOOL_WITH_CACHE */
2902
2903
2904	/**
2905	* Frees a user record associated with a page.
2906	*
2907	* This does not clear the entry in the user table, it simply replaces the
2908	* user record to the chain of free records.
2909	*
2910	* @param pPool The pool.
2911	* @param HCPhys The HC physical address of the shadow page.
2912	* @param iUser The shadow page pool index of the user table.
2913	* @param iUserTable The index into the user table (shadowed).
2914	*/
2915	static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
2916	{
2917	/*
2918	* Unlink and free the specified user entry.
2919	*/
2920	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
2921
2922	Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
2923	/* Special: For PAE and 32-bit paging, there is usually no more than one user. */
2924	uint16_t i = pPage->iUserHead;
2925	if ( i != NIL_PGMPOOL_USER_INDEX
2926	&& paUsers[i].iUser == iUser
2927	&& paUsers[i].iUserTable == iUserTable)
2928	{
2929	pPage->iUserHead = paUsers[i].iNext;
2930
2931	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2932	paUsers[i].iNext = pPool->iUserFreeHead;
2933	pPool->iUserFreeHead = i;
2934	return;
2935	}
2936
2937	/* General: Linear search. */
2938	uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
2939	while (i != NIL_PGMPOOL_USER_INDEX)
2940	{
2941	if ( paUsers[i].iUser == iUser
2942	&& paUsers[i].iUserTable == iUserTable)
2943	{
2944	if (iPrev != NIL_PGMPOOL_USER_INDEX)
2945	paUsers[iPrev].iNext = paUsers[i].iNext;
2946	else
2947	pPage->iUserHead = paUsers[i].iNext;
2948
2949	paUsers[i].iUser = NIL_PGMPOOL_IDX;
2950	paUsers[i].iNext = pPool->iUserFreeHead;
2951	pPool->iUserFreeHead = i;
2952	return;
2953	}
2954	iPrev = i;
2955	i = paUsers[i].iNext;
2956	}
2957
2958	/* Fatal: didn't find it */
2959	AssertFatalMsgFailed(("Didn't find the user entry! iUser=%#x iUserTable=%#x GCPhys=%RGp\n",
2960	iUser, iUserTable, pPage->GCPhys));
2961	}
2962
2963
2964	/**
2965	* Gets the entry size of a shadow table.
2966	*
2967	* @param enmKind The kind of page.
2968	*
2969	* @returns The size of the entry in bytes. That is, 4 or 8.
2970	* @returns If the kind is not for a table, an assertion is raised and 0 is
2971	* returned.
2972	*/
2973	DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
2974	{
2975	switch (enmKind)
2976	{
2977	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
2978	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
2979	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
2980	case PGMPOOLKIND_32BIT_PD:
2981	case PGMPOOLKIND_32BIT_PD_PHYS:
2982	return 4;
2983
2984	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
2985	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
2986	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
2987	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
2988	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
2989	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
2990	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
2991	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
2992	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
2993	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
2994	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
2995	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
2996	case PGMPOOLKIND_64BIT_PML4:
2997	case PGMPOOLKIND_PAE_PDPT:
2998	case PGMPOOLKIND_ROOT_NESTED:
2999	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3000	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3001	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3002	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3003	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3004	case PGMPOOLKIND_PAE_PD_PHYS:
3005	case PGMPOOLKIND_PAE_PDPT_PHYS:
3006	return 8;
3007
3008	default:
3009	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3010	}
3011	}
3012
3013
3014	/**
3015	* Gets the entry size of a guest table.
3016	*
3017	* @param enmKind The kind of page.
3018	*
3019	* @returns The size of the entry in bytes. That is, 0, 4 or 8.
3020	* @returns If the kind is not for a table, an assertion is raised and 0 is
3021	* returned.
3022	*/
3023	DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
3024	{
3025	switch (enmKind)
3026	{
3027	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3028	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3029	case PGMPOOLKIND_32BIT_PD:
3030	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3031	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3032	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3033	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3034	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3035	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3036	return 4;
3037
3038	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3039	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3040	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3041	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3042	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3043	case PGMPOOLKIND_64BIT_PML4:
3044	case PGMPOOLKIND_PAE_PDPT:
3045	return 8;
3046
3047	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3048	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3049	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3050	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3051	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3052	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3053	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3054	case PGMPOOLKIND_ROOT_NESTED:
3055	case PGMPOOLKIND_PAE_PD_PHYS:
3056	case PGMPOOLKIND_PAE_PDPT_PHYS:
3057	case PGMPOOLKIND_32BIT_PD_PHYS:
3058	/** @todo can we return 0? (nobody is calling this...) */
3059	AssertFailed();
3060	return 0;
3061
3062	default:
3063	AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
3064	}
3065	}
3066
3067	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3068
3069	/**
3070	* Scans one shadow page table for mappings of a physical page.
3071	*
3072	* @param pVM The VM handle.
3073	* @param pPhysPage The guest page in question.
3074	* @param iShw The shadow page table.
3075	* @param cRefs The number of references made in that PT.
3076	*/
3077	static void pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
3078	{
3079	LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
3080	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3081
3082	/*
3083	* Assert sanity.
3084	*/
3085	Assert(cRefs == 1);
3086	AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
3087	PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
3088
3089	/*
3090	* Then, clear the actual mappings to the page in the shadow PT.
3091	*/
3092	switch (pPage->enmKind)
3093	{
3094	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3095	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3096	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3097	{
3098	const uint32_t u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3099	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3100	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3101	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3102	{
3103	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32 cRefs=%#x\n", i, pPT->a[i], cRefs));
3104	pPT->a[i].u = 0;
3105	cRefs--;
3106	if (!cRefs)
3107	return;
3108	}
3109	#ifdef LOG_ENABLED
3110	Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3111	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3112	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3113	{
3114	Log(("i=%d cRefs=%d\n", i, cRefs--));
3115	}
3116	#endif
3117	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3118	break;
3119	}
3120
3121	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3122	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3123	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3124	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3125	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3126	{
3127	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3128	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3129	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3130	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
3131	{
3132	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
3133	pPT->a[i].u = 0;
3134	cRefs--;
3135	if (!cRefs)
3136	return;
3137	}
3138	#ifdef LOG_ENABLED
3139	Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3140	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3141	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
3142	{
3143	Log(("i=%d cRefs=%d\n", i, cRefs--));
3144	}
3145	#endif
3146	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d u64=%RX64\n", cRefs, pPage->iFirstPresent, pPage->cPresent, u64));
3147	break;
3148	}
3149
3150	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
3151	{
3152	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3153	PEPTPT pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3154	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3155	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
3156	{
3157	Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64 cRefs=%#x\n", i, pPT->a[i], cRefs));
3158	pPT->a[i].u = 0;
3159	cRefs--;
3160	if (!cRefs)
3161	return;
3162	}
3163	#ifdef LOG_ENABLED
3164	Log(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3165	for (unsigned i = 0; i < RT_ELEMENTS(pPT->a); i++)
3166	if ((pPT->a[i].u & (EPT_PTE_PG_MASK \| X86_PTE_P)) == u64)
3167	{
3168	Log(("i=%d cRefs=%d\n", i, cRefs--));
3169	}
3170	#endif
3171	AssertFatalMsgFailed(("cRefs=%d iFirstPresent=%d cPresent=%d\n", cRefs, pPage->iFirstPresent, pPage->cPresent));
3172	break;
3173	}
3174
3175	default:
3176	AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
3177	}
3178	}
3179
3180
3181	/**
3182	* Scans one shadow page table for mappings of a physical page.
3183	*
3184	* @param pVM The VM handle.
3185	* @param pPhysPage The guest page in question.
3186	* @param iShw The shadow page table.
3187	* @param cRefs The number of references made in that PT.
3188	*/
3189	void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, uint16_t iShw, uint16_t cRefs)
3190	{
3191	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
3192	LogFlow(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%R[pgmpage] iShw=%d cRefs=%d\n", pPhysPage, iShw, cRefs));
3193	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
3194	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, iShw, cRefs);
3195	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3196	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
3197	}
3198
3199
3200	/**
3201	* Flushes a list of shadow page tables mapping the same physical page.
3202	*
3203	* @param pVM The VM handle.
3204	* @param pPhysPage The guest page in question.
3205	* @param iPhysExt The physical cross reference extent list to flush.
3206	*/
3207	void pgmPoolTrackFlushGCPhysPTs(PVM pVM, PPGMPAGE pPhysPage, uint16_t iPhysExt)
3208	{
3209	Assert(PGMIsLockOwner(pVM));
3210	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3211	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
3212	LogFlow(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%R[pgmpage] iPhysExt\n", pPhysPage, iPhysExt));
3213
3214	const uint16_t iPhysExtStart = iPhysExt;
3215	PPGMPOOLPHYSEXT pPhysExt;
3216	do
3217	{
3218	Assert(iPhysExt < pPool->cMaxPhysExts);
3219	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3220	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3221	if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
3222	{
3223	pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, pPhysExt->aidx[i], 1);
3224	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3225	}
3226
3227	/* next */
3228	iPhysExt = pPhysExt->iNext;
3229	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3230
3231	/* insert the list into the free list and clear the ram range entry. */
3232	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3233	pPool->iPhysExtFreeHead = iPhysExtStart;
3234	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3235
3236	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
3237	}
3238
3239	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
3240
3241	/**
3242	* Flushes all shadow page table mappings of the given guest page.
3243	*
3244	* This is typically called when the host page backing the guest one has been
3245	* replaced or when the page protection was changed due to an access handler.
3246	*
3247	* @returns VBox status code.
3248	* @retval VINF_SUCCESS if all references has been successfully cleared.
3249	* @retval VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
3250	* pool cleaning. FF and sync flags are set.
3251	*
3252	* @param pVM The VM handle.
3253	* @param pPhysPage The guest page in question.
3254	* @param pfFlushTLBs This is set to @a true if the shadow TLBs should be
3255	* flushed, it is NOT touched if this isn't necessary.
3256	* The caller MUST initialized this to @a false.
3257	*/
3258	int pgmPoolTrackFlushGCPhys(PVM pVM, PPGMPAGE pPhysPage, bool *pfFlushTLBs)
3259	{
3260	PVMCPU pVCpu = VMMGetCpu(pVM);
3261	pgmLock(pVM);
3262	int rc = VINF_SUCCESS;
3263	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3264	const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
3265	if (u16)
3266	{
3267	/*
3268	* The zero page is currently screwing up the tracking and we'll
3269	* have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
3270	* is defined, zero pages won't normally be mapped. Some kind of solution
3271	* will be needed for this problem of course, but it will have to wait...
3272	*/
3273	if (PGM_PAGE_IS_ZERO(pPhysPage))
3274	rc = VINF_PGM_GCPHYS_ALIASED;
3275	else
3276	{
3277	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3278	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow and
3279	pgmPoolTrackFlushGCPhysPTs will/may kill the pool otherwise. */
3280	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3281	# endif
3282
3283	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3284	pgmPoolTrackFlushGCPhysPT(pVM,
3285	pPhysPage,
3286	PGMPOOL_TD_GET_IDX(u16),
3287	PGMPOOL_TD_GET_CREFS(u16));
3288	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3289	pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, PGMPOOL_TD_GET_IDX(u16));
3290	else
3291	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3292	*pfFlushTLBs = true;
3293
3294	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3295	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3296	# endif
3297	}
3298	}
3299
3300	#elif defined(PGMPOOL_WITH_CACHE)
3301	if (PGM_PAGE_IS_ZERO(pPhysPage))
3302	rc = VINF_PGM_GCPHYS_ALIASED;
3303	else
3304	{
3305	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3306	/* Start a subset here because pgmPoolTrackFlushGCPhysPTsSlow kills the pool otherwise. */
3307	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
3308	# endif
3309	rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
3310	if (rc == VINF_SUCCESS)
3311	*pfFlushTLBs = true;
3312	}
3313
3314	# ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3315	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
3316	# endif
3317
3318	#else
3319	rc = VINF_PGM_GCPHYS_ALIASED;
3320	#endif
3321
3322	if (rc == VINF_PGM_GCPHYS_ALIASED)
3323	{
3324	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3325	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3326	rc = VINF_PGM_SYNC_CR3;
3327	}
3328	pgmUnlock(pVM);
3329	return rc;
3330	}
3331
3332
3333	/**
3334	* Scans all shadow page tables for mappings of a physical page.
3335	*
3336	* This may be slow, but it's most likely more efficient than cleaning
3337	* out the entire page pool / cache.
3338	*
3339	* @returns VBox status code.
3340	* @retval VINF_SUCCESS if all references has been successfully cleared.
3341	* @retval VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
3342	* a page pool cleaning.
3343	*
3344	* @param pVM The VM handle.
3345	* @param pPhysPage The guest page in question.
3346	*/
3347	int pgmPoolTrackFlushGCPhysPTsSlow(PVM pVM, PPGMPAGE pPhysPage)
3348	{
3349	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3350	STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3351	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
3352	pPool->cUsedPages, pPool->cPresent, pPhysPage));
3353
3354	#if 1
3355	/*
3356	* There is a limit to what makes sense.
3357	*/
3358	if (pPool->cPresent > 1024)
3359	{
3360	LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
3361	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3362	return VINF_PGM_GCPHYS_ALIASED;
3363	}
3364	#endif
3365
3366	/*
3367	* Iterate all the pages until we've encountered all that in use.
3368	* This is simple but not quite optimal solution.
3369	*/
3370	const uint64_t u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) \| X86_PTE_P;
3371	const uint32_t u32 = u64;
3372	unsigned cLeft = pPool->cUsedPages;
3373	unsigned iPage = pPool->cCurPages;
3374	while (--iPage >= PGMPOOL_IDX_FIRST)
3375	{
3376	PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
3377	if (pPage->GCPhys != NIL_RTGCPHYS)
3378	{
3379	switch (pPage->enmKind)
3380	{
3381	/*
3382	* We only care about shadow page tables.
3383	*/
3384	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
3385	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
3386	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
3387	{
3388	unsigned cPresent = pPage->cPresent;
3389	PX86PT pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3390	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3391	if (pPT->a[i].n.u1Present)
3392	{
3393	if ((pPT->a[i].u & (X86_PTE_PG_MASK \| X86_PTE_P)) == u32)
3394	{
3395	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
3396	pPT->a[i].u = 0;
3397	}
3398	if (!--cPresent)
3399	break;
3400	}
3401	break;
3402	}
3403
3404	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
3405	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
3406	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
3407	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
3408	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
3409	{
3410	unsigned cPresent = pPage->cPresent;
3411	PX86PTPAE pPT = (PX86PTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
3412	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
3413	if (pPT->a[i].n.u1Present)
3414	{
3415	if ((pPT->a[i].u & (X86_PTE_PAE_PG_MASK \| X86_PTE_P)) == u64)
3416	{
3417	//Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
3418	pPT->a[i].u = 0;
3419	}
3420	if (!--cPresent)
3421	break;
3422	}
3423	break;
3424	}
3425	}
3426	if (!--cLeft)
3427	break;
3428	}
3429	}
3430
3431	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3432	STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
3433	return VINF_SUCCESS;
3434	}
3435
3436
3437	/**
3438	* Clears the user entry in a user table.
3439	*
3440	* This is used to remove all references to a page when flushing it.
3441	*/
3442	static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
3443	{
3444	Assert(pUser->iUser != NIL_PGMPOOL_IDX);
3445	Assert(pUser->iUser < pPool->cCurPages);
3446	uint32_t iUserTable = pUser->iUserTable;
3447
3448	/*
3449	* Map the user page.
3450	*/
3451	PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
3452	union
3453	{
3454	uint64_t *pau64;
3455	uint32_t *pau32;
3456	} u;
3457	u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
3458
3459	LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
3460
3461	/* Safety precaution in case we change the paging for other modes too in the future. */
3462	Assert(!pgmPoolIsPageLocked(&pPool->CTX_SUFF(pVM)->pgm.s, pPage));
3463
3464	#ifdef VBOX_STRICT
3465	/*
3466	* Some sanity checks.
3467	*/
3468	switch (pUserPage->enmKind)
3469	{
3470	case PGMPOOLKIND_32BIT_PD:
3471	case PGMPOOLKIND_32BIT_PD_PHYS:
3472	Assert(iUserTable < X86_PG_ENTRIES);
3473	break;
3474	case PGMPOOLKIND_PAE_PDPT:
3475	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3476	case PGMPOOLKIND_PAE_PDPT_PHYS:
3477	Assert(iUserTable < 4);
3478	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3479	break;
3480	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3481	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3482	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3483	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3484	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3485	case PGMPOOLKIND_PAE_PD_PHYS:
3486	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3487	break;
3488	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3489	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3490	Assert(!(u.pau64[iUserTable] & PGM_PDFLAGS_MAPPING));
3491	break;
3492	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3493	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3494	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3495	break;
3496	case PGMPOOLKIND_64BIT_PML4:
3497	Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
3498	/* GCPhys >> PAGE_SHIFT is the index here */
3499	break;
3500	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3501	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3502	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3503	break;
3504
3505	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3506	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3507	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3508	break;
3509
3510	case PGMPOOLKIND_ROOT_NESTED:
3511	Assert(iUserTable < X86_PG_PAE_ENTRIES);
3512	break;
3513
3514	default:
3515	AssertMsgFailed(("enmKind=%d\n", pUserPage->enmKind));
3516	break;
3517	}
3518	#endif /* VBOX_STRICT */
3519
3520	/*
3521	* Clear the entry in the user page.
3522	*/
3523	switch (pUserPage->enmKind)
3524	{
3525	/* 32-bit entries */
3526	case PGMPOOLKIND_32BIT_PD:
3527	case PGMPOOLKIND_32BIT_PD_PHYS:
3528	u.pau32[iUserTable] = 0;
3529	break;
3530
3531	/* 64-bit entries */
3532	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
3533	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
3534	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
3535	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
3536	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
3537	#if defined(IN_RC)
3538	/* In 32 bits PAE mode we must invalidate the TLB when changing a PDPT entry; the CPU fetches them only during cr3 load, so any
3539	* non-present PDPT will continue to cause page faults.
3540	*/
3541	ASMReloadCR3();
3542	#endif
3543	/* no break */
3544	case PGMPOOLKIND_PAE_PD_PHYS:
3545	case PGMPOOLKIND_PAE_PDPT_PHYS:
3546	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
3547	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
3548	case PGMPOOLKIND_64BIT_PML4:
3549	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
3550	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
3551	case PGMPOOLKIND_PAE_PDPT:
3552	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
3553	case PGMPOOLKIND_ROOT_NESTED:
3554	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
3555	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
3556	u.pau64[iUserTable] = 0;
3557	break;
3558
3559	default:
3560	AssertFatalMsgFailed(("enmKind=%d iUser=%#x iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
3561	}
3562	}
3563
3564
3565	/**
3566	* Clears all users of a page.
3567	*/
3568	static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
3569	{
3570	/*
3571	* Free all the user records.
3572	*/
3573	LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
3574
3575	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
3576	uint16_t i = pPage->iUserHead;
3577	while (i != NIL_PGMPOOL_USER_INDEX)
3578	{
3579	/* Clear enter in user table. */
3580	pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
3581
3582	/* Free it. */
3583	const uint16_t iNext = paUsers[i].iNext;
3584	paUsers[i].iUser = NIL_PGMPOOL_IDX;
3585	paUsers[i].iNext = pPool->iUserFreeHead;
3586	pPool->iUserFreeHead = i;
3587
3588	/* Next. */
3589	i = iNext;
3590	}
3591	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
3592	}
3593
3594	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
3595
3596	/**
3597	* Allocates a new physical cross reference extent.
3598	*
3599	* @returns Pointer to the allocated extent on success. NULL if we're out of them.
3600	* @param pVM The VM handle.
3601	* @param piPhysExt Where to store the phys ext index.
3602	*/
3603	PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVM pVM, uint16_t *piPhysExt)
3604	{
3605	Assert(PGMIsLockOwner(pVM));
3606	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3607	uint16_t iPhysExt = pPool->iPhysExtFreeHead;
3608	if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
3609	{
3610	STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
3611	return NULL;
3612	}
3613	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3614	pPool->iPhysExtFreeHead = pPhysExt->iNext;
3615	pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
3616	*piPhysExt = iPhysExt;
3617	return pPhysExt;
3618	}
3619
3620
3621	/**
3622	* Frees a physical cross reference extent.
3623	*
3624	* @param pVM The VM handle.
3625	* @param iPhysExt The extent to free.
3626	*/
3627	void pgmPoolTrackPhysExtFree(PVM pVM, uint16_t iPhysExt)
3628	{
3629	Assert(PGMIsLockOwner(pVM));
3630	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3631	Assert(iPhysExt < pPool->cMaxPhysExts);
3632	PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3633	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3634	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3635	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3636	pPool->iPhysExtFreeHead = iPhysExt;
3637	}
3638
3639
3640	/**
3641	* Frees a physical cross reference extent.
3642	*
3643	* @param pVM The VM handle.
3644	* @param iPhysExt The extent to free.
3645	*/
3646	void pgmPoolTrackPhysExtFreeList(PVM pVM, uint16_t iPhysExt)
3647	{
3648	Assert(PGMIsLockOwner(pVM));
3649	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3650
3651	const uint16_t iPhysExtStart = iPhysExt;
3652	PPGMPOOLPHYSEXT pPhysExt;
3653	do
3654	{
3655	Assert(iPhysExt < pPool->cMaxPhysExts);
3656	pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
3657	for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
3658	pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
3659
3660	/* next */
3661	iPhysExt = pPhysExt->iNext;
3662	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3663
3664	pPhysExt->iNext = pPool->iPhysExtFreeHead;
3665	pPool->iPhysExtFreeHead = iPhysExtStart;
3666	}
3667
3668
3669	/**
3670	* Insert a reference into a list of physical cross reference extents.
3671	*
3672	* @returns The new tracking data for PGMPAGE.
3673	*
3674	* @param pVM The VM handle.
3675	* @param iPhysExt The physical extent index of the list head.
3676	* @param iShwPT The shadow page table index.
3677	*
3678	*/
3679	static uint16_t pgmPoolTrackPhysExtInsert(PVM pVM, uint16_t iPhysExt, uint16_t iShwPT)
3680	{
3681	Assert(PGMIsLockOwner(pVM));
3682	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
3683	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3684
3685	/* special common case. */
3686	if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
3687	{
3688	paPhysExts[iPhysExt].aidx[2] = iShwPT;
3689	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3690	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d}\n", iPhysExt, iShwPT));
3691	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3692	}
3693
3694	/* general treatment. */
3695	const uint16_t iPhysExtStart = iPhysExt;
3696	unsigned cMax = 15;
3697	for (;;)
3698	{
3699	Assert(iPhysExt < pPool->cMaxPhysExts);
3700	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3701	if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
3702	{
3703	paPhysExts[iPhysExt].aidx[i] = iShwPT;
3704	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedMany);
3705	LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d} i=%d cMax=%d\n", iPhysExt, iShwPT, i, cMax));
3706	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
3707	}
3708	if (!--cMax)
3709	{
3710	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3711	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3712	LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
3713	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3714	}
3715	}
3716
3717	/* add another extent to the list. */
3718	PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3719	if (!pNew)
3720	{
3721	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackOverflows);
3722	pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
3723	LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
3724	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3725	}
3726	pNew->iNext = iPhysExtStart;
3727	pNew->aidx[0] = iShwPT;
3728	LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d}->%d\n", iPhysExt, iShwPT, iPhysExtStart));
3729	return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3730	}
3731
3732
3733	/**
3734	* Add a reference to guest physical page where extents are in use.
3735	*
3736	* @returns The new tracking data for PGMPAGE.
3737	*
3738	* @param pVM The VM handle.
3739	* @param u16 The ram range flags (top 16-bits).
3740	* @param iShwPT The shadow page table index.
3741	*/
3742	uint16_t pgmPoolTrackPhysExtAddref(PVM pVM, uint16_t u16, uint16_t iShwPT)
3743	{
3744	pgmLock(pVM);
3745	if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
3746	{
3747	/*
3748	* Convert to extent list.
3749	*/
3750	Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
3751	uint16_t iPhysExt;
3752	PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
3753	if (pPhysExt)
3754	{
3755	LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
3756	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliased);
3757	pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
3758	pPhysExt->aidx[1] = iShwPT;
3759	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
3760	}
3761	else
3762	u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
3763	}
3764	else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
3765	{
3766	/*
3767	* Insert into the extent list.
3768	*/
3769	u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT);
3770	}
3771	else
3772	STAM_COUNTER_INC(&pVM->pgm.s.StatTrackAliasedLots);
3773	pgmUnlock(pVM);
3774	return u16;
3775	}
3776
3777
3778	/**
3779	* Clear references to guest physical memory.
3780	*
3781	* @param pPool The pool.
3782	* @param pPage The page.
3783	* @param pPhysPage Pointer to the aPages entry in the ram range.
3784	*/
3785	void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage)
3786	{
3787	const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
3788	AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3789
3790	uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
3791	if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
3792	{
3793	PVM pVM = pPool->CTX_SUFF(pVM);
3794	pgmLock(pVM);
3795
3796	uint16_t iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
3797	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
3798	do
3799	{
3800	Assert(iPhysExt < pPool->cMaxPhysExts);
3801
3802	/*
3803	* Look for the shadow page and check if it's all freed.
3804	*/
3805	for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3806	{
3807	if (paPhysExts[iPhysExt].aidx[i] == pPage->idx)
3808	{
3809	paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
3810
3811	for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
3812	if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
3813	{
3814	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3815	pgmUnlock(pVM);
3816	return;
3817	}
3818
3819	/* we can free the node. */
3820	const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
3821	if ( iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
3822	&& iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
3823	{
3824	/* lonely node */
3825	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3826	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
3827	PGM_PAGE_SET_TRACKING(pPhysPage, 0);
3828	}
3829	else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
3830	{
3831	/* head */
3832	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
3833	PGM_PAGE_SET_TRACKING(pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
3834	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3835	}
3836	else
3837	{
3838	/* in list */
3839	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
3840	paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
3841	pgmPoolTrackPhysExtFree(pVM, iPhysExt);
3842	}
3843	iPhysExt = iPhysExtNext;
3844	pgmUnlock(pVM);
3845	return;
3846	}
3847	}
3848
3849	/* next */
3850	iPhysExtPrev = iPhysExt;
3851	iPhysExt = paPhysExts[iPhysExt].iNext;
3852	} while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
3853
3854	pgmUnlock(pVM);
3855	AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
3856	}
3857	else /* nothing to do */
3858	Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
3859	}
3860
3861
3862	/**
3863	* Clear references to guest physical memory.
3864	*
3865	* This is the same as pgmPoolTracDerefGCPhys except that the guest physical address
3866	* is assumed to be correct, so the linear search can be skipped and we can assert
3867	* at an earlier point.
3868	*
3869	* @param pPool The pool.
3870	* @param pPage The page.
3871	* @param HCPhys The host physical address corresponding to the guest page.
3872	* @param GCPhys The guest physical address corresponding to HCPhys.
3873	*/
3874	static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys)
3875	{
3876	/*
3877	* Walk range list.
3878	*/
3879	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3880	while (pRam)
3881	{
3882	RTGCPHYS off = GCPhys - pRam->GCPhys;
3883	if (off < pRam->cb)
3884	{
3885	/* does it match? */
3886	const unsigned iPage = off >> PAGE_SHIFT;
3887	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3888	#ifdef LOG_ENABLED
3889	RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]);
3890	Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
3891	#endif
3892	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3893	{
3894	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3895	return;
3896	}
3897	break;
3898	}
3899	pRam = pRam->CTX_SUFF(pNext);
3900	}
3901	AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
3902	}
3903
3904
3905	/**
3906	* Clear references to guest physical memory.
3907	*
3908	* @param pPool The pool.
3909	* @param pPage The page.
3910	* @param HCPhys The host physical address corresponding to the guest page.
3911	* @param GCPhysHint The guest physical address which may corresponding to HCPhys.
3912	*/
3913	void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint)
3914	{
3915	Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
3916
3917	/*
3918	* Walk range list.
3919	*/
3920	PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3921	while (pRam)
3922	{
3923	RTGCPHYS off = GCPhysHint - pRam->GCPhys;
3924	if (off < pRam->cb)
3925	{
3926	/* does it match? */
3927	const unsigned iPage = off >> PAGE_SHIFT;
3928	Assert(PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]));
3929	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3930	{
3931	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3932	return;
3933	}
3934	break;
3935	}
3936	pRam = pRam->CTX_SUFF(pNext);
3937	}
3938
3939	/*
3940	* Damn, the hint didn't work. We'll have to do an expensive linear search.
3941	*/
3942	STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
3943	pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRanges);
3944	while (pRam)
3945	{
3946	unsigned iPage = pRam->cb >> PAGE_SHIFT;
3947	while (iPage-- > 0)
3948	{
3949	if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
3950	{
3951	Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
3952	HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
3953	pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage]);
3954	return;
3955	}
3956	}
3957	pRam = pRam->CTX_SUFF(pNext);
3958	}
3959
3960	AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp\n", HCPhys, GCPhysHint));
3961	}
3962
3963
3964	/**
3965	* Clear references to guest physical memory in a 32-bit / 32-bit page table.
3966	*
3967	* @param pPool The pool.
3968	* @param pPage The page.
3969	* @param pShwPT The shadow page table (mapping of the page).
3970	* @param pGstPT The guest page table.
3971	*/
3972	DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
3973	{
3974	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3975	if (pShwPT->a[i].n.u1Present)
3976	{
3977	Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
3978	i, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
3979	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
3980	if (!--pPage->cPresent)
3981	break;
3982	}
3983	}
3984
3985
3986	/**
3987	* Clear references to guest physical memory in a PAE / 32-bit page table.
3988	*
3989	* @param pPool The pool.
3990	* @param pPage The page.
3991	* @param pShwPT The shadow page table (mapping of the page).
3992	* @param pGstPT The guest page table (just a half one).
3993	*/
3994	DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PT pGstPT)
3995	{
3996	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
3997	if (pShwPT->a[i].n.u1Present)
3998	{
3999	Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
4000	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
4001	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK);
4002	if (!--pPage->cPresent)
4003	break;
4004	}
4005	}
4006
4007
4008	/**
4009	* Clear references to guest physical memory in a PAE / PAE page table.
4010	*
4011	* @param pPool The pool.
4012	* @param pPage The page.
4013	* @param pShwPT The shadow page table (mapping of the page).
4014	* @param pGstPT The guest page table.
4015	*/
4016	DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT, PCX86PTPAE pGstPT)
4017	{
4018	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
4019	if (pShwPT->a[i].n.u1Present)
4020	{
4021	Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
4022	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
4023	pgmPoolTracDerefGCPhysHint(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK);
4024	if (!--pPage->cPresent)
4025	break;
4026	}
4027	}
4028
4029
4030	/**
4031	* Clear references to guest physical memory in a 32-bit / 4MB page table.
4032	*
4033	* @param pPool The pool.
4034	* @param pPage The page.
4035	* @param pShwPT The shadow page table (mapping of the page).
4036	*/
4037	DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
4038	{
4039	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4040	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4041	if (pShwPT->a[i].n.u1Present)
4042	{
4043	Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
4044	i, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys));
4045	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PG_MASK, GCPhys);
4046	if (!--pPage->cPresent)
4047	break;
4048	}
4049	}
4050
4051
4052	/**
4053	* Clear references to guest physical memory in a PAE / 2/4MB page table.
4054	*
4055	* @param pPool The pool.
4056	* @param pPage The page.
4057	* @param pShwPT The shadow page table (mapping of the page).
4058	*/
4059	DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PTPAE pShwPT)
4060	{
4061	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4062	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4063	if (pShwPT->a[i].n.u1Present)
4064	{
4065	Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
4066	i, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys));
4067	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & X86_PTE_PAE_PG_MASK, GCPhys);
4068	if (!--pPage->cPresent)
4069	break;
4070	}
4071	}
4072
4073
4074	/**
4075	* Clear references to shadowed pages in an EPT page table.
4076	*
4077	* @param pPool The pool.
4078	* @param pPage The page.
4079	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4080	*/
4081	DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
4082	{
4083	RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
4084	for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
4085	if (pShwPT->a[i].n.u1Present)
4086	{
4087	Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
4088	i, pShwPT->a[i].u & EPT_PTE_PG_MASK, pPage->GCPhys));
4089	pgmPoolTracDerefGCPhys(pPool, pPage, pShwPT->a[i].u & EPT_PTE_PG_MASK, GCPhys);
4090	if (!--pPage->cPresent)
4091	break;
4092	}
4093	}
4094
4095	#endif /* PGMPOOL_WITH_GCPHYS_TRACKING */
4096
4097
4098	/**
4099	* Clear references to shadowed pages in a 32 bits page directory.
4100	*
4101	* @param pPool The pool.
4102	* @param pPage The page.
4103	* @param pShwPD The shadow page directory (mapping of the page).
4104	*/
4105	DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
4106	{
4107	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4108	{
4109	if ( pShwPD->a[i].n.u1Present
4110	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
4111	)
4112	{
4113	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
4114	if (pSubPage)
4115	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4116	else
4117	AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
4118	}
4119	}
4120	}
4121
4122	/**
4123	* Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
4124	*
4125	* @param pPool The pool.
4126	* @param pPage The page.
4127	* @param pShwPD The shadow page directory (mapping of the page).
4128	*/
4129	DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
4130	{
4131	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4132	{
4133	if ( pShwPD->a[i].n.u1Present
4134	&& !(pShwPD->a[i].u & PGM_PDFLAGS_MAPPING)
4135	)
4136	{
4137	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PAE_PG_MASK);
4138	if (pSubPage)
4139	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4140	else
4141	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & X86_PDE_PAE_PG_MASK));
4142	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4143	}
4144	}
4145	}
4146
4147	/**
4148	* Clear references to shadowed pages in a PAE page directory pointer table.
4149	*
4150	* @param pPool The pool.
4151	* @param pPage The page.
4152	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4153	*/
4154	DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4155	{
4156	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
4157	{
4158	if ( pShwPDPT->a[i].n.u1Present
4159	&& !(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING)
4160	)
4161	{
4162	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4163	if (pSubPage)
4164	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4165	else
4166	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4167	}
4168	}
4169	}
4170
4171
4172	/**
4173	* Clear references to shadowed pages in a 64-bit page directory pointer table.
4174	*
4175	* @param pPool The pool.
4176	* @param pPage The page.
4177	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4178	*/
4179	DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
4180	{
4181	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4182	{
4183	Assert(!(pShwPDPT->a[i].u & PGM_PLXFLAGS_MAPPING));
4184	if (pShwPDPT->a[i].n.u1Present)
4185	{
4186	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & X86_PDPE_PG_MASK);
4187	if (pSubPage)
4188	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4189	else
4190	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & X86_PDPE_PG_MASK));
4191	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4192	}
4193	}
4194	}
4195
4196
4197	/**
4198	* Clear references to shadowed pages in a 64-bit level 4 page table.
4199	*
4200	* @param pPool The pool.
4201	* @param pPage The page.
4202	* @param pShwPML4 The shadow page directory pointer table (mapping of the page).
4203	*/
4204	DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
4205	{
4206	for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
4207	{
4208	if (pShwPML4->a[i].n.u1Present)
4209	{
4210	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPML4->a[i].u & X86_PDPE_PG_MASK);
4211	if (pSubPage)
4212	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4213	else
4214	AssertFatalMsgFailed(("%RX64\n", pShwPML4->a[i].u & X86_PML4E_PG_MASK));
4215	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4216	}
4217	}
4218	}
4219
4220
4221	/**
4222	* Clear references to shadowed pages in an EPT page directory.
4223	*
4224	* @param pPool The pool.
4225	* @param pPage The page.
4226	* @param pShwPD The shadow page directory (mapping of the page).
4227	*/
4228	DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
4229	{
4230	for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
4231	{
4232	if (pShwPD->a[i].n.u1Present)
4233	{
4234	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & EPT_PDE_PG_MASK);
4235	if (pSubPage)
4236	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4237	else
4238	AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
4239	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4240	}
4241	}
4242	}
4243
4244
4245	/**
4246	* Clear references to shadowed pages in an EPT page directory pointer table.
4247	*
4248	* @param pPool The pool.
4249	* @param pPage The page.
4250	* @param pShwPDPT The shadow page directory pointer table (mapping of the page).
4251	*/
4252	DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
4253	{
4254	for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
4255	{
4256	if (pShwPDPT->a[i].n.u1Present)
4257	{
4258	PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK);
4259	if (pSubPage)
4260	pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
4261	else
4262	AssertFatalMsgFailed(("%RX64\n", pShwPDPT->a[i].u & EPT_PDPTE_PG_MASK));
4263	/** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
4264	}
4265	}
4266	}
4267
4268
4269	/**
4270	* Clears all references made by this page.
4271	*
4272	* This includes other shadow pages and GC physical addresses.
4273	*
4274	* @param pPool The pool.
4275	* @param pPage The page.
4276	*/
4277	static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4278	{
4279	/*
4280	* Map the shadow page and take action according to the page kind.
4281	*/
4282	void *pvShw = PGMPOOL_PAGE_2_LOCKED_PTR(pPool->CTX_SUFF(pVM), pPage);
4283	switch (pPage->enmKind)
4284	{
4285	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4286	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4287	{
4288	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4289	void *pvGst;
4290	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4291	pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
4292	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4293	break;
4294	}
4295
4296	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4297	{
4298	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4299	void *pvGst;
4300	int rc = PGM_GCPHYS_2_PTR_EX(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4301	pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PT)pvGst);
4302	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4303	break;
4304	}
4305
4306	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4307	{
4308	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4309	void *pvGst;
4310	int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
4311	pgmPoolTrackDerefPTPaePae(pPool, pPage, (PX86PTPAE)pvShw, (PCX86PTPAE)pvGst);
4312	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4313	break;
4314	}
4315
4316	case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
4317	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4318	{
4319	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4320	pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
4321	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4322	break;
4323	}
4324
4325	case PGMPOOLKIND_PAE_PT_FOR_PHYS: /* treat it like a 2 MB page */
4326	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4327	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4328	{
4329	STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
4330	pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PX86PTPAE)pvShw);
4331	STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
4332	break;
4333	}
4334
4335	#else /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4336	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4337	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4338	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4339	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4340	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4341	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4342	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4343	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4344	break;
4345	#endif /* !PGMPOOL_WITH_GCPHYS_TRACKING */
4346
4347	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4348	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4349	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4350	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4351	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4352	case PGMPOOLKIND_PAE_PD_PHYS:
4353	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4354	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4355	pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
4356	break;
4357
4358	case PGMPOOLKIND_32BIT_PD_PHYS:
4359	case PGMPOOLKIND_32BIT_PD:
4360	pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
4361	break;
4362
4363	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4364	case PGMPOOLKIND_PAE_PDPT:
4365	case PGMPOOLKIND_PAE_PDPT_PHYS:
4366	pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
4367	break;
4368
4369	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4370	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4371	pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
4372	break;
4373
4374	case PGMPOOLKIND_64BIT_PML4:
4375	pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
4376	break;
4377
4378	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4379	pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
4380	break;
4381
4382	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4383	pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
4384	break;
4385
4386	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4387	pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
4388	break;
4389
4390	default:
4391	AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
4392	}
4393
4394	/* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
4395	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4396	ASMMemZeroPage(pvShw);
4397	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4398	pPage->fZeroed = true;
4399	PGMPOOL_UNLOCK_PTR(pPool->CTX_SUFF(pVM), pvShw);
4400	}
4401	#endif /* PGMPOOL_WITH_USER_TRACKING */
4402
4403	/**
4404	* Flushes a pool page.
4405	*
4406	* This moves the page to the free list after removing all user references to it.
4407	*
4408	* @returns VBox status code.
4409	* @retval VINF_SUCCESS on success.
4410	* @param pPool The pool.
4411	* @param HCPhys The HC physical address of the shadow page.
4412	*/
4413	int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
4414	{
4415	PVM pVM = pPool->CTX_SUFF(pVM);
4416
4417	int rc = VINF_SUCCESS;
4418	STAM_PROFILE_START(&pPool->StatFlushPage, f);
4419	LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
4420	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
4421
4422	/*
4423	* Quietly reject any attempts at flushing any of the special root pages.
4424	*/
4425	if (pPage->idx < PGMPOOL_IDX_FIRST)
4426	{
4427	AssertFailed(); /* can no longer happen */
4428	Log(("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4429	return VINF_SUCCESS;
4430	}
4431
4432	pgmLock(pVM);
4433
4434	/*
4435	* Quietly reject any attempts at flushing the currently active shadow CR3 mapping
4436	*/
4437	if (pgmPoolIsPageLocked(&pVM->pgm.s, pPage))
4438	{
4439	AssertMsg( pPage->enmKind == PGMPOOLKIND_64BIT_PML4
4440	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT
4441	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
4442	\|\| pPage->enmKind == PGMPOOLKIND_32BIT_PD
4443	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4444	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
4445	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
4446	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
4447	\|\| pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD,
4448	("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
4449	Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
4450	pgmUnlock(pVM);
4451	return VINF_SUCCESS;
4452	}
4453
4454	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4455	/* Start a subset so we won't run out of mapping space. */
4456	PVMCPU pVCpu = VMMGetCpu(pVM);
4457	uint32_t iPrevSubset = PGMDynMapPushAutoSubset(pVCpu);
4458	#endif
4459
4460	/*
4461	* Mark the page as being in need of an ASMMemZeroPage().
4462	*/
4463	pPage->fZeroed = false;
4464
4465	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4466	if (pPage->fDirty)
4467	pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirty, true /* force removal */);
4468	#endif
4469
4470	#ifdef PGMPOOL_WITH_USER_TRACKING
4471	/*
4472	* Clear the page.
4473	*/
4474	pgmPoolTrackClearPageUsers(pPool, pPage);
4475	STAM_PROFILE_START(&pPool->StatTrackDeref,a);
4476	pgmPoolTrackDeref(pPool, pPage);
4477	STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
4478	#endif
4479
4480	#ifdef PGMPOOL_WITH_CACHE
4481	/*
4482	* Flush it from the cache.
4483	*/
4484	pgmPoolCacheFlushPage(pPool, pPage);
4485	#endif /* PGMPOOL_WITH_CACHE */
4486
4487	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
4488	/* Heavy stuff done. */
4489	PGMDynMapPopAutoSubset(pVCpu, iPrevSubset);
4490	#endif
4491
4492	#ifdef PGMPOOL_WITH_MONITORING
4493	/*
4494	* Deregistering the monitoring.
4495	*/
4496	if (pPage->fMonitored)
4497	rc = pgmPoolMonitorFlush(pPool, pPage);
4498	#endif
4499
4500	/*
4501	* Free the page.
4502	*/
4503	Assert(pPage->iNext == NIL_PGMPOOL_IDX);
4504	pPage->iNext = pPool->iFreeHead;
4505	pPool->iFreeHead = pPage->idx;
4506	pPage->enmKind = PGMPOOLKIND_FREE;
4507	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4508	pPage->GCPhys = NIL_RTGCPHYS;
4509	pPage->fReusedFlushPending = false;
4510
4511	pPool->cUsedPages--;
4512	pgmUnlock(pVM);
4513	STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
4514	return rc;
4515	}
4516
4517
4518	/**
4519	* Frees a usage of a pool page.
4520	*
4521	* The caller is responsible to updating the user table so that it no longer
4522	* references the shadow page.
4523	*
4524	* @param pPool The pool.
4525	* @param HCPhys The HC physical address of the shadow page.
4526	* @param iUser The shadow page pool index of the user table.
4527	* @param iUserTable The index into the user table (shadowed).
4528	*/
4529	void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
4530	{
4531	PVM pVM = pPool->CTX_SUFF(pVM);
4532
4533	STAM_PROFILE_START(&pPool->StatFree, a);
4534	LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%#x iUserTable=%#x\n",
4535	pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
4536	Assert(pPage->idx >= PGMPOOL_IDX_FIRST);
4537	pgmLock(pVM);
4538	#ifdef PGMPOOL_WITH_USER_TRACKING
4539	pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
4540	#endif
4541	#ifdef PGMPOOL_WITH_CACHE
4542	if (!pPage->fCached)
4543	#endif
4544	pgmPoolFlushPage(pPool, pPage);
4545	pgmUnlock(pVM);
4546	STAM_PROFILE_STOP(&pPool->StatFree, a);
4547	}
4548
4549
4550	/**
4551	* Makes one or more free page free.
4552	*
4553	* @returns VBox status code.
4554	* @retval VINF_SUCCESS on success.
4555	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4556	*
4557	* @param pPool The pool.
4558	* @param enmKind Page table kind
4559	* @param iUser The user of the page.
4560	*/
4561	static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
4562	{
4563	PVM pVM = pPool->CTX_SUFF(pVM);
4564
4565	LogFlow(("pgmPoolMakeMoreFreePages: iUser=%#x\n", iUser));
4566
4567	/*
4568	* If the pool isn't full grown yet, expand it.
4569	*/
4570	if ( pPool->cCurPages < pPool->cMaxPages
4571	#if defined(IN_RC)
4572	/* Hack alert: we can't deal with jumps to ring 3 when called from MapCR3 and allocating pages for PAE PDs. */
4573	&& enmKind != PGMPOOLKIND_PAE_PD_FOR_PAE_PD
4574	&& (enmKind < PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD \|\| enmKind > PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD)
4575	#endif
4576	)
4577	{
4578	STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
4579	#ifdef IN_RING3
4580	int rc = PGMR3PoolGrow(pVM);
4581	#else
4582	int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_PGM_POOL_GROW, 0);
4583	#endif
4584	if (RT_FAILURE(rc))
4585	return rc;
4586	STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
4587	if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
4588	return VINF_SUCCESS;
4589	}
4590
4591	#ifdef PGMPOOL_WITH_CACHE
4592	/*
4593	* Free one cached page.
4594	*/
4595	return pgmPoolCacheFreeOne(pPool, iUser);
4596	#else
4597	/*
4598	* Flush the pool.
4599	*
4600	* If we have tracking enabled, it should be possible to come up with
4601	* a cheap replacement strategy...
4602	*/
4603	/* @todo This path no longer works (CR3 root pages will be flushed)!! */
4604	AssertCompileFailed();
4605	Assert(!CPUMIsGuestInLongMode(pVM));
4606	pgmPoolFlushAllInt(pPool);
4607	return VERR_PGM_POOL_FLUSHED;
4608	#endif
4609	}
4610
4611	/**
4612	* Allocates a page from the pool.
4613	*
4614	* This page may actually be a cached page and not in need of any processing
4615	* on the callers part.
4616	*
4617	* @returns VBox status code.
4618	* @retval VINF_SUCCESS if a NEW page was allocated.
4619	* @retval VINF_PGM_CACHED_PAGE if a CACHED page was returned.
4620	* @retval VERR_PGM_POOL_FLUSHED if the pool was flushed.
4621	* @param pVM The VM handle.
4622	* @param GCPhys The GC physical address of the page we're gonna shadow.
4623	* For 4MB and 2MB PD entries, it's the first address the
4624	* shadow PT is covering.
4625	* @param enmKind The kind of mapping.
4626	* @param enmAccess Access type for the mapping (only relevant for big pages)
4627	* @param iUser The shadow page pool index of the user table.
4628	* @param iUserTable The index into the user table (shadowed).
4629	* @param ppPage Where to store the pointer to the page. NULL is stored here on failure.
4630	* @param fLockPage Lock the page
4631	*/
4632	int pgmPoolAllocEx(PVM pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage, bool fLockPage)
4633	{
4634	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4635	STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
4636	LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%#x iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
4637	*ppPage = NULL;
4638	/** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
4639	* (TRPMR3SyncIDT) because of FF priority. Try fix that?
4640	* Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
4641
4642	pgmLock(pVM);
4643
4644	#ifdef PGMPOOL_WITH_CACHE
4645	if (pPool->fCacheEnabled)
4646	{
4647	int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, iUser, iUserTable, ppPage);
4648	if (RT_SUCCESS(rc2))
4649	{
4650	if (fLockPage)
4651	pgmPoolLockPage(pPool, *ppPage);
4652	pgmUnlock(pVM);
4653	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4654	LogFlow(("pgmPoolAlloc: cached returns %Rrc ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, ppPage, (ppPage)->Core.Key, (ppPage)->idx));
4655	return rc2;
4656	}
4657	}
4658	#endif
4659
4660	/*
4661	* Allocate a new one.
4662	*/
4663	int rc = VINF_SUCCESS;
4664	uint16_t iNew = pPool->iFreeHead;
4665	if (iNew == NIL_PGMPOOL_IDX)
4666	{
4667	rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
4668	if (RT_FAILURE(rc))
4669	{
4670	pgmUnlock(pVM);
4671	Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
4672	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4673	return rc;
4674	}
4675	iNew = pPool->iFreeHead;
4676	AssertReleaseReturn(iNew != NIL_PGMPOOL_IDX, VERR_INTERNAL_ERROR);
4677	}
4678
4679	/* unlink the free head */
4680	PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
4681	pPool->iFreeHead = pPage->iNext;
4682	pPage->iNext = NIL_PGMPOOL_IDX;
4683
4684	/*
4685	* Initialize it.
4686	*/
4687	pPool->cUsedPages++; /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
4688	pPage->enmKind = enmKind;
4689	pPage->enmAccess = enmAccess;
4690	pPage->GCPhys = GCPhys;
4691	pPage->fSeenNonGlobal = false; /* Set this to 'true' to disable this feature. */
4692	pPage->fMonitored = false;
4693	pPage->fCached = false;
4694	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4695	pPage->fDirty = false;
4696	#endif
4697	pPage->fReusedFlushPending = false;
4698	#ifdef PGMPOOL_WITH_MONITORING
4699	pPage->cModifications = 0;
4700	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4701	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4702	#else
4703	pPage->fCR3Mix = false;
4704	#endif
4705	#ifdef PGMPOOL_WITH_USER_TRACKING
4706	pPage->cPresent = 0;
4707	pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
4708	pPage->pvLastAccessHandlerFault = 0;
4709	pPage->cLastAccessHandlerCount = 0;
4710	pPage->pvLastAccessHandlerRip = 0;
4711
4712	/*
4713	* Insert into the tracking and cache. If this fails, free the page.
4714	*/
4715	int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
4716	if (RT_FAILURE(rc3))
4717	{
4718	pPool->cUsedPages--;
4719	pPage->enmKind = PGMPOOLKIND_FREE;
4720	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4721	pPage->GCPhys = NIL_RTGCPHYS;
4722	pPage->iNext = pPool->iFreeHead;
4723	pPool->iFreeHead = pPage->idx;
4724	pgmUnlock(pVM);
4725	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4726	Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
4727	return rc3;
4728	}
4729	#endif /* PGMPOOL_WITH_USER_TRACKING */
4730
4731	/*
4732	* Commit the allocation, clear the page and return.
4733	*/
4734	#ifdef VBOX_WITH_STATISTICS
4735	if (pPool->cUsedPages > pPool->cUsedPagesHigh)
4736	pPool->cUsedPagesHigh = pPool->cUsedPages;
4737	#endif
4738
4739	if (!pPage->fZeroed)
4740	{
4741	STAM_PROFILE_START(&pPool->StatZeroPage, z);
4742	void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
4743	ASMMemZeroPage(pv);
4744	STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
4745	}
4746
4747	*ppPage = pPage;
4748	if (fLockPage)
4749	pgmPoolLockPage(pPool, pPage);
4750	pgmUnlock(pVM);
4751	LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
4752	rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
4753	STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
4754	return rc;
4755	}
4756
4757
4758	/**
4759	* Frees a usage of a pool page.
4760	*
4761	* @param pVM The VM handle.
4762	* @param HCPhys The HC physical address of the shadow page.
4763	* @param iUser The shadow page pool index of the user table.
4764	* @param iUserTable The index into the user table (shadowed).
4765	*/
4766	void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
4767	{
4768	LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%#x iUserTable=%#x\n", HCPhys, iUser, iUserTable));
4769	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4770	pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
4771	}
4772
4773	/**
4774	* Internal worker for finding a 'in-use' shadow page give by it's physical address.
4775	*
4776	* @returns Pointer to the shadow page structure.
4777	* @param pPool The pool.
4778	* @param HCPhys The HC physical address of the shadow page.
4779	*/
4780	PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
4781	{
4782	PVM pVM = pPool->CTX_SUFF(pVM);
4783
4784	Assert(PGMIsLockOwner(pVM));
4785
4786	/*
4787	* Look up the page.
4788	*/
4789	pgmLock(pVM);
4790	PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
4791	pgmUnlock(pVM);
4792
4793	AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
4794	return pPage;
4795	}
4796
4797	#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
4798	/**
4799	* Flush the specified page if present
4800	*
4801	* @param pVM The VM handle.
4802	* @param GCPhys Guest physical address of the page to flush
4803	*/
4804	void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
4805	{
4806	#ifdef PGMPOOL_WITH_CACHE
4807	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4808
4809	VM_ASSERT_EMT(pVM);
4810
4811	/*
4812	* Look up the GCPhys in the hash.
4813	*/
4814	GCPhys = GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1);
4815	unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
4816	if (i == NIL_PGMPOOL_IDX)
4817	return;
4818
4819	do
4820	{
4821	PPGMPOOLPAGE pPage = &pPool->aPages[i];
4822	if (pPage->GCPhys - GCPhys < PAGE_SIZE)
4823	{
4824	switch (pPage->enmKind)
4825	{
4826	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
4827	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
4828	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
4829	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
4830	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
4831	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
4832	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
4833	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
4834	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
4835	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
4836	case PGMPOOLKIND_64BIT_PML4:
4837	case PGMPOOLKIND_32BIT_PD:
4838	case PGMPOOLKIND_PAE_PDPT:
4839	{
4840	Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
4841	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4842	if (pPage->fDirty)
4843	STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
4844	else
4845	#endif
4846	STAM_COUNTER_INC(&pPool->StatForceFlushPage);
4847	Assert(!pgmPoolIsPageLocked(&pVM->pgm.s, pPage));
4848	pgmPoolMonitorChainFlush(pPool, pPage);
4849	return;
4850	}
4851
4852	/* ignore, no monitoring. */
4853	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
4854	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
4855	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
4856	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
4857	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
4858	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
4859	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
4860	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
4861	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
4862	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
4863	case PGMPOOLKIND_ROOT_NESTED:
4864	case PGMPOOLKIND_PAE_PD_PHYS:
4865	case PGMPOOLKIND_PAE_PDPT_PHYS:
4866	case PGMPOOLKIND_32BIT_PD_PHYS:
4867	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
4868	break;
4869
4870	default:
4871	AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
4872	}
4873	}
4874
4875	/* next */
4876	i = pPage->iNext;
4877	} while (i != NIL_PGMPOOL_IDX);
4878	#endif
4879	return;
4880	}
4881	#endif /* IN_RING3 */
4882
4883	#ifdef IN_RING3
4884	/**
4885	* Flushes the entire cache.
4886	*
4887	* It will assert a global CR3 flush (FF) and assumes the caller is aware of this
4888	* and execute this CR3 flush.
4889	*
4890	* @param pPool The pool.
4891	*/
4892	void pgmR3PoolReset(PVM pVM)
4893	{
4894	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
4895
4896	Assert(PGMIsLockOwner(pVM));
4897	STAM_PROFILE_START(&pPool->StatFlushAllInt, a);
4898	LogFlow(("pgmPoolFlushAllInt:\n"));
4899
4900	/*
4901	* If there are no pages in the pool, there is nothing to do.
4902	*/
4903	if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
4904	{
4905	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
4906	return;
4907	}
4908
4909	/*
4910	* Exit the shadow mode since we're going to clear everything,
4911	* including the root page.
4912	*/
4913	for (unsigned i=0;i<pVM->cCPUs;i++)
4914	{
4915	PVMCPU pVCpu = &pVM->aCpus[i];
4916	pgmR3ExitShadowModeBeforePoolFlush(pVM, pVCpu);
4917	}
4918
4919	/*
4920	* Nuke the free list and reinsert all pages into it.
4921	*/
4922	for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
4923	{
4924	PPGMPOOLPAGE pPage = &pPool->aPages[i];
4925
4926	Assert(pPage->Core.Key == MMPage2Phys(pVM, pPage->pvPageR3));
4927	#ifdef PGMPOOL_WITH_MONITORING
4928	if (pPage->fMonitored)
4929	pgmPoolMonitorFlush(pPool, pPage);
4930	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
4931	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
4932	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
4933	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
4934	pPage->cModifications = 0;
4935	#endif
4936	pPage->GCPhys = NIL_RTGCPHYS;
4937	pPage->enmKind = PGMPOOLKIND_FREE;
4938	pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
4939	Assert(pPage->idx == i);
4940	pPage->iNext = i + 1;
4941	pPage->fZeroed = false; /* This could probably be optimized, but better safe than sorry. */
4942	pPage->fSeenNonGlobal = false;
4943	pPage->fMonitored = false;
4944	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
4945	pPage->fDirty = false;
4946	#endif
4947	pPage->fCached = false;
4948	pPage->fReusedFlushPending = false;
4949	#ifdef PGMPOOL_WITH_USER_TRACKING
4950	pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
4951	#else
4952	pPage->fCR3Mix = false;
4953	#endif
4954	#ifdef PGMPOOL_WITH_CACHE
4955	pPage->iAgeNext = NIL_PGMPOOL_IDX;
4956	pPage->iAgePrev = NIL_PGMPOOL_IDX;
4957	#endif
4958	pPage->cLocked = 0;
4959	}
4960	pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
4961	pPool->iFreeHead = PGMPOOL_IDX_FIRST;
4962	pPool->cUsedPages = 0;
4963
4964	#ifdef PGMPOOL_WITH_USER_TRACKING
4965	/*
4966	* Zap and reinitialize the user records.
4967	*/
4968	pPool->cPresent = 0;
4969	pPool->iUserFreeHead = 0;
4970	PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
4971	const unsigned cMaxUsers = pPool->cMaxUsers;
4972	for (unsigned i = 0; i < cMaxUsers; i++)
4973	{
4974	paUsers[i].iNext = i + 1;
4975	paUsers[i].iUser = NIL_PGMPOOL_IDX;
4976	paUsers[i].iUserTable = 0xfffffffe;
4977	}
4978	paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
4979	#endif
4980
4981	#ifdef PGMPOOL_WITH_GCPHYS_TRACKING
4982	/*
4983	* Clear all the GCPhys links and rebuild the phys ext free list.
4984	*/
4985	for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRanges);
4986	pRam;
4987	pRam = pRam->CTX_SUFF(pNext))
4988	{
4989	unsigned iPage = pRam->cb >> PAGE_SHIFT;
4990	while (iPage-- > 0)
4991	PGM_PAGE_SET_TRACKING(&pRam->aPages[iPage], 0);
4992	}
4993
4994	pPool->iPhysExtFreeHead = 0;
4995	PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
4996	const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
4997	for (unsigned i = 0; i < cMaxPhysExts; i++)
4998	{
4999	paPhysExts[i].iNext = i + 1;
5000	paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
5001	paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
5002	paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
5003	}
5004	paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
5005	#endif
5006
5007	#ifdef PGMPOOL_WITH_MONITORING
5008	/*
5009	* Just zap the modified list.
5010	*/
5011	pPool->cModifiedPages = 0;
5012	pPool->iModifiedHead = NIL_PGMPOOL_IDX;
5013	#endif
5014
5015	#ifdef PGMPOOL_WITH_CACHE
5016	/*
5017	* Clear the GCPhys hash and the age list.
5018	*/
5019	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
5020	pPool->aiHash[i] = NIL_PGMPOOL_IDX;
5021	pPool->iAgeHead = NIL_PGMPOOL_IDX;
5022	pPool->iAgeTail = NIL_PGMPOOL_IDX;
5023	#endif
5024
5025	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5026	/* Clear all dirty pages. */
5027	pPool->idxFreeDirtyPage = 0;
5028	pPool->cDirtyPages = 0;
5029	for (unsigned i = 0; i < RT_ELEMENTS(pPool->aIdxDirtyPages); i++)
5030	pPool->aIdxDirtyPages[i] = NIL_PGMPOOL_IDX;
5031	#endif
5032
5033	/*
5034	* Reinsert active pages into the hash and ensure monitoring chains are correct.
5035	*/
5036	for (unsigned i = PGMPOOL_IDX_FIRST_SPECIAL; i < PGMPOOL_IDX_FIRST; i++)
5037	{
5038	PPGMPOOLPAGE pPage = &pPool->aPages[i];
5039	pPage->iNext = NIL_PGMPOOL_IDX;
5040	#ifdef PGMPOOL_WITH_MONITORING
5041	pPage->iModifiedNext = NIL_PGMPOOL_IDX;
5042	pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
5043	pPage->cModifications = 0;
5044	/* ASSUMES that we're not sharing with any of the other special pages (safe for now). */
5045	pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
5046	pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
5047	if (pPage->fMonitored)
5048	{
5049	int rc = PGMHandlerPhysicalChangeCallbacks(pVM, pPage->GCPhys & ~(RTGCPHYS)(PAGE_SIZE - 1),
5050	pPool->pfnAccessHandlerR3, MMHyperCCToR3(pVM, pPage),
5051	pPool->pfnAccessHandlerR0, MMHyperCCToR0(pVM, pPage),
5052	pPool->pfnAccessHandlerRC, MMHyperCCToRC(pVM, pPage),
5053	pPool->pszAccessHandler);
5054	AssertFatalRCSuccess(rc);
5055	# ifdef PGMPOOL_WITH_CACHE
5056	pgmPoolHashInsert(pPool, pPage);
5057	# endif
5058	}
5059	#endif
5060	#ifdef PGMPOOL_WITH_USER_TRACKING
5061	Assert(pPage->iUserHead == NIL_PGMPOOL_USER_INDEX); /* for now */
5062	#endif
5063	#ifdef PGMPOOL_WITH_CACHE
5064	Assert(pPage->iAgeNext == NIL_PGMPOOL_IDX);
5065	Assert(pPage->iAgePrev == NIL_PGMPOOL_IDX);
5066	#endif
5067	}
5068
5069	for (unsigned i=0;i<pVM->cCPUs;i++)
5070	{
5071	PVMCPU pVCpu = &pVM->aCpus[i];
5072	/*
5073	* Re-enter the shadowing mode and assert Sync CR3 FF.
5074	*/
5075	pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
5076	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
5077	}
5078
5079	STAM_PROFILE_STOP(&pPool->StatFlushAllInt, a);
5080	}
5081	#endif /* IN_RING3 */
5082
5083	#ifdef LOG_ENABLED
5084	static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
5085	{
5086	switch(enmKind)
5087	{
5088	case PGMPOOLKIND_INVALID:
5089	return "PGMPOOLKIND_INVALID";
5090	case PGMPOOLKIND_FREE:
5091	return "PGMPOOLKIND_FREE";
5092	case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
5093	return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
5094	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
5095	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
5096	case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
5097	return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
5098	case PGMPOOLKIND_PAE_PT_FOR_PHYS:
5099	return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
5100	case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
5101	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
5102	case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
5103	return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
5104	case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
5105	return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
5106	case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
5107	return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
5108	case PGMPOOLKIND_32BIT_PD:
5109	return "PGMPOOLKIND_32BIT_PD";
5110	case PGMPOOLKIND_32BIT_PD_PHYS:
5111	return "PGMPOOLKIND_32BIT_PD_PHYS";
5112	case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
5113	return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
5114	case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
5115	return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
5116	case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
5117	return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
5118	case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
5119	return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
5120	case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
5121	return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
5122	case PGMPOOLKIND_PAE_PD_PHYS:
5123	return "PGMPOOLKIND_PAE_PD_PHYS";
5124	case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
5125	return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
5126	case PGMPOOLKIND_PAE_PDPT:
5127	return "PGMPOOLKIND_PAE_PDPT";
5128	case PGMPOOLKIND_PAE_PDPT_PHYS:
5129	return "PGMPOOLKIND_PAE_PDPT_PHYS";
5130	case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
5131	return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
5132	case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
5133	return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
5134	case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
5135	return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
5136	case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
5137	return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
5138	case PGMPOOLKIND_64BIT_PML4:
5139	return "PGMPOOLKIND_64BIT_PML4";
5140	case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
5141	return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
5142	case PGMPOOLKIND_EPT_PD_FOR_PHYS:
5143	return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
5144	case PGMPOOLKIND_EPT_PT_FOR_PHYS:
5145	return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
5146	case PGMPOOLKIND_ROOT_NESTED:
5147	return "PGMPOOLKIND_ROOT_NESTED";
5148	}
5149	return "Unknown kind!";
5150	}
5151	#endif /* LOG_ENABLED*/

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPool.cpp@ 22788

以其他格式下載: