IEMN8veRecompilerTlbLookup.h@ 106286

最後變更在這個檔案從106286是 106286,由 vboxsync 提交於 5 月前
VMM/IEM: Tested the expand down limit checking code. bugref:10371
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 63.5 KB

行
1	/* $Id: IEMN8veRecompilerTlbLookup.h 106286 2024-10-10 10:48:48Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Native Recompiler TLB Lookup Code Emitter.
4	*/
5
6	/*
7	* Copyright (C) 2023-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.alldomusa.eu.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
29	#define VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34	#include "IEMN8veRecompiler.h"
35	#include "IEMN8veRecompilerEmit.h"
36
37
38	/** @defgroup grp_iem_n8ve_re_tlblookup Native Recompiler TLB Lookup Code Emitter
39	* @ingroup grp_iem_n8ve_re
40	* @{
41	*/
42
43	/*
44	* TLB Lookup config.
45	*/
46	#if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
47	# define IEMNATIVE_WITH_TLB_LOOKUP
48	#endif
49	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
50	# define IEMNATIVE_WITH_TLB_LOOKUP_FETCH
51	#endif
52	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
53	# define IEMNATIVE_WITH_TLB_LOOKUP_STORE
54	#endif
55	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
56	# define IEMNATIVE_WITH_TLB_LOOKUP_MAPPED
57	#endif
58	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
59	# define IEMNATIVE_WITH_TLB_LOOKUP_PUSH
60	#endif
61	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
62	# define IEMNATIVE_WITH_TLB_LOOKUP_POP
63	#endif
64
65
66	/**
67	* This must be instantiate before branching off to the lookup code,
68	* so that register spilling and whatnot happens for everyone.
69	*/
70	typedef struct IEMNATIVEEMITTLBSTATE
71	{
72	bool const fSkip;
73	uint8_t const idxRegPtrHlp; /*< We don't support immediate variables with register assignment, so this a tmp reg alloc. /
74	uint8_t const idxRegPtr;
75	uint8_t const idxRegSegBase;
76	uint8_t const idxRegSegLimit;
77	uint8_t const idxRegSegAttrib;
78	uint8_t const idxReg1;
79	uint8_t const idxReg2;
80	#if defined(RT_ARCH_ARM64)
81	uint8_t const idxReg3;
82	/** @def IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
83	* Use LDP and STDP to reduce number of instructions accessing memory at the
84	* cost of using more registers. This will typically reduce the number of
85	* instructions emitted as well.
86	* @todo Profile this and ensure that it performs the same or better.
87	*/
88	# define IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
89	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
90	uint8_t const idxReg4;
91	uint8_t const idxReg5;
92	# endif
93	#endif
94	uint64_t const uAbsPtr;
95
96	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint32_t *a_poff, uint8_t a_idxVarGCPtrMem,
97	uint8_t a_iSegReg, uint8_t a_cbMem, uint8_t a_offDisp = 0)
98	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
99	/* 32-bit and 64-bit wraparound will require special handling, so skip these for absolute addresses. */
100	: fSkip( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
101	== kIemNativeVarKind_Immediate
102	&& ( (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT
103	? (uint64_t)(UINT32_MAX - a_cbMem - a_offDisp)
104	: (uint64_t)(UINT64_MAX - a_cbMem - a_offDisp))
105	< a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
106	#else
107	: fSkip(true)
108	#endif
109	#if defined(RT_ARCH_AMD64) /* got good immediate encoding, otherwise we just load the address in a reg immediately. */
110	, idxRegPtrHlp(UINT8_MAX)
111	#else
112	, idxRegPtrHlp( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
113	!= kIemNativeVarKind_Immediate
114	\|\| fSkip
115	? UINT8_MAX
116	: iemNativeRegAllocTmpImm(a_pReNative, a_poff,
117	a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue))
118	#endif
119	, idxRegPtr( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
120	!= kIemNativeVarKind_Immediate
121	&& !fSkip
122	? iemNativeVarRegisterAcquire(a_pReNative, a_idxVarGCPtrMem, a_poff,
123	true /fInitialized/, IEMNATIVE_CALL_ARG2_GREG)
124	: idxRegPtrHlp)
125	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
126	? UINT8_MAX
127	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
128	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
129	? UINT8_MAX
130	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
131	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
132	? UINT8_MAX
133	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
134	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
135	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
136	#if defined(RT_ARCH_ARM64)
137	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
138	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
139	, idxReg4(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
140	, idxReg5(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
141	# endif
142	#endif
143	, uAbsPtr( a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].enmKind
144	!= kIemNativeVarKind_Immediate
145	\|\| fSkip
146	? UINT64_MAX
147	: a_pReNative->Core.aVars[IEMNATIVE_VAR_IDX_UNPACK(a_idxVarGCPtrMem)].u.uValue)
148
149	{
150	RT_NOREF(a_cbMem, a_offDisp);
151	}
152
153	/* Alternative constructor for PUSH and POP where we don't have a GCPtrMem
154	variable, only a register derived from the guest RSP. */
155	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, uint8_t a_idxRegPtr, uint32_t *a_poff,
156	uint8_t a_iSegReg, uint8_t a_cbMem)
157	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
158	: fSkip(false)
159	#else
160	: fSkip(true)
161	#endif
162	, idxRegPtrHlp(UINT8_MAX)
163	, idxRegPtr(a_idxRegPtr)
164	, idxRegSegBase(a_iSegReg == UINT8_MAX \|\| fSkip
165	? UINT8_MAX
166	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(a_iSegReg)))
167	, idxRegSegLimit((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
168	? UINT8_MAX
169	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(a_iSegReg)))
170	, idxRegSegAttrib((a_iSegReg == UINT8_MAX \|\| (a_pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT) \|\| fSkip
171	? UINT8_MAX
172	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_ATTRIB(a_iSegReg)))
173	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
174	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
175	#if defined(RT_ARCH_ARM64)
176	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
177	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
178	, idxReg4(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
179	, idxReg5(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
180	# endif
181	#endif
182	, uAbsPtr(UINT64_MAX)
183
184	{
185	RT_NOREF_PV(a_cbMem);
186	}
187
188	/* Alternative constructor for the code TLB lookups where we implictly use RIP
189	variable, only a register derived from the guest RSP. */
190	IEMNATIVEEMITTLBSTATE(PIEMRECOMPILERSTATE a_pReNative, bool a_fFlat, uint32_t *a_poff)
191	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
192	: fSkip(false)
193	#else
194	: fSkip(true)
195	#endif
196	, idxRegPtrHlp(UINT8_MAX)
197	, idxRegPtr(iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, kIemNativeGstReg_Pc))
198	, idxRegSegBase(a_fFlat \|\| fSkip
199	? UINT8_MAX
200	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_BASE(X86_SREG_CS)))
201	, idxRegSegLimit(/*a_fFlat \|\| fSkip
202	? UINT8_MAX
203	: iemNativeRegAllocTmpForGuestReg(a_pReNative, a_poff, IEMNATIVEGSTREG_SEG_LIMIT(X86_SREG_CS))*/
204	UINT8_MAX)
205	, idxRegSegAttrib(UINT8_MAX)
206	, idxReg1(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
207	, idxReg2(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
208	#if defined(RT_ARCH_ARM64)
209	, idxReg3(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
210	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
211	, idxReg4(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
212	, idxReg5(!fSkip ? iemNativeRegAllocTmp(a_pReNative, a_poff) : UINT8_MAX)
213	# endif
214	#endif
215	, uAbsPtr(UINT64_MAX)
216
217	{
218	}
219
220	void freeRegsAndReleaseVars(PIEMRECOMPILERSTATE a_pReNative, uint8_t idxVarGCPtrMem = UINT8_MAX, bool fIsCode = false) const
221	{
222	if (!fIsCode)
223	{
224	if (idxRegPtr != UINT8_MAX)
225	{
226	if (idxRegPtrHlp == UINT8_MAX)
227	{
228	if (idxVarGCPtrMem != UINT8_MAX)
229	iemNativeVarRegisterRelease(a_pReNative, idxVarGCPtrMem);
230	}
231	else
232	{
233	Assert(idxRegPtrHlp == idxRegPtr);
234	iemNativeRegFreeTmpImm(a_pReNative, idxRegPtrHlp);
235	}
236	}
237	else
238	Assert(idxRegPtrHlp == UINT8_MAX);
239	}
240	else
241	{
242	Assert(idxVarGCPtrMem == UINT8_MAX);
243	Assert(idxRegPtrHlp == UINT8_MAX);
244	iemNativeRegFreeTmp(a_pReNative, idxRegPtr); /* RIP */
245	}
246	if (idxRegSegBase != UINT8_MAX)
247	iemNativeRegFreeTmp(a_pReNative, idxRegSegBase);
248	if (idxRegSegLimit != UINT8_MAX)
249	iemNativeRegFreeTmp(a_pReNative, idxRegSegLimit);
250	if (idxRegSegAttrib != UINT8_MAX)
251	iemNativeRegFreeTmp(a_pReNative, idxRegSegAttrib);
252	#if defined(RT_ARCH_ARM64)
253	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
254	iemNativeRegFreeTmp(a_pReNative, idxReg5);
255	iemNativeRegFreeTmp(a_pReNative, idxReg4);
256	# endif
257	iemNativeRegFreeTmp(a_pReNative, idxReg3);
258	#endif
259	iemNativeRegFreeTmp(a_pReNative, idxReg2);
260	iemNativeRegFreeTmp(a_pReNative, idxReg1);
261
262	}
263
264	uint32_t getRegsNotToSave() const
265	{
266	if (!fSkip)
267	return RT_BIT_32(idxReg1)
268	\| RT_BIT_32(idxReg2)
269	#if defined(RT_ARCH_ARM64)
270	\| RT_BIT_32(idxReg3)
271	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
272	\| RT_BIT_32(idxReg4)
273	\| RT_BIT_32(idxReg5)
274	# endif
275	#endif
276	;
277	return 0;
278	}
279
280	/** This is only for avoid assertions. */
281	uint32_t getActiveRegsWithShadows(bool fCode = false) const
282	{
283	#ifdef VBOX_STRICT
284	if (!fSkip)
285	return (idxRegSegBase != UINT8_MAX ? RT_BIT_32(idxRegSegBase) : 0)
286	\| (idxRegSegLimit != UINT8_MAX ? RT_BIT_32(idxRegSegLimit) : 0)
287	\| (idxRegSegAttrib != UINT8_MAX ? RT_BIT_32(idxRegSegAttrib) : 0)
288	\| (fCode ? RT_BIT_32(idxRegPtr) : 0);
289	#else
290	RT_NOREF_PV(fCode);
291	#endif
292	return 0;
293	}
294	} IEMNATIVEEMITTLBSTATE;
295
296	DECLASM(void) iemNativeHlpAsmSafeWrapCheckTlbLookup(void);
297
298
299	#ifdef IEMNATIVE_WITH_TLB_LOOKUP
300	/**
301	*
302	* @returns New @a off value.
303	* @param pReNative .
304	* @param off .
305	* @param pTlbState .
306	* @param iSegReg .
307	* @param cbMem .
308	* @param fAlignMaskAndCtl The low 8-bit is the alignment mask, ie. a
309	* 128-bit aligned access passes 15. This is only
310	* applied to ring-3 code, when dictated by the
311	* control bits and for atomic accesses.
312	*
313	* The other bits are used for alignment control:
314	* - IEM_MEMMAP_F_ALIGN_GP
315	* - IEM_MEMMAP_F_ALIGN_SSE
316	* - IEM_MEMMAP_F_ALIGN_GP_OR_AC
317	* Any non-zero upper bits means we will go to
318	* tlbmiss on anything out of alignment according
319	* to the mask in the low 8 bits.
320	* @param fAccess .
321	* @param idxLabelTlbLookup .
322	* @param idxLabelTlbMiss .
323	* @param idxRegMemResult .
324	* @param offDisp .
325	* @tparam a_fDataTlb .
326	* @tparam a_fNoReturn .
327	*/
328	template<bool const a_fDataTlb, bool const a_fNoReturn = false>
329	DECL_INLINE_THROW(uint32_t)
330	iemNativeEmitTlbLookup(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVEEMITTLBSTATE const * const pTlbState,
331	uint8_t iSegReg, uint8_t cbMem, uint32_t fAlignMaskAndCtl, uint32_t fAccess,
332	uint32_t idxLabelTlbLookup, uint32_t idxLabelTlbMiss, uint8_t idxRegMemResult,
333	uint8_t offDisp = 0)
334	{
335	Assert(!pTlbState->fSkip);
336	uint32_t const offVCpuTlb = a_fDataTlb ? RT_UOFFSETOF(VMCPUCC, iem.s.DataTlb) : RT_UOFFSETOF(VMCPUCC, iem.s.CodeTlb);
337	# if defined(RT_ARCH_AMD64)
338	uint8_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 512);
339	# elif defined(RT_ARCH_ARM64)
340	uint32_t * const pCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 96);
341	# endif
342
343	/*
344	* The expand down check isn't use all that much, so we emit here to keep
345	* the lookup straighter.
346	*/
347	/* check_expand_down: ; complicted! */
348	uint32_t const offCheckExpandDown = off;
349	uint32_t offFixupLimitDone = 0;
350	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
351	{
352	/* cmp seglim, regptr */
353	if (pTlbState->idxRegPtr != UINT8_MAX && offDisp == 0)
354	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxRegPtr);
355	else if (pTlbState->idxRegPtr == UINT8_MAX)
356	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
357	(uint32_t)(pTlbState->uAbsPtr + offDisp));
358	else if (cbMem == 1)
359	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, pTlbState->idxReg2);
360	else
361	{ /* use idxRegMemResult to calc the displaced address. */
362	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxRegPtr, offDisp);
363	off = iemNativeEmitCmpGpr32WithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit, idxRegMemResult);
364	}
365	/* ja tlbmiss */
366	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
367
368	/* reg1 = segattr & X86DESCATTR_D (0x4000) */
369	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib, X86DESCATTR_D);
370	/* xor reg1, X86DESCATTR_D */
371	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_D);
372	/* shl reg1, 2 (16 - 14) */
373	AssertCompile((X86DESCATTR_D << 2) == UINT32_C(0x10000));
374	off = iemNativeEmitShiftGpr32LeftEx(pCodeBuf, off, pTlbState->idxReg1, 2);
375	/* dec reg1 (=> 0xffff if D=0; 0xffffffff if D=1) */
376	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, 1);
377	/* cmp reg1, reg2 (64-bit) / imm (32-bit) */
378	if (pTlbState->idxRegPtr != UINT8_MAX)
379	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1,
380	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
381	else
382	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1,
383	(uint32_t)(pTlbState->uAbsPtr + offDisp + cbMem - 1)); /* fSkip=true on overflow. */
384	/* jbe tlbmiss */
385	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
386	/* jmp limitdone */
387	offFixupLimitDone = off;
388	off = iemNativeEmitJmpToFixedEx(pCodeBuf, off, off + 256 /* force near */);
389	}
390
391	/*
392	* Snippet for checking whether misaligned accesses are within the
393	* page (see step 2).
394	*
395	* This sequence is 1 instruction longer than the strict alignment test,
396	* and since most accesses are correctly aligned it is better to do it
397	* this way. Runs of r163597 seems to indicate there was a regression
398	* when placing this code in the main code flow.
399	*/
400	uint8_t const idxRegFlatPtr = iSegReg != UINT8_MAX \|\| pTlbState->idxRegPtr == UINT8_MAX \|\| offDisp != 0
401	? idxRegMemResult : pTlbState->idxRegPtr; /* (not immediately ready for tlblookup use) */
402	uint8_t const fAlignMask = a_fDataTlb ? (uint8_t)fAlignMaskAndCtl : 0;
403	if (a_fDataTlb)
404	{
405	Assert(!(fAlignMaskAndCtl & ~(UINT32_C(0xff) \| IEM_MEMMAP_F_ALIGN_SSE \| IEM_MEMMAP_F_ALIGN_GP \| IEM_MEMMAP_F_ALIGN_GP_OR_AC)));
406	Assert(RT_IS_POWER_OF_TWO(fAlignMask + 1U));
407	Assert(cbMem == fAlignMask + 1U \|\| !(fAccess & IEM_ACCESS_ATOMIC));
408	Assert(cbMem < 128); /* alignment test assumptions */
409	}
410
411	uint32_t offMisalignedAccess = UINT32_MAX;
412	uint32_t offFixupMisalignedAccessJmpBack = UINT32_MAX;
413	if ( a_fDataTlb
414	&& !(fAlignMaskAndCtl & ~UINT32_C(0xff))
415	&& !(fAccess & IEM_ACCESS_ATOMIC)
416	&& cbMem > 1
417	&& RT_IS_POWER_OF_TWO(cbMem)
418	&& !(pReNative->fExec & IEM_F_X86_AC))
419	{
420	/* tlbmisaligned: */
421	offMisalignedAccess = off;
422	/* reg1 = regflat & 0xfff */
423	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,/=/ idxRegFlatPtr,/&/ GUEST_PAGE_OFFSET_MASK);
424	/* cmp reg1, GUEST_PAGE_SIZE - cbMem */
425	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE - cbMem);
426	/* jbe short jmpback */
427	offFixupMisalignedAccessJmpBack = off;
428	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 256 /near/, kIemNativeInstrCond_be);
429	# ifdef IEM_WITH_TLB_STATISTICS
430	off = iemNativeEmitIncU32CounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
431	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissCrossPage));
432	# endif
433	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
434	}
435
436	/* The ODD TLB entry is checked last when CR4.PGE=0 or when not in ring-0. */
437	bool const fEvenFirst = (pReNative->fExec & IEM_F_X86_CPL_MASK) != 0
438	\|\| !(pReNative->pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE);
439	bool const fIncCheckAltTlbe = (pReNative->fExec & IEM_F_X86_CPL_MASK) == 0;
440
441	/*
442	* Snippet for checking the alternative TLBE entry when CR4.PGE=1 and
443	* for doing statistics.
444	*
445	* This code assists step 3c, so look down there for register assignments.
446	*/
447	/* checkalttlbe_and_missedtagstats: */
448	uint32_t const offCheckAltTlbeAndMissedTagStats = off;
449	uint32_t offFixupCheckAltTlbeJmpBack = UINT32_MAX / 2;
450	if (fIncCheckAltTlbe)
451	{
452	# ifdef RT_ARCH_AMD64
453	/* Update pTlbe: reg2 = fEvenFirst ? reg2 + sizeof(IEMTLBENTRY) : reg2 - sizeof(IEMTLBENTRY); */
454	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_R \| X86_OP_REX_B);
455	pCodeBuf[off++] = 0x8d; /* LEA r64,m64 */
456	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg2,
457	fEvenFirst ? (int32_t)sizeof(IEMTLBENTRY) : -(int32_t)sizeof(IEMTLBENTRY));
458
459	/* reg1 = reg1 & ~IEMTLB_REVISION_MASK; */
460	off = iemNativeEmitShiftGprLeftEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
461	off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
462	/* or reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevisionGlobal/uTlbRevision] */
463	pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
464	pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
465	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1,
466	fEvenFirst ? offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevisionGlobal)
467	: offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
468
469	/* cmp reg1, [reg2] */
470	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
471	pCodeBuf[off++] = 0x3b;
472	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
473
474	# elif defined(RT_ARCH_ARM64)
475	/* reg3 = uTlbRevision/uTlbRevisionGlobal; (We've ditched reg4 already, so have to get it via pVCpu.) */
476	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3,
477	fEvenFirst ? offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevisionGlobal)
478	: offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision));
479
480	/* reg1 = reg1 & ~IEMTLB_REVISION_MASK; */
481	AssertCompile(UINT64_C(0x0000000fffffffff) == ~IEMTLB_REVISION_MASK);
482	Assert(Armv8A64ConvertImmRImmS2Mask64(0x63, 0) == ~IEMTLB_REVISION_MASK);
483	pCodeBuf[off++] = Armv8A64MkInstrAndImm(pTlbState->idxReg1, pTlbState->idxReg1, 0x63, 0);
484
485	/* reg1 \|= reg3 (uTlbRevision/uTlbRevisionGlobal); */
486	pCodeBuf[off++] = Armv8A64MkInstrOrr(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
487
488	/* reg2 = reg2 +/- sizeof(IEMTLBENTRY); via preindexing.
489	reg3 = uTag; [pair: reg4 = fFlagsAndPhysRev;] */
490	AssertCompileMemberOffset(IEMTLBENTRY, uTag, 0);
491	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
492	AssertCompileAdjacentMembers(IEMTLBENTRY, uTag, fFlagsAndPhysRev);
493	pCodeBuf[off++] = Armv8A64MkInstrLdPairGpr(pTlbState->idxReg3, pTlbState->idxReg4, pTlbState->idxReg2,
494	fEvenFirst ? (int)sizeof(IEMTLBENTRY) / 8 : -(int)sizeof(IEMTLBENTRY) / 8,
495	kArm64InstrStLdPairType_PreIndex);
496	# else
497	pCodeBuf[off++] = Armv8A64MkInstrStrLdrPreIndex9(kArmv8A64InstrLdStType_Ld_Dword, pTlbState->idxReg3, pTlbState->idxReg2,
498	fEvenFirst ? (int)sizeof(IEMTLBENTRY) / 8 : -(int)sizeof(IEMTLBENTRY) / 8);
499	# endif
500	/* cmp reg1, reg3; (uRev \| Hash(FlatPtr), IEMTLBENTRY::uTag)*/
501	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
502
503	# else
504	# error "portme"
505	# endif
506	/* je near jumpback_checkalttlbe */
507	offFixupCheckAltTlbeJmpBack = off;
508	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 256, kIemNativeInstrCond_e);
509	}
510
511	# ifdef IEM_WITH_TLB_STATISTICS
512	/* inc stat */
513	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
514	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissTag));
515	# endif
516	# ifndef IEM_WITH_TLB_STATISTICS
517	if (fIncCheckAltTlbe)
518	# endif
519	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
520	off = iemNativeEmitBrkEx(pCodeBuf, off, 0x7679);
521
522	/*
523	* tlblookup:
524	*/
525	iemNativeLabelDefine(pReNative, idxLabelTlbLookup, off);
526	# if defined(RT_ARCH_ARM64) && 0
527	off = iemNativeEmitBrkEx(pCodeBuf, off, 0);
528	# endif
529
530	/*
531	* 1. Segmentation.
532	*
533	* 1a. Check segment limit and attributes if non-flat 32-bit code. This is complicated.
534	*
535	* This can be skipped for code TLB lookups because limit is checked by jmp, call,
536	* ret, and iret prior to making it. It is also checked by the helpers prior to
537	* doing TLB loading.
538	*/
539	if (a_fDataTlb && iSegReg != UINT8_MAX && (pReNative->fExec & IEM_F_MODE_CPUMODE_MASK) != IEMMODE_64BIT)
540	{
541	/* Check that we've got a segment loaded and that it allows the access.
542	For write access this means a writable data segment.
543	For read-only accesses this means a readable code segment or any data segment. */
544	if (fAccess & IEM_ACCESS_TYPE_WRITE)
545	{
546	uint32_t const fMustBe1 = X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_WRITE;
547	uint32_t const fMustBe0 = X86DESCATTR_UNUSABLE \| X86_SEL_TYPE_CODE;
548	/* reg1 = segattrs & (must1\|must0) */
549	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
550	pTlbState->idxRegSegAttrib, fMustBe1 \| fMustBe0);
551	/* cmp reg1, must1 */
552	AssertCompile(fMustBe1 <= UINT16_MAX);
553	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, fMustBe1);
554	/* jne tlbmiss */
555	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
556	}
557	else
558	{
559	/* U \| !P \|!DT \|!CD \| RW \|
560	16 \| 8 \| 4 \| 3 \| 1 \|
561	-------------------------------
562	0 \| 0 \| 0 \| 0 \| 0 \| execute-only code segment. - must be excluded
563	0 \| 0 \| 0 \| 0 \| 1 \| execute-read code segment.
564	0 \| 0 \| 0 \| 1 \| 0 \| read-only data segment.
565	0 \| 0 \| 0 \| 1 \| 1 \| read-write data segment. - last valid combination
566	*/
567	/* reg1 = segattrs & (relevant attributes) */
568	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxRegSegAttrib,
569	X86DESCATTR_UNUSABLE \| X86DESCATTR_P \| X86DESCATTR_DT
570	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE);
571	/* xor reg1, X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE ; place C=1 RW=0 at the bottom & limit the range.
572	; EO-code=0, ER-code=2, RO-data=8, RW-data=10 */
573	#ifdef RT_ARCH_ARM64
574	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
575	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1, X86DESCATTR_P);
576	#else
577	off = iemNativeEmitXorGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg1,
578	X86DESCATTR_P \| X86DESCATTR_DT \| X86_SEL_TYPE_CODE);
579	#endif
580	/* sub reg1, X86_SEL_TYPE_WRITE ; EO-code=-2, ER-code=0, RO-data=6, RW-data=8 */
581	off = iemNativeEmitSubGpr32ImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_WRITE /* ER-code */);
582	/* cmp reg1, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE */
583	AssertCompile(X86_SEL_TYPE_CODE == 8);
584	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_CODE);
585	/* ja tlbmiss */
586	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
587	}
588
589	/* If we're accessing more than one byte or if we're working with a non-zero offDisp,
590	put the last address we'll be accessing in idxReg2 (64-bit). */
591	if ((cbMem > 1 \|\| offDisp != 0) && pTlbState->idxRegPtr != UINT8_MAX)
592	{
593	if (!offDisp)
594	/* reg2 = regptr + cbMem - 1; 64-bit result so we can fend of wraparounds/overflows. */
595	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ cbMem - 1);
596	else
597	{
598	/* reg2 = (uint32_t)(regptr + offDisp) + cbMem - 1;. */
599	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off,
600	pTlbState->idxReg2,/=/ pTlbState->idxRegPtr,/+/ + offDisp);
601	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, cbMem - 1);
602	}
603	}
604
605	/*
606	* Check the limit. If this is a write access, we know that it's a
607	* data segment and includes the expand_down bit. For read-only accesses
608	* we need to check that code/data=0 and expanddown=1 before continuing.
609	*/
610	if (fAccess & IEM_ACCESS_TYPE_WRITE)
611	{
612	/* test segattrs, X86_SEL_TYPE_DOWN */
613	AssertCompile(X86_SEL_TYPE_DOWN < 128);
614	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, pTlbState->idxRegSegAttrib, X86_SEL_TYPE_DOWN);
615	/* jnz check_expand_down */
616	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_ne);
617	}
618	else
619	{
620	/* reg1 = segattr & (code \| down) */
621	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,
622	pTlbState->idxRegSegAttrib, X86_SEL_TYPE_CODE \| X86_SEL_TYPE_DOWN);
623	/* cmp reg1, down */
624	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, X86_SEL_TYPE_DOWN);
625	/* je check_expand_down */
626	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckExpandDown, kIemNativeInstrCond_e);
627	}
628
629	/* expand_up:
630	cmp seglim, regptr/reg2/imm */
631	if (pTlbState->idxRegPtr != UINT8_MAX)
632	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
633	cbMem > 1 \|\| offDisp != 0 ? pTlbState->idxReg2 : pTlbState->idxRegPtr);
634	else
635	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxRegSegLimit,
636	(uint32_t)pTlbState->uAbsPtr + offDisp + cbMem - 1U); /* fSkip=true on overflow. */
637	/* jbe tlbmiss */
638	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_be);
639
640	/* limitdone: */
641	iemNativeFixupFixedJump(pReNative, offFixupLimitDone, off);
642	}
643
644	/* 1b. Add the segment base. We use idxRegMemResult for the ptr register if
645	this step is required or if the address is a constant (simplicity) or
646	if offDisp is non-zero. */
647	if (iSegReg != UINT8_MAX)
648	{
649	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
650	/* regflat = segbase + regptr/imm */
651	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
652	{
653	Assert(iSegReg >= X86_SREG_FS);
654	if (pTlbState->idxRegPtr != UINT8_MAX)
655	{
656	off = iemNativeEmitGprEqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
657	if (offDisp != 0)
658	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
659	}
660	else
661	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase,
662	pTlbState->uAbsPtr + offDisp);
663	}
664	else if (pTlbState->idxRegPtr != UINT8_MAX)
665	{
666	off = iemNativeEmitGpr32EqGprPlusGprEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegSegBase, pTlbState->idxRegPtr);
667	if (offDisp != 0)
668	off = iemNativeEmitAddGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, offDisp);
669	}
670	else
671	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr,
672	pTlbState->idxRegSegBase, (uint32_t)pTlbState->uAbsPtr + offDisp);
673	}
674	else if (pTlbState->idxRegPtr == UINT8_MAX)
675	{
676	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
677	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->uAbsPtr + offDisp);
678	else
679	off = iemNativeEmitLoadGpr32ImmEx(pCodeBuf, off, idxRegFlatPtr, (uint32_t)pTlbState->uAbsPtr + offDisp);
680	}
681	else if (offDisp != 0)
682	{
683	Assert(idxRegFlatPtr != pTlbState->idxRegPtr);
684	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
685	off = iemNativeEmitGprEqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
686	else
687	off = iemNativeEmitGpr32EqGprPlusImmEx(pCodeBuf, off, idxRegFlatPtr, pTlbState->idxRegPtr, offDisp);
688	}
689	else
690	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
691
692	/*
693	* 2. Check that the address doesn't cross a page boundrary and doesn't
694	* have alignment issues (not applicable to code).
695	*
696	* For regular accesses (non-SSE/AVX & atomic stuff) we only need to
697	* check for #AC in ring-3 code. To simplify this, the need for AC
698	* checking is indicated by IEM_F_X86_AC in IEMCPU::fExec.
699	*
700	* The caller informs us about about SSE/AVX aligned accesses via the
701	* upper bits of fAlignMaskAndCtl and atomic accesses via fAccess.
702	*/
703	if (a_fDataTlb)
704	{
705	if (offMisalignedAccess != UINT32_MAX)
706	{
707	#ifdef RT_ARCH_ARM64
708	if (cbMem == 2)
709	{
710	/* tbnz regflatptr, #0, tlbmiss */
711	pCodeBuf[off++] = Armv8A64MkInstrTbnz((int32_t)offMisalignedAccess - (int32_t)off, idxRegFlatPtr, 0);
712	}
713	else
714	#endif
715	{
716	/* test regflat, fAlignMask */
717	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, idxRegFlatPtr, cbMem - 1);
718	/* jnz tlbmiss */
719	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offMisalignedAccess, kIemNativeInstrCond_ne);
720	}
721	/** @todo ARM64: two byte access checks can be reduced to single instruction */
722	iemNativeFixupFixedJump(pReNative, offFixupMisalignedAccessJmpBack, off);
723	}
724	else
725	{
726	/*
727	* 2a. Strict alignment check using fAlignMask for atomic, strictly
728	* aligned stuff (SSE & AVX) and AC=1 (ring-3).
729	*/
730	bool const fStrictAlignmentCheck = fAlignMask
731	&& ( (fAlignMaskAndCtl & ~UINT32_C(0xff))
732	\|\| (fAccess & IEM_ACCESS_ATOMIC)
733	\|\| (pReNative->fExec & IEM_F_X86_AC) );
734	if (fStrictAlignmentCheck)
735	{
736	/* test regflat, fAlignMask */
737	off = iemNativeEmitTestAnyBitsInGpr8Ex(pCodeBuf, off, idxRegFlatPtr, fAlignMask);
738
739	#ifndef IEM_WITH_TLB_STATISTICS
740	/* jnz tlbmiss */
741	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
742	#else
743	/* jz 1F; inc stat; jmp tlbmiss */
744	uint32_t const offFixup1 = off;
745	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 16, kIemNativeInstrCond_e);
746	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
747	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissAlignment));
748	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
749	iemNativeFixupFixedJump(pReNative, offFixup1, off);
750	#endif
751	}
752
753	/*
754	* 2b. Check that it's not crossing page a boundrary if the access is
755	* larger than the aligment mask or if we didn't do the strict
756	* alignment check above.
757	*/
758	if ( cbMem > 1
759	&& ( !fStrictAlignmentCheck
760	\|\| cbMem > fAlignMask + 1U))
761	{
762	/* reg1 = regflat & 0xfff */
763	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1,/=/ idxRegFlatPtr,/&/ GUEST_PAGE_OFFSET_MASK);
764	/* cmp reg1, GUEST_PAGE_SIZE - cbMem */
765	off = iemNativeEmitCmpGpr32WithImmEx(pCodeBuf, off, pTlbState->idxReg1, GUEST_PAGE_SIZE - cbMem);
766	#ifndef IEM_WITH_TLB_STATISTICS
767	/* ja tlbmiss */
768	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
769	#else
770	/* jbe 1F; inc stat; jmp tlbmiss */
771	uint32_t const offFixup1 = off;
772	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 16, kIemNativeInstrCond_be);
773	off = iemNativeEmitIncU32CounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
774	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissCrossPage));
775	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
776	iemNativeFixupFixedJump(pReNative, offFixup1, off);
777	#endif
778	}
779	}
780	}
781	else
782	Assert(fAlignMaskAndCtl == 0);
783
784	/*
785	* 3. TLB lookup.
786	*
787	* 3a. Calculate the TLB tag value (IEMTLB_CALC_TAG_NO_REV).
788	* In 64-bit mode we will also check for non-canonical addresses here.
789	*/
790	if ((pReNative->fExec & IEM_F_MODE_MASK) == IEM_F_MODE_X86_64BIT)
791	{
792	# if defined(RT_ARCH_AMD64)
793	/* mov reg1, regflat */
794	off = iemNativeEmitLoadGprFromGprEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr);
795	/* rol reg1, 16 */
796	off = iemNativeEmitRotateGprLeftEx(pCodeBuf, off, pTlbState->idxReg1, 16);
797	/** @todo Would 'movsx reg2, word reg1' and working on reg2 in dwords be faster? */
798	/* inc word reg1 */
799	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
800	if (pTlbState->idxReg1 >= 8)
801	pCodeBuf[off++] = X86_OP_REX_B;
802	pCodeBuf[off++] = 0xff;
803	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, pTlbState->idxReg1 & 7);
804	/* cmp word reg1, 1 */
805	pCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
806	if (pTlbState->idxReg1 >= 8)
807	pCodeBuf[off++] = X86_OP_REX_B;
808	pCodeBuf[off++] = 0x83;
809	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, pTlbState->idxReg1 & 7);
810	pCodeBuf[off++] = 1;
811	# ifndef IEM_WITH_TLB_STATISTICS
812	/* ja tlbmiss */
813	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_nbe);
814	# else
815	/* jbe 1F; inc stat; jmp tlbmiss */
816	uint32_t const offFixup1 = off;
817	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 16, kIemNativeInstrCond_be);
818	off = iemNativeEmitIncU32CounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
819	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissNonCanonical));
820	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
821	iemNativeFixupFixedJump(pReNative, offFixup1, off);
822	# endif
823	/* shr reg1, 16 + GUEST_PAGE_SHIFT */
824	off = iemNativeEmitShiftGprRightEx(pCodeBuf, off, pTlbState->idxReg1, 16 + GUEST_PAGE_SHIFT);
825
826	# elif defined(RT_ARCH_ARM64)
827	/* lsr reg1, regflat, #48 */
828	pCodeBuf[off++] = Armv8A64MkInstrLsrImm(pTlbState->idxReg1, idxRegFlatPtr, 48);
829	/* add reg1, reg1, #1 */
830	pCodeBuf[off++] = Armv8A64MkInstrAddUImm12(pTlbState->idxReg1, pTlbState->idxReg1, 1, false /f64Bit/);
831	/* tst reg1, #0xfffe */
832	Assert(Armv8A64ConvertImmRImmS2Mask32(14, 31) == 0xfffe);
833	pCodeBuf[off++] = Armv8A64MkInstrTstImm(pTlbState->idxReg1, 14, 31, false /f64Bit/);
834	# ifndef IEM_WITH_TLB_STATISTICS
835	/* b.ne tlbmiss */
836	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
837	# else
838	/* b.eq 1F; inc stat; jmp tlbmiss */
839	uint32_t const offFixup1 = off;
840	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 16, kIemNativeInstrCond_e);
841	off = iemNativeEmitIncU32CounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
842	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissNonCanonical));
843	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
844	iemNativeFixupFixedJump(pReNative, offFixup1, off);
845	# endif
846
847	/* ubfx reg1, regflat, #12, #36 */
848	pCodeBuf[off++] = Armv8A64MkInstrUbfx(pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT, 48 - GUEST_PAGE_SHIFT);
849	# else
850	# error "Port me"
851	# endif
852	}
853	else
854	{
855	/* reg1 = (uint32_t)(regflat >> 12) */
856	off = iemNativeEmitGpr32EqGprShiftRightImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, GUEST_PAGE_SHIFT);
857	}
858
859	/* or reg1, [qword pVCpu->iem.s.DataTlb.uTlbRevision] */
860	# if defined(RT_ARCH_AMD64)
861	pCodeBuf[off++] = pTlbState->idxReg1 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_R;
862	pCodeBuf[off++] = 0x0b; /* OR r64,r/m64 */
863	off = iemNativeEmitGprByVCpuDisp(pCodeBuf, off, pTlbState->idxReg1,
864	fEvenFirst ? offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision)
865	: offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevisionGlobal));
866	# else
867	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
868	/* Load uTlbRevision[Global] into reg3 and uTlbPhysRev into reg5.
869	We load the pointer for IEMTLB::aEntries[!fEvenFirst] into reg4 and use
870	it for addressing here and later when calculating pTble (saves one
871	instruction, simplifies odd-first). */
872	AssertCompileMemberAlignment(IEMTLB, uTlbRevision, 16); /* It is said that misaligned pair loads doesn't perform well. */
873	AssertCompileAdjacentMembers(IEMTLB, uTlbRevision, uTlbPhysRev);
874	AssertCompileAdjacentMembers(IEMTLB, uTlbPhysRev, uTlbRevisionGlobal);
875	AssertCompile(RTASSERT_OFFSET_OF(IEMTLB, uTlbPhysRev) < RTASSERT_OFFSET_OF(IEMTLB, aEntries));
876	AssertCompile(RTASSERT_OFFSET_OF(VMCPUCC, iem.s.DataTlb.aEntries) < _64K);
877	uint32_t const offEntries = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries) + (fEvenFirst ? 0 : sizeof(IEMTLBENTRY));
878	if (offEntries < _64K)
879	{
880	pCodeBuf[off++] = Armv8A64MkInstrMovZ(pTlbState->idxReg4, offEntries);
881	pCodeBuf[off++] = Armv8A64MkInstrAddReg(pTlbState->idxReg4, IEMNATIVE_REG_FIXED_PVMCPU, pTlbState->idxReg4);
882	}
883	else
884	{
885	AssertCompileMemberAlignment(VMCPUCC, iem.s.CodeTlb.aEntries, 32);
886	AssertCompileMemberAlignment(IEMTLB, aEntries, 32);
887	AssertCompileSizeAlignment(IEMTLBENTRY, 32);
888	# if IEMTLB_ENTRY_COUNT <= 16384 /?/
889	AssertCompile(RTASSERT_OFFSET_OF(VMCPUCC, iem.s.CodeTlb.aEntries) < _64K*32U);
890	pCodeBuf[off++] = Armv8A64MkInstrMovZ(pTlbState->idxReg4, offEntries >> 5);
891	pCodeBuf[off++] = Armv8A64MkInstrAddReg(pTlbState->idxReg4, IEMNATIVE_REG_FIXED_PVMCPU, pTlbState->idxReg4,
892	true /64Bit/, false /fSetFlags/, 5 /cShift/, kArmv8A64InstrShift_Lsl);
893	# else
894	AssertCompile(RTASSERT_OFFSET_OF(VMCPUCC, iem.s.CodeTlb.aEntries) >= _64K*32U);
895	pCodeBuf[off++] = Armv8A64MkInstrMovZ(pTlbState->idxReg4, offEntries & UINT16_MAX);
896	pCodeBuf[off++] = Armv8A64MkInstrMovK(pTlbState->idxReg4, offEntries >> 16, 1);
897	pCodeBuf[off++] = Armv8A64MkInstrAddReg(pTlbState->idxReg4, IEMNATIVE_REG_FIXED_PVMCPU, pTlbState->idxReg4);
898	# endif
899	}
900	AssertCompile(RTASSERT_OFFSET_OF(IEMTLB, aEntries) < 64U*8U - sizeof(IEMTLBENTRY));
901	if (fEvenFirst)
902	pCodeBuf[off++] = Armv8A64MkInstrLdPairGpr(pTlbState->idxReg3, pTlbState->idxReg5, pTlbState->idxReg4,
903	(RT_OFFSETOF(IEMTLB, uTlbRevision) - RT_OFFSETOF(IEMTLB, aEntries)) / 8);
904	else /* This isn't 128-bit aligned, hope that doesn't hurt too much... */
905	pCodeBuf[off++] = Armv8A64MkInstrLdPairGpr(pTlbState->idxReg5, pTlbState->idxReg3, pTlbState->idxReg4,
906	( RT_OFFSETOF(IEMTLB, uTlbPhysRev) - RT_OFFSETOF(IEMTLB, aEntries)
907	- (int)sizeof(IEMTLBENTRY)) / 8);
908	# else
909	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3,
910	fEvenFirst ? offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevision)
911	: offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbRevisionGlobal));
912	# endif
913	off = iemNativeEmitOrGprByGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
914	# endif
915
916	/*
917	* 3b. Calc pTlbe.
918	*/
919	# if !defined(RT_ARCH_ARM64) \|\| !defined(IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR)
920	uint32_t const offTlbEntriesAdjusted = offVCpuTlb + RT_UOFFSETOF(IEMTLB, aEntries) + (fEvenFirst ? 0 : sizeof(IEMTLBENTRY));
921	# endif
922	# if defined(RT_ARCH_AMD64)
923	# if IEMTLB_ENTRY_COUNT == 256
924	/* movzx reg2, byte reg1 */
925	off = iemNativeEmitLoadGprFromGpr8Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
926	# else
927	/* mov reg2, reg1 */
928	off = iemNativeEmitLoadGprFromGpr32Ex(pCodeBuf, off, pTlbState->idxReg2, pTlbState->idxReg1);
929	/* and reg2, IEMTLB_ENTRY_COUNT - 1U */
930	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, pTlbState->idxReg2, IEMTLB_ENTRY_COUNT - 1U);
931	# endif
932	/* shl reg2, 6 ; reg2 = sizeof(IEMTLBENTRY) 2 */
933	AssertCompileSize(IEMTLBENTRY, 32);
934	off = iemNativeEmitShiftGprLeftEx(pCodeBuf, off, pTlbState->idxReg2, 6);
935	/* lea reg2, [&pVCpu->iem.s.DataTlb.aEntries[!fEvenFirst] + reg2] */
936	AssertCompile(IEMNATIVE_REG_FIXED_PVMCPU < 8);
937	pCodeBuf[off++] = pTlbState->idxReg2 < 8 ? X86_OP_REX_W : X86_OP_REX_W \| X86_OP_REX_X \| X86_OP_REX_R;
938	pCodeBuf[off++] = 0x8d;
939	pCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, pTlbState->idxReg2 & 7, 4 /SIB/);
940	pCodeBuf[off++] = X86_SIB_MAKE(IEMNATIVE_REG_FIXED_PVMCPU & 7, pTlbState->idxReg2 & 7, 0);
941	pCodeBuf[off++] = RT_BYTE1(offTlbEntriesAdjusted);
942	pCodeBuf[off++] = RT_BYTE2(offTlbEntriesAdjusted);
943	pCodeBuf[off++] = RT_BYTE3(offTlbEntriesAdjusted);
944	pCodeBuf[off++] = RT_BYTE4(offTlbEntriesAdjusted);
945
946	# elif defined(RT_ARCH_ARM64)
947	/* reg2 = (reg1 & tlbmask) << 6 */
948	AssertCompileSize(IEMTLBENTRY, 32);
949	pCodeBuf[off++] = Armv8A64MkInstrUbfiz(pTlbState->idxReg2, pTlbState->idxReg1, 6, IEMTLB_ENTRY_COUNT_AS_POWER_OF_TWO);
950	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
951	/* reg2 += &pVCpu->iem.s.[Data\|Code]Tlb.aEntries[!fEvenFirst] */
952	pCodeBuf[off++] = Armv8A64MkInstrAddReg(pTlbState->idxReg2, pTlbState->idxReg2, pTlbState->idxReg4);
953	# else
954	/* reg2 += offsetof(VMCPUCC, iem.s.DataTlb.aEntries[!fEvenFirst]) */
955	off = iemNativeEmitAddGprImmEx(pCodeBuf, off, pTlbState->idxReg2, offTlbEntriesAdjusted, pTlbState->idxReg3 /iGprTmp/);
956	/* reg2 += pVCpu */
957	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, pTlbState->idxReg2, IEMNATIVE_REG_FIXED_PVMCPU);
958	# endif
959	# else
960	# error "Port me"
961	# endif
962
963	/*
964	* 3c. Compare the TLBE.uTag with the one from 2a (reg1).
965	*/
966	# if defined(RT_ARCH_AMD64)
967	/* cmp reg1, [reg2] */
968	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
969	pCodeBuf[off++] = 0x3b;
970	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
971	# elif defined(RT_ARCH_ARM64)
972	/* reg3 = uTag; [pair: reg4 = fFlagsAndPhysRev;] */
973	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
974	AssertCompileMemberAlignment(IEMTLBENTRY, uTag, 16); /* It is said that misaligned pair loads doesn't perform well. */
975	AssertCompile(RT_UOFFSETOF(IEMTLBENTRY, uTag) + sizeof(uint64_t) == RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
976	pCodeBuf[off++] = Armv8A64MkInstrLdPairGpr(pTlbState->idxReg3, pTlbState->idxReg4,
977	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag) / 8);
978	# else
979	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3, pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, uTag));
980	# endif
981	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
982	# else
983	# error "Port me"
984	# endif
985	/* jne checkalttlbe_and_missedtagstats */
986	# ifndef IEM_WITH_TLB_STATISTICS
987	if (!fIncCheckAltTlbe)
988	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
989	else
990	# endif
991	{
992	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, offCheckAltTlbeAndMissedTagStats, kIemNativeInstrCond_ne);
993	if (fIncCheckAltTlbe)
994	iemNativeFixupFixedJump(pReNative, offFixupCheckAltTlbeJmpBack, off);
995	}
996
997	/*
998	* 4. Check TLB page table level access flags and physical page revision #.
999	*/
1000	/* mov reg1, mask */
1001	AssertCompile(IEMTLBE_F_PT_NO_USER == 4);
1002	uint64_t const fNoUser = (((pReNative->fExec >> IEM_F_X86_CPL_SHIFT) & IEM_F_X86_CPL_SMASK) + 1) & IEMTLBE_F_PT_NO_USER;
1003	uint64_t fTlbe = IEMTLBE_F_PHYS_REV \| IEMTLBE_F_NO_MAPPINGR3 \| IEMTLBE_F_PG_UNASSIGNED \| IEMTLBE_F_PT_NO_ACCESSED
1004	\| fNoUser;
1005	if (fAccess & IEM_ACCESS_TYPE_EXEC)
1006	fTlbe \|= IEMTLBE_F_PT_NO_EXEC /\| IEMTLBE_F_PG_NO_READ?/;
1007	if (fAccess & IEM_ACCESS_TYPE_READ)
1008	fTlbe \|= IEMTLBE_F_PG_NO_READ;
1009	if (fAccess & IEM_ACCESS_TYPE_WRITE)
1010	fTlbe \|= IEMTLBE_F_PT_NO_WRITE \| IEMTLBE_F_PG_NO_WRITE \| IEMTLBE_F_PT_NO_DIRTY;
1011	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, fTlbe);
1012	# if defined(RT_ARCH_AMD64)
1013	/* and reg1, [reg2->fFlagsAndPhysRev] */
1014	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R) \| (pTlbState->idxReg2 < 8 ? 0 : X86_OP_REX_B);
1015	pCodeBuf[off++] = 0x23;
1016	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1,
1017	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
1018
1019	/* cmp reg1, [pVCpu->iem.s.DataTlb.uTlbPhysRev] */
1020	pCodeBuf[off++] = X86_OP_REX_W \| (pTlbState->idxReg1 < 8 ? 0 : X86_OP_REX_R);
1021	pCodeBuf[off++] = 0x3b;
1022	off = iemNativeEmitGprByGprDisp(pCodeBuf, off, pTlbState->idxReg1, IEMNATIVE_REG_FIXED_PVMCPU,
1023	offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
1024	# elif defined(RT_ARCH_ARM64)
1025	# ifdef IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR
1026	pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg4);
1027	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg5);
1028	# else
1029	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg3,
1030	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, fFlagsAndPhysRev));
1031	pCodeBuf[off++] = Armv8A64MkInstrAnd(pTlbState->idxReg1, pTlbState->idxReg1, pTlbState->idxReg3);
1032	off = iemNativeEmitLoadGprFromVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg3, offVCpuTlb + RT_UOFFSETOF(IEMTLB, uTlbPhysRev));
1033	off = iemNativeEmitCmpGprWithGprEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg3);
1034	# endif
1035	# else
1036	# error "Port me"
1037	# endif
1038	# ifndef IEM_WITH_TLB_STATISTICS
1039	/* jne tlbmiss */
1040	off = iemNativeEmitJccToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss, kIemNativeInstrCond_ne);
1041	# else
1042	/* je 2F; inc stat; jmp tlbmiss */
1043	uint32_t const offFixup2 = off;
1044	off = iemNativeEmitJccToFixedEx(pCodeBuf, off, off + 16, kIemNativeInstrCond_e);
1045	off = iemNativeEmitIncStamCounterInVCpuEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->idxReg2,
1046	offVCpuTlb + RT_UOFFSETOF(IEMTLB, cTlbNativeMissFlagsAndPhysRev));
1047	off = iemNativeEmitJmpToLabelEx(pReNative, pCodeBuf, off, idxLabelTlbMiss);
1048	iemNativeFixupFixedJump(pReNative, offFixup2, off);
1049	# endif
1050
1051	/*
1052	* 5. Check that pbMappingR3 isn't NULL (paranoia) and calculate the
1053	* resulting pointer.
1054	*
1055	* For code TLB lookups we have some more work to do here to set various
1056	* IEMCPU members and we return a GCPhys address rather than a host pointer.
1057	*/
1058	# if defined(RT_ARCH_ARM64)
1059	uint8_t const idxRegMappingPtr = a_fDataTlb && idxRegFlatPtr != idxRegMemResult /* See step 1b. */
1060	? idxRegMemResult /* saves one instruction */ : pTlbState->idxReg1;
1061	# else
1062	uint8_t const idxRegMappingPtr = pTlbState->idxReg1; /** @todo optimize the AMD64 case as well. */
1063	# endif
1064	# if defined(RT_ARCH_ARM64) && defined(IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR)
1065	if (!a_fDataTlb)
1066	{
1067	/* ldp reg4, reg1, [reg2->GCPhys+pbMappingR3] */
1068	AssertCompileMemberAlignment(IEMTLBENTRY, GCPhys, 16);
1069	AssertCompileAdjacentMembers(IEMTLBENTRY, GCPhys, pbMappingR3);
1070	pCodeBuf[off++] = Armv8A64MkInstrLdPairGpr(pTlbState->idxReg4, idxRegMappingPtr,
1071	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys) / 8);
1072	}
1073	else
1074	# endif
1075	{
1076	/* mov reg1, [reg2->pbMappingR3] */
1077	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, idxRegMappingPtr, pTlbState->idxReg2,
1078	RT_UOFFSETOF(IEMTLBENTRY, pbMappingR3));
1079	}
1080
1081	if (a_fDataTlb)
1082	{
1083	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
1084	{
1085	Assert(idxRegMappingPtr == pTlbState->idxReg1);
1086	/* and result, 0xfff */
1087	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
1088	/* add result, reg1 */
1089	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, idxRegMappingPtr);
1090	}
1091	else
1092	{
1093	Assert(idxRegFlatPtr == pTlbState->idxRegPtr);
1094	# if defined(RT_ARCH_ARM64)
1095	Assert(idxRegMappingPtr == idxRegMemResult);
1096	AssertCompile(GUEST_PAGE_SIZE <= HOST_PAGE_SIZE);
1097	pCodeBuf[off++] = Armv8A64MkInstrBfxil(idxRegMemResult, idxRegFlatPtr, 0, GUEST_PAGE_SHIFT);
1098	# else
1099	Assert(idxRegMappingPtr == pTlbState->idxReg1);
1100	/* result = regflat & 0xfff */
1101	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
1102	/* add result, reg1 */
1103	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
1104	# endif
1105	}
1106	}
1107	else
1108	{
1109	/*
1110	* Code TLB use a la iemOpcodeFetchBytesJmp - keep reg2 pointing to the TLBE.
1111	*
1112	* Note. We do not need to set offCurInstrStart or offInstrNextByte.
1113	*/
1114	# if !defined(RT_ARCH_ARM64) \|\| !defined(IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR)
1115	# ifdef RT_ARCH_AMD64
1116	uint8_t const idxReg3 = UINT8_MAX;
1117	# else
1118	uint8_t const idxReg3 = pTlbState->idxReg3;
1119	# endif
1120	/* Set pbInstrBuf first since we've got it loaded already. */
1121	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
1122	RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf), idxReg3);
1123	/* Set uInstrBufPc to (FlatPC & ~GUEST_PAGE_OFFSET_MASK). */
1124	off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg1, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
1125	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
1126	RT_UOFFSETOF(VMCPUCC, iem.s.uInstrBufPc), idxReg3);
1127	/* Set cbInstrBufTotal to GUEST_PAGE_SIZE. / /* @todo this is a simplifications. Calc right size using CS.LIM and EIP? */
1128	off = iemNativeEmitStoreImmToVCpuU16Ex(pCodeBuf, off, GUEST_PAGE_SIZE, RT_UOFFSETOF(VMCPUCC, iem.s.cbInstrBufTotal),
1129	pTlbState->idxReg1, idxReg3);
1130	/* Now set GCPhysInstrBuf last as we'll be returning it in idxRegMemResult. */
1131	# if defined(RT_ARCH_ARM64) && defined(IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR)
1132	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg4,
1133	RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
1134	# else
1135	off = iemNativeEmitLoadGprByGprU64Ex(pCodeBuf, off, pTlbState->idxReg1,
1136	pTlbState->idxReg2, RT_UOFFSETOF(IEMTLBENTRY, GCPhys));
1137	off = iemNativeEmitStoreGprToVCpuU64Ex(pCodeBuf, off, pTlbState->idxReg1,
1138	RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf), idxReg3);
1139	# endif
1140	# else
1141	/* ARM64: Same as above but using STP. This ASSUMES that we can trash
1142	the 6 bytes following iem.s.cbInstrBufTotal! */
1143	AssertCompileMemberAlignment(VMCPUCC, iem.s.pbInstrBuf, 16);
1144	AssertCompileAdjacentMembers(VMCPUCC, iem.s.pbInstrBuf, iem.s.uInstrBufPc);
1145	AssertCompile(RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf) < 512);
1146	/* idxReg1 = reg2->pbMappingR3 (see previous LDP) */
1147	/* idxReg3 = FlatPC & ~GUEST_PAGE_OFFSET_MASK. */
1148	off = iemNativeEmitGprEqGprAndImmEx(pCodeBuf, off, pTlbState->idxReg3, idxRegFlatPtr, ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK);
1149	pCodeBuf[off++] = Armv8A64MkInstrStPairGpr(pTlbState->idxReg1, pTlbState->idxReg3,
1150	IEMNATIVE_REG_FIXED_PVMCPU, RT_UOFFSETOF(VMCPUCC, iem.s.pbInstrBuf) / 8);
1151
1152	AssertCompileMemberAlignment(VMCPUCC, iem.s.GCPhysInstrBuf, 16);
1153	AssertCompileAdjacentMembers(VMCPUCC, iem.s.GCPhysInstrBuf, iem.s.cbInstrBufTotal);
1154	AssertCompile(RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf) < 512);
1155	# ifndef IEM_WITH_OPAQUE_DECODER_STATE
1156	AssertCompileAdjacentMembers(VMCPUCC, iem.s.cbInstrBufTotal, iem.s.offCurInstrStart);
1157	AssertCompileAdjacentMembers(VMCPUCC, iem.s.offCurInstrStart, iem.s.fPrefixes); /* these two will be set to ~0. */
1158	# endif
1159	/* idxReg4 = reg2->GCPhys (see previous LDP) */
1160	/* idxReg3 = GUEST_PAGE_SIZE \| UINT64_C(0xffffffffffff0000) */
1161	pCodeBuf[off++] = Armv8A64MkInstrMovN(pTlbState->idxReg3, ~GUEST_PAGE_SIZE & 0xffff);
1162	pCodeBuf[off++] = Armv8A64MkInstrStPairGpr(pTlbState->idxReg4, pTlbState->idxReg3,
1163	IEMNATIVE_REG_FIXED_PVMCPU, RT_UOFFSETOF(VMCPUCC, iem.s.GCPhysInstrBuf) / 8);
1164	# endif
1165	if (!a_fNoReturn) /* (We skip this for iemNativeEmitBltLoadTlbAfterBranch.) */
1166	{
1167	/* Set idxRegMemResult. */
1168	if (idxRegFlatPtr == idxRegMemResult) /* See step 1b. */
1169	off = iemNativeEmitAndGpr32ByImmEx(pCodeBuf, off, idxRegMemResult, GUEST_PAGE_OFFSET_MASK);
1170	else
1171	off = iemNativeEmitGpr32EqGprAndImmEx(pCodeBuf, off, idxRegMemResult, idxRegFlatPtr, GUEST_PAGE_OFFSET_MASK);
1172	# if defined(RT_ARCH_ARM64) && defined(IEMNATIVE_WITH_TLB_LOOKUP_LOAD_STORE_PAIR)
1173	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg4);
1174	# else
1175	off = iemNativeEmitAddTwoGprsEx(pCodeBuf, off, idxRegMemResult, pTlbState->idxReg1);
1176	# endif
1177	}
1178	}
1179
1180	# if 0
1181	/*
1182	* To verify the result we call iemNativeHlpCheckTlbLookup via a wrapper.
1183	*
1184	* It's like the state logging, so parameters are passed on the stack.
1185	* iemNativeHlpAsmSafeWrapCheckTlbLookup(pVCpu, result, addr, seg \| (cbMem << 8) \| (fAccess << 16))
1186	*/
1187	if (a_fDataTlb)
1188	{
1189	# ifdef RT_ARCH_AMD64
1190	if (!offDisp && !(fAccess & 0x8000))
1191	{
1192	/* push seg \| (cbMem << 8) \| (fAccess << 16) */
1193	pCodeBuf[off++] = 0x68;
1194	pCodeBuf[off++] = iSegReg;
1195	pCodeBuf[off++] = cbMem;
1196	pCodeBuf[off++] = RT_BYTE1(fAccess);
1197	pCodeBuf[off++] = RT_BYTE2(fAccess);
1198	}
1199	else
1200	{
1201	/* mov reg1, seg \| (cbMem << 8) \| (fAccess << 16) \| (offDisp << 32) */
1202	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1,
1203	iSegReg \| ((uint32_t)cbMem << 8) \| (fAccess << 16) \| ((uint64_t)offDisp << 32));
1204	/* push reg1 */
1205	if (pTlbState->idxReg1 >= 8)
1206	pCodeBuf[off++] = X86_OP_REX_B;
1207	pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
1208	}
1209	/* push pTlbState->idxRegPtr / immediate address. */
1210	if (pTlbState->idxRegPtr != UINT8_MAX)
1211	{
1212	if (pTlbState->idxRegPtr >= 8)
1213	pCodeBuf[off++] = X86_OP_REX_B;
1214	pCodeBuf[off++] = 0x50 + (pTlbState->idxRegPtr & 7);
1215	}
1216	else
1217	{
1218	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg1, pTlbState->uAbsPtr);
1219	if (pTlbState->idxReg1 >= 8)
1220	pCodeBuf[off++] = X86_OP_REX_B;
1221	pCodeBuf[off++] = 0x50 + (pTlbState->idxReg1 & 7);
1222	}
1223	/* push idxRegMemResult */
1224	if (idxRegMemResult >= 8)
1225	pCodeBuf[off++] = X86_OP_REX_B;
1226	pCodeBuf[off++] = 0x50 + (idxRegMemResult & 7);
1227	/* push pVCpu */
1228	pCodeBuf[off++] = 0x50 + IEMNATIVE_REG_FIXED_PVMCPU;
1229	/* reg1 = helper; call reg1 */
1230	off = iemNativeEmitCallImmEx(pCodeBuf, off, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup, pTlbState->idxReg1);
1231	/* The stack is cleaned up by the helper function. */
1232
1233	# elif defined(RT_ARCH_ARM64)
1234	/* Use the temporary registers for setting up the "call frame" and making the call. */
1235	/* reg1 = seg \| (cbMem << 8) \| (fAccess << 16) */
1236	pCodeBuf[off++] = Armv8A64MkInstrMovZ(pTlbState->idxReg1, RT_MAKE_U16(iSegReg, cbMem));
1237	pCodeBuf[off++] = Armv8A64MkInstrMovK(pTlbState->idxReg1, RT_LO_U16(fAccess), 1);
1238	if (offDisp)
1239	pCodeBuf[off++] = Armv8A64MkInstrMovK(pTlbState->idxReg1, offDisp, 2);
1240	if (pTlbState->idxRegPtr != UINT8_MAX)
1241	{
1242	/* stp idxRegPtr, reg1, [sp, #-16]! */
1243	pCodeBuf[off++] = Armv8A64MkInstrStPairGpr(pTlbState->idxRegPtr, pTlbState->idxReg1,
1244	ARMV8_A64_REG_SP, -2, kArm64InstrStLdPairType_PreIndex);
1245	}
1246	else
1247	{
1248	/* reg2 = immediate address */
1249	off = iemNativeEmitLoadGprImmEx(pCodeBuf, off, pTlbState->idxReg2, pTlbState->uAbsPtr);
1250	/* stp reg2, reg1, [sp, #-16]! */
1251	pCodeBuf[off++] = Armv8A64MkInstrStPairGpr(pTlbState->idxReg2, pTlbState->idxReg1,
1252	ARMV8_A64_REG_SP, -2, kArm64InstrStLdPairType_PreIndex);
1253	}
1254	/* stp pVCpu, idxRegMemResult, [sp, #-16]! (we don't need pVCpu, but push it for stack alignment) */
1255	pCodeBuf[off++] = Armv8A64MkInstrStPairGpr(IEMNATIVE_REG_FIXED_PVMCPU, idxRegMemResult,
1256	ARMV8_A64_REG_SP, -2, kArm64InstrStLdPairType_PreIndex);
1257	/* reg1 = helper; brl reg1 */
1258	off = iemNativeEmitCallImmEx(pCodeBuf, off, (uintptr_t)iemNativeHlpAsmSafeWrapCheckTlbLookup, pTlbState->idxReg1);
1259	/* The stack is cleaned up by the helper function. */
1260
1261	# else
1262	# error "Port me"
1263	# endif
1264	}
1265
1266	# endif
1267
1268	IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);
1269
1270	return off;
1271	}
1272	#endif /* IEMNATIVE_WITH_TLB_LOOKUP */
1273
1274
1275	/** @} */
1276
1277	#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerTlbLookup_h */
1278

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompilerTlbLookup.h@ 106286

以其他格式下載: