IEMMc.h@ 105295

最後變更在這個檔案從105295是 105283,由 vboxsync 提交於 8 月前
VMM/IEM: Implement vcmpps, vcmppd, vcmpss, vcmpsd instruction emulations, bugref:9898
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 177.4 KB

行
1	/* $Id: IEMMc.h 105283 2024-07-11 20:26:27Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.alldomusa.eu.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Dummy MC that prevents native recompilation. */
59	#define IEM_MC_NO_NATIVE_RECOMPILE() ((void)0)
60
61	/** Advances RIP, finishes the instruction and returns.
62	* This may include raising debug exceptions and such. */
63	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
65	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
66	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
67	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
68	* @note only usable in 16-bit op size mode. */
69	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
70	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
73	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
74	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
75	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) \
76	return iemRegRipJumpU16AndFinishClearingRF((pVCpu), (a_u16NewIP), IEM_GET_INSTR_LEN(pVCpu))
77	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
78	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) \
79	return iemRegRipJumpU32AndFinishClearingRF((pVCpu), (a_u32NewIP), IEM_GET_INSTR_LEN(pVCpu))
80	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
81	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) \
82	return iemRegRipJumpU64AndFinishClearingRF((pVCpu), (a_u64NewIP), IEM_GET_INSTR_LEN(pVCpu))
83
84	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
85	* @note only usable in 16-bit op size mode. */
86	#define IEM_MC_REL_CALL_S16_AND_FINISH(a_i16) \
87	return iemRegRipRelativeCallS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
88	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
89	#define IEM_MC_REL_CALL_S32_AND_FINISH(a_i32) \
90	return iemRegEip32RelativeCallS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32))
91	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
92	#define IEM_MC_REL_CALL_S64_AND_FINISH(a_i64) \
93	return iemRegRip64RelativeCallS64AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i64))
94	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
95	#define IEM_MC_IND_CALL_U16_AND_FINISH(a_u16NewIP) \
96	return iemRegIp16IndirectCallU16AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16NewIP))
97	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
98	#define IEM_MC_IND_CALL_U32_AND_FINISH(a_u32NewIP) \
99	return iemRegEip32IndirectCallU32AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u32NewIP))
100	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
101	#define IEM_MC_IND_CALL_U64_AND_FINISH(a_u64NewIP) \
102	return iemRegRip64IndirectCallU64AndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u64NewIP))
103
104
105	/** Fetches the near return address from the stack, sets RIP and RSP (may trigger
106	* \#GP or \#SS), finishes the instruction and returns. */
107	#define IEM_MC_RETN_AND_FINISH(a_u16Pop) \
108	return iemRegRipNearReturnAndFinishClearingRF((pVCpu), IEM_GET_INSTR_LEN(pVCpu), (a_u16Pop), pVCpu->iem.s.enmEffOpSize)
109
110
111	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
112	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
113	do { \
114	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
115	{ /* probable */ } \
116	else return iemRaiseDeviceNotAvailable(pVCpu); \
117	} while (0)
118	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
119	do { \
120	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
121	{ /* probable */ } \
122	else return iemRaiseDeviceNotAvailable(pVCpu); \
123	} while (0)
124	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
125	do { \
126	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
127	{ /* probable */ } \
128	else return iemRaiseMathFault(pVCpu); \
129	} while (0)
130	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
131	do { \
132	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
133	be reduced to a single compare branch in the more probably code path. */ \
134	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
135	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
136	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
137	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
138	{ /* probable */ } \
139	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
140	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
141	return iemRaiseUndefinedOpcode(pVCpu); \
142	else \
143	return iemRaiseDeviceNotAvailable(pVCpu); \
144	} while (0)
145	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
146	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
147	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
148	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
149	do { \
150	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
151	single compare branch in the more probable code path. */ \
152	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
153	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
154	== X86_CR4_OSFXSR)) \
155	{ /* likely */ } \
156	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
157	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
158	return iemRaiseUndefinedOpcode(pVCpu); \
159	else \
160	return iemRaiseDeviceNotAvailable(pVCpu); \
161	} while (0)
162	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
163	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
164	do { \
165	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
166	single compare branch in the more probable code path. */ \
167	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
168	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
169	{ /* probable */ } \
170	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
171	return iemRaiseUndefinedOpcode(pVCpu); \
172	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
173	return iemRaiseDeviceNotAvailable(pVCpu); \
174	else \
175	return iemRaiseMathFault(pVCpu); \
176	} while (0)
177	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
178	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
179	* count the CPL into the TB key. However it is safe enough for now, as
180	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
181	* emitted for it. */
182	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
183	do { \
184	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
185	else return iemRaiseGeneralProtectionFault0(pVCpu); \
186	} while (0)
187	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
188	do { \
189	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
190	else return iemRaiseGeneralProtectionFault0(pVCpu); \
191	} while (0)
192	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
193	do { \
194	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
195	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
196	{ /* probable */ } \
197	else return iemRaiseUndefinedOpcode(pVCpu); \
198	} while (0)
199	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
200	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
201	do { \
202	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
203	else return iemRaiseGeneralProtectionFault0(pVCpu); \
204	} while (0)
205	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
206	do { \
207	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
208	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
209	{ /* probable */ } \
210	else \
211	{ \
212	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
213	return iemRaiseSimdFpException(pVCpu); \
214	return iemRaiseUndefinedOpcode(pVCpu); \
215	} \
216	} while (0)
217
218
219	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
220	#define IEM_MC_LOCAL_ASSIGN(a_Type, a_Name, a_Value) a_Type a_Name = (a_Value)
221	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
222	#define IEM_MC_NOREF(a_Name) RT_NOREF_PV(a_Name) /* NOP/liveness hack */
223	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
224	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
225	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
226	/** @note IEMAllInstPython.py duplicates the expansion. */
227	#define IEM_MC_ARG_EFLAGS(a_Name, a_iArg) uint32_t const a_Name = pVCpu->cpum.GstCtx.eflags.u
228	/** @note IEMAllInstPython.py duplicates the expansion. */
229	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
230	uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u; \
231	uint32_t *a_pName = &a_Name
232	/** @note IEMAllInstPython.py duplicates the expansion. */
233	#define IEM_MC_LOCAL_EFLAGS(a_Name) uint32_t a_Name = pVCpu->cpum.GstCtx.eflags.u
234	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
235	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
236	#define IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) do { \
237	AssertMsg((pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) == ((a_EFlags) & ~(a_fEflOutput)), \
238	("eflags.u=%#x (%#x) vs %s=%#x (%#x) - diff %#x (a_fEflOutput=%#x)\n", \
239	pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput), pVCpu->cpum.GstCtx.eflags.u, #a_EFlags, \
240	(a_EFlags) & ~(a_fEflOutput), (a_EFlags), \
241	(pVCpu->cpum.GstCtx.eflags.u & ~(a_fEflOutput)) ^ ((a_EFlags) & ~(a_fEflOutput)), a_fEflOutput)); \
242	pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); \
243	Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); \
244	} while (0)
245	#define IEM_MC_COMMIT_EFLAGS_OPT(a_EFlags) IEM_MC_COMMIT_EFLAGS(a_EFlags)
246	#define IEM_MC_COMMIT_EFLAGS_OPT_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_COMMIT_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput)
247
248	/** ASSUMES the source variable not used after this statement. */
249	#define IEM_MC_ASSIGN_TO_SMALLER(a_VarDst, a_VarSrcEol) (a_VarDst) = (a_VarSrcEol)
250
251	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
252	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
253	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
254	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
255	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
256	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
257	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
258	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
259	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
260	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
261	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
262	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
263	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
264	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
265	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
266	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
267	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
268	#define IEM_MC_FETCH_GREG_PAIR_U32(a_u64Dst, a_iGRegLo, a_iGRegHi) do { \
269	(a_u64Dst).s.Lo = iemGRegFetchU32(pVCpu, (a_iGRegLo)); \
270	(a_u64Dst).s.Hi = iemGRegFetchU32(pVCpu, (a_iGRegHi)); \
271	} while(0)
272	#define IEM_MC_FETCH_GREG_PAIR_U64(a_u128Dst, a_iGRegLo, a_iGRegHi) do { \
273	(a_u128Dst).s.Lo = iemGRegFetchU64(pVCpu, (a_iGRegLo)); \
274	(a_u128Dst).s.Hi = iemGRegFetchU64(pVCpu, (a_iGRegHi)); \
275	} while(0)
276	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
277	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
278	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
279	} while (0)
280	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
281	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
282	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
283	} while (0)
284	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
285	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
286	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
287	} while (0)
288	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
289	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
290	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
291	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
292	} while (0)
293	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
294	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
295	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
296	} while (0)
297	/** @note Not for IOPL or IF testing or modification. */
298	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
299	#define IEM_MC_FETCH_EFLAGS_EX(a_EFlags, a_fEflInput, a_fEflOutput) IEM_MC_FETCH_EFLAGS(a_EFlags)
300	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u /* (only LAHF) */
301	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
302	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
303
304	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
305	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
306	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
307	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
308	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
309	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
310	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
311	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
312	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
313	#define IEM_MC_STORE_GREG_PAIR_U32(a_iGRegLo, a_iGRegHi, a_u64Value) do { \
314	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint32_t)(a_u64Value).s.Lo; \
315	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint32_t)(a_u64Value).s.Hi; \
316	} while(0)
317	#define IEM_MC_STORE_GREG_PAIR_U64(a_iGRegLo, a_iGRegHi, a_u128Value) do { \
318	*iemGRegRefU64(pVCpu, (a_iGRegLo)) = (uint64_t)(a_u128Value).s.Lo; \
319	*iemGRegRefU64(pVCpu, (a_iGRegHi)) = (uint64_t)(a_u128Value).s.Hi; \
320	} while(0)
321	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
322
323	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
324	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
325	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
326	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
327	} while (0)
328	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
329	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
330	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
331	} while (0)
332	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
333	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
334
335
336	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
337	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
338	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
339	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
340	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
341	* Use IEM_MC_CLEAR_HIGH_GREG_U64! */
342	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
343	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
344	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
345	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
346	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
347	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
348	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
349	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
350	/** @note Not for IOPL or IF testing or modification.
351	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
352	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
353	#define IEM_MC_REF_EFLAGS_EX(a_pEFlags, a_fEflInput, a_fEflOutput) IEM_MC_REF_EFLAGS(a_pEFlags)
354
355	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
356	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
357	do { \
358	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
359	*pu32Reg += (a_u32Value); \
360	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
361	} while (0)
362	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
363
364	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
365	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
366	do { \
367	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
368	*pu32Reg -= (a_u8Const); \
369	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
370	} while (0)
371	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
372	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
373
374	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
375	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
376	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
377	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
378	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
379	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
380	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
381
382	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
383	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
384	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
385	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
386
387	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
388	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
389	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
390
391	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
392	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
393	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
394
395	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
396	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
397	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
398
399	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
400
401	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
402	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
403	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
404
405	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
406
407	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
408
409	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
410	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
411	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
412	do { \
413	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
414	*pu32Reg &= (a_u32Value); \
415	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
416	} while (0)
417	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
418
419	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
420	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
421	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
422	do { \
423	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
424	*pu32Reg \|= (a_u32Value); \
425	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
426	} while (0)
427	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
428
429	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
430	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
431	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
432
433	/** @note Not for IOPL or IF modification. */
434	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
435	/** @note Not for IOPL or IF modification. */
436	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
437	/** @note Not for IOPL or IF modification. */
438	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
439
440	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
441
442	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
443	#define IEM_MC_FPU_TO_MMX_MODE() do { \
444	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
445	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
446	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
447	} while (0)
448
449	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
450	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
451	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
452	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
453	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
454	} while (0)
455
456	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
457	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
458	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg, a_iDWord) \
459	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[a_iDWord]; } while (0)
460	#define IEM_MC_FETCH_MREG_U16(a_u16Value, a_iMReg, a_iWord) \
461	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[a_iWord]; } while (0)
462	#define IEM_MC_FETCH_MREG_U8(a_u8Value, a_iMReg, a_iByte) \
463	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[a_iByte]; } while (0)
464	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) \
465	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
466	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
467	} while (0)
468	#define IEM_MC_STORE_MREG_U32(a_iMReg, a_iDword, a_u32Value) \
469	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[(a_iDword)] = (a_u32Value); \
470	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
471	} while (0)
472	#define IEM_MC_STORE_MREG_U16(a_iMReg, a_iWord, a_u16Value) \
473	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au16[(a_iWord)] = (a_u16Value); \
474	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
475	} while (0)
476	#define IEM_MC_STORE_MREG_U8(a_iMReg, a_iByte, a_u8Value) \
477	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au8[(a_iByte)] = (a_u8Value); \
478	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
479	} while (0)
480	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) \
481	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
482	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
483	} while (0)
484	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
485	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
486	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
487	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
488	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
489	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
490	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
491	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
492	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
493	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
494
495	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
496	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
497	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
498	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
499	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
500	} while (0)
501	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
502	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
503	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
504	} while (0)
505	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
506	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
507	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
508	} while (0)
509	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
510	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
511	#define IEM_MC_FETCH_XREG_R64(a_r64Value, a_iXReg, a_iQWord) \
512	do { (a_r64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[(a_iQWord)]; } while (0)
513	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
514	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
515	#define IEM_MC_FETCH_XREG_R32(a_r32Value, a_iXReg, a_iDWord) \
516	do { (a_r32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[(a_iDWord)]; } while (0)
517	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
518	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
519	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
520	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)]; } while (0)
521	#define IEM_MC_FETCH_XREG_PAIR_U128(a_Dst, a_iXReg1, a_iXReg2) \
522	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
523	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
524	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
525	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
526	} while (0)
527	#define IEM_MC_FETCH_XREG_PAIR_XMM(a_Dst, a_iXReg1, a_iXReg2) \
528	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
529	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
530	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
531	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
532	} while (0)
533	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iXReg2) \
534	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
535	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
536	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
537	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
538	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
539	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
540	} while (0)
541	#define IEM_MC_FETCH_XREG_PAIR_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iXReg2) \
542	do { (a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
543	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
544	(a_Dst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[0]; \
545	(a_Dst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg2)].au64[1]; \
546	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
547	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
548	} while (0)
549	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
550	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
551	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
552	} while (0)
553	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
554	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
555	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
556	} while (0)
557	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
558	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
559	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
560	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
561	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
562	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
563	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
564	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
565	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
566	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)] = (a_u16Value); } while (0)
567	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
568	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au8[(a_iByte)] = (a_u8Value); } while (0)
569
570	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
571	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
572	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
573	} while (0)
574
575	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
576	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
577	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
578	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
579	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
580	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
581	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
582	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
583	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
584	} while (0)
585
586	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
587	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
588	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
589	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
590	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
591	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
592	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
593	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
597	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
598	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
599	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
600	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
601	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
602	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
603	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
604	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
605	} while (0)
606	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
607	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
608	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
609	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
610	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
612	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
613	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
614	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
615	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
616	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
617	} while (0)
618	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
619	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
620	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
621	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
622	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
623	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
624	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
625	} while (0)
626	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
627	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
628	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
630	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
631	} while (0)
632
633	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
634	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
635	#define IEM_MC_REF_XREG_XMM(a_pXmmDst, a_iXReg) \
636	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
637	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
638	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
639	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
640	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
641	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
642	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
643	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
644	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
645	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
646	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
647	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
648	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
649	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
650	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
651	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
652	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
653	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
654	} while (0)
655
656	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
657	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
658	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
659	} while (0)
660	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc, a_iQWord) \
661	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
662	if ((a_iQWord) < 2) \
663	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[(a_iQWord)]; \
664	else \
665	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[(a_iQWord) - 2]; \
666	} while (0)
667	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc, a_iDQword) \
668	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
669	if ((a_iDQword) == 0) \
670	{ \
671	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[0]; \
672	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegSrcTmp)].au64[1]; \
673	} \
674	else \
675	{ \
676	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[0]; \
677	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegSrcTmp)].au64[1]; \
678	} \
679	} while (0)
680	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
681	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
682	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
683	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
684	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
685	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
686	} while (0)
687	#define IEM_MC_FETCH_YREG_YMM(a_uYmmDst, a_iYRegSrc) \
688	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
689	(a_uYmmDst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
690	(a_uYmmDst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
691	(a_uYmmDst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
692	(a_uYmmDst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
693	} while (0)
694	#define IEM_MC_FETCH_YREG_PAIR_YMM(a_uYmmDst, a_iYRegSrc1, a_iYRegSrc2) \
695	do { uintptr_t const iYRegSrc1Tmp = (a_iYRegSrc1); \
696	uintptr_t const iYRegSrc2Tmp = (a_iYRegSrc2); \
697	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[0]; \
698	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc1Tmp].au64[1]; \
699	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[0]; \
700	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc1Tmp].au64[1]; \
701	(a_uYmmDst).uSrc2.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[0]; \
702	(a_uYmmDst).uSrc2.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc2Tmp].au64[1]; \
703	(a_uYmmDst).uSrc2.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[0]; \
704	(a_uYmmDst).uSrc2.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrc2Tmp].au64[1]; \
705	} while (0)
706
707	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
708	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
709	if ((a_iDQword) == 0) \
710	{ \
711	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
712	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
713	} \
714	else \
715	{ \
716	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
717	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
718	} \
719	} while (0)
720
721	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
722	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
723	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
724	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
725	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
726	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
727	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
728	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
729	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
730	} while (0)
731	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
732	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
733	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
734	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
735	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
736	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
737	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
738	} while (0)
739	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
740	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
741	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
742	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
743	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
744	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
745	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
746	} while (0)
747	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
748	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
749	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
750	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
751	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
752	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
753	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
754	} while (0)
755	#define IEM_MC_STORE_YREG_YMM_ZX_VLMAX(a_iYRegDst, a_uYmmSrc) \
756	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
757	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[0]; \
758	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[1]; \
759	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_uYmmSrc).au64[2]; \
760	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_uYmmSrc).au64[3]; \
761	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
762	} while (0)
763	#define IEM_MC_STORE_YREG_U32_U256(a_iYRegDst, a_iDwDst, a_u256Value, a_iDwSrc) \
764	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
765	if ((a_iDwDst) < 4) \
766	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au32[(a_iDwDst)] = (a_u256Value).au32[(a_iDwSrc)]; \
767	else \
768	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au32[(a_iDwDst) - 4] = (a_u256Value).au32[(a_iDwSrc)]; \
769	} while (0)
770	#define IEM_MC_STORE_YREG_U64_U256(a_iYRegDst, a_iQwDst, a_u256Value, a_iQwSrc) \
771	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
772	if ((a_iQwDst) < 2) \
773	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQwDst)] = (a_u256Value).au64[(a_iQwSrc)]; \
774	else \
775	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQwDst) - 2] = (a_u256Value).au64[(a_iQwSrc)]; \
776	} while (0)
777	#define IEM_MC_STORE_YREG_U64(a_iYRegDst, a_iQword, a_u64Value) \
778	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
779	if ((a_iQword) < 2) \
780	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[(a_iQword)] = (a_u64Value); \
781	else \
782	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[(a_iQword) - 2] = (a_u64Value); \
783	} while (0)
784
785	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
786	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
789	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
790	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
791	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
792	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
793	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
794	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
795	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
798	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
799	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
800	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
801	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
802	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
803	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
804	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
805	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
806	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
809	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
810	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
811	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
812	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
813	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
814	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
815	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
816	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
817	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
818	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
819	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
820	} while (0)
821	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
822	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
823	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
824	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
825	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
826	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
827	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
828	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
829	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
830	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
831	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
832	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
833	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
834	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
835	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
836	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
837	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
838	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
839	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
840	} while (0)
841	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
842	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
843	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
844	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
845	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
846	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
847	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
848	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
849	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
850	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
851	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
852	} while (0)
853	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
854	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
855	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
856	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
857	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
858	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
859	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
860	} while (0)
861	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
862	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
863	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
864	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
865	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
866	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
867	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
868	} while (0)
869
870	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
871	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
872	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
873	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
874	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
875	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
876	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
877	do { uintptr_t const iYRegTmp = (a_iYReg); \
878	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
879	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
880	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
881	} while (0)
882
883	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
884	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
885	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
886	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
887	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
888	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
889	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
890	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
891	} while (0)
892	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
893	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
894	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
895	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
896	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
897	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
898	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
899	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
900	} while (0)
901	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
902	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
903	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
904	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
905	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
906	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
907	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
908	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
909	} while (0)
910
911	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
912	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
913	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
914	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
915	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
916	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
917	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
918	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
919	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
920	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
921	} while (0)
922	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
923	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
924	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
925	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
926	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
927	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
928	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
929	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
930	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
931	} while (0)
932	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
933	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
934	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
935	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
936	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
937	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
938	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
939	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
940	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
941	} while (0)
942	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
943	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
944	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
945	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
946	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
947	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
948	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
949	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
950	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
951	} while (0)
952	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
953	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
954	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
955	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
956	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
957	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
958	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
959	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
960	} while (0)
961	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
962	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
963	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
964	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
965	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
966	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
967	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
968	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
969	} while (0)
970
971	#define IEM_MC_CLEAR_ZREG_256_UP(a_iYReg) \
972	do { IEM_MC_INT_CLEAR_ZMM_256_UP(a_iYReg); } while (0)
973
974	#ifndef IEM_WITH_SETJMP
975	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
976	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
977	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
978	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
979	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
980	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
981	#else
982	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
983	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
984	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
985	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
986	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
987	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
988
989	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
990	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
991	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
992	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
993	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
994	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
995	#endif
996
997	#ifndef IEM_WITH_SETJMP
998	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
999	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
1000	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1001	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1002	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1003	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
1004	#else
1005	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1006	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1007	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1008	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1009	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
1010	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1011
1012	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
1013	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1014	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
1015	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1016	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
1017	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1018	#endif
1019
1020	#ifndef IEM_WITH_SETJMP
1021	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1022	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
1023	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1024	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1025	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1026	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
1027	#else
1028	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1029	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1030	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1031	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1032	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
1033	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1034
1035	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
1036	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1037	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
1038	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1039	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
1040	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1041	#endif
1042
1043	#ifndef IEM_WITH_SETJMP
1044	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1045	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1046	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1047	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1048	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1049	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
1050	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1051	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
1052	#else
1053	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1054	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1055	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
1056	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
1057	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
1058	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1059	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
1060	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1061
1062	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
1063	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1064	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
1065	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
1066	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
1067	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
1068	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
1069	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1070	#endif
1071
1072	#ifndef IEM_WITH_SETJMP
1073	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1074	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
1075	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1076	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
1077	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1078	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
1079	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1080	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
1081	#else
1082	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
1083	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1084	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
1085	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1086	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
1087	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
1088	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
1089	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
1090
1091	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
1092	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1093	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
1094	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
1095	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
1096	iemMemFlatFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_GCPtrMem))
1097	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
1098	iemMemFlatFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_GCPtrMem))
1099	#endif
1100
1101	#ifndef IEM_WITH_SETJMP
1102	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1103	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1104	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1105	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1106	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1107	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
1108
1109	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1110	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1111	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1112	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
1113
1114	# define IEM_MC_FETCH_MEM_U128_NO_AC_AND_XREG_U128(a_u128Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1115	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128NoAc(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1116	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1117	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1118	} while (0)
1119
1120	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1121	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2))); \
1122	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1123	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1124	} while (0)
1125
1126	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1127	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1128	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1129	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1130	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1131	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1132	} while (0)
1133
1134	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1135	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1136	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)], (a_iSeg2), (a_GCPtrMem2))); \
1137	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1138	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1139	} while (0)
1140
1141	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1142	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1143	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1144	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1145	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1146	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1147	} while (0)
1148	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1149	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2))); \
1150	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1151	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1152	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1153	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1154	} while (0)
1155
1156	#else
1157	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
1158	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1159	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
1160	iemMemFetchDataU128NoAcJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1161	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
1162	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
1163
1164	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
1165	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1166	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
1167	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1168	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
1169	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
1170
1171	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
1172	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1173	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
1174	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1175	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
1176	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_GCPtrMem))
1177
1178	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
1179	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1180	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
1181	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1182	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
1183	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_GCPtrMem))
1184
1185	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1186	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1187	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1188	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1189	} while (0)
1190	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1191	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1192	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1193	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1194	} while (0)
1195
1196	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1197	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_iSeg2), (a_GCPtrMem2)); \
1198	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1199	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1200	} while (0)
1201	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE_AND_XREG_XMM(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1202	iemMemFlatFetchDataU128AlignedSseJmp(pVCpu, &(a_Dst).uSrc2.uXmm, (a_GCPtrMem2)); \
1203	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1204	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1205	} while (0)
1206
1207	# define IEM_MC_FETCH_MEM_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_iSeg2, a_GCPtrMem2) do { \
1208	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1209	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1210	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1211	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1212	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1213	} while (0)
1214	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32_AND_XREG_XMM(a_Dst, a_iXReg1, a_iDWord2, a_GCPtrMem2) do { \
1215	(a_Dst).uSrc2.uXmm.au64[0] = 0; \
1216	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1217	(a_Dst).uSrc2.uXmm.au32[(a_iDWord2)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem2)); \
1218	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1219	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1220	} while (0)
1221
1222	# define IEM_MC_FETCH_MEM_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_iSeg2, a_GCPtrMem2) do { \
1223	(a_Dst).uSrc2.uXmm.au64[!(a_iQWord2)] = 0; \
1224	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg2), (a_GCPtrMem2)); \
1225	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1226	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1227	} while (0)
1228	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64_AND_XREG_XMM(a_Dst, a_iXReg1, a_iQWord2, a_GCPtrMem2) do { \
1229	(a_Dst).uSrc2.uXmm.au64[1] = 0; \
1230	(a_Dst).uSrc2.uXmm.au64[(a_iQWord2)] = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem2)); \
1231	(a_Dst).uSrc1.uXmm.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1232	(a_Dst).uSrc1.uXmm.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1233	} while (0)
1234
1235
1236	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1237	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1238	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1239	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1240	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1241	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1242	} while (0)
1243	# define IEM_MC_FETCH_MEM_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_iSeg2, a_GCPtrMem2) do { \
1244	iemMemFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_iSeg2), (a_GCPtrMem2)); \
1245	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1246	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1247	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1248	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1249	} while (0)
1250
1251	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_RAX_RDX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1252	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1253	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1254	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1255	(a_Dst).u64Rax = pVCpu->cpum.GstCtx.rax; \
1256	(a_Dst).u64Rdx = pVCpu->cpum.GstCtx.rdx; \
1257	} while (0)
1258	# define IEM_MC_FETCH_MEM_FLAT_U128_AND_XREG_U128_AND_EAX_EDX_U32_SX_U64(a_Dst, a_iXReg1, a_GCPtrMem2) do { \
1259	iemMemFlatFetchDataU128Jmp(pVCpu, &(a_Dst).uSrc2, (a_GCPtrMem2)); \
1260	(a_Dst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[0]; \
1261	(a_Dst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg1)].au64[1]; \
1262	(a_Dst).u64Rax = (int64_t)(int32_t)pVCpu->cpum.GstCtx.eax; \
1263	(a_Dst).u64Rdx = (int64_t)(int32_t)pVCpu->cpum.GstCtx.edx; \
1264	} while (0)
1265
1266	#endif
1267
1268	#ifndef IEM_WITH_SETJMP
1269	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1270	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1271	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1272	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1273	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1274	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1275
1276	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1277	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1278	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1279	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256NoAc(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1280	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1281	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1282
1283	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1284	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1285	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedAvx(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2))); \
1286	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1287	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1288	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1289	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1290	} while (0)
1291
1292	#else
1293	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1294	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1295	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1296	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1297	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1298	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1299
1300	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1301	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1302	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1303	iemMemFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1304	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1305	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1306
1307	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_iSeg2, a_GCPtrMem2) do { \
1308	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1309	iemMemFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_iSeg2), (a_GCPtrMem2)); \
1310	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1311	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1312	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1313	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1314	} while (0)
1315
1316	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1317	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1318	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1319	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1320	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1321	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_u256Dst), (a_GCPtrMem))
1322
1323	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1324	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1325	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1326	iemMemFlatFetchDataU256NoAcJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1327	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1328	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_YmmDst).ymm, (a_GCPtrMem))
1329
1330	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX_AND_YREG_YMM(a_uYmmDst, a_iYRegSrc1, a_GCPtrMem2) do { \
1331	uintptr_t const a_iYRegSrc1Tmp = (a_iYRegSrc1); \
1332	iemMemFlatFetchDataU256AlignedAvxJmp(pVCpu, &(a_uYmmDst).uSrc2.ymm, (a_GCPtrMem2)); \
1333	(a_uYmmDst).uSrc1.au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[0]; \
1334	(a_uYmmDst).uSrc1.au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[a_iYRegSrc1Tmp].au64[1]; \
1335	(a_uYmmDst).uSrc1.au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[0]; \
1336	(a_uYmmDst).uSrc1.au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[a_iYRegSrc1Tmp].au64[1]; \
1337	} while (0)
1338
1339	#endif
1340
1341
1342
1343	#ifndef IEM_WITH_SETJMP
1344	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1345	do { \
1346	uint8_t u8Tmp; \
1347	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1348	(a_u16Dst) = u8Tmp; \
1349	} while (0)
1350	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1351	do { \
1352	uint8_t u8Tmp; \
1353	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1354	(a_u32Dst) = u8Tmp; \
1355	} while (0)
1356	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1357	do { \
1358	uint8_t u8Tmp; \
1359	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1360	(a_u64Dst) = u8Tmp; \
1361	} while (0)
1362	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1363	do { \
1364	uint16_t u16Tmp; \
1365	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1366	(a_u32Dst) = u16Tmp; \
1367	} while (0)
1368	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1369	do { \
1370	uint16_t u16Tmp; \
1371	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1372	(a_u64Dst) = u16Tmp; \
1373	} while (0)
1374	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1375	do { \
1376	uint32_t u32Tmp; \
1377	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1378	(a_u64Dst) = u32Tmp; \
1379	} while (0)
1380	#else /* IEM_WITH_SETJMP */
1381	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1382	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1383	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1384	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1385	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1386	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1387	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1388	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1389	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1390	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1391	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1392	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1393
1394	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1395	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1396	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1397	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1398	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1399	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1400	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1401	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1402	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1403	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1404	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1405	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1406	#endif /* IEM_WITH_SETJMP */
1407
1408	#ifndef IEM_WITH_SETJMP
1409	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1410	do { \
1411	uint8_t u8Tmp; \
1412	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1413	(a_u16Dst) = (int8_t)u8Tmp; \
1414	} while (0)
1415	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1416	do { \
1417	uint8_t u8Tmp; \
1418	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1419	(a_u32Dst) = (int8_t)u8Tmp; \
1420	} while (0)
1421	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1422	do { \
1423	uint8_t u8Tmp; \
1424	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1425	(a_u64Dst) = (int8_t)u8Tmp; \
1426	} while (0)
1427	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1428	do { \
1429	uint16_t u16Tmp; \
1430	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1431	(a_u32Dst) = (int16_t)u16Tmp; \
1432	} while (0)
1433	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1434	do { \
1435	uint16_t u16Tmp; \
1436	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1437	(a_u64Dst) = (int16_t)u16Tmp; \
1438	} while (0)
1439	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1440	do { \
1441	uint32_t u32Tmp; \
1442	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1443	(a_u64Dst) = (int32_t)u32Tmp; \
1444	} while (0)
1445	#else /* IEM_WITH_SETJMP */
1446	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1447	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1448	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1449	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1450	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1451	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1452	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1453	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1454	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1455	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1456	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1457	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1458
1459	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1460	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1461	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1462	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1463	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1464	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1465	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1466	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1467	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1468	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1469	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1470	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1471	#endif /* IEM_WITH_SETJMP */
1472
1473	#ifndef IEM_WITH_SETJMP
1474	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1475	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1476	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1477	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1478	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1479	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1480	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1481	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1482	#else
1483	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1484	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1485	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1486	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1487	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1488	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1489	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1490	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1491
1492	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1493	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1494	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1495	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1496	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1497	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1498	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1499	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1500	#endif
1501
1502	#ifndef IEM_WITH_SETJMP
1503	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1504	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1505	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1506	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1507	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1508	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1509	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1510	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1511	#else
1512	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1513	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1514	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1515	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1516	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1517	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1518	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1519	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1520
1521	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1522	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1523	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1524	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1525	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1526	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1527	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1528	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1529	#endif
1530
1531	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1532	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1533	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1534	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1535	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1536	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1537	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1538	do { \
1539	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1540	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1541	} while (0)
1542	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1543	do { \
1544	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1545	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1546	} while (0)
1547
1548	#ifndef IEM_WITH_SETJMP
1549	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1550	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1551	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1552	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value)))
1553	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1554	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1555	#else
1556	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1557	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1558	# define IEM_MC_STORE_MEM_U128_NO_AC(a_iSeg, a_GCPtrMem, a_u128Value) \
1559	iemMemStoreDataU128NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u128Value))
1560	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1561	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1562
1563	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1564	iemMemFlatStoreDataU128Jmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1565	# define IEM_MC_STORE_MEM_FLAT_U128_NO_AC(a_GCPtrMem, a_u128Value) \
1566	iemMemFlatStoreDataU128NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u128Value))
1567	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1568	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1569	#endif
1570
1571	#ifndef IEM_WITH_SETJMP
1572	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1573	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1574	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1575	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256NoAc(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1576	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1577	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1578	#else
1579	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1580	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1581	# define IEM_MC_STORE_MEM_U256_NO_AC(a_iSeg, a_GCPtrMem, a_u256Value) \
1582	iemMemStoreDataU256NoAcJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1583	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1584	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1585
1586	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1587	iemMemFlatStoreDataU256Jmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1588	# define IEM_MC_STORE_MEM_FLAT_U256_NO_AC(a_GCPtrMem, a_u256Value) \
1589	iemMemFlatStoreDataU256NoAcJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1590	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1591	iemMemFlatStoreDataU256AlignedAvxJmp(pVCpu, (a_GCPtrMem), &(a_u256Value))
1592	#endif
1593
1594	/* Regular stack push and pop: */
1595	#ifndef IEM_WITH_SETJMP
1596	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1597	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1598	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1599	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1600
1601	# define IEM_MC_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1602	# define IEM_MC_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1603	# define IEM_MC_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1604	#else
1605	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1606	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1607	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1608	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1609
1610	# define IEM_MC_POP_GREG_U16(a_iGReg) iemMemStackPopGRegU16Jmp(pVCpu, (a_iGReg))
1611	# define IEM_MC_POP_GREG_U32(a_iGReg) iemMemStackPopGRegU32Jmp(pVCpu, (a_iGReg))
1612	# define IEM_MC_POP_GREG_U64(a_iGReg) iemMemStackPopGRegU64Jmp(pVCpu, (a_iGReg))
1613	#endif
1614
1615	/* 32-bit flat stack push and pop: */
1616	#ifndef IEM_WITH_SETJMP
1617	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1618	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1619	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1620
1621	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1622	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU32(pVCpu, (a_iGReg)))
1623	#else
1624	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1625	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1626	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1627
1628	# define IEM_MC_FLAT32_POP_GREG_U16(a_iGReg) iemMemFlat32StackPopGRegU16Jmp(pVCpu, a_iGReg))
1629	# define IEM_MC_FLAT32_POP_GREG_U32(a_iGReg) iemMemFlat32StackPopGRegU32Jmp(pVCpu, a_iGReg))
1630	#endif
1631
1632	/* 64-bit flat stack push and pop: */
1633	#ifndef IEM_WITH_SETJMP
1634	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1635	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1636
1637	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU16(pVCpu, (a_iGReg)))
1638	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopGRegU64(pVCpu, (a_iGReg)))
1639	#else
1640	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1641	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1642
1643	# define IEM_MC_FLAT64_POP_GREG_U16(a_iGReg) iemMemFlat64StackPopGRegU16Jmp(pVCpu, (a_iGReg))
1644	# define IEM_MC_FLAT64_POP_GREG_U64(a_iGReg) iemMemFlat64StackPopGRegU64Jmp(pVCpu, (a_iGReg))
1645	#endif
1646
1647
1648	/* 8-bit */
1649
1650	/**
1651	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1652	* acccess, for atomic operations.
1653	*
1654	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1655	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1656	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1657	* @param[in] a_GCPtrMem The memory address.
1658	* @remarks Will return/long jump on errors.
1659	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1660	*/
1661	#ifndef IEM_WITH_SETJMP
1662	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1663	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1664	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1665	#else
1666	# define IEM_MC_MEM_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1667	(a_pu8Mem) = iemMemMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1668	#endif
1669
1670	/**
1671	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1672	*
1673	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1674	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1675	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1676	* @param[in] a_GCPtrMem The memory address.
1677	* @remarks Will return/long jump on errors.
1678	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1679	*/
1680	#ifndef IEM_WITH_SETJMP
1681	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1682	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1683	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1684	#else
1685	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1686	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1687	#endif
1688
1689	/**
1690	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1691	*
1692	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1693	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1694	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1695	* @param[in] a_GCPtrMem The memory address.
1696	* @remarks Will return/long jump on errors.
1697	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1698	*/
1699	#ifndef IEM_WITH_SETJMP
1700	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1701	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1702	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1703	#else
1704	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1705	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1706	#endif
1707
1708	/**
1709	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1710	*
1711	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1712	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1713	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1714	* @param[in] a_GCPtrMem The memory address.
1715	* @remarks Will return/long jump on errors.
1716	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1717	*/
1718	#ifndef IEM_WITH_SETJMP
1719	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1720	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), (a_iSeg), \
1721	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1722	#else
1723	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1724	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1725	#endif
1726
1727	/**
1728	* Maps guest memory for byte atomic read+write direct (or bounce) buffer
1729	* acccess, flat address variant.
1730	*
1731	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1732	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1733	* @param[in] a_GCPtrMem The memory address.
1734	* @remarks Will return/long jump on errors.
1735	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1736	*/
1737	#ifndef IEM_WITH_SETJMP
1738	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1739	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1740	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, 0))
1741	#else
1742	# define IEM_MC_MEM_FLAT_MAP_U8_ATOMIC(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1743	(a_pu8Mem) = iemMemFlatMapDataU8AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1744	#endif
1745
1746	/**
1747	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1748	* address variant.
1749	*
1750	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1751	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1752	* @param[in] a_GCPtrMem The memory address.
1753	* @remarks Will return/long jump on errors.
1754	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1755	*/
1756	#ifndef IEM_WITH_SETJMP
1757	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1758	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1759	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0))
1760	#else
1761	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1762	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1763	#endif
1764
1765	/**
1766	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1767	* address variant.
1768	*
1769	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1770	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1771	* @param[in] a_GCPtrMem The memory address.
1772	* @remarks Will return/long jump on errors.
1773	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1774	*/
1775	#ifndef IEM_WITH_SETJMP
1776	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1777	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1778	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0))
1779	#else
1780	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1781	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1782	#endif
1783
1784	/**
1785	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1786	* address variant.
1787	*
1788	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1789	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1790	* @param[in] a_GCPtrMem The memory address.
1791	* @remarks Will return/long jump on errors.
1792	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1793	*/
1794	#ifndef IEM_WITH_SETJMP
1795	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1796	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), &(a_bUnmapInfo), sizeof(uint8_t), UINT8_MAX, \
1797	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0))
1798	#else
1799	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1800	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1801	#endif
1802
1803
1804	/* 16-bit */
1805
1806	/**
1807	* Maps guest memory for word atomic read+write direct (or bounce) buffer acccess.
1808	*
1809	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1810	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1811	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1812	* @param[in] a_GCPtrMem The memory address.
1813	* @remarks Will return/long jump on errors.
1814	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1815	*/
1816	#ifndef IEM_WITH_SETJMP
1817	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1818	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1819	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1820	#else
1821	# define IEM_MC_MEM_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1822	(a_pu16Mem) = iemMemMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1823	#endif
1824
1825	/**
1826	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1827	*
1828	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1829	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1830	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1831	* @param[in] a_GCPtrMem The memory address.
1832	* @remarks Will return/long jump on errors.
1833	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1834	*/
1835	#ifndef IEM_WITH_SETJMP
1836	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1837	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1838	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1839	#else
1840	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1841	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1842	#endif
1843
1844	/**
1845	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1846	*
1847	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1848	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1849	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1850	* @param[in] a_GCPtrMem The memory address.
1851	* @remarks Will return/long jump on errors.
1852	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1853	*/
1854	#ifndef IEM_WITH_SETJMP
1855	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1856	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1857	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1858	#else
1859	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1860	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1861	#endif
1862
1863	/**
1864	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1865	*
1866	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1867	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1868	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1869	* @param[in] a_GCPtrMem The memory address.
1870	* @remarks Will return/long jump on errors.
1871	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1872	*/
1873	#ifndef IEM_WITH_SETJMP
1874	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1875	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), (a_iSeg), \
1876	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1877	#else
1878	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1879	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1880	#endif
1881
1882	/**
1883	* Maps guest memory for word atomic read+write direct (or bounce) buffer
1884	* acccess, flat address variant.
1885	*
1886	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1887	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1888	* @param[in] a_GCPtrMem The memory address.
1889	* @remarks Will return/long jump on errors.
1890	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1891	*/
1892	#ifndef IEM_WITH_SETJMP
1893	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1894	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1895	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint16_t) - 1))
1896	#else
1897	# define IEM_MC_MEM_FLAT_MAP_U16_ATOMIC(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1898	(a_pu16Mem) = iemMemFlatMapDataU16AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1899	#endif
1900
1901	/**
1902	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1903	* address variant.
1904	*
1905	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1906	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1907	* @param[in] a_GCPtrMem The memory address.
1908	* @remarks Will return/long jump on errors.
1909	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1910	*/
1911	#ifndef IEM_WITH_SETJMP
1912	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1913	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1914	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1))
1915	#else
1916	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1917	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1918	#endif
1919
1920	/**
1921	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1922	* address variant.
1923	*
1924	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1925	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1926	* @param[in] a_GCPtrMem The memory address.
1927	* @remarks Will return/long jump on errors.
1928	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1929	*/
1930	#ifndef IEM_WITH_SETJMP
1931	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1932	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1933	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1))
1934	#else
1935	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1936	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1937	#endif
1938
1939	/**
1940	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1941	* address variant.
1942	*
1943	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1944	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1945	* @param[in] a_GCPtrMem The memory address.
1946	* @remarks Will return/long jump on errors.
1947	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1948	*/
1949	#ifndef IEM_WITH_SETJMP
1950	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1951	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), &(a_bUnmapInfo), sizeof(uint16_t), UINT8_MAX, \
1952	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1))
1953	#else
1954	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1955	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1956	#endif
1957
1958	/** int16_t alias. */
1959	#ifndef IEM_WITH_SETJMP
1960	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1961	IEM_MC_MEM_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
1962	#else
1963	# define IEM_MC_MEM_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1964	(a_pi16Mem) = (int16_t *)iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1965	#endif
1966
1967	/** Flat int16_t alias. */
1968	#ifndef IEM_WITH_SETJMP
1969	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1970	IEM_MC_MEM_FLAT_MAP_U16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem)
1971	#else
1972	# define IEM_MC_MEM_FLAT_MAP_I16_WO(a_pi16Mem, a_bUnmapInfo, a_GCPtrMem) \
1973	(a_pi16Mem) = (int16_t *)iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1974	#endif
1975
1976
1977	/* 32-bit */
1978
1979	/**
1980	* Maps guest memory for dword atomic read+write direct (or bounce) buffer acccess.
1981	*
1982	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1983	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1984	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1985	* @param[in] a_GCPtrMem The memory address.
1986	* @remarks Will return/long jump on errors.
1987	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
1988	*/
1989	#ifndef IEM_WITH_SETJMP
1990	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1991	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
1992	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
1993	#else
1994	# define IEM_MC_MEM_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1995	(a_pu32Mem) = iemMemMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1996	#endif
1997
1998	/**
1999	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
2000	*
2001	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2002	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2003	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2004	* @param[in] a_GCPtrMem The memory address.
2005	* @remarks Will return/long jump on errors.
2006	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2007	*/
2008	#ifndef IEM_WITH_SETJMP
2009	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2010	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2011	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2012	#else
2013	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2014	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2015	#endif
2016
2017	/**
2018	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
2019	*
2020	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2021	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2022	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2023	* @param[in] a_GCPtrMem The memory address.
2024	* @remarks Will return/long jump on errors.
2025	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2026	*/
2027	#ifndef IEM_WITH_SETJMP
2028	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2029	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2030	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2031	#else
2032	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2033	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2034	#endif
2035
2036	/**
2037	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
2038	*
2039	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2040	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2041	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2042	* @param[in] a_GCPtrMem The memory address.
2043	* @remarks Will return/long jump on errors.
2044	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2045	*/
2046	#ifndef IEM_WITH_SETJMP
2047	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2048	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), (a_iSeg), \
2049	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2050	#else
2051	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2052	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2053	#endif
2054
2055	/**
2056	* Maps guest memory for dword atomic read+write direct (or bounce) buffer
2057	* acccess, flat address variant.
2058	*
2059	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2060	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2061	* @param[in] a_GCPtrMem The memory address.
2062	* @remarks Will return/long jump on errors.
2063	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2064	*/
2065	#ifndef IEM_WITH_SETJMP
2066	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2067	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2068	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint32_t) - 1))
2069	#else
2070	# define IEM_MC_MEM_FLAT_MAP_U32_ATOMIC(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2071	(a_pu32Mem) = iemMemFlatMapDataU32AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2072	#endif
2073
2074	/**
2075	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
2076	* flat address variant.
2077	*
2078	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2079	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2080	* @param[in] a_GCPtrMem The memory address.
2081	* @remarks Will return/long jump on errors.
2082	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2083	*/
2084	#ifndef IEM_WITH_SETJMP
2085	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2086	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2087	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1))
2088	#else
2089	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2090	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2091	#endif
2092
2093	/**
2094	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
2095	* address variant.
2096	*
2097	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2098	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2099	* @param[in] a_GCPtrMem The memory address.
2100	* @remarks Will return/long jump on errors.
2101	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2102	*/
2103	#ifndef IEM_WITH_SETJMP
2104	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2105	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2106	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1))
2107	#else
2108	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2109	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2110	#endif
2111
2112	/**
2113	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
2114	* address variant.
2115	*
2116	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
2117	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2118	* @param[in] a_GCPtrMem The memory address.
2119	* @remarks Will return/long jump on errors.
2120	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2121	*/
2122	#ifndef IEM_WITH_SETJMP
2123	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2124	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), &(a_bUnmapInfo), sizeof(uint32_t), UINT8_MAX, \
2125	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1))
2126	#else
2127	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
2128	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2129	#endif
2130
2131	/** int32_t alias. */
2132	#ifndef IEM_WITH_SETJMP
2133	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2134	IEM_MC_MEM_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2135	#else
2136	# define IEM_MC_MEM_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2137	(a_pi32Mem) = (int32_t *)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2138	#endif
2139
2140	/** Flat int32_t alias. */
2141	#ifndef IEM_WITH_SETJMP
2142	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2143	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem)
2144	#else
2145	# define IEM_MC_MEM_FLAT_MAP_I32_WO(a_pi32Mem, a_bUnmapInfo, a_GCPtrMem) \
2146	(a_pi32Mem) = (int32_t *)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2147	#endif
2148
2149	/** RTFLOAT32U alias. */
2150	#ifndef IEM_WITH_SETJMP
2151	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2152	IEM_MC_MEM_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2153	#else
2154	# define IEM_MC_MEM_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2155	(a_pr32Mem) = (PRTFLOAT32U)iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2156	#endif
2157
2158	/** Flat RTFLOAT32U alias. */
2159	#ifndef IEM_WITH_SETJMP
2160	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2161	IEM_MC_MEM_FLAT_MAP_U32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem)
2162	#else
2163	# define IEM_MC_MEM_FLAT_MAP_R32_WO(a_pr32Mem, a_bUnmapInfo, a_GCPtrMem) \
2164	(a_pr32Mem) = (PRTFLOAT32U)iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2165	#endif
2166
2167
2168	/* 64-bit */
2169
2170	/**
2171	* Maps guest memory for qword atomic read+write direct (or bounce) buffer acccess.
2172	*
2173	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2174	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2175	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2176	* @param[in] a_GCPtrMem The memory address.
2177	* @remarks Will return/long jump on errors.
2178	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2179	*/
2180	#ifndef IEM_WITH_SETJMP
2181	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2182	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2183	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2184	#else
2185	# define IEM_MC_MEM_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2186	(a_pu64Mem) = iemMemMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2187	#endif
2188
2189	/**
2190	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
2191	*
2192	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2193	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2194	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2195	* @param[in] a_GCPtrMem The memory address.
2196	* @remarks Will return/long jump on errors.
2197	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2198	*/
2199	#ifndef IEM_WITH_SETJMP
2200	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2201	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2202	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2203	#else
2204	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2205	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2206	#endif
2207
2208	/**
2209	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
2210	*
2211	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2212	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2213	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2214	* @param[in] a_GCPtrMem The memory address.
2215	* @remarks Will return/long jump on errors.
2216	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2217	*/
2218	#ifndef IEM_WITH_SETJMP
2219	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2220	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2221	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2222	#else
2223	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2224	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2225	#endif
2226
2227	/**
2228	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
2229	*
2230	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2231	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2232	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2233	* @param[in] a_GCPtrMem The memory address.
2234	* @remarks Will return/long jump on errors.
2235	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2236	*/
2237	#ifndef IEM_WITH_SETJMP
2238	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2239	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), (a_iSeg), \
2240	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2241	#else
2242	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2243	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2244	#endif
2245
2246	/**
2247	* Maps guest memory for qword atomic read+write direct (or bounce) buffer
2248	* acccess, flat address variant.
2249	*
2250	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2251	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2252	* @param[in] a_GCPtrMem The memory address.
2253	* @remarks Will return/long jump on errors.
2254	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2255	*/
2256	#ifndef IEM_WITH_SETJMP
2257	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2258	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2259	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(uint64_t) - 1))
2260	#else
2261	# define IEM_MC_MEM_FLAT_MAP_U64_ATOMIC(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2262	(a_pu64Mem) = iemMemFlatMapDataU64AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2263	#endif
2264
2265	/**
2266	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
2267	* flat address variant.
2268	*
2269	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2270	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2271	* @param[in] a_GCPtrMem The memory address.
2272	* @remarks Will return/long jump on errors.
2273	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2274	*/
2275	#ifndef IEM_WITH_SETJMP
2276	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2277	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2278	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1))
2279	#else
2280	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2281	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2282	#endif
2283
2284	/**
2285	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
2286	* address variant.
2287	*
2288	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2289	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2290	* @param[in] a_GCPtrMem The memory address.
2291	* @remarks Will return/long jump on errors.
2292	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2293	*/
2294	#ifndef IEM_WITH_SETJMP
2295	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2296	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2297	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2298	#else
2299	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2300	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2301	#endif
2302
2303	/**
2304	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
2305	* address variant.
2306	*
2307	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
2308	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2309	* @param[in] a_GCPtrMem The memory address.
2310	* @remarks Will return/long jump on errors.
2311	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2312	*/
2313	#ifndef IEM_WITH_SETJMP
2314	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2315	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), &(a_bUnmapInfo), sizeof(uint64_t), UINT8_MAX, \
2316	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1))
2317	#else
2318	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
2319	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2320	#endif
2321
2322	/** int64_t alias. */
2323	#ifndef IEM_WITH_SETJMP
2324	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2325	IEM_MC_MEM_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2326	#else
2327	# define IEM_MC_MEM_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2328	(a_pi64Mem) = (int64_t *)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2329	#endif
2330
2331	/** Flat int64_t alias. */
2332	#ifndef IEM_WITH_SETJMP
2333	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2334	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem)
2335	#else
2336	# define IEM_MC_MEM_FLAT_MAP_I64_WO(a_pi64Mem, a_bUnmapInfo, a_GCPtrMem) \
2337	(a_pi64Mem) = (int64_t *)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2338	#endif
2339
2340	/** RTFLOAT64U alias. */
2341	#ifndef IEM_WITH_SETJMP
2342	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2343	IEM_MC_MEM_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem)
2344	#else
2345	# define IEM_MC_MEM_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2346	(a_pr64Mem) = (PRTFLOAT64U)iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2347	#endif
2348
2349	/** Flat RTFLOAT64U alias. */
2350	#ifndef IEM_WITH_SETJMP
2351	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2352	IEM_MC_MEM_FLAT_MAP_U64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem)
2353	#else
2354	# define IEM_MC_MEM_FLAT_MAP_R64_WO(a_pr64Mem, a_bUnmapInfo, a_GCPtrMem) \
2355	(a_pr64Mem) = (PRTFLOAT64U)iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2356	#endif
2357
2358
2359	/* 128-bit */
2360
2361	/**
2362	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer acccess.
2363	*
2364	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2365	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2366	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2367	* @param[in] a_GCPtrMem The memory address.
2368	* @remarks Will return/long jump on errors.
2369	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2370	*/
2371	#ifndef IEM_WITH_SETJMP
2372	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2373	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2374	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128U) - 1))
2375	#else
2376	# define IEM_MC_MEM_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2377	(a_pu128Mem) = iemMemMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2378	#endif
2379
2380	/**
2381	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess.
2382	*
2383	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2384	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2385	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2386	* @param[in] a_GCPtrMem The memory address.
2387	* @remarks Will return/long jump on errors.
2388	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2389	*/
2390	#ifndef IEM_WITH_SETJMP
2391	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2392	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128U), (a_iSeg), \
2393	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128U) - 1))
2394	#else
2395	# define IEM_MC_MEM_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2396	(a_pu128Mem) = iemMemMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2397	#endif
2398
2399	/**
2400	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess.
2401	*
2402	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2403	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2404	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2405	* @param[in] a_GCPtrMem The memory address.
2406	* @remarks Will return/long jump on errors.
2407	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2408	*/
2409	#ifndef IEM_WITH_SETJMP
2410	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2411	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2412	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2413	#else
2414	# define IEM_MC_MEM_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2415	(a_pu128Mem) = iemMemMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2416	#endif
2417
2418	/**
2419	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess.
2420	*
2421	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2422	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2423	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2424	* @param[in] a_GCPtrMem The memory address.
2425	* @remarks Will return/long jump on errors.
2426	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2427	*/
2428	#ifndef IEM_WITH_SETJMP
2429	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2430	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), (a_iSeg), \
2431	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2432	#else
2433	# define IEM_MC_MEM_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2434	(a_pu128Mem) = iemMemMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2435	#endif
2436
2437	/**
2438	* Maps guest memory for dqword atomic read+write direct (or bounce) buffer
2439	* access, flat address variant.
2440	*
2441	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2442	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2443	* @param[in] a_GCPtrMem The memory address.
2444	* @remarks Will return/long jump on errors.
2445	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC
2446	*/
2447	#ifndef IEM_WITH_SETJMP
2448	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2449	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2450	(a_GCPtrMem), IEM_ACCESS_DATA_ATOMIC, sizeof(RTUINT128) - 1))
2451	#else
2452	# define IEM_MC_MEM_FLAT_MAP_U128_ATOMIC(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2453	(a_pu128Mem) = iemMemFlatMapDataU128AtJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2454	#endif
2455
2456	/**
2457	* Maps guest memory for dqword read+write direct (or bounce) buffer acccess,
2458	* flat address variant.
2459	*
2460	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2461	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2462	* @param[in] a_GCPtrMem The memory address.
2463	* @remarks Will return/long jump on errors.
2464	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
2465	*/
2466	#ifndef IEM_WITH_SETJMP
2467	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2468	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2469	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(RTUINT128) - 1))
2470	#else
2471	# define IEM_MC_MEM_FLAT_MAP_U128_RW(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2472	(a_pu128Mem) = iemMemFlatMapDataU128RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2473	#endif
2474
2475	/**
2476	* Maps guest memory for dqword writeonly direct (or bounce) buffer acccess,
2477	* flat address variant.
2478	*
2479	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2480	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2481	* @param[in] a_GCPtrMem The memory address.
2482	* @remarks Will return/long jump on errors.
2483	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2484	*/
2485	#ifndef IEM_WITH_SETJMP
2486	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2487	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2488	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(RTUINT128) - 1))
2489	#else
2490	# define IEM_MC_MEM_FLAT_MAP_U128_WO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2491	(a_pu128Mem) = iemMemFlatMapDataU128WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2492	#endif
2493
2494	/**
2495	* Maps guest memory for dqword readonly direct (or bounce) buffer acccess, flat
2496	* address variant.
2497	*
2498	* @param[out] a_pu128Mem Where to return the pointer to the mapping.
2499	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2500	* @param[in] a_GCPtrMem The memory address.
2501	* @remarks Will return/long jump on errors.
2502	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
2503	*/
2504	#ifndef IEM_WITH_SETJMP
2505	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2506	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu128Mem), &(a_bUnmapInfo), sizeof(RTUINT128), UINT8_MAX, \
2507	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(RTUINT128) - 1))
2508	#else
2509	# define IEM_MC_MEM_FLAT_MAP_U128_RO(a_pu128Mem, a_bUnmapInfo, a_GCPtrMem) \
2510	(a_pu128Mem) = iemMemFlatMapDataU128RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2511	#endif
2512
2513
2514	/* misc */
2515
2516	/**
2517	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2518	*
2519	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2520	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2521	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2522	* @param[in] a_GCPtrMem The memory address.
2523	* @remarks Will return/long jump on errors.
2524	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2525	*/
2526	#ifndef IEM_WITH_SETJMP
2527	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2528	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2529	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2530	#else
2531	# define IEM_MC_MEM_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2532	(a_pr80Mem) = iemMemMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2533	#endif
2534
2535	/**
2536	* Maps guest memory for 80-bit float writeonly direct (or bounce) buffer acccess.
2537	*
2538	* @param[out] a_pr80Mem Where to return the pointer to the mapping.
2539	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2540	* @param[in] a_GCPtrMem The memory address.
2541	* @remarks Will return/long jump on errors.
2542	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2543	*/
2544	#ifndef IEM_WITH_SETJMP
2545	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2546	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pr80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2547	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2548	#else
2549	# define IEM_MC_MEM_FLAT_MAP_R80_WO(a_pr80Mem, a_bUnmapInfo, a_GCPtrMem) \
2550	(a_pr80Mem) = iemMemFlatMapDataR80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2551	#endif
2552
2553
2554	/**
2555	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2556	*
2557	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2558	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2559	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
2560	* @param[in] a_GCPtrMem The memory address.
2561	* @remarks Will return/long jump on errors.
2562	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2563	*/
2564	#ifndef IEM_WITH_SETJMP
2565	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2566	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), (a_iSeg), \
2567	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2568	#else
2569	# define IEM_MC_MEM_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
2570	(a_pd80Mem) = iemMemMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
2571	#endif
2572
2573	/**
2574	* Maps guest memory for 80-bit BCD writeonly direct (or bounce) buffer acccess.
2575	*
2576	* @param[out] a_pd80Mem Where to return the pointer to the mapping.
2577	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
2578	* @param[in] a_GCPtrMem The memory address.
2579	* @remarks Will return/long jump on errors.
2580	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
2581	*/
2582	#ifndef IEM_WITH_SETJMP
2583	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2584	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pd80Mem), &(a_bUnmapInfo), sizeof(RTFLOAT80U), UINT8_MAX, \
2585	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1))
2586	#else
2587	# define IEM_MC_MEM_FLAT_MAP_D80_WO(a_pd80Mem, a_bUnmapInfo, a_GCPtrMem) \
2588	(a_pd80Mem) = iemMemFlatMapDataD80WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
2589	#endif
2590
2591
2592
2593	/* commit + unmap */
2594
2595	/** Commits the memory and unmaps guest memory previously mapped RW.
2596	* @remarks May return.
2597	* @note Implictly frees the a_bMapInfo variable.
2598	*/
2599	#ifndef IEM_WITH_SETJMP
2600	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2601	#else
2602	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2603	#endif
2604
2605	/** Commits the memory and unmaps guest memory previously mapped ATOMIC.
2606	* @remarks May return.
2607	* @note Implictly frees the a_bMapInfo variable.
2608	*/
2609	#ifndef IEM_WITH_SETJMP
2610	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2611	#else
2612	# define IEM_MC_MEM_COMMIT_AND_UNMAP_ATOMIC(a_bMapInfo) iemMemCommitAndUnmapRwJmp(pVCpu, (a_bMapInfo))
2613	#endif
2614
2615	/** Commits the memory and unmaps guest memory previously mapped W.
2616	* @remarks May return.
2617	* @note Implictly frees the a_bMapInfo variable.
2618	*/
2619	#ifndef IEM_WITH_SETJMP
2620	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2621	#else
2622	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_bMapInfo) iemMemCommitAndUnmapWoJmp(pVCpu, (a_bMapInfo))
2623	#endif
2624
2625	/** Commits the memory and unmaps guest memory previously mapped R.
2626	* @remarks May return.
2627	* @note Implictly frees the a_bMapInfo variable.
2628	*/
2629	#ifndef IEM_WITH_SETJMP
2630	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo))
2631	#else
2632	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_bMapInfo) iemMemCommitAndUnmapRoJmp(pVCpu, (a_bMapInfo))
2633	#endif
2634
2635
2636	/** Commits the memory and unmaps the guest memory unless the FPU status word
2637	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
2638	* that would cause FLD not to store.
2639	*
2640	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
2641	* store, while \#P will not.
2642	*
2643	* @remarks May in theory return - for now.
2644	* @note Implictly frees both the a_bMapInfo and a_u16FSW variables.
2645	*/
2646	#ifndef IEM_WITH_SETJMP
2647	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2648	if ( !(a_u16FSW & X86_FSW_ES) \
2649	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2650	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2651	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, a_bMapInfo)); \
2652	else \
2653	iemMemRollbackAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W); \
2654	} while (0)
2655	#else
2656	# define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE_WO(a_bMapInfo, a_u16FSW) do { \
2657	if ( !(a_u16FSW & X86_FSW_ES) \
2658	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
2659	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
2660	iemMemCommitAndUnmapWoJmp(pVCpu, a_bMapInfo); \
2661	else \
2662	iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo); \
2663	} while (0)
2664	#endif
2665
2666	/** Rolls back (conceptually only, assumes no writes) and unmaps the guest memory.
2667	* @note Implictly frees the a_bMapInfo variable. */
2668	#ifndef IEM_WITH_SETJMP
2669	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmap(pVCpu, a_bMapInfo)
2670	#else
2671	# define IEM_MC_MEM_ROLLBACK_AND_UNMAP_WO(a_bMapInfo) iemMemRollbackAndUnmapWo(pVCpu, a_bMapInfo)
2672	#endif
2673
2674
2675
2676	/** Calculate efficient address from R/M. */
2677	#ifndef IEM_WITH_SETJMP
2678	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2679	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
2680	#else
2681	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
2682	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
2683	#endif
2684
2685
2686	/** The @a a_fSupportedHosts mask are ORed together RT_ARCH_VAL_XXX values. */
2687	#define IEM_MC_NATIVE_IF(a_fSupportedHosts) if (false) {
2688	#define IEM_MC_NATIVE_ELSE() } else {
2689	#define IEM_MC_NATIVE_ENDIF() } ((void)0)
2690
2691	#define IEM_MC_NATIVE_EMIT_0(a_fnEmitter)
2692	#define IEM_MC_NATIVE_EMIT_1(a_fnEmitter, a0) (void)(a0)
2693	#define IEM_MC_NATIVE_EMIT_2(a_fnEmitter, a0, a1) (void)(a0), (void)(a1)
2694	#define IEM_MC_NATIVE_EMIT_3(a_fnEmitter, a0, a1, a2) (void)(a0), (void)(a1), (void)(a2)
2695	#define IEM_MC_NATIVE_EMIT_4(a_fnEmitter, a0, a1, a2, a3) (void)(a0), (void)(a1), (void)(a2), (void)(a3)
2696	#define IEM_MC_NATIVE_EMIT_5(a_fnEmitter, a0, a1, a2, a3, a4) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4)
2697	#define IEM_MC_NATIVE_EMIT_6(a_fnEmitter, a0, a1, a2, a3, a4, a5) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5)
2698	#define IEM_MC_NATIVE_EMIT_7(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6)
2699	#define IEM_MC_NATIVE_EMIT_8(a_fnEmitter, a0, a1, a2, a3, a4, a5, a6, a7) (void)(a0), (void)(a1), (void)(a2), (void)(a3), (void)(a4), (void)(a5), (void)(a6), (void)(a7)
2700
2701	/** This can be used to direct the register allocator when dealing with
2702	* x86/AMD64 instructions (like SHL reg,CL) that takes fixed registers. */
2703	#define IEM_MC_NATIVE_SET_AMD64_HOST_REG_FOR_LOCAL(a_VarNm, a_idxHostReg) ((void)0)
2704
2705
2706	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
2707	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
2708	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
2709	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
2710	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
2711	#define IEM_MC_CALL_AIMPL_3(a_rcType, a_rc, a_pfn, a0, a1, a2) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2))
2712	#define IEM_MC_CALL_AIMPL_4(a_rcType, a_rc, a_pfn, a0, a1, a2, a3) a_rcType const a_rc = (a_pfn)((a0), (a1), (a2), (a3))
2713
2714
2715	/** @def IEM_MC_CALL_CIMPL_HLP_RET
2716	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
2717	*/
2718	#ifdef VBOX_STRICT
2719	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
2720	do { \
2721	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
2722	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
2723	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
2724	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
2725	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
2726	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
2727	if (rcStrictHlp == VINF_SUCCESS) \
2728	{ \
2729	uint64_t const fRipMask = (pVCpu->iem.s.fExec & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_64BIT ? UINT64_MAX : UINT32_MAX; \
2730	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
2731	\|\| ( ((uRipBefore + cbInstr) & fRipMask) == pVCpu->cpum.GstCtx.rip \
2732	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
2733	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
2734	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
2735	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
2736	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
2737	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, (uRipBefore + cbInstr) & fRipMask)); \
2738	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
2739	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
2740	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
2741	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2742	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2743	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2744	else \
2745	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2746	== (fEflBefore & ~(X86_EFL_RF)), \
2747	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2748	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2749	{ \
2750	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2751	AssertMsg( fExecBefore == fExecRecalc \
2752	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2753	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2754	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2755	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2756	} \
2757	} \
2758	return rcStrictHlp; \
2759	} while (0)
2760	#else
2761	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2762	#endif
2763
2764	/**
2765	* Defers the rest of the instruction emulation to a C implementation routine
2766	* and returns, only taking the standard parameters.
2767	*
2768	* @param a_fFlags IEM_CIMPL_F_XXX.
2769	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2770	* in the native recompiler.
2771	* @param a_pfnCImpl The pointer to the C routine.
2772	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2773	*/
2774	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2775	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2776
2777	/**
2778	* Defers the rest of instruction emulation to a C implementation routine and
2779	* returns, taking one argument in addition to the standard ones.
2780	*
2781	* @param a_fFlags IEM_CIMPL_F_XXX.
2782	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2783	* in the native recompiler.
2784	* @param a_pfnCImpl The pointer to the C routine.
2785	* @param a0 The argument.
2786	*/
2787	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2788	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2789
2790	/**
2791	* Defers the rest of the instruction emulation to a C implementation routine
2792	* and returns, taking two arguments in addition to the standard ones.
2793	*
2794	* @param a_fFlags IEM_CIMPL_F_XXX.
2795	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2796	* in the native recompiler.
2797	* @param a_pfnCImpl The pointer to the C routine.
2798	* @param a0 The first extra argument.
2799	* @param a1 The second extra argument.
2800	*/
2801	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2802	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2803
2804	/**
2805	* Defers the rest of the instruction emulation to a C implementation routine
2806	* and returns, taking three arguments in addition to the standard ones.
2807	*
2808	* @param a_fFlags IEM_CIMPL_F_XXX.
2809	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2810	* in the native recompiler.
2811	* @param a_pfnCImpl The pointer to the C routine.
2812	* @param a0 The first extra argument.
2813	* @param a1 The second extra argument.
2814	* @param a2 The third extra argument.
2815	*/
2816	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2817	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2818
2819	/**
2820	* Defers the rest of the instruction emulation to a C implementation routine
2821	* and returns, taking four arguments in addition to the standard ones.
2822	*
2823	* @param a_fFlags IEM_CIMPL_F_XXX.
2824	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2825	* in the native recompiler.
2826	* @param a_pfnCImpl The pointer to the C routine.
2827	* @param a0 The first extra argument.
2828	* @param a1 The second extra argument.
2829	* @param a2 The third extra argument.
2830	* @param a3 The fourth extra argument.
2831	*/
2832	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3) \
2833	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2834
2835	/**
2836	* Defers the rest of the instruction emulation to a C implementation routine
2837	* and returns, taking two arguments in addition to the standard ones.
2838	*
2839	* @param a_fFlags IEM_CIMPL_F_XXX.
2840	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2841	* in the native recompiler.
2842	* @param a_pfnCImpl The pointer to the C routine.
2843	* @param a0 The first extra argument.
2844	* @param a1 The second extra argument.
2845	* @param a2 The third extra argument.
2846	* @param a3 The fourth extra argument.
2847	* @param a4 The fifth extra argument.
2848	*/
2849	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2, a3, a4) \
2850	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2851
2852	/**
2853	* Defers the entire instruction emulation to a C implementation routine and
2854	* returns, only taking the standard parameters.
2855	*
2856	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2857	*
2858	* @param a_fFlags IEM_CIMPL_F_XXX.
2859	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2860	* in the native recompiler.
2861	* @param a_pfnCImpl The pointer to the C routine.
2862	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2863	*/
2864	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl) \
2865	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2866
2867	/**
2868	* Defers the entire instruction emulation to a C implementation routine and
2869	* returns, taking one argument in addition to the standard ones.
2870	*
2871	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2872	*
2873	* @param a_fFlags IEM_CIMPL_F_XXX.
2874	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2875	* in the native recompiler.
2876	* @param a_pfnCImpl The pointer to the C routine.
2877	* @param a0 The argument.
2878	*/
2879	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0) \
2880	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2881
2882	/**
2883	* Defers the entire instruction emulation to a C implementation routine and
2884	* returns, taking two arguments in addition to the standard ones.
2885	*
2886	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2887	*
2888	* @param a_fFlags IEM_CIMPL_F_XXX.
2889	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2890	* in the native recompiler.
2891	* @param a_pfnCImpl The pointer to the C routine.
2892	* @param a0 The first extra argument.
2893	* @param a1 The second extra argument.
2894	*/
2895	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1) \
2896	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2897
2898	/**
2899	* Defers the entire instruction emulation to a C implementation routine and
2900	* returns, taking three arguments in addition to the standard ones.
2901	*
2902	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2903	*
2904	* @param a_fFlags IEM_CIMPL_F_XXX.
2905	* @param a_fGstShwFlush Guest shadow register copies needing to be flushed
2906	* in the native recompiler.
2907	* @param a_pfnCImpl The pointer to the C routine.
2908	* @param a0 The first extra argument.
2909	* @param a1 The second extra argument.
2910	* @param a2 The third extra argument.
2911	*/
2912	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_fGstShwFlush, a_pfnCImpl, a0, a1, a2) \
2913	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2914
2915
2916	/**
2917	* Calls a FPU assembly implementation taking one visible argument.
2918	*
2919	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2920	* @param a0 The first extra argument.
2921	*/
2922	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2923	do { \
2924	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2925	} while (0)
2926
2927	/**
2928	* Calls a FPU assembly implementation taking two visible arguments.
2929	*
2930	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2931	* @param a0 The first extra argument.
2932	* @param a1 The second extra argument.
2933	*/
2934	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2935	do { \
2936	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2937	} while (0)
2938
2939	/**
2940	* Calls a FPU assembly implementation taking three visible arguments.
2941	*
2942	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2943	* @param a0 The first extra argument.
2944	* @param a1 The second extra argument.
2945	* @param a2 The third extra argument.
2946	*/
2947	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2948	do { \
2949	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2950	} while (0)
2951
2952	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2953	do { \
2954	(a_FpuData).FSW = (a_FSW); \
2955	(a_FpuData).r80Result = *(a_pr80Value); \
2956	} while (0)
2957
2958	/** Pushes FPU result onto the stack. */
2959	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2960	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2961	/** Pushes FPU result onto the stack and sets the FPUDP. */
2962	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2963	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2964
2965	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2966	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2967	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2968
2969	/** Stores FPU result in a stack register. */
2970	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2971	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2972	/** Stores FPU result in a stack register and pops the stack. */
2973	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2974	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2975	/** Stores FPU result in a stack register and sets the FPUDP. */
2976	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2977	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2978	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2979	* stack. */
2980	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2981	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2982
2983	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2984	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2985	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2986	/** Free a stack register (for FFREE and FFREEP). */
2987	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2988	iemFpuStackFree(pVCpu, a_iStReg)
2989	/** Increment the FPU stack pointer. */
2990	#define IEM_MC_FPU_STACK_INC_TOP() \
2991	iemFpuStackIncTop(pVCpu)
2992	/** Decrement the FPU stack pointer. */
2993	#define IEM_MC_FPU_STACK_DEC_TOP() \
2994	iemFpuStackDecTop(pVCpu)
2995
2996	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2997	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2998	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2999	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
3000	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
3001	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
3002	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
3003	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3004	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3005	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
3006	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
3007	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3008	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
3009	* stack. */
3010	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3011	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3012	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
3013	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
3014	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
3015
3016	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
3017	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
3018	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
3019	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3020	* stack. */
3021	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
3022	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
3023	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3024	* FPUDS. */
3025	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3026	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3027	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
3028	* FPUDS. Pops stack. */
3029	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3030	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3031	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
3032	* stack twice. */
3033	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
3034	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
3035	/** Raises a FPU stack underflow exception for an instruction pushing a result
3036	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
3037	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
3038	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
3039	/** Raises a FPU stack underflow exception for an instruction pushing a result
3040	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
3041	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
3042	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
3043
3044	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3045	* FPUIP, FPUCS and FOP. */
3046	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
3047	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
3048	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
3049	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
3050	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
3051	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
3052	/** Prepares for using the FPU state.
3053	* Ensures that we can use the host FPU in the current context (RC+R0.
3054	* Ensures the guest FPU state in the CPUMCTX is up to date. */
3055	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
3056	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
3057	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
3058	/** Actualizes the guest FPU state so it can be accessed and modified. */
3059	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
3060
3061	/** Prepares for using the SSE state.
3062	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
3063	* Ensures the guest SSE state in the CPUMCTX is up to date. */
3064	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
3065	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3066	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
3067	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
3068	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
3069
3070	/** Prepares for using the AVX state.
3071	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
3072	* Ensures the guest AVX state in the CPUMCTX is up to date.
3073	* @note This will include the AVX512 state too when support for it is added
3074	* due to the zero extending feature of VEX instruction. */
3075	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
3076	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
3077	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
3078	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
3079	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
3080
3081	/**
3082	* Calls a MMX assembly implementation taking two visible arguments.
3083	*
3084	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3085	* @param a0 The first extra argument.
3086	* @param a1 The second extra argument.
3087	*/
3088	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
3089	do { \
3090	IEM_MC_PREPARE_FPU_USAGE(); \
3091	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
3092	} while (0)
3093
3094	/**
3095	* Calls a MMX assembly implementation taking three visible arguments.
3096	*
3097	* @param a_pfnAImpl Pointer to the assembly MMX routine.
3098	* @param a0 The first extra argument.
3099	* @param a1 The second extra argument.
3100	* @param a2 The third extra argument.
3101	*/
3102	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3103	do { \
3104	IEM_MC_PREPARE_FPU_USAGE(); \
3105	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
3106	} while (0)
3107
3108
3109	/**
3110	* Calls a SSE assembly implementation taking two visible arguments.
3111	*
3112	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3113	* @param a0 The first extra argument.
3114	* @param a1 The second extra argument.
3115	*/
3116	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
3117	do { \
3118	IEM_MC_PREPARE_SSE_USAGE(); \
3119	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3120	(a0), (a1)); \
3121	} while (0)
3122
3123	/**
3124	* Calls a SSE assembly implementation taking three visible arguments.
3125	*
3126	* @param a_pfnAImpl Pointer to the assembly SSE routine.
3127	* @param a0 The first extra argument.
3128	* @param a1 The second extra argument.
3129	* @param a2 The third extra argument.
3130	*/
3131	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3132	do { \
3133	IEM_MC_PREPARE_SSE_USAGE(); \
3134	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3135	(a0), (a1), (a2)); \
3136	} while (0)
3137
3138
3139	/**
3140	* Calls a AVX assembly implementation taking two visible arguments.
3141	*
3142	* There is one implicit zero'th argument, a pointer to the extended state.
3143	*
3144	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3145	* @param a0 The first extra argument.
3146	* @param a1 The second extra argument.
3147	*/
3148	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a0, a1) \
3149	do { \
3150	IEM_MC_PREPARE_AVX_USAGE(); \
3151	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3152	(a0), (a1)); \
3153	} while (0)
3154
3155	/**
3156	* Calls a AVX assembly implementation taking three visible arguments.
3157	*
3158	* There is one implicit zero'th argument, a pointer to the extended state.
3159	*
3160	* @param a_pfnAImpl Pointer to the assembly AVX routine.
3161	* @param a0 The first extra argument.
3162	* @param a1 The second extra argument.
3163	* @param a2 The third extra argument.
3164	*/
3165	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
3166	do { \
3167	IEM_MC_PREPARE_AVX_USAGE(); \
3168	pVCpu->cpum.GstCtx.XState.x87.MXCSR = a_pfnAImpl(pVCpu->cpum.GstCtx.XState.x87.MXCSR & ~X86_MXCSR_XCPT_FLAGS, \
3169	(a0), (a1), (a2)); \
3170	} while (0)
3171
3172	/** @note Not for IOPL or IF testing. */
3173	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
3174	/** @note Not for IOPL or IF testing. */
3175	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
3176	/** @note Not for IOPL or IF testing. */
3177	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
3178	/** @note Not for IOPL or IF testing. */
3179	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
3180	/** @note Not for IOPL or IF testing. */
3181	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
3182	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3183	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3184	/** @note Not for IOPL or IF testing. */
3185	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
3186	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3187	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3188	/** @note Not for IOPL or IF testing. */
3189	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
3190	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3191	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3192	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3193	/** @note Not for IOPL or IF testing. */
3194	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
3195	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
3196	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
3197	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
3198	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
3199	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
3200	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
3201	#define IEM_MC_IF_CX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.cx != 1) {
3202	#define IEM_MC_IF_ECX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.ecx != 1) {
3203	#define IEM_MC_IF_RCX_IS_NOT_ONE() if (pVCpu->cpum.GstCtx.rcx != 1) {
3204	/** @note Not for IOPL or IF testing. */
3205	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3206	if ( pVCpu->cpum.GstCtx.cx != 1 \
3207	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3208	/** @note Not for IOPL or IF testing. */
3209	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3210	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3211	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3212	/** @note Not for IOPL or IF testing. */
3213	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_SET(a_fBit) \
3214	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3215	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3216	/** @note Not for IOPL or IF testing. */
3217	#define IEM_MC_IF_CX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3218	if ( pVCpu->cpum.GstCtx.cx != 1 \
3219	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3220	/** @note Not for IOPL or IF testing. */
3221	#define IEM_MC_IF_ECX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3222	if ( pVCpu->cpum.GstCtx.ecx != 1 \
3223	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3224	/** @note Not for IOPL or IF testing. */
3225	#define IEM_MC_IF_RCX_IS_NOT_ONE_AND_EFL_BIT_NOT_SET(a_fBit) \
3226	if ( pVCpu->cpum.GstCtx.rcx != 1 \
3227	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
3228	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
3229	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
3230
3231	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
3232	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
3233	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
3234	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
3235	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
3236	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
3237	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
3238	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
3239	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
3240	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
3241	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
3242	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
3243	#define IEM_MC_IF_FCW_IM() \
3244	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
3245
3246	#define IEM_MC_ELSE() } else {
3247	#define IEM_MC_ENDIF() } do {} while (0)
3248
3249
3250	/** Recompiler debugging: Flush guest register shadow copies. */
3251	#define IEM_MC_HINT_FLUSH_GUEST_SHADOW(g_fGstShwFlush) ((void)0)
3252
3253	/** @} */
3254
3255	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
3256

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 105295

以其他格式下載: