IEMMc.h@ 101587

最後變更在這個檔案從101587是 101587,由 vboxsync 提交於 17 月前
VMM/IEM: Native IEM_MC_SUB_GREG_U32 and IEM_MC_SUB_GREG_U64. Covers 32-bit and 64-bit loop instructions. Some cleanups. bugref:10371
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 134.4 KB

行
1	/* $Id: IEMMc.h 101587 2023-10-25 12:25:11Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - IEM_MC_XXX.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.alldomusa.eu.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMMc_h
29	#define VMM_INCLUDED_SRC_include_IEMMc_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35	/** @name "Microcode" macros.
36	*
37	* The idea is that we should be able to use the same code to interpret
38	* instructions as well as recompiler instructions. Thus this obfuscation.
39	*
40	* @{
41	*/
42
43	#define IEM_MC_BEGIN(a_cArgs, a_cLocals, a_fMcFlags, a_fCImplFlags) {
44	#define IEM_MC_END() }
45
46	/** Internal macro. */
47	#define IEM_MC_RETURN_ON_FAILURE(a_Expr) \
48	do \
49	{ \
50	VBOXSTRICTRC rcStrict2 = a_Expr; \
51	if (rcStrict2 == VINF_SUCCESS) \
52	{ /* likely */ } \
53	else \
54	return rcStrict2; \
55	} while (0)
56
57
58	/** Advances RIP, finishes the instruction and returns.
59	* This may include raising debug exceptions and such. */
60	#define IEM_MC_ADVANCE_RIP_AND_FINISH() return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu))
61	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
62	#define IEM_MC_REL_JMP_S8_AND_FINISH(a_i8) \
63	return iemRegRipRelativeJumpS8AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i8), pVCpu->iem.s.enmEffOpSize)
64	/** Sets RIP (may trigger \#GP), finishes the instruction and returns.
65	* @note only usable in 16-bit op size mode. */
66	#define IEM_MC_REL_JMP_S16_AND_FINISH(a_i16) \
67	return iemRegRipRelativeJumpS16AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i16))
68	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
69	#define IEM_MC_REL_JMP_S32_AND_FINISH(a_i32) \
70	return iemRegRipRelativeJumpS32AndFinishClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu), (a_i32), pVCpu->iem.s.enmEffOpSize)
71	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
72	#define IEM_MC_SET_RIP_U16_AND_FINISH(a_u16NewIP) return iemRegRipJumpU16AndFinishClearningRF((pVCpu), (a_u16NewIP))
73	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
74	#define IEM_MC_SET_RIP_U32_AND_FINISH(a_u32NewIP) return iemRegRipJumpU32AndFinishClearningRF((pVCpu), (a_u32NewIP))
75	/** Sets RIP (may trigger \#GP), finishes the instruction and returns. */
76	#define IEM_MC_SET_RIP_U64_AND_FINISH(a_u64NewIP) return iemRegRipJumpU64AndFinishClearningRF((pVCpu), (a_u64NewIP))
77
78	#define IEM_MC_RAISE_DIVIDE_ERROR() return iemRaiseDivideError(pVCpu)
79	#define IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE() \
80	do { \
81	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)))) \
82	{ /* probable */ } \
83	else return iemRaiseDeviceNotAvailable(pVCpu); \
84	} while (0)
85	#define IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE() \
86	do { \
87	if (RT_LIKELY(!((pVCpu->cpum.GstCtx.cr0 & (X86_CR0_MP \| X86_CR0_TS)) == (X86_CR0_MP \| X86_CR0_TS)))) \
88	{ /* probable */ } \
89	else return iemRaiseDeviceNotAvailable(pVCpu); \
90	} while (0)
91	#define IEM_MC_MAYBE_RAISE_FPU_XCPT() \
92	do { \
93	if (RT_LIKELY(!(pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))) \
94	{ /* probable */ } \
95	else return iemRaiseMathFault(pVCpu); \
96	} while (0)
97	#define IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT() \
98	do { \
99	/* Since none of the bits we compare from XCR0, CR4 and CR0 overlap, it can \
100	be reduced to a single compare branch in the more probably code path. */ \
101	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) \
102	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE) \
103	\| (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)) \
104	== (XSAVE_C_YMM \| XSAVE_C_SSE \| X86_CR4_OSXSAVE))) \
105	{ /* probable */ } \
106	else if ( (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_YMM \| XSAVE_C_SSE)) != (XSAVE_C_YMM \| XSAVE_C_SSE) \
107	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)) \
108	return iemRaiseUndefinedOpcode(pVCpu); \
109	else \
110	return iemRaiseDeviceNotAvailable(pVCpu); \
111	} while (0)
112	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR4_OSXSAVE));
113	AssertCompile(!((XSAVE_C_YMM \| XSAVE_C_SSE) & X86_CR0_TS));
114	AssertCompile(!(X86_CR4_OSXSAVE & X86_CR0_TS));
115	#define IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT() \
116	do { \
117	/* Since the CR4 and CR0 bits doesn't overlap, it can be reduced to a
118	single compare branch in the more probable code path. */ \
119	if (RT_LIKELY( ( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
120	\| (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
121	== X86_CR4_OSFXSR)) \
122	{ /* likely */ } \
123	else if ( (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
124	\|\| !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR)) \
125	return iemRaiseUndefinedOpcode(pVCpu); \
126	else \
127	return iemRaiseDeviceNotAvailable(pVCpu); \
128	} while (0)
129	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_CR4_OSFXSR));
130	#define IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT() \
131	do { \
132	/* Since the two CR0 bits doesn't overlap with FSW.ES, this can be reduced to a
133	single compare branch in the more probable code path. */ \
134	if (RT_LIKELY(!( (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM \| X86_CR0_TS)) \
135	\| (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES)))) \
136	{ /* probable */ } \
137	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM) \
138	return iemRaiseUndefinedOpcode(pVCpu); \
139	else if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS) \
140	return iemRaiseDeviceNotAvailable(pVCpu); \
141	else \
142	return iemRaiseMathFault(pVCpu); \
143	} while (0)
144	AssertCompile(!((X86_CR0_EM \| X86_CR0_TS) & X86_FSW_ES));
145	/** @todo recomp: this one is slightly problematic as the recompiler doesn't
146	* count the CPL into the TB key. However it is safe enough for now, as
147	* it calls iemRaiseGeneralProtectionFault0 directly so no calls will be
148	* emitted for it. */
149	#define IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO() \
150	do { \
151	if (RT_LIKELY(IEM_GET_CPL(pVCpu) == 0)) { /* probable */ } \
152	else return iemRaiseGeneralProtectionFault0(pVCpu); \
153	} while (0)
154	#define IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(a_EffAddr, a_cbAlign) \
155	do { \
156	if (!((a_EffAddr) & ((a_cbAlign) - 1))) { /* likely */ } \
157	else return iemRaiseGeneralProtectionFault0(pVCpu); \
158	} while (0)
159	#define IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT() \
160	do { \
161	if (RT_LIKELY( ((pVCpu->cpum.GstCtx.cr4 & X86_CR4_FSGSBASE) \| IEM_GET_CPU_MODE(pVCpu)) \
162	== (X86_CR4_FSGSBASE \| IEMMODE_64BIT))) \
163	{ /* probable */ } \
164	else return iemRaiseUndefinedOpcode(pVCpu); \
165	} while (0)
166	AssertCompile(X86_CR4_FSGSBASE > UINT8_MAX);
167	#define IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(a_u64Addr) \
168	do { \
169	if (RT_LIKELY(IEM_IS_CANONICAL(a_u64Addr))) { /* likely */ } \
170	else return iemRaiseGeneralProtectionFault0(pVCpu); \
171	} while (0)
172	#define IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
173	do { \
174	if (RT_LIKELY(( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
175	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)) \
176	{ /* probable */ } \
177	else \
178	{ \
179	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT) \
180	return iemRaiseSimdFpException(pVCpu); \
181	return iemRaiseUndefinedOpcode(pVCpu); \
182	} \
183	} while (0)
184	#define IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT() \
185	do { \
186	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXMMEEXCPT)\
187	return iemRaiseSimdFpException(pVCpu); \
188	return iemRaiseUndefinedOpcode(pVCpu); \
189	} while (0)
190
191
192	#define IEM_MC_LOCAL(a_Type, a_Name) a_Type a_Name
193	#define IEM_MC_LOCAL_CONST(a_Type, a_Name, a_Value) a_Type const a_Name = (a_Value)
194	#define IEM_MC_REF_LOCAL(a_pRefArg, a_Local) (a_pRefArg) = &(a_Local)
195	#define IEM_MC_ARG(a_Type, a_Name, a_iArg) a_Type a_Name
196	#define IEM_MC_ARG_CONST(a_Type, a_Name, a_Value, a_iArg) a_Type const a_Name = (a_Value)
197	#define IEM_MC_ARG_LOCAL_REF(a_Type, a_Name, a_Local, a_iArg) a_Type const a_Name = &(a_Local)
198	/** @note IEMAllInstPython.py duplicates the expansion. */
199	#define IEM_MC_ARG_LOCAL_EFLAGS(a_pName, a_Name, a_iArg) \
200	uint32_t a_Name; \
201	uint32_t *a_pName = &a_Name
202	#define IEM_MC_COMMIT_EFLAGS(a_EFlags) \
203	do { pVCpu->cpum.GstCtx.eflags.u = (a_EFlags); Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_1); } while (0)
204
205	#define IEM_MC_ASSIGN(a_VarOrArg, a_CVariableOrConst) (a_VarOrArg) = (a_CVariableOrConst)
206	#define IEM_MC_ASSIGN_TO_SMALLER IEM_MC_ASSIGN
207	#define IEM_MC_ASSIGN_U8_SX_U64(a_u64VarOrArg, a_u8CVariableOrConst) \
208	(a_u64VarOrArg) = (int8_t)(a_u8CVariableOrConst)
209	#define IEM_MC_ASSIGN_U32_SX_U64(a_u64VarOrArg, a_u32CVariableOrConst) \
210	(a_u64VarOrArg) = (int32_t)(a_u32CVariableOrConst)
211
212	#define IEM_MC_FETCH_GREG_U8(a_u8Dst, a_iGReg) (a_u8Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
213	#define IEM_MC_FETCH_GREG_U8_ZX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
214	#define IEM_MC_FETCH_GREG_U8_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
215	#define IEM_MC_FETCH_GREG_U8_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU8(pVCpu, (a_iGReg))
216	#define IEM_MC_FETCH_GREG_U8_SX_U16(a_u16Dst, a_iGReg) (a_u16Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
217	#define IEM_MC_FETCH_GREG_U8_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
218	#define IEM_MC_FETCH_GREG_U8_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int8_t)iemGRegFetchU8(pVCpu, (a_iGReg))
219	#define IEM_MC_FETCH_GREG_U16(a_u16Dst, a_iGReg) (a_u16Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
220	#define IEM_MC_FETCH_GREG_U16_ZX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
221	#define IEM_MC_FETCH_GREG_U16_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU16(pVCpu, (a_iGReg))
222	#define IEM_MC_FETCH_GREG_U16_SX_U32(a_u32Dst, a_iGReg) (a_u32Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
223	#define IEM_MC_FETCH_GREG_U16_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int16_t)iemGRegFetchU16(pVCpu, (a_iGReg))
224	#define IEM_MC_FETCH_GREG_U32(a_u32Dst, a_iGReg) (a_u32Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
225	#define IEM_MC_FETCH_GREG_U32_ZX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU32(pVCpu, (a_iGReg))
226	#define IEM_MC_FETCH_GREG_U32_SX_U64(a_u64Dst, a_iGReg) (a_u64Dst) = (int32_t)iemGRegFetchU32(pVCpu, (a_iGReg))
227	#define IEM_MC_FETCH_GREG_U64(a_u64Dst, a_iGReg) (a_u64Dst) = iemGRegFetchU64(pVCpu, (a_iGReg))
228	#define IEM_MC_FETCH_GREG_U64_ZX_U64 IEM_MC_FETCH_GREG_U64
229	#define IEM_MC_FETCH_SREG_U16(a_u16Dst, a_iSReg) do { \
230	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
231	(a_u16Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
232	} while (0)
233	#define IEM_MC_FETCH_SREG_ZX_U32(a_u32Dst, a_iSReg) do { \
234	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
235	(a_u32Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
236	} while (0)
237	#define IEM_MC_FETCH_SREG_ZX_U64(a_u64Dst, a_iSReg) do { \
238	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
239	(a_u64Dst) = iemSRegFetchU16(pVCpu, (a_iSReg)); \
240	} while (0)
241	/** @todo IEM_MC_FETCH_SREG_BASE_U64 & IEM_MC_FETCH_SREG_BASE_U32 probably aren't worth it... */
242	#define IEM_MC_FETCH_SREG_BASE_U64(a_u64Dst, a_iSReg) do { \
243	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
244	(a_u64Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
245	} while (0)
246	#define IEM_MC_FETCH_SREG_BASE_U32(a_u32Dst, a_iSReg) do { \
247	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
248	(a_u32Dst) = iemSRegBaseFetchU64(pVCpu, (a_iSReg)); \
249	} while (0)
250	/** @note Not for IOPL or IF testing or modification. */
251	#define IEM_MC_FETCH_EFLAGS(a_EFlags) (a_EFlags) = pVCpu->cpum.GstCtx.eflags.u
252	#define IEM_MC_FETCH_EFLAGS_U8(a_EFlags) (a_EFlags) = (uint8_t)pVCpu->cpum.GstCtx.eflags.u
253	#define IEM_MC_FETCH_FSW(a_u16Fsw) (a_u16Fsw) = pVCpu->cpum.GstCtx.XState.x87.FSW
254	#define IEM_MC_FETCH_FCW(a_u16Fcw) (a_u16Fcw) = pVCpu->cpum.GstCtx.XState.x87.FCW
255
256	#define IEM_MC_STORE_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) = (a_u8Value)
257	#define IEM_MC_STORE_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) = (a_u16Value)
258	#define IEM_MC_STORE_GREG_U32(a_iGReg, a_u32Value) iemGRegRefU64(pVCpu, (a_iGReg)) = (uint32_t)(a_u32Value) / clear high bits. */
259	#define IEM_MC_STORE_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) = (a_u64Value)
260	#define IEM_MC_STORE_GREG_I64(a_iGReg, a_i64Value) *iemGRegRefI64(pVCpu, (a_iGReg)) = (a_i64Value)
261	#define IEM_MC_STORE_GREG_U8_CONST IEM_MC_STORE_GREG_U8
262	#define IEM_MC_STORE_GREG_U16_CONST IEM_MC_STORE_GREG_U16
263	#define IEM_MC_STORE_GREG_U32_CONST IEM_MC_STORE_GREG_U32
264	#define IEM_MC_STORE_GREG_U64_CONST IEM_MC_STORE_GREG_U64
265	#define IEM_MC_CLEAR_HIGH_GREG_U64(a_iGReg) *iemGRegRefU64(pVCpu, (a_iGReg)) &= UINT32_MAX
266	#define IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(a_pu32Dst) do { (a_pu32Dst)[1] = 0; } while (0)
267	/** @todo IEM_MC_STORE_SREG_BASE_U64 & IEM_MC_STORE_SREG_BASE_U32 aren't worth it... */
268	#define IEM_MC_STORE_SREG_BASE_U64(a_iSReg, a_u64Value) do { \
269	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
270	*iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (a_u64Value); \
271	} while (0)
272	#define IEM_MC_STORE_SREG_BASE_U32(a_iSReg, a_u32Value) do { \
273	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(a_iSReg)); \
274	iemSRegBaseRefU64(pVCpu, (a_iSReg)) = (uint32_t)(a_u32Value); / clear high bits. */ \
275	} while (0)
276	#define IEM_MC_STORE_FPUREG_R80_SRC_REF(a_iSt, a_pr80Src) \
277	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80 = *(a_pr80Src); } while (0)
278
279
280	#define IEM_MC_REF_GREG_U8(a_pu8Dst, a_iGReg) (a_pu8Dst) = iemGRegRefU8( pVCpu, (a_iGReg))
281	#define IEM_MC_REF_GREG_U8_CONST(a_pu8Dst, a_iGReg) (a_pu8Dst) = (uint8_t const *)iemGRegRefU8( pVCpu, (a_iGReg))
282	#define IEM_MC_REF_GREG_U16(a_pu16Dst, a_iGReg) (a_pu16Dst) = iemGRegRefU16(pVCpu, (a_iGReg))
283	#define IEM_MC_REF_GREG_U16_CONST(a_pu16Dst, a_iGReg) (a_pu16Dst) = (uint16_t const *)iemGRegRefU16(pVCpu, (a_iGReg))
284	/** @todo User of IEM_MC_REF_GREG_U32 needs to clear the high bits on commit.
285	* Use IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF! */
286	#define IEM_MC_REF_GREG_U32(a_pu32Dst, a_iGReg) (a_pu32Dst) = iemGRegRefU32(pVCpu, (a_iGReg))
287	#define IEM_MC_REF_GREG_U32_CONST(a_pu32Dst, a_iGReg) (a_pu32Dst) = (uint32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
288	#define IEM_MC_REF_GREG_I32(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t *)iemGRegRefU32(pVCpu, (a_iGReg))
289	#define IEM_MC_REF_GREG_I32_CONST(a_pi32Dst, a_iGReg) (a_pi32Dst) = (int32_t const *)iemGRegRefU32(pVCpu, (a_iGReg))
290	#define IEM_MC_REF_GREG_U64(a_pu64Dst, a_iGReg) (a_pu64Dst) = iemGRegRefU64(pVCpu, (a_iGReg))
291	#define IEM_MC_REF_GREG_U64_CONST(a_pu64Dst, a_iGReg) (a_pu64Dst) = (uint64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
292	#define IEM_MC_REF_GREG_I64(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t *)iemGRegRefU64(pVCpu, (a_iGReg))
293	#define IEM_MC_REF_GREG_I64_CONST(a_pi64Dst, a_iGReg) (a_pi64Dst) = (int64_t const *)iemGRegRefU64(pVCpu, (a_iGReg))
294	/** @note Not for IOPL or IF testing or modification.
295	* @note Must preserve any undefined bits, see CPUMX86EFLAGS! */
296	#define IEM_MC_REF_EFLAGS(a_pEFlags) (a_pEFlags) = &pVCpu->cpum.GstCtx.eflags.uBoth
297	#define IEM_MC_REF_MXCSR(a_pfMxcsr) (a_pfMxcsr) = &pVCpu->cpum.GstCtx.XState.x87.MXCSR
298
299	#define IEM_MC_ADD_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) += (a_u16Value)
300	#define IEM_MC_ADD_GREG_U32(a_iGReg, a_u32Value) \
301	do { \
302	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
303	*pu32Reg += (a_u32Value); \
304	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
305	} while (0)
306	#define IEM_MC_ADD_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) += (a_u64Value)
307
308	#define IEM_MC_SUB_GREG_U16(a_iGReg, a_u8Const) *iemGRegRefU16(pVCpu, (a_iGReg)) -= (a_u8Const)
309	#define IEM_MC_SUB_GREG_U32(a_iGReg, a_u8Const) \
310	do { \
311	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
312	*pu32Reg -= (a_u8Const); \
313	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
314	} while (0)
315	#define IEM_MC_SUB_GREG_U64(a_iGReg, a_u8Const) *iemGRegRefU64(pVCpu, (a_iGReg)) -= (a_u8Const)
316	#define IEM_MC_SUB_LOCAL_U16(a_u16Value, a_u16Const) do { (a_u16Value) -= a_u16Const; } while (0)
317
318	#define IEM_MC_ADD_GREG_U8_TO_LOCAL(a_u8Value, a_iGReg) do { (a_u8Value) += iemGRegFetchU8( pVCpu, (a_iGReg)); } while (0)
319	#define IEM_MC_ADD_GREG_U16_TO_LOCAL(a_u16Value, a_iGReg) do { (a_u16Value) += iemGRegFetchU16(pVCpu, (a_iGReg)); } while (0)
320	#define IEM_MC_ADD_GREG_U32_TO_LOCAL(a_u32Value, a_iGReg) do { (a_u32Value) += iemGRegFetchU32(pVCpu, (a_iGReg)); } while (0)
321	#define IEM_MC_ADD_GREG_U64_TO_LOCAL(a_u64Value, a_iGReg) do { (a_u64Value) += iemGRegFetchU64(pVCpu, (a_iGReg)); } while (0)
322	#define IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(a_EffAddr, a_i16) do { (a_EffAddr) += (a_i16); } while (0)
323	#define IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(a_EffAddr, a_i32) do { (a_EffAddr) += (a_i32); } while (0)
324	#define IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(a_EffAddr, a_i64) do { (a_EffAddr) += (a_i64); } while (0)
325
326	#define IEM_MC_AND_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) &= (a_u8Mask); } while (0)
327	#define IEM_MC_AND_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) &= (a_u16Mask); } while (0)
328	#define IEM_MC_AND_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
329	#define IEM_MC_AND_LOCAL_U64(a_u64Local, a_u64Mask) do { (a_u64Local) &= (a_u64Mask); } while (0)
330
331	#define IEM_MC_AND_ARG_U16(a_u16Arg, a_u16Mask) do { (a_u16Arg) &= (a_u16Mask); } while (0)
332	#define IEM_MC_AND_ARG_U32(a_u32Arg, a_u32Mask) do { (a_u32Arg) &= (a_u32Mask); } while (0)
333	#define IEM_MC_AND_ARG_U64(a_u64Arg, a_u64Mask) do { (a_u64Arg) &= (a_u64Mask); } while (0)
334
335	#define IEM_MC_OR_LOCAL_U8(a_u8Local, a_u8Mask) do { (a_u8Local) \|= (a_u8Mask); } while (0)
336	#define IEM_MC_OR_LOCAL_U16(a_u16Local, a_u16Mask) do { (a_u16Local) \|= (a_u16Mask); } while (0)
337	#define IEM_MC_OR_LOCAL_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
338
339	#define IEM_MC_SAR_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) >>= (a_cShift); } while (0)
340	#define IEM_MC_SAR_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) >>= (a_cShift); } while (0)
341	#define IEM_MC_SAR_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) >>= (a_cShift); } while (0)
342
343	#define IEM_MC_SHR_LOCAL_U8(a_u8Local, a_cShift) do { (a_u8Local) >>= (a_cShift); } while (0)
344
345	#define IEM_MC_SHL_LOCAL_S16(a_i16Local, a_cShift) do { (a_i16Local) <<= (a_cShift); } while (0)
346	#define IEM_MC_SHL_LOCAL_S32(a_i32Local, a_cShift) do { (a_i32Local) <<= (a_cShift); } while (0)
347	#define IEM_MC_SHL_LOCAL_S64(a_i64Local, a_cShift) do { (a_i64Local) <<= (a_cShift); } while (0)
348
349	#define IEM_MC_AND_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) &= (a_u32Mask); } while (0)
350
351	#define IEM_MC_OR_2LOCS_U32(a_u32Local, a_u32Mask) do { (a_u32Local) \|= (a_u32Mask); } while (0)
352
353	#define IEM_MC_AND_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) &= (a_u8Value)
354	#define IEM_MC_AND_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) &= (a_u16Value)
355	#define IEM_MC_AND_GREG_U32(a_iGReg, a_u32Value) \
356	do { \
357	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
358	*pu32Reg &= (a_u32Value); \
359	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
360	} while (0)
361	#define IEM_MC_AND_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) &= (a_u64Value)
362
363	#define IEM_MC_OR_GREG_U8(a_iGReg, a_u8Value) *iemGRegRefU8( pVCpu, (a_iGReg)) \|= (a_u8Value)
364	#define IEM_MC_OR_GREG_U16(a_iGReg, a_u16Value) *iemGRegRefU16(pVCpu, (a_iGReg)) \|= (a_u16Value)
365	#define IEM_MC_OR_GREG_U32(a_iGReg, a_u32Value) \
366	do { \
367	uint32_t *pu32Reg = iemGRegRefU32(pVCpu, (a_iGReg)); \
368	*pu32Reg \|= (a_u32Value); \
369	pu32Reg[1] = 0; /* implicitly clear the high bit. */ \
370	} while (0)
371	#define IEM_MC_OR_GREG_U64(a_iGReg, a_u64Value) *iemGRegRefU64(pVCpu, (a_iGReg)) \|= (a_u64Value)
372
373	#define IEM_MC_BSWAP_LOCAL_U16(a_u16Local) (a_u16Local) = RT_BSWAP_U16((a_u16Local));
374	#define IEM_MC_BSWAP_LOCAL_U32(a_u32Local) (a_u32Local) = RT_BSWAP_U32((a_u32Local));
375	#define IEM_MC_BSWAP_LOCAL_U64(a_u64Local) (a_u64Local) = RT_BSWAP_U64((a_u64Local));
376
377	/** @note Not for IOPL or IF modification. */
378	#define IEM_MC_SET_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u \|= (a_fBit); } while (0)
379	/** @note Not for IOPL or IF modification. */
380	#define IEM_MC_CLEAR_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u &= ~(a_fBit); } while (0)
381	/** @note Not for IOPL or IF modification. */
382	#define IEM_MC_FLIP_EFL_BIT(a_fBit) do { pVCpu->cpum.GstCtx.eflags.u ^= (a_fBit); } while (0)
383
384	#define IEM_MC_CLEAR_FSW_EX() do { pVCpu->cpum.GstCtx.XState.x87.FSW &= X86_FSW_C_MASK \| X86_FSW_TOP_MASK; } while (0)
385
386	/** Switches the FPU state to MMX mode (FSW.TOS=0, FTW=0) if necessary. */
387	#define IEM_MC_FPU_TO_MMX_MODE() do { \
388	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
389	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
390	pVCpu->cpum.GstCtx.XState.x87.FTW = 0xff; \
391	} while (0)
392
393	/** Switches the FPU state from MMX mode (FSW.TOS=0, FTW=0xffff). */
394	#define IEM_MC_FPU_FROM_MMX_MODE() do { \
395	iemFpuRotateStackSetTop(&pVCpu->cpum.GstCtx.XState.x87, 0); \
396	pVCpu->cpum.GstCtx.XState.x87.FSW &= ~X86_FSW_TOP_MASK; \
397	pVCpu->cpum.GstCtx.XState.x87.FTW = 0; \
398	} while (0)
399
400	#define IEM_MC_FETCH_MREG_U64(a_u64Value, a_iMReg) \
401	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx; } while (0)
402	#define IEM_MC_FETCH_MREG_U32(a_u32Value, a_iMReg) \
403	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[0]; } while (0)
404	#define IEM_MC_STORE_MREG_U64(a_iMReg, a_u64Value) do { \
405	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (a_u64Value); \
406	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
407	} while (0)
408	#define IEM_MC_STORE_MREG_U32_ZX_U64(a_iMReg, a_u32Value) do { \
409	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx = (uint32_t)(a_u32Value); \
410	pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; \
411	} while (0)
412	#define IEM_MC_REF_MREG_U64(a_pu64Dst, a_iMReg) /** @todo need to set high word to 0xffff on commit (see IEM_MC_STORE_MREG_U64) */ \
413	(a_pu64Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
414	#define IEM_MC_REF_MREG_U64_CONST(a_pu64Dst, a_iMReg) \
415	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
416	#define IEM_MC_REF_MREG_U32_CONST(a_pu32Dst, a_iMReg) \
417	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].mmx)
418	#define IEM_MC_MODIFIED_MREG(a_iMReg) \
419	do { pVCpu->cpum.GstCtx.XState.x87.aRegs[(a_iMReg)].au32[2] = 0xffff; } while (0)
420	#define IEM_MC_MODIFIED_MREG_BY_REF(a_pu64Dst) \
421	do { ((uint32_t *)(a_pu64Dst))[2] = 0xffff; } while (0)
422
423	#define IEM_MC_CLEAR_XREG_U32_MASK(a_iXReg, a_bMask) \
424	do { if ((a_bMask) & (1 << 0)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0] = 0; \
425	if ((a_bMask) & (1 << 1)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[1] = 0; \
426	if ((a_bMask) & (1 << 2)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[2] = 0; \
427	if ((a_bMask) & (1 << 3)) pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[3] = 0; \
428	} while (0)
429	#define IEM_MC_FETCH_XREG_U128(a_u128Value, a_iXReg) \
430	do { (a_u128Value).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
431	(a_u128Value).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
432	} while (0)
433	#define IEM_MC_FETCH_XREG_XMM(a_XmmValue, a_iXReg) \
434	do { (a_XmmValue).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0]; \
435	(a_XmmValue).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1]; \
436	} while (0)
437	#define IEM_MC_FETCH_XREG_U64(a_u64Value, a_iXReg, a_iQWord) \
438	do { (a_u64Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQWord)]; } while (0)
439	#define IEM_MC_FETCH_XREG_U32(a_u32Value, a_iXReg, a_iDWord) \
440	do { (a_u32Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDWord)]; } while (0)
441	#define IEM_MC_FETCH_XREG_U16(a_u16Value, a_iXReg, a_iWord) \
442	do { (a_u16Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iWord)]; } while (0)
443	#define IEM_MC_FETCH_XREG_U8( a_u8Value, a_iXReg, a_iByte) \
444	do { (a_u8Value) = pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au16[(a_iByte)]; } while (0)
445	#define IEM_MC_STORE_XREG_U128(a_iXReg, a_u128Value) \
446	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u128Value).au64[0]; \
447	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u128Value).au64[1]; \
448	} while (0)
449	#define IEM_MC_STORE_XREG_XMM(a_iXReg, a_XmmValue) \
450	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_XmmValue).au64[0]; \
451	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_XmmValue).au64[1]; \
452	} while (0)
453	#define IEM_MC_STORE_XREG_XMM_U32(a_iXReg, a_iDword, a_XmmValue) \
454	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_XmmValue).au32[(a_iDword)]; } while (0)
455	#define IEM_MC_STORE_XREG_XMM_U64(a_iXReg, a_iQword, a_XmmValue) \
456	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_XmmValue).au64[(a_iQword)]; } while (0)
457	#define IEM_MC_STORE_XREG_U64(a_iXReg, a_iQword, a_u64Value) \
458	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[(a_iQword)] = (a_u64Value); } while (0)
459	#define IEM_MC_STORE_XREG_U32(a_iXReg, a_iDword, a_u32Value) \
460	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDword)] = (a_u32Value); } while (0)
461	#define IEM_MC_STORE_XREG_U16(a_iXReg, a_iWord, a_u16Value) \
462	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iWord)] = (a_u16Value); } while (0)
463	#define IEM_MC_STORE_XREG_U8(a_iXReg, a_iByte, a_u8Value) \
464	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iByte)] = (a_u8Value); } while (0)
465
466	#define IEM_MC_STORE_XREG_U64_ZX_U128(a_iXReg, a_u64Value) \
467	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (a_u64Value); \
468	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
469	} while (0)
470
471	#define IEM_MC_STORE_XREG_U32_U128(a_iXReg, a_iDwDst, a_u128Value, a_iDwSrc) \
472	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[(a_iDwDst)] = (a_u128Value).au32[(a_iDwSrc)]; } while (0)
473	#define IEM_MC_STORE_XREG_R32(a_iXReg, a_r32Value) \
474	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0] = (a_r32Value); } while (0)
475	#define IEM_MC_STORE_XREG_R64(a_iXReg, a_r64Value) \
476	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0] = (a_r64Value); } while (0)
477	#define IEM_MC_STORE_XREG_U32_ZX_U128(a_iXReg, a_u32Value) \
478	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0] = (uint32_t)(a_u32Value); \
479	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = 0; \
480	} while (0)
481	#define IEM_MC_STORE_XREG_HI_U64(a_iXReg, a_u64Value) \
482	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[1] = (a_u64Value); } while (0)
483
484	#define IEM_MC_BROADCAST_XREG_U8_ZX_VLMAX(a_iXRegDst, a_u8Src) \
485	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
486	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[0] = (a_u8Src); \
487	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[1] = (a_u8Src); \
488	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[2] = (a_u8Src); \
489	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[3] = (a_u8Src); \
490	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[4] = (a_u8Src); \
491	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[5] = (a_u8Src); \
492	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[6] = (a_u8Src); \
493	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[7] = (a_u8Src); \
494	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[8] = (a_u8Src); \
495	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[9] = (a_u8Src); \
496	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[10] = (a_u8Src); \
497	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[11] = (a_u8Src); \
498	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[12] = (a_u8Src); \
499	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[13] = (a_u8Src); \
500	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[14] = (a_u8Src); \
501	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au8[15] = (a_u8Src); \
502	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
503	} while (0)
504	#define IEM_MC_BROADCAST_XREG_U16_ZX_VLMAX(a_iXRegDst, a_u16Src) \
505	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
506	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[0] = (a_u16Src); \
507	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[1] = (a_u16Src); \
508	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[2] = (a_u16Src); \
509	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[3] = (a_u16Src); \
510	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[4] = (a_u16Src); \
511	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[5] = (a_u16Src); \
512	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[6] = (a_u16Src); \
513	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au16[7] = (a_u16Src); \
514	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
515	} while (0)
516	#define IEM_MC_BROADCAST_XREG_U32_ZX_VLMAX(a_iXRegDst, a_u32Src) \
517	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
518	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[0] = (a_u32Src); \
519	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[1] = (a_u32Src); \
520	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[2] = (a_u32Src); \
521	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au32[3] = (a_u32Src); \
522	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
523	} while (0)
524	#define IEM_MC_BROADCAST_XREG_U64_ZX_VLMAX(a_iXRegDst, a_u64Src) \
525	do { uintptr_t const iXRegDstTmp = (a_iXRegDst); \
526	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[0] = (a_u64Src); \
527	pVCpu->cpum.GstCtx.XState.x87.aXMM[iXRegDstTmp].au64[1] = (a_u64Src); \
528	IEM_MC_CLEAR_YREG_128_UP(iXRegDstTmp); \
529	} while (0)
530
531	#define IEM_MC_REF_XREG_U128(a_pu128Dst, a_iXReg) \
532	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
533	#define IEM_MC_REF_XREG_U128_CONST(a_pu128Dst, a_iXReg) \
534	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].uXmm)
535	#define IEM_MC_REF_XREG_XMM_CONST(a_pXmmDst, a_iXReg) \
536	(a_pXmmDst) = (&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)])
537	#define IEM_MC_REF_XREG_U32_CONST(a_pu32Dst, a_iXReg) \
538	(a_pu32Dst) = ((uint32_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au32[0])
539	#define IEM_MC_REF_XREG_U64_CONST(a_pu64Dst, a_iXReg) \
540	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].au64[0])
541	#define IEM_MC_REF_XREG_R32_CONST(a_pr32Dst, a_iXReg) \
542	(a_pr32Dst) = ((RTFLOAT32U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar32[0])
543	#define IEM_MC_REF_XREG_R64_CONST(a_pr64Dst, a_iXReg) \
544	(a_pr64Dst) = ((RTFLOAT64U const *)&pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXReg)].ar64[0])
545	#define IEM_MC_COPY_XREG_U128(a_iXRegDst, a_iXRegSrc) \
546	do { pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[0] \
547	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[0]; \
548	pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegDst)].au64[1] \
549	= pVCpu->cpum.GstCtx.XState.x87.aXMM[(a_iXRegSrc)].au64[1]; \
550	} while (0)
551
552	#define IEM_MC_FETCH_YREG_U32(a_u32Dst, a_iYRegSrc) \
553	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
554	(a_u32Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au32[0]; \
555	} while (0)
556	#define IEM_MC_FETCH_YREG_U64(a_u64Dst, a_iYRegSrc) \
557	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
558	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
559	} while (0)
560	#define IEM_MC_FETCH_YREG_2ND_U64(a_u64Dst, a_iYRegSrc) \
561	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
562	(a_u64Dst) = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
563	} while (0)
564	#define IEM_MC_FETCH_YREG_U128(a_u128Dst, a_iYRegSrc) \
565	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
566	(a_u128Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
567	(a_u128Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
568	} while (0)
569	#define IEM_MC_FETCH_YREG_U256(a_u256Dst, a_iYRegSrc) \
570	do { uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
571	(a_u256Dst).au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
572	(a_u256Dst).au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
573	(a_u256Dst).au64[2] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
574	(a_u256Dst).au64[3] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
575	} while (0)
576
577	#define IEM_MC_STORE_YREG_U128(a_iYRegDst, a_iDQword, a_u128Value) \
578	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
579	if ((a_iDQword) == 0) \
580	{ \
581	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
582	pVCpu->cpum.GstCtx.XState.x87.aXMM[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
583	} \
584	else \
585	{ \
586	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[0] = (a_u128Value).au64[0]; \
587	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[(iYRegDstTmp)].au64[1] = (a_u128Value).au64[1]; \
588	} \
589	} while (0)
590
591	#define IEM_MC_INT_CLEAR_ZMM_256_UP(a_iXRegDst) do { /* For AVX512 and AVX1024 support. */ } while (0)
592	#define IEM_MC_STORE_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
593	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
594	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
595	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = 0; \
596	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
597	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
598	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
599	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
600	} while (0)
601	#define IEM_MC_STORE_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
602	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
603	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
604	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
605	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
606	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
607	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
608	} while (0)
609	#define IEM_MC_STORE_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
610	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
611	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
612	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
613	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
614	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
615	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
616	} while (0)
617	#define IEM_MC_STORE_YREG_U256_ZX_VLMAX(a_iYRegDst, a_u256Src) \
618	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
619	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u256Src).au64[0]; \
620	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u256Src).au64[1]; \
621	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u256Src).au64[2]; \
622	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u256Src).au64[3]; \
623	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
624	} while (0)
625
626	#define IEM_MC_BROADCAST_YREG_U8_ZX_VLMAX(a_iYRegDst, a_u8Src) \
627	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
628	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[0] = (a_u8Src); \
629	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[1] = (a_u8Src); \
630	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[2] = (a_u8Src); \
631	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[3] = (a_u8Src); \
632	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[4] = (a_u8Src); \
633	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[5] = (a_u8Src); \
634	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[6] = (a_u8Src); \
635	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[7] = (a_u8Src); \
636	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[8] = (a_u8Src); \
637	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[9] = (a_u8Src); \
638	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[10] = (a_u8Src); \
639	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[11] = (a_u8Src); \
640	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[12] = (a_u8Src); \
641	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[13] = (a_u8Src); \
642	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[14] = (a_u8Src); \
643	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au8[15] = (a_u8Src); \
644	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[0] = (a_u8Src); \
645	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[1] = (a_u8Src); \
646	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[2] = (a_u8Src); \
647	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[3] = (a_u8Src); \
648	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[4] = (a_u8Src); \
649	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[5] = (a_u8Src); \
650	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[6] = (a_u8Src); \
651	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[7] = (a_u8Src); \
652	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[8] = (a_u8Src); \
653	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[9] = (a_u8Src); \
654	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[10] = (a_u8Src); \
655	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[11] = (a_u8Src); \
656	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[12] = (a_u8Src); \
657	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[13] = (a_u8Src); \
658	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[14] = (a_u8Src); \
659	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au8[15] = (a_u8Src); \
660	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
661	} while (0)
662	#define IEM_MC_BROADCAST_YREG_U16_ZX_VLMAX(a_iYRegDst, a_u16Src) \
663	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
664	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[0] = (a_u16Src); \
665	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[1] = (a_u16Src); \
666	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[2] = (a_u16Src); \
667	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[3] = (a_u16Src); \
668	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[4] = (a_u16Src); \
669	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[5] = (a_u16Src); \
670	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[6] = (a_u16Src); \
671	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au16[7] = (a_u16Src); \
672	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[0] = (a_u16Src); \
673	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[1] = (a_u16Src); \
674	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[2] = (a_u16Src); \
675	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[3] = (a_u16Src); \
676	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[4] = (a_u16Src); \
677	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[5] = (a_u16Src); \
678	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[6] = (a_u16Src); \
679	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au16[7] = (a_u16Src); \
680	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
681	} while (0)
682	#define IEM_MC_BROADCAST_YREG_U32_ZX_VLMAX(a_iYRegDst, a_u32Src) \
683	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
684	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = (a_u32Src); \
685	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = (a_u32Src); \
686	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[2] = (a_u32Src); \
687	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[3] = (a_u32Src); \
688	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[0] = (a_u32Src); \
689	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[1] = (a_u32Src); \
690	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[2] = (a_u32Src); \
691	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au32[3] = (a_u32Src); \
692	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
693	} while (0)
694	#define IEM_MC_BROADCAST_YREG_U64_ZX_VLMAX(a_iYRegDst, a_u64Src) \
695	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
696	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Src); \
697	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Src); \
698	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u64Src); \
699	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u64Src); \
700	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
701	} while (0)
702	#define IEM_MC_BROADCAST_YREG_U128_ZX_VLMAX(a_iYRegDst, a_u128Src) \
703	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
704	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
705	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
706	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = (a_u128Src).au64[0]; \
707	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = (a_u128Src).au64[1]; \
708	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
709	} while (0)
710
711	#define IEM_MC_REF_YREG_U128(a_pu128Dst, a_iYReg) \
712	(a_pu128Dst) = (&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
713	#define IEM_MC_REF_YREG_U128_CONST(a_pu128Dst, a_iYReg) \
714	(a_pu128Dst) = ((PCRTUINT128U)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].uXmm)
715	#define IEM_MC_REF_YREG_U64_CONST(a_pu64Dst, a_iYReg) \
716	(a_pu64Dst) = ((uint64_t const *)&pVCpu->cpum.GstCtx.XState.x87.aYMM[(a_iYReg)].au64[0])
717	#define IEM_MC_CLEAR_YREG_128_UP(a_iYReg) \
718	do { uintptr_t const iYRegTmp = (a_iYReg); \
719	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[0] = 0; \
720	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegTmp].au64[1] = 0; \
721	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegTmp); \
722	} while (0)
723
724	#define IEM_MC_COPY_YREG_U256_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
725	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
726	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
727	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
728	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
729	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[0]; \
730	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegSrcTmp].au64[1]; \
731	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
732	} while (0)
733	#define IEM_MC_COPY_YREG_U128_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
734	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
735	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
736	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
737	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[1]; \
738	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
739	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
740	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
741	} while (0)
742	#define IEM_MC_COPY_YREG_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc) \
743	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
744	uintptr_t const iYRegSrcTmp = (a_iYRegSrc); \
745	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcTmp].au64[0]; \
746	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = 0; \
747	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
748	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
749	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
750	} while (0)
751
752	#define IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(a_iYRegDst, a_iYRegSrc32, a_iYRegSrcHx) \
753	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
754	uintptr_t const iYRegSrc32Tmp = (a_iYRegSrc32); \
755	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
756	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc32Tmp].au32[0]; \
757	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au32[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au32[1]; \
758	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
759	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
760	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
761	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
762	} while (0)
763	#define IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) \
764	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
765	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
766	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
767	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
768	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
769	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
770	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
771	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
772	} while (0)
773	#define IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
774	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
775	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
776	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
777	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[0]; \
778	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
779	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
780	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
781	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
782	} while (0)
783	#define IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(a_iYRegDst, a_iYRegSrc64, a_iYRegSrcHx) /* for vmovhlps */ \
784	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
785	uintptr_t const iYRegSrc64Tmp = (a_iYRegSrc64); \
786	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
787	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrc64Tmp].au64[1]; \
788	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
789	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
790	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
791	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
792	} while (0)
793	#define IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(a_iYRegDst, a_iYRegSrcHx, a_u64Local) \
794	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
795	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
796	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[0]; \
797	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = (a_u64Local); \
798	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
799	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
800	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
801	} while (0)
802	#define IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(a_iYRegDst, a_u64Local, a_iYRegSrcHx) \
803	do { uintptr_t const iYRegDstTmp = (a_iYRegDst); \
804	uintptr_t const iYRegSrcHxTmp = (a_iYRegSrcHx); \
805	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[0] = (a_u64Local); \
806	pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegDstTmp].au64[1] = pVCpu->cpum.GstCtx.XState.x87.aXMM[iYRegSrcHxTmp].au64[1]; \
807	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[0] = 0; \
808	pVCpu->cpum.GstCtx.XState.u.YmmHi.aYmmHi[iYRegDstTmp].au64[1] = 0; \
809	IEM_MC_INT_CLEAR_ZMM_256_UP(iYRegDstTmp); \
810	} while (0)
811
812	#ifndef IEM_WITH_SETJMP
813	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
814	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem)))
815	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
816	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem16)))
817	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
818	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &(a_u8Dst), (a_iSeg), (a_GCPtrMem32)))
819	#else
820	# define IEM_MC_FETCH_MEM_U8(a_u8Dst, a_iSeg, a_GCPtrMem) \
821	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
822	# define IEM_MC_FETCH_MEM16_U8(a_u8Dst, a_iSeg, a_GCPtrMem16) \
823	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem16)))
824	# define IEM_MC_FETCH_MEM32_U8(a_u8Dst, a_iSeg, a_GCPtrMem32) \
825	((a_u8Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem32)))
826
827	# define IEM_MC_FETCH_MEM_FLAT_U8(a_u8Dst, a_GCPtrMem) \
828	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
829	# define IEM_MC_FETCH_MEM16_FLAT_U8(a_u8Dst, a_GCPtrMem16) \
830	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem16)))
831	# define IEM_MC_FETCH_MEM32_FLAT_U8(a_u8Dst, a_GCPtrMem32) \
832	((a_u8Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem32)))
833	#endif
834
835	#ifndef IEM_WITH_SETJMP
836	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
837	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem)))
838	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
839	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &(a_u16Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
840	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
841	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, (uint16_t *)&(a_i16Dst), (a_iSeg), (a_GCPtrMem)))
842	#else
843	# define IEM_MC_FETCH_MEM_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
844	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
845	# define IEM_MC_FETCH_MEM_U16_DISP(a_u16Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
846	((a_u16Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
847	# define IEM_MC_FETCH_MEM_I16(a_i16Dst, a_iSeg, a_GCPtrMem) \
848	((a_i16Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
849
850	# define IEM_MC_FETCH_MEM_FLAT_U16(a_u16Dst, a_GCPtrMem) \
851	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
852	# define IEM_MC_FETCH_MEM_FLAT_U16_DISP(a_u16Dst, a_GCPtrMem, a_offDisp) \
853	((a_u16Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
854	# define IEM_MC_FETCH_MEM_FLAT_I16(a_i16Dst, a_GCPtrMem) \
855	((a_i16Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
856	#endif
857
858	#ifndef IEM_WITH_SETJMP
859	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
860	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem)))
861	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
862	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_u32Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
863	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
864	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, (uint32_t *)&(a_i32Dst), (a_iSeg), (a_GCPtrMem)))
865	#else
866	# define IEM_MC_FETCH_MEM_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
867	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
868	# define IEM_MC_FETCH_MEM_U32_DISP(a_u32Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
869	((a_u32Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
870	# define IEM_MC_FETCH_MEM_I32(a_i32Dst, a_iSeg, a_GCPtrMem) \
871	((a_i32Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
872
873	# define IEM_MC_FETCH_MEM_FLAT_U32(a_u32Dst, a_GCPtrMem) \
874	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
875	# define IEM_MC_FETCH_MEM_FLAT_U32_DISP(a_u32Dst, a_GCPtrMem, a_offDisp) \
876	((a_u32Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
877	# define IEM_MC_FETCH_MEM_FLAT_I32(a_i32Dst, a_GCPtrMem) \
878	((a_i32Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
879	#endif
880
881	#ifdef SOME_UNUSED_FUNCTION
882	# define IEM_MC_FETCH_MEM_S32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
883	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataS32SxU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
884	#endif
885
886	#ifndef IEM_WITH_SETJMP
887	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
888	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
889	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
890	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
891	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
892	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64AlignedU128(pVCpu, &(a_u64Dst), (a_iSeg), (a_GCPtrMem)))
893	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
894	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, (uint64_t *)&(a_i64Dst), (a_iSeg), (a_GCPtrMem)))
895	#else
896	# define IEM_MC_FETCH_MEM_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
897	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
898	# define IEM_MC_FETCH_MEM_U64_DISP(a_u64Dst, a_iSeg, a_GCPtrMem, a_offDisp) \
899	((a_u64Dst) = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem) + (a_offDisp)))
900	# define IEM_MC_FETCH_MEM_U64_ALIGN_U128(a_u64Dst, a_iSeg, a_GCPtrMem) \
901	((a_u64Dst) = iemMemFetchDataU64AlignedU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
902	# define IEM_MC_FETCH_MEM_I64(a_i64Dst, a_iSeg, a_GCPtrMem) \
903	((a_i64Dst) = (int64_t)iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
904
905	# define IEM_MC_FETCH_MEM_FLAT_U64(a_u64Dst, a_GCPtrMem) \
906	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
907	# define IEM_MC_FETCH_MEM_FLAT_U64_DISP(a_u64Dst, a_GCPtrMem, a_offDisp) \
908	((a_u64Dst) = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem) + (a_offDisp)))
909	# define IEM_MC_FETCH_MEM_FLAT_U64_ALIGN_U128(a_u64Dst, a_GCPtrMem) \
910	((a_u64Dst) = iemMemFlatFetchDataU64AlignedU128Jmp(pVCpu, (a_GCPtrMem)))
911	# define IEM_MC_FETCH_MEM_FLAT_I64(a_i64Dst, a_GCPtrMem) \
912	((a_i64Dst) = (int64_t)iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
913	#endif
914
915	#ifndef IEM_WITH_SETJMP
916	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
917	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_r32Dst).u, (a_iSeg), (a_GCPtrMem)))
918	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
919	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_r64Dst).u, (a_iSeg), (a_GCPtrMem)))
920	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
921	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataR80(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem)))
922	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
923	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataD80(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem)))
924	#else
925	# define IEM_MC_FETCH_MEM_R32(a_r32Dst, a_iSeg, a_GCPtrMem) \
926	((a_r32Dst).u = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
927	# define IEM_MC_FETCH_MEM_R64(a_r64Dst, a_iSeg, a_GCPtrMem) \
928	((a_r64Dst).u = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
929	# define IEM_MC_FETCH_MEM_R80(a_r80Dst, a_iSeg, a_GCPtrMem) \
930	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), (a_iSeg), (a_GCPtrMem))
931	# define IEM_MC_FETCH_MEM_D80(a_d80Dst, a_iSeg, a_GCPtrMem) \
932	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), (a_iSeg), (a_GCPtrMem))
933
934	# define IEM_MC_FETCH_MEM_FLAT_R32(a_r32Dst, a_GCPtrMem) \
935	((a_r32Dst).u = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
936	# define IEM_MC_FETCH_MEM_FLAT_R64(a_r64Dst, a_GCPtrMem) \
937	((a_r64Dst).u = iemMemFlatFetchDataU64Jmp(pVCpu, (a_GCPtrMem)))
938	# define IEM_MC_FETCH_MEM_FLAT_R80(a_r80Dst, a_GCPtrMem) \
939	iemMemFetchDataR80Jmp(pVCpu, &(a_r80Dst), UINT8_MAX, (a_GCPtrMem))
940	# define IEM_MC_FETCH_MEM_FLAT_D80(a_d80Dst, a_GCPtrMem) \
941	iemMemFetchDataD80Jmp(pVCpu, &(a_d80Dst), UINT8_MAX, (a_GCPtrMem))
942	#endif
943
944	#ifndef IEM_WITH_SETJMP
945	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
946	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
947	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
948	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
949	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
950	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem)))
951
952	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
953	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
954	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
955	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
956	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
957	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU128AlignedSse(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem)))
958	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
959	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &(a_XmmDst).au32[(a_iDWord)], (a_iSeg), (a_GCPtrMem)))
960	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
961	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU64(pVCpu, &(a_XmmDst).au64[(a_iQWord)], (a_iSeg), (a_GCPtrMem)))
962	#else
963	# define IEM_MC_FETCH_MEM_U128(a_u128Dst, a_iSeg, a_GCPtrMem) \
964	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
965	# define IEM_MC_FETCH_MEM_U128_NO_AC(a_u128Dst, a_iSeg, a_GCPtrMem) \
966	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
967	# define IEM_MC_FETCH_MEM_U128_ALIGN_SSE(a_u128Dst, a_iSeg, a_GCPtrMem) \
968	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), (a_iSeg), (a_GCPtrMem))
969
970	# define IEM_MC_FETCH_MEM_XMM(a_XmmDst, a_iSeg, a_GCPtrMem) \
971	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
972	# define IEM_MC_FETCH_MEM_XMM_NO_AC(a_XmmDst, a_iSeg, a_GCPtrMem) \
973	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
974	# define IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(a_XmmDst, a_iSeg, a_GCPtrMem) \
975	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, (a_iSeg), (a_GCPtrMem))
976	# define IEM_MC_FETCH_MEM_XMM_U32(a_XmmDst, a_iDWord, a_iSeg, a_GCPtrMem) \
977	(a_XmmDst).au32[(a_iDWord)] = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
978	# define IEM_MC_FETCH_MEM_XMM_U64(a_XmmDst, a_iQWord, a_iSeg, a_GCPtrMem) \
979	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem))
980
981	# define IEM_MC_FETCH_MEM_FLAT_U128(a_u128Dst, a_GCPtrMem) \
982	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
983	# define IEM_MC_FETCH_MEM_FLAT_U128_NO_AC(a_u128Dst, a_GCPtrMem) \
984	iemMemFetchDataU128Jmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
985	# define IEM_MC_FETCH_MEM_FLAT_U128_ALIGN_SSE(a_u128Dst, a_GCPtrMem) \
986	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_u128Dst), UINT8_MAX, (a_GCPtrMem))
987
988	# define IEM_MC_FETCH_MEM_FLAT_XMM(a_XmmDst, a_GCPtrMem) \
989	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
990	# define IEM_MC_FETCH_MEM_FLAT_XMM_NO_AC(a_XmmDst, a_GCPtrMem) \
991	iemMemFetchDataU128Jmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
992	# define IEM_MC_FETCH_MEM_FLAT_XMM_ALIGN_SSE(a_XmmDst, a_GCPtrMem) \
993	iemMemFetchDataU128AlignedSseJmp(pVCpu, &(a_XmmDst).uXmm, UINT8_MAX, (a_GCPtrMem))
994	# define IEM_MC_FETCH_MEM_FLAT_XMM_U32(a_XmmDst, a_iDWord, a_GCPtrMem) \
995	(a_XmmDst).au32[(a_iDWord)] = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem))
996	# define IEM_MC_FETCH_MEM_FLAT_XMM_U64(a_XmmDst, a_iQWord, a_GCPtrMem) \
997	(a_XmmDst).au64[(a_iQWord)] = iemMemFetchDataU64Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem))
998	#endif
999
1000	#ifndef IEM_WITH_SETJMP
1001	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1002	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1003	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1004	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1005	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1006	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem)))
1007
1008	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1009	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1010	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1011	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1012	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1013	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU256AlignedSse(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem)))
1014	#else
1015	# define IEM_MC_FETCH_MEM_U256(a_u256Dst, a_iSeg, a_GCPtrMem) \
1016	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1017	# define IEM_MC_FETCH_MEM_U256_NO_AC(a_u256Dst, a_iSeg, a_GCPtrMem) \
1018	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1019	# define IEM_MC_FETCH_MEM_U256_ALIGN_AVX(a_u256Dst, a_iSeg, a_GCPtrMem) \
1020	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), (a_iSeg), (a_GCPtrMem))
1021
1022	# define IEM_MC_FETCH_MEM_YMM(a_YmmDst, a_iSeg, a_GCPtrMem) \
1023	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1024	# define IEM_MC_FETCH_MEM_YMM_NO_AC(a_YmmDst, a_iSeg, a_GCPtrMem) \
1025	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1026	# define IEM_MC_FETCH_MEM_YMM_ALIGN_AVX(a_YmmDst, a_iSeg, a_GCPtrMem) \
1027	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, (a_iSeg), (a_GCPtrMem))
1028
1029	# define IEM_MC_FETCH_MEM_FLAT_U256(a_u256Dst, a_GCPtrMem) \
1030	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1031	# define IEM_MC_FETCH_MEM_FLAT_U256_NO_AC(a_u256Dst, a_GCPtrMem) \
1032	iemMemFetchDataU256Jmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1033	# define IEM_MC_FETCH_MEM_FLAT_U256_ALIGN_AVX(a_u256Dst, a_GCPtrMem) \
1034	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_u256Dst), UINT8_MAX, (a_GCPtrMem))
1035
1036	# define IEM_MC_FETCH_MEM_FLAT_YMM(a_YmmDst, a_GCPtrMem) \
1037	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1038	# define IEM_MC_FETCH_MEM_FLAT_YMM_NO_AC(a_YmmDst, a_GCPtrMem) \
1039	iemMemFetchDataU256Jmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1040	# define IEM_MC_FETCH_MEM_FLAT_YMM_ALIGN_AVX(a_YmmDst, a_GCPtrMem) \
1041	iemMemFetchDataU256AlignedSseJmp(pVCpu, &(a_YmmDst).ymm, UINT8_MAX, (a_GCPtrMem))
1042	#endif
1043
1044
1045
1046	#ifndef IEM_WITH_SETJMP
1047	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1048	do { \
1049	uint8_t u8Tmp; \
1050	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1051	(a_u16Dst) = u8Tmp; \
1052	} while (0)
1053	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1054	do { \
1055	uint8_t u8Tmp; \
1056	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1057	(a_u32Dst) = u8Tmp; \
1058	} while (0)
1059	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1060	do { \
1061	uint8_t u8Tmp; \
1062	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1063	(a_u64Dst) = u8Tmp; \
1064	} while (0)
1065	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1066	do { \
1067	uint16_t u16Tmp; \
1068	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1069	(a_u32Dst) = u16Tmp; \
1070	} while (0)
1071	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1072	do { \
1073	uint16_t u16Tmp; \
1074	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1075	(a_u64Dst) = u16Tmp; \
1076	} while (0)
1077	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1078	do { \
1079	uint32_t u32Tmp; \
1080	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1081	(a_u64Dst) = u32Tmp; \
1082	} while (0)
1083	#else /* IEM_WITH_SETJMP */
1084	# define IEM_MC_FETCH_MEM_U8_ZX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1085	((a_u16Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1086	# define IEM_MC_FETCH_MEM_U8_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1087	((a_u32Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1088	# define IEM_MC_FETCH_MEM_U8_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1089	((a_u64Dst) = iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1090	# define IEM_MC_FETCH_MEM_U16_ZX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1091	((a_u32Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1092	# define IEM_MC_FETCH_MEM_U16_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1093	((a_u64Dst) = iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1094	# define IEM_MC_FETCH_MEM_U32_ZX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1095	((a_u64Dst) = iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1096
1097	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U16(a_u16Dst, a_GCPtrMem) \
1098	((a_u16Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1099	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U32(a_u32Dst, a_GCPtrMem) \
1100	((a_u32Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1101	# define IEM_MC_FETCH_MEM_FLAT_U8_ZX_U64(a_u64Dst, a_GCPtrMem) \
1102	((a_u64Dst) = iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1103	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U32(a_u32Dst, a_GCPtrMem) \
1104	((a_u32Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1105	# define IEM_MC_FETCH_MEM_FLAT_U16_ZX_U64(a_u64Dst, a_GCPtrMem) \
1106	((a_u64Dst) = iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1107	# define IEM_MC_FETCH_MEM_FLAT_U32_ZX_U64(a_u64Dst, a_GCPtrMem) \
1108	((a_u64Dst) = iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1109	#endif /* IEM_WITH_SETJMP */
1110
1111	#ifndef IEM_WITH_SETJMP
1112	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1113	do { \
1114	uint8_t u8Tmp; \
1115	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1116	(a_u16Dst) = (int8_t)u8Tmp; \
1117	} while (0)
1118	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1119	do { \
1120	uint8_t u8Tmp; \
1121	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1122	(a_u32Dst) = (int8_t)u8Tmp; \
1123	} while (0)
1124	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1125	do { \
1126	uint8_t u8Tmp; \
1127	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU8(pVCpu, &u8Tmp, (a_iSeg), (a_GCPtrMem))); \
1128	(a_u64Dst) = (int8_t)u8Tmp; \
1129	} while (0)
1130	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1131	do { \
1132	uint16_t u16Tmp; \
1133	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1134	(a_u32Dst) = (int16_t)u16Tmp; \
1135	} while (0)
1136	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1137	do { \
1138	uint16_t u16Tmp; \
1139	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU16(pVCpu, &u16Tmp, (a_iSeg), (a_GCPtrMem))); \
1140	(a_u64Dst) = (int16_t)u16Tmp; \
1141	} while (0)
1142	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1143	do { \
1144	uint32_t u32Tmp; \
1145	IEM_MC_RETURN_ON_FAILURE(iemMemFetchDataU32(pVCpu, &u32Tmp, (a_iSeg), (a_GCPtrMem))); \
1146	(a_u64Dst) = (int32_t)u32Tmp; \
1147	} while (0)
1148	#else /* IEM_WITH_SETJMP */
1149	# define IEM_MC_FETCH_MEM_U8_SX_U16(a_u16Dst, a_iSeg, a_GCPtrMem) \
1150	((a_u16Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1151	# define IEM_MC_FETCH_MEM_U8_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1152	((a_u32Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1153	# define IEM_MC_FETCH_MEM_U8_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1154	((a_u64Dst) = (int8_t)iemMemFetchDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1155	# define IEM_MC_FETCH_MEM_U16_SX_U32(a_u32Dst, a_iSeg, a_GCPtrMem) \
1156	((a_u32Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1157	# define IEM_MC_FETCH_MEM_U16_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1158	((a_u64Dst) = (int16_t)iemMemFetchDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1159	# define IEM_MC_FETCH_MEM_U32_SX_U64(a_u64Dst, a_iSeg, a_GCPtrMem) \
1160	((a_u64Dst) = (int32_t)iemMemFetchDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem)))
1161
1162	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U16(a_u16Dst, a_GCPtrMem) \
1163	((a_u16Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1164	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U32(a_u32Dst, a_GCPtrMem) \
1165	((a_u32Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1166	# define IEM_MC_FETCH_MEM_FLAT_U8_SX_U64(a_u64Dst, a_GCPtrMem) \
1167	((a_u64Dst) = (int8_t)iemMemFlatFetchDataU8Jmp(pVCpu, (a_GCPtrMem)))
1168	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U32(a_u32Dst, a_GCPtrMem) \
1169	((a_u32Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1170	# define IEM_MC_FETCH_MEM_FLAT_U16_SX_U64(a_u64Dst, a_GCPtrMem) \
1171	((a_u64Dst) = (int16_t)iemMemFlatFetchDataU16Jmp(pVCpu, (a_GCPtrMem)))
1172	# define IEM_MC_FETCH_MEM_FLAT_U32_SX_U64(a_u64Dst, a_GCPtrMem) \
1173	((a_u64Dst) = (int32_t)iemMemFlatFetchDataU32Jmp(pVCpu, (a_GCPtrMem)))
1174	#endif /* IEM_WITH_SETJMP */
1175
1176	#ifndef IEM_WITH_SETJMP
1177	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1178	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value)))
1179	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1180	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value)))
1181	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1182	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value)))
1183	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1184	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value)))
1185	#else
1186	# define IEM_MC_STORE_MEM_U8(a_iSeg, a_GCPtrMem, a_u8Value) \
1187	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8Value))
1188	# define IEM_MC_STORE_MEM_U16(a_iSeg, a_GCPtrMem, a_u16Value) \
1189	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16Value))
1190	# define IEM_MC_STORE_MEM_U32(a_iSeg, a_GCPtrMem, a_u32Value) \
1191	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32Value))
1192	# define IEM_MC_STORE_MEM_U64(a_iSeg, a_GCPtrMem, a_u64Value) \
1193	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64Value))
1194
1195	# define IEM_MC_STORE_MEM_FLAT_U8(a_GCPtrMem, a_u8Value) \
1196	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8Value))
1197	# define IEM_MC_STORE_MEM_FLAT_U16(a_GCPtrMem, a_u16Value) \
1198	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16Value))
1199	# define IEM_MC_STORE_MEM_FLAT_U32(a_GCPtrMem, a_u32Value) \
1200	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32Value))
1201	# define IEM_MC_STORE_MEM_FLAT_U64(a_GCPtrMem, a_u64Value) \
1202	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64Value))
1203	#endif
1204
1205	#ifndef IEM_WITH_SETJMP
1206	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1207	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU8(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C)))
1208	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1209	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU16(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C)))
1210	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1211	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU32(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C)))
1212	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1213	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU64(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C)))
1214	#else
1215	# define IEM_MC_STORE_MEM_U8_CONST(a_iSeg, a_GCPtrMem, a_u8C) \
1216	iemMemStoreDataU8Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u8C))
1217	# define IEM_MC_STORE_MEM_U16_CONST(a_iSeg, a_GCPtrMem, a_u16C) \
1218	iemMemStoreDataU16Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u16C))
1219	# define IEM_MC_STORE_MEM_U32_CONST(a_iSeg, a_GCPtrMem, a_u32C) \
1220	iemMemStoreDataU32Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u32C))
1221	# define IEM_MC_STORE_MEM_U64_CONST(a_iSeg, a_GCPtrMem, a_u64C) \
1222	iemMemStoreDataU64Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u64C))
1223
1224	# define IEM_MC_STORE_MEM_FLAT_U8_CONST(a_GCPtrMem, a_u8C) \
1225	iemMemFlatStoreDataU8Jmp(pVCpu, (a_GCPtrMem), (a_u8C))
1226	# define IEM_MC_STORE_MEM_FLAT_U16_CONST(a_GCPtrMem, a_u16C) \
1227	iemMemFlatStoreDataU16Jmp(pVCpu, (a_GCPtrMem), (a_u16C))
1228	# define IEM_MC_STORE_MEM_FLAT_U32_CONST(a_GCPtrMem, a_u32C) \
1229	iemMemFlatStoreDataU32Jmp(pVCpu, (a_GCPtrMem), (a_u32C))
1230	# define IEM_MC_STORE_MEM_FLAT_U64_CONST(a_GCPtrMem, a_u64C) \
1231	iemMemFlatStoreDataU64Jmp(pVCpu, (a_GCPtrMem), (a_u64C))
1232	#endif
1233
1234	#define IEM_MC_STORE_MEM_I8_CONST_BY_REF( a_pi8Dst, a_i8C) *(a_pi8Dst) = (a_i8C)
1235	#define IEM_MC_STORE_MEM_I16_CONST_BY_REF(a_pi16Dst, a_i16C) *(a_pi16Dst) = (a_i16C)
1236	#define IEM_MC_STORE_MEM_I32_CONST_BY_REF(a_pi32Dst, a_i32C) *(a_pi32Dst) = (a_i32C)
1237	#define IEM_MC_STORE_MEM_I64_CONST_BY_REF(a_pi64Dst, a_i64C) *(a_pi64Dst) = (a_i64C)
1238	#define IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(a_pr32Dst) (a_pr32Dst)->u = UINT32_C(0xffc00000)
1239	#define IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(a_pr64Dst) (a_pr64Dst)->u = UINT64_C(0xfff8000000000000)
1240	#define IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(a_pr80Dst) \
1241	do { \
1242	(a_pr80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1243	(a_pr80Dst)->au16[4] = UINT16_C(0xffff); \
1244	} while (0)
1245	#define IEM_MC_STORE_MEM_INDEF_D80_BY_REF(a_pd80Dst) \
1246	do { \
1247	(a_pd80Dst)->au64[0] = UINT64_C(0xc000000000000000); \
1248	(a_pd80Dst)->au16[4] = UINT16_C(0xffff); \
1249	} while (0)
1250
1251	#ifndef IEM_WITH_SETJMP
1252	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1253	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1254	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1255	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU128AlignedSse(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value)))
1256	#else
1257	# define IEM_MC_STORE_MEM_U128(a_iSeg, a_GCPtrMem, a_u128Value) \
1258	iemMemStoreDataU128Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1259	# define IEM_MC_STORE_MEM_U128_ALIGN_SSE(a_iSeg, a_GCPtrMem, a_u128Value) \
1260	iemMemStoreDataU128AlignedSseJmp(pVCpu, (a_iSeg), (a_GCPtrMem), (a_u128Value))
1261
1262	# define IEM_MC_STORE_MEM_FLAT_U128(a_GCPtrMem, a_u128Value) \
1263	iemMemStoreDataU128Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1264	# define IEM_MC_STORE_MEM_FLAT_U128_ALIGN_SSE(a_GCPtrMem, a_u128Value) \
1265	iemMemStoreDataU128AlignedSseJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), (a_u128Value))
1266	#endif
1267
1268	#ifndef IEM_WITH_SETJMP
1269	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1270	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1271	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1272	IEM_MC_RETURN_ON_FAILURE(iemMemStoreDataU256AlignedAvx(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value)))
1273	#else
1274	# define IEM_MC_STORE_MEM_U256(a_iSeg, a_GCPtrMem, a_u256Value) \
1275	iemMemStoreDataU256Jmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1276	# define IEM_MC_STORE_MEM_U256_ALIGN_AVX(a_iSeg, a_GCPtrMem, a_u256Value) \
1277	iemMemStoreDataU256AlignedAvxJmp(pVCpu, (a_iSeg), (a_GCPtrMem), &(a_u256Value))
1278
1279	# define IEM_MC_STORE_MEM_FLAT_U256(a_GCPtrMem, a_u256Value) \
1280	iemMemStoreDataU256Jmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1281	# define IEM_MC_STORE_MEM_FLAT_U256_ALIGN_AVX(a_GCPtrMem, a_u256Value) \
1282	iemMemStoreDataU256AlignedAvxJmp(pVCpu, UINT8_MAX, (a_GCPtrMem), &(a_u256Value))
1283	#endif
1284
1285	/* Regular stack push and pop: */
1286	#ifndef IEM_WITH_SETJMP
1287	# define IEM_MC_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1288	# define IEM_MC_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1289	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1290	# define IEM_MC_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1291
1292	# define IEM_MC_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1293	# define IEM_MC_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1294	# define IEM_MC_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1295	#else
1296	# define IEM_MC_PUSH_U16(a_u16Value) iemMemStackPushU16Jmp(pVCpu, (a_u16Value))
1297	# define IEM_MC_PUSH_U32(a_u32Value) iemMemStackPushU32Jmp(pVCpu, (a_u32Value))
1298	# define IEM_MC_PUSH_U32_SREG(a_uSegVal) iemMemStackPushU32SRegJmp(pVCpu, (a_uSegVal))
1299	# define IEM_MC_PUSH_U64(a_u64Value) iemMemStackPushU64Jmp(pVCpu, (a_u64Value))
1300
1301	# define IEM_MC_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemStackPopU16Jmp(pVCpu))
1302	# define IEM_MC_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemStackPopU32Jmp(pVCpu))
1303	# define IEM_MC_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemStackPopU64Jmp(pVCpu))
1304	#endif
1305
1306	/* 32-bit flat stack push and pop: */
1307	#ifndef IEM_WITH_SETJMP
1308	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1309	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32(pVCpu, (a_u32Value)))
1310	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU32SReg(pVCpu, (a_uSegVal)))
1311
1312	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1313	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU32(pVCpu, (a_pu32Value)))
1314	#else
1315	# define IEM_MC_FLAT32_PUSH_U16(a_u16Value) iemMemFlat32StackPushU16Jmp(pVCpu, (a_u16Value))
1316	# define IEM_MC_FLAT32_PUSH_U32(a_u32Value) iemMemFlat32StackPushU32Jmp(pVCpu, (a_u32Value))
1317	# define IEM_MC_FLAT32_PUSH_U32_SREG(a_uSegVal) iemMemFlat32StackPushU32SRegJmp(pVCpu, (a_uSegVal))
1318
1319	# define IEM_MC_FLAT32_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat32StackPopU16Jmp(pVCpu))
1320	# define IEM_MC_FLAT32_POP_U32(a_pu32Value) (*(a_pu32Value) = iemMemFlat32StackPopU32Jmp(pVCpu))
1321	#endif
1322
1323	/* 64-bit flat stack push and pop: */
1324	#ifndef IEM_WITH_SETJMP
1325	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU16(pVCpu, (a_u16Value)))
1326	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPushU64(pVCpu, (a_u64Value)))
1327
1328	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU16(pVCpu, (a_pu16Value)))
1329	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) IEM_MC_RETURN_ON_FAILURE(iemMemStackPopU64(pVCpu, (a_pu64Value)))
1330	#else
1331	# define IEM_MC_FLAT64_PUSH_U16(a_u16Value) iemMemFlat64StackPushU16Jmp(pVCpu, (a_u16Value))
1332	# define IEM_MC_FLAT64_PUSH_U64(a_u64Value) iemMemFlat64StackPushU64Jmp(pVCpu, (a_u64Value))
1333
1334	# define IEM_MC_FLAT64_POP_U16(a_pu16Value) (*(a_pu16Value) = iemMemFlat64StackPopU16Jmp(pVCpu))
1335	# define IEM_MC_FLAT64_POP_U64(a_pu64Value) (*(a_pu64Value) = iemMemFlat64StackPopU64Jmp(pVCpu))
1336	#endif
1337
1338
1339	/** Maps guest memory for direct or bounce buffered access.
1340	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1341	* @remarks May return.
1342	* @deprecated
1343	*/
1344	#define IEM_MC_MEM_MAP(a_pMem, a_fAccess, a_iSeg, a_GCPtrMem, a_iArg) \
1345	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), (a_iSeg), \
1346	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1347
1348	/** Flat variant of IEM_MC_MEM_MAP.
1349	* @deprecated
1350	*/
1351	#define IEM_MC_MEM_FLAT_MAP(a_pMem, a_fAccess, a_GCPtrMem, a_iArg) \
1352	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void *)&(a_pMem), sizeof((a_pMem)), UINT8_MAX, \
1353	(a_GCPtrMem), (a_fAccess), sizeof(*(a_pMem)) - 1))
1354
1355	/** Maps guest memory for direct or bounce buffered access.
1356	* The purpose is to pass it to an operand implementation, thus the a_iArg.
1357	* @remarks May return.
1358	* @deprecated
1359	*/
1360	#define IEM_MC_MEM_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_iSeg, a_GCPtrMem, a_cbAlign, a_iArg) \
1361	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), (a_iSeg), \
1362	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1363
1364	/** Flat variant of IEM_MC_MEM_MAP_EX.
1365	* @deprecated
1366	*/
1367	#define IEM_MC_MEM_FLAT_MAP_EX(a_pvMem, a_fAccess, a_cbMem, a_GCPtrMem, a_cbAlign, a_iArg) \
1368	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pvMem), (a_cbMem), UINT8_MAX, \
1369	(a_GCPtrMem), (a_fAccess), (a_cbAlign)))
1370
1371	/** Commits the memory and unmaps the guest memory.
1372	* @remarks May return.
1373	* @deprecated
1374	*/
1375	#define IEM_MC_MEM_COMMIT_AND_UNMAP(a_pvMem, a_fAccess) \
1376	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess)))
1377
1378
1379	/* 8-bit */
1380
1381	/**
1382	* Maps guest memory for byte read+write direct (or bounce) buffer acccess.
1383	*
1384	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1385	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1386	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1387	* @param[in] a_GCPtrMem The memory address.
1388	* @remarks Will return/long jump on errors.
1389	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1390	*/
1391	#ifndef IEM_WITH_SETJMP
1392	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1393	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1394	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1395	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1396	} while (0)
1397	#else
1398	# define IEM_MC_MEM_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1399	(a_pu8Mem) = iemMemMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1400	#endif
1401
1402	/**
1403	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess.
1404	*
1405	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1406	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1407	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1408	* @param[in] a_GCPtrMem The memory address.
1409	* @remarks Will return/long jump on errors.
1410	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1411	*/
1412	#ifndef IEM_WITH_SETJMP
1413	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1414	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1415	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1416	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1417	} while (0)
1418	#else
1419	# define IEM_MC_MEM_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1420	(a_pu8Mem) = iemMemMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1421	#endif
1422
1423	/**
1424	* Maps guest memory for byte readonly direct (or bounce) buffer acccess.
1425	*
1426	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1427	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1428	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1429	* @param[in] a_GCPtrMem The memory address.
1430	* @remarks Will return/long jump on errors.
1431	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1432	*/
1433	#ifndef IEM_WITH_SETJMP
1434	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1435	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), (a_iSeg), \
1436	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1437	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1438	} while (0)
1439	#else
1440	# define IEM_MC_MEM_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1441	(a_pu8Mem) = iemMemMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1442	#endif
1443
1444	/**
1445	* Maps guest memory for byte read+write direct (or bounce) buffer acccess, flat
1446	* address variant.
1447	*
1448	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1449	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1450	* @param[in] a_GCPtrMem The memory address.
1451	* @remarks Will return/long jump on errors.
1452	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1453	*/
1454	#ifndef IEM_WITH_SETJMP
1455	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1456	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1457	(a_GCPtrMem), IEM_ACCESS_DATA_RW, 0)); \
1458	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1459	} while (0)
1460	#else
1461	# define IEM_MC_MEM_FLAT_MAP_U8_RW(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1462	(a_pu8Mem) = iemMemFlatMapDataU8RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1463	#endif
1464
1465	/**
1466	* Maps guest memory for byte writeonly direct (or bounce) buffer acccess, flat
1467	* address variant.
1468	*
1469	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1470	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1471	* @param[in] a_GCPtrMem The memory address.
1472	* @remarks Will return/long jump on errors.
1473	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1474	*/
1475	#ifndef IEM_WITH_SETJMP
1476	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1477	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1478	(a_GCPtrMem), IEM_ACCESS_DATA_W, 0)); \
1479	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1480	} while (0)
1481	#else
1482	# define IEM_MC_MEM_FLAT_MAP_U8_WO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1483	(a_pu8Mem) = iemMemFlatMapDataU8WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1484	#endif
1485
1486	/**
1487	* Maps guest memory for byte readonly direct (or bounce) buffer acccess, flat
1488	* address variant.
1489	*
1490	* @param[out] a_pu8Mem Where to return the pointer to the mapping.
1491	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1492	* @param[in] a_GCPtrMem The memory address.
1493	* @remarks Will return/long jump on errors.
1494	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1495	*/
1496	#ifndef IEM_WITH_SETJMP
1497	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1498	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu8Mem), sizeof(uint8_t), UINT8_MAX, \
1499	(a_GCPtrMem), IEM_ACCESS_DATA_R, 0)); \
1500	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1501	} while (0)
1502	#else
1503	# define IEM_MC_MEM_FLAT_MAP_U8_RO(a_pu8Mem, a_bUnmapInfo, a_GCPtrMem) \
1504	(a_pu8Mem) = iemMemFlatMapDataU8RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1505	#endif
1506
1507
1508	/* 16-bit */
1509
1510	/**
1511	* Maps guest memory for word read+write direct (or bounce) buffer acccess.
1512	*
1513	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1514	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1515	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1516	* @param[in] a_GCPtrMem The memory address.
1517	* @remarks Will return/long jump on errors.
1518	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1519	*/
1520	#ifndef IEM_WITH_SETJMP
1521	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1522	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1523	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1524	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1525	} while (0)
1526	#else
1527	# define IEM_MC_MEM_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1528	(a_pu16Mem) = iemMemMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1529	#endif
1530
1531	/**
1532	* Maps guest memory for word writeonly direct (or bounce) buffer acccess.
1533	*
1534	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1535	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1536	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1537	* @param[in] a_GCPtrMem The memory address.
1538	* @remarks Will return/long jump on errors.
1539	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1540	*/
1541	#ifndef IEM_WITH_SETJMP
1542	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1543	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1544	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1545	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1546	} while (0)
1547	#else
1548	# define IEM_MC_MEM_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1549	(a_pu16Mem) = iemMemMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1550	#endif
1551
1552	/**
1553	* Maps guest memory for word readonly direct (or bounce) buffer acccess.
1554	*
1555	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1556	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1557	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1558	* @param[in] a_GCPtrMem The memory address.
1559	* @remarks Will return/long jump on errors.
1560	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1561	*/
1562	#ifndef IEM_WITH_SETJMP
1563	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1564	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), (a_iSeg), \
1565	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1566	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1567	} while (0)
1568	#else
1569	# define IEM_MC_MEM_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1570	(a_pu16Mem) = iemMemMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1571	#endif
1572
1573	/**
1574	* Maps guest memory for word read+write direct (or bounce) buffer acccess, flat
1575	* address variant.
1576	*
1577	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1578	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1579	* @param[in] a_GCPtrMem The memory address.
1580	* @remarks Will return/long jump on errors.
1581	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1582	*/
1583	#ifndef IEM_WITH_SETJMP
1584	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1585	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1586	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint16_t) - 1)); \
1587	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1588	} while (0)
1589	#else
1590	# define IEM_MC_MEM_FLAT_MAP_U16_RW(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1591	(a_pu16Mem) = iemMemFlatMapDataU16RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1592	#endif
1593
1594	/**
1595	* Maps guest memory for word writeonly direct (or bounce) buffer acccess, flat
1596	* address variant.
1597	*
1598	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1599	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1600	* @param[in] a_GCPtrMem The memory address.
1601	* @remarks Will return/long jump on errors.
1602	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1603	*/
1604	#ifndef IEM_WITH_SETJMP
1605	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1606	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1607	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint16_t) - 1)); \
1608	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1609	} while (0)
1610	#else
1611	# define IEM_MC_MEM_FLAT_MAP_U16_WO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1612	(a_pu16Mem) = iemMemFlatMapDataU16WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1613	#endif
1614
1615	/**
1616	* Maps guest memory for word readonly direct (or bounce) buffer acccess, flat
1617	* address variant.
1618	*
1619	* @param[out] a_pu16Mem Where to return the pointer to the mapping.
1620	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1621	* @param[in] a_GCPtrMem The memory address.
1622	* @remarks Will return/long jump on errors.
1623	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1624	*/
1625	#ifndef IEM_WITH_SETJMP
1626	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1627	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu16Mem), sizeof(uint16_t), UINT8_MAX, \
1628	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint16_t) - 1)); \
1629	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1630	} while (0)
1631	#else
1632	# define IEM_MC_MEM_FLAT_MAP_U16_RO(a_pu16Mem, a_bUnmapInfo, a_GCPtrMem) \
1633	(a_pu16Mem) = iemMemFlatMapDataU16RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1634	#endif
1635
1636
1637	/* 32-bit */
1638
1639	/**
1640	* Maps guest memory for dword read+write direct (or bounce) buffer acccess.
1641	*
1642	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1643	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1644	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1645	* @param[in] a_GCPtrMem The memory address.
1646	* @remarks Will return/long jump on errors.
1647	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1648	*/
1649	#ifndef IEM_WITH_SETJMP
1650	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1651	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1652	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1653	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1654	} while (0)
1655	#else
1656	# define IEM_MC_MEM_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1657	(a_pu32Mem) = iemMemMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1658	#endif
1659
1660	/**
1661	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess.
1662	*
1663	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1664	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1665	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1666	* @param[in] a_GCPtrMem The memory address.
1667	* @remarks Will return/long jump on errors.
1668	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1669	*/
1670	#ifndef IEM_WITH_SETJMP
1671	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1672	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1673	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1674	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1675	} while (0)
1676	#else
1677	# define IEM_MC_MEM_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1678	(a_pu32Mem) = iemMemMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1679	#endif
1680
1681	/**
1682	* Maps guest memory for dword readonly direct (or bounce) buffer acccess.
1683	*
1684	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1685	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1686	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1687	* @param[in] a_GCPtrMem The memory address.
1688	* @remarks Will return/long jump on errors.
1689	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1690	*/
1691	#ifndef IEM_WITH_SETJMP
1692	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1693	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), (a_iSeg), \
1694	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1695	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1696	} while (0)
1697	#else
1698	# define IEM_MC_MEM_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1699	(a_pu32Mem) = iemMemMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1700	#endif
1701
1702	/**
1703	* Maps guest memory for dword read+write direct (or bounce) buffer acccess,
1704	* flat address variant.
1705	*
1706	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1707	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1708	* @param[in] a_GCPtrMem The memory address.
1709	* @remarks Will return/long jump on errors.
1710	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1711	*/
1712	#ifndef IEM_WITH_SETJMP
1713	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1714	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1715	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint32_t) - 1)); \
1716	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1717	} while (0)
1718	#else
1719	# define IEM_MC_MEM_FLAT_MAP_U32_RW(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1720	(a_pu32Mem) = iemMemFlatMapDataU32RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1721	#endif
1722
1723	/**
1724	* Maps guest memory for dword writeonly direct (or bounce) buffer acccess, flat
1725	* address variant.
1726	*
1727	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1728	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1729	* @param[in] a_GCPtrMem The memory address.
1730	* @remarks Will return/long jump on errors.
1731	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1732	*/
1733	#ifndef IEM_WITH_SETJMP
1734	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1735	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1736	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint32_t) - 1)); \
1737	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1738	} while (0)
1739	#else
1740	# define IEM_MC_MEM_FLAT_MAP_U32_WO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1741	(a_pu32Mem) = iemMemFlatMapDataU32WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1742	#endif
1743
1744	/**
1745	* Maps guest memory for dword readonly direct (or bounce) buffer acccess, flat
1746	* address variant.
1747	*
1748	* @param[out] a_pu32Mem Where to return the pointer to the mapping.
1749	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1750	* @param[in] a_GCPtrMem The memory address.
1751	* @remarks Will return/long jump on errors.
1752	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1753	*/
1754	#ifndef IEM_WITH_SETJMP
1755	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1756	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu32Mem), sizeof(uint32_t), UINT8_MAX, \
1757	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint32_t) - 1)); \
1758	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1759	} while (0)
1760	#else
1761	# define IEM_MC_MEM_FLAT_MAP_U32_RO(a_pu32Mem, a_bUnmapInfo, a_GCPtrMem) \
1762	(a_pu32Mem) = iemMemFlatMapDataU32RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1763	#endif
1764
1765
1766	/* 64-bit */
1767
1768	/**
1769	* Maps guest memory for qword read+write direct (or bounce) buffer acccess.
1770	*
1771	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1772	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1773	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1774	* @param[in] a_GCPtrMem The memory address.
1775	* @remarks Will return/long jump on errors.
1776	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1777	*/
1778	#ifndef IEM_WITH_SETJMP
1779	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1780	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1781	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1782	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1783	} while (0)
1784	#else
1785	# define IEM_MC_MEM_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1786	(a_pu64Mem) = iemMemMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1787	#endif
1788
1789	/**
1790	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess.
1791	*
1792	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1793	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1794	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1795	* @param[in] a_GCPtrMem The memory address.
1796	* @remarks Will return/long jump on errors.
1797	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1798	*/
1799	#ifndef IEM_WITH_SETJMP
1800	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1801	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1802	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1803	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1804	} while (0)
1805	#else
1806	# define IEM_MC_MEM_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1807	(a_pu64Mem) = iemMemMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1808	#endif
1809
1810	/**
1811	* Maps guest memory for qword readonly direct (or bounce) buffer acccess.
1812	*
1813	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1814	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1815	* @param[in] a_iSeg The segment register to access via. No UINT8_MAX!
1816	* @param[in] a_GCPtrMem The memory address.
1817	* @remarks Will return/long jump on errors.
1818	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1819	*/
1820	#ifndef IEM_WITH_SETJMP
1821	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) do { \
1822	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), (a_iSeg), \
1823	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1824	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1825	} while (0)
1826	#else
1827	# define IEM_MC_MEM_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_iSeg, a_GCPtrMem) \
1828	(a_pu64Mem) = iemMemMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_iSeg), (a_GCPtrMem))
1829	#endif
1830
1831	/**
1832	* Maps guest memory for qword read+write direct (or bounce) buffer acccess,
1833	* flat address variant.
1834	*
1835	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1836	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1837	* @param[in] a_GCPtrMem The memory address.
1838	* @remarks Will return/long jump on errors.
1839	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RW
1840	*/
1841	#ifndef IEM_WITH_SETJMP
1842	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1843	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1844	(a_GCPtrMem), IEM_ACCESS_DATA_RW, sizeof(uint64_t) - 1)); \
1845	a_bUnmapInfo = 1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4); \
1846	} while (0)
1847	#else
1848	# define IEM_MC_MEM_FLAT_MAP_U64_RW(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1849	(a_pu64Mem) = iemMemFlatMapDataU64RwJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1850	#endif
1851
1852	/**
1853	* Maps guest memory for qword writeonly direct (or bounce) buffer acccess, flat
1854	* address variant.
1855	*
1856	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1857	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1858	* @param[in] a_GCPtrMem The memory address.
1859	* @remarks Will return/long jump on errors.
1860	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_WO
1861	*/
1862	#ifndef IEM_WITH_SETJMP
1863	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1864	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1865	(a_GCPtrMem), IEM_ACCESS_DATA_W, sizeof(uint64_t) - 1)); \
1866	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_WRITE << 4); \
1867	} while (0)
1868	#else
1869	# define IEM_MC_MEM_FLAT_MAP_U64_WO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1870	(a_pu64Mem) = iemMemFlatMapDataU64WoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1871	#endif
1872
1873	/**
1874	* Maps guest memory for qword readonly direct (or bounce) buffer acccess, flat
1875	* address variant.
1876	*
1877	* @param[out] a_pu64Mem Where to return the pointer to the mapping.
1878	* @param[out] a_bUnmapInfo Where to return umapping instructions. uint8_t.
1879	* @param[in] a_GCPtrMem The memory address.
1880	* @remarks Will return/long jump on errors.
1881	* @see IEM_MC_MEM_COMMIT_AND_UNMAP_RO
1882	*/
1883	#ifndef IEM_WITH_SETJMP
1884	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) do { \
1885	IEM_MC_RETURN_ON_FAILURE(iemMemMap(pVCpu, (void **)&(a_pu64Mem), sizeof(uint64_t), UINT8_MAX, \
1886	(a_GCPtrMem), IEM_ACCESS_DATA_R, sizeof(uint64_t) - 1)); \
1887	a_bUnmapInfo = 1 \| (IEM_ACCESS_TYPE_READ << 4); \
1888	} while (0)
1889	#else
1890	# define IEM_MC_MEM_FLAT_MAP_U64_RO(a_pu64Mem, a_bUnmapInfo, a_GCPtrMem) \
1891	(a_pu64Mem) = iemMemFlatMapDataU64RoJmp(pVCpu, &(a_bUnmapInfo), (a_GCPtrMem))
1892	#endif
1893
1894
1895	/* commit + unmap */
1896
1897	/** Commits the memory and unmaps guest memory previously mapped RW.
1898	* @remarks May return.
1899	*/
1900	#ifndef IEM_WITH_SETJMP
1901	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) do { \
1902	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| ((IEM_ACCESS_TYPE_READ \| IEM_ACCESS_TYPE_WRITE) << 4)) ); \
1903	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_RW)); \
1904	} while (0)
1905	#else
1906	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RW(a_pvMem, a_bMapInfo) \
1907	iemMemCommitAndUnmapRwJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1908	#endif
1909
1910	/** Commits the memory and unmaps guest memory previously mapped W.
1911	* @remarks May return.
1912	*/
1913	#ifndef IEM_WITH_SETJMP
1914	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) do { \
1915	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_WRITE << 4)) ); \
1916	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), IEM_ACCESS_DATA_W)); \
1917	} while (0)
1918	#else
1919	# define IEM_MC_MEM_COMMIT_AND_UNMAP_WO(a_pvMem, a_bMapInfo) \
1920	iemMemCommitAndUnmapWoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1921	#endif
1922
1923	/** Commits the memory and unmaps guest memory previously mapped R.
1924	* @remarks May return.
1925	*/
1926	#ifndef IEM_WITH_SETJMP
1927	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) do { \
1928	RT_NOREF_PV(a_bMapInfo); Assert(a_bMapInfo == (1 \| (IEM_ACCESS_TYPE_READ << 4)) ); \
1929	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (void *)(a_pvMem), IEM_ACCESS_DATA_R)); \
1930	} while (0)
1931	#else
1932	# define IEM_MC_MEM_COMMIT_AND_UNMAP_RO(a_pvMem, a_bMapInfo) \
1933	iemMemCommitAndUnmapRoJmp(pVCpu, (a_pvMem), (a_bMapInfo))
1934	#endif
1935
1936
1937	/** Commits the memory and unmaps the guest memory unless the FPU status word
1938	* indicates (@a a_u16FSW) and FPU control word indicates a pending exception
1939	* that would cause FLD not to store.
1940	*
1941	* The current understanding is that \#O, \#U, \#IA and \#IS will prevent a
1942	* store, while \#P will not.
1943	*
1944	* @remarks May in theory return - for now.
1945	*/
1946	#define IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(a_pvMem, a_fAccess, a_u16FSW) \
1947	do { \
1948	if ( !(a_u16FSW & X86_FSW_ES) \
1949	\|\| !( (a_u16FSW & (X86_FSW_UE \| X86_FSW_OE \| X86_FSW_IE)) \
1950	& ~(pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_MASK_ALL) ) ) \
1951	IEM_MC_RETURN_ON_FAILURE(iemMemCommitAndUnmap(pVCpu, (a_pvMem), (a_fAccess))); \
1952	} while (0)
1953
1954
1955
1956	/** Calculate efficient address from R/M. */
1957	#ifndef IEM_WITH_SETJMP
1958	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1959	IEM_MC_RETURN_ON_FAILURE(iemOpHlpCalcRmEffAddr(pVCpu, (a_bRm), (a_cbImmAndRspOffset), &(a_GCPtrEff)))
1960	#else
1961	# define IEM_MC_CALC_RM_EFF_ADDR(a_GCPtrEff, a_bRm, a_cbImmAndRspOffset) \
1962	((a_GCPtrEff) = iemOpHlpCalcRmEffAddrJmp(pVCpu, (a_bRm), (a_cbImmAndRspOffset)))
1963	#endif
1964
1965	#define IEM_MC_CALL_VOID_AIMPL_0(a_pfn) (a_pfn)()
1966	#define IEM_MC_CALL_VOID_AIMPL_1(a_pfn, a0) (a_pfn)((a0))
1967	#define IEM_MC_CALL_VOID_AIMPL_2(a_pfn, a0, a1) (a_pfn)((a0), (a1))
1968	#define IEM_MC_CALL_VOID_AIMPL_3(a_pfn, a0, a1, a2) (a_pfn)((a0), (a1), (a2))
1969	#define IEM_MC_CALL_VOID_AIMPL_4(a_pfn, a0, a1, a2, a3) (a_pfn)((a0), (a1), (a2), (a3))
1970	#define IEM_MC_CALL_AIMPL_3(a_rc, a_pfn, a0, a1, a2) (a_rc) = (a_pfn)((a0), (a1), (a2))
1971	#define IEM_MC_CALL_AIMPL_4(a_rc, a_pfn, a0, a1, a2, a3) (a_rc) = (a_pfn)((a0), (a1), (a2), (a3))
1972
1973
1974	/** @def IEM_MC_CALL_CIMPL_HLP_RET
1975	* Helper macro for check that all important IEM_CIMPL_F_XXX bits are set.
1976	*/
1977	#ifdef VBOX_STRICT
1978	#define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) \
1979	do { \
1980	uint8_t const cbInstr = IEM_GET_INSTR_LEN(pVCpu); /* may be flushed */ \
1981	uint16_t const uCsBefore = pVCpu->cpum.GstCtx.cs.Sel; \
1982	uint64_t const uRipBefore = pVCpu->cpum.GstCtx.rip; \
1983	uint32_t const fEflBefore = pVCpu->cpum.GstCtx.eflags.u; \
1984	uint32_t const fExecBefore = pVCpu->iem.s.fExec; \
1985	VBOXSTRICTRC const rcStrictHlp = a_CallExpr; \
1986	if (rcStrictHlp == VINF_SUCCESS) \
1987	{ \
1988	AssertMsg( ((a_fFlags) & IEM_CIMPL_F_BRANCH_ANY) \
1989	\|\| ( uRipBefore + cbInstr == pVCpu->cpum.GstCtx.rip \
1990	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel) \
1991	\|\| ( ((a_fFlags) & IEM_CIMPL_F_REP) \
1992	&& uRipBefore == pVCpu->cpum.GstCtx.rip \
1993	&& uCsBefore == pVCpu->cpum.GstCtx.cs.Sel), \
1994	("CS:RIP=%04x:%08RX64 + %x -> %04x:%08RX64, expected %04x:%08RX64\n", uCsBefore, uRipBefore, cbInstr, \
1995	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uCsBefore, uRipBefore + cbInstr)); \
1996	if ((a_fFlags) & IEM_CIMPL_F_RFLAGS) \
1997	{ /* No need to check fEflBefore */ Assert(!((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS)); } \
1998	else if ((a_fFlags) & IEM_CIMPL_F_STATUS_FLAGS) \
1999	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)) \
2000	== (fEflBefore & ~(X86_EFL_STATUS_BITS \| X86_EFL_RF)), \
2001	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2002	else \
2003	AssertMsg( (pVCpu->cpum.GstCtx.eflags.u & ~(X86_EFL_RF)) \
2004	== (fEflBefore & ~(X86_EFL_RF)), \
2005	("EFL=%#RX32 -> %#RX32\n", fEflBefore, pVCpu->cpum.GstCtx.eflags.u)); \
2006	if (!((a_fFlags) & IEM_CIMPL_F_MODE)) \
2007	{ \
2008	uint32_t fExecRecalc = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS); \
2009	AssertMsg( fExecBefore == fExecRecalc \
2010	/* in case ES, DS or SS was external initially (happens alot with HM): */ \
2011	\|\| ( fExecBefore == (fExecRecalc & ~IEM_F_MODE_X86_FLAT_OR_PRE_386_MASK) \
2012	&& (fExecRecalc & IEM_F_MODE_CPUMODE_MASK) == IEMMODE_32BIT), \
2013	("fExec=%#x -> %#x (diff %#x)\n", fExecBefore, fExecRecalc, fExecBefore ^ fExecRecalc)); \
2014	} \
2015	} \
2016	return rcStrictHlp; \
2017	} while (0)
2018	#else
2019	# define IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, a_CallExpr) return a_CallExpr
2020	#endif
2021
2022	/**
2023	* Defers the rest of the instruction emulation to a C implementation routine
2024	* and returns, only taking the standard parameters.
2025	*
2026	* @param a_fFlags IEM_CIMPL_F_XXX.
2027	* @param a_pfnCImpl The pointer to the C routine.
2028	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2029	*/
2030	#define IEM_MC_CALL_CIMPL_0(a_fFlags, a_pfnCImpl) \
2031	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2032
2033	/**
2034	* Defers the rest of instruction emulation to a C implementation routine and
2035	* returns, taking one argument in addition to the standard ones.
2036	*
2037	* @param a_fFlags IEM_CIMPL_F_XXX.
2038	* @param a_pfnCImpl The pointer to the C routine.
2039	* @param a0 The argument.
2040	*/
2041	#define IEM_MC_CALL_CIMPL_1(a_fFlags, a_pfnCImpl, a0) \
2042	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2043
2044	/**
2045	* Defers the rest of the instruction emulation to a C implementation routine
2046	* and returns, taking two arguments in addition to the standard ones.
2047	*
2048	* @param a_fFlags IEM_CIMPL_F_XXX.
2049	* @param a_pfnCImpl The pointer to the C routine.
2050	* @param a0 The first extra argument.
2051	* @param a1 The second extra argument.
2052	*/
2053	#define IEM_MC_CALL_CIMPL_2(a_fFlags, a_pfnCImpl, a0, a1) \
2054	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2055
2056	/**
2057	* Defers the rest of the instruction emulation to a C implementation routine
2058	* and returns, taking three arguments in addition to the standard ones.
2059	*
2060	* @param a_fFlags IEM_CIMPL_F_XXX.
2061	* @param a_pfnCImpl The pointer to the C routine.
2062	* @param a0 The first extra argument.
2063	* @param a1 The second extra argument.
2064	* @param a2 The third extra argument.
2065	*/
2066	#define IEM_MC_CALL_CIMPL_3(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2067	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2068
2069	/**
2070	* Defers the rest of the instruction emulation to a C implementation routine
2071	* and returns, taking four arguments in addition to the standard ones.
2072	*
2073	* @param a_fFlags IEM_CIMPL_F_XXX.
2074	* @param a_pfnCImpl The pointer to the C routine.
2075	* @param a0 The first extra argument.
2076	* @param a1 The second extra argument.
2077	* @param a2 The third extra argument.
2078	* @param a3 The fourth extra argument.
2079	*/
2080	#define IEM_MC_CALL_CIMPL_4(a_fFlags, a_pfnCImpl, a0, a1, a2, a3) \
2081	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3))
2082
2083	/**
2084	* Defers the rest of the instruction emulation to a C implementation routine
2085	* and returns, taking two arguments in addition to the standard ones.
2086	*
2087	* @param a_fFlags IEM_CIMPL_F_XXX.
2088	* @param a_pfnCImpl The pointer to the C routine.
2089	* @param a0 The first extra argument.
2090	* @param a1 The second extra argument.
2091	* @param a2 The third extra argument.
2092	* @param a3 The fourth extra argument.
2093	* @param a4 The fifth extra argument.
2094	*/
2095	#define IEM_MC_CALL_CIMPL_5(a_fFlags, a_pfnCImpl, a0, a1, a2, a3, a4) \
2096	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2, a3, a4))
2097
2098	/**
2099	* Defers the entire instruction emulation to a C implementation routine and
2100	* returns, only taking the standard parameters.
2101	*
2102	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2103	*
2104	* @param a_fFlags IEM_CIMPL_F_XXX.
2105	* @param a_pfnCImpl The pointer to the C routine.
2106	* @sa IEM_DECL_IMPL_C_TYPE_0 and IEM_CIMPL_DEF_0.
2107	*/
2108	#define IEM_MC_DEFER_TO_CIMPL_0_RET(a_fFlags, a_pfnCImpl) \
2109	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu)))
2110
2111	/**
2112	* Defers the entire instruction emulation to a C implementation routine and
2113	* returns, taking one argument in addition to the standard ones.
2114	*
2115	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2116	*
2117	* @param a_fFlags IEM_CIMPL_F_XXX.
2118	* @param a_pfnCImpl The pointer to the C routine.
2119	* @param a0 The argument.
2120	*/
2121	#define IEM_MC_DEFER_TO_CIMPL_1_RET(a_fFlags, a_pfnCImpl, a0) \
2122	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0))
2123
2124	/**
2125	* Defers the entire instruction emulation to a C implementation routine and
2126	* returns, taking two arguments in addition to the standard ones.
2127	*
2128	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2129	*
2130	* @param a_fFlags IEM_CIMPL_F_XXX.
2131	* @param a_pfnCImpl The pointer to the C routine.
2132	* @param a0 The first extra argument.
2133	* @param a1 The second extra argument.
2134	*/
2135	#define IEM_MC_DEFER_TO_CIMPL_2_RET(a_fFlags, a_pfnCImpl, a0, a1) \
2136	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1))
2137
2138	/**
2139	* Defers the entire instruction emulation to a C implementation routine and
2140	* returns, taking three arguments in addition to the standard ones.
2141	*
2142	* This shall be used without any IEM_MC_BEGIN or IEM_END macro surrounding it.
2143	*
2144	* @param a_fFlags IEM_CIMPL_F_XXX.
2145	* @param a_pfnCImpl The pointer to the C routine.
2146	* @param a0 The first extra argument.
2147	* @param a1 The second extra argument.
2148	* @param a2 The third extra argument.
2149	*/
2150	#define IEM_MC_DEFER_TO_CIMPL_3_RET(a_fFlags, a_pfnCImpl, a0, a1, a2) \
2151	IEM_MC_CALL_CIMPL_HLP_RET(a_fFlags, (a_pfnCImpl)(pVCpu, IEM_GET_INSTR_LEN(pVCpu), a0, a1, a2))
2152
2153
2154	/**
2155	* Calls a FPU assembly implementation taking one visible argument.
2156	*
2157	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2158	* @param a0 The first extra argument.
2159	*/
2160	#define IEM_MC_CALL_FPU_AIMPL_1(a_pfnAImpl, a0) \
2161	do { \
2162	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0)); \
2163	} while (0)
2164
2165	/**
2166	* Calls a FPU assembly implementation taking two visible arguments.
2167	*
2168	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2169	* @param a0 The first extra argument.
2170	* @param a1 The second extra argument.
2171	*/
2172	#define IEM_MC_CALL_FPU_AIMPL_2(a_pfnAImpl, a0, a1) \
2173	do { \
2174	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2175	} while (0)
2176
2177	/**
2178	* Calls a FPU assembly implementation taking three visible arguments.
2179	*
2180	* @param a_pfnAImpl Pointer to the assembly FPU routine.
2181	* @param a0 The first extra argument.
2182	* @param a1 The second extra argument.
2183	* @param a2 The third extra argument.
2184	*/
2185	#define IEM_MC_CALL_FPU_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2186	do { \
2187	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2188	} while (0)
2189
2190	#define IEM_MC_SET_FPU_RESULT(a_FpuData, a_FSW, a_pr80Value) \
2191	do { \
2192	(a_FpuData).FSW = (a_FSW); \
2193	(a_FpuData).r80Result = *(a_pr80Value); \
2194	} while (0)
2195
2196	/** Pushes FPU result onto the stack. */
2197	#define IEM_MC_PUSH_FPU_RESULT(a_FpuData, a_uFpuOpcode) \
2198	iemFpuPushResult(pVCpu, &a_FpuData, a_uFpuOpcode)
2199	/** Pushes FPU result onto the stack and sets the FPUDP. */
2200	#define IEM_MC_PUSH_FPU_RESULT_MEM_OP(a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2201	iemFpuPushResultWithMemOp(pVCpu, &a_FpuData, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2202
2203	/** Replaces ST0 with value one and pushes value 2 onto the FPU stack. */
2204	#define IEM_MC_PUSH_FPU_RESULT_TWO(a_FpuDataTwo, a_uFpuOpcode) \
2205	iemFpuPushResultTwo(pVCpu, &a_FpuDataTwo, a_uFpuOpcode)
2206
2207	/** Stores FPU result in a stack register. */
2208	#define IEM_MC_STORE_FPU_RESULT(a_FpuData, a_iStReg, a_uFpuOpcode) \
2209	iemFpuStoreResult(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2210	/** Stores FPU result in a stack register and pops the stack. */
2211	#define IEM_MC_STORE_FPU_RESULT_THEN_POP(a_FpuData, a_iStReg, a_uFpuOpcode) \
2212	iemFpuStoreResultThenPop(pVCpu, &a_FpuData, a_iStReg, a_uFpuOpcode)
2213	/** Stores FPU result in a stack register and sets the FPUDP. */
2214	#define IEM_MC_STORE_FPU_RESULT_MEM_OP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2215	iemFpuStoreResultWithMemOp(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2216	/** Stores FPU result in a stack register, sets the FPUDP, and pops the
2217	* stack. */
2218	#define IEM_MC_STORE_FPU_RESULT_WITH_MEM_OP_THEN_POP(a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2219	iemFpuStoreResultWithMemOpThenPop(pVCpu, &a_FpuData, a_iStReg, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2220
2221	/** Only update the FOP, FPUIP, and FPUCS. (For FNOP.) */
2222	#define IEM_MC_UPDATE_FPU_OPCODE_IP(a_uFpuOpcode) \
2223	iemFpuUpdateOpcodeAndIp(pVCpu, a_uFpuOpcode)
2224	/** Free a stack register (for FFREE and FFREEP). */
2225	#define IEM_MC_FPU_STACK_FREE(a_iStReg) \
2226	iemFpuStackFree(pVCpu, a_iStReg)
2227	/** Increment the FPU stack pointer. */
2228	#define IEM_MC_FPU_STACK_INC_TOP() \
2229	iemFpuStackIncTop(pVCpu)
2230	/** Decrement the FPU stack pointer. */
2231	#define IEM_MC_FPU_STACK_DEC_TOP() \
2232	iemFpuStackDecTop(pVCpu)
2233
2234	/** Updates the FSW, FOP, FPUIP, and FPUCS. */
2235	#define IEM_MC_UPDATE_FSW(a_u16FSW, a_uFpuOpcode) \
2236	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2237	/** Updates the FSW with a constant value as well as FOP, FPUIP, and FPUCS. */
2238	#define IEM_MC_UPDATE_FSW_CONST(a_u16FSW, a_uFpuOpcode) \
2239	iemFpuUpdateFSW(pVCpu, a_u16FSW, a_uFpuOpcode)
2240	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS. */
2241	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2242	iemFpuUpdateFSWWithMemOp(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2243	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack. */
2244	#define IEM_MC_UPDATE_FSW_THEN_POP(a_u16FSW, a_uFpuOpcode) \
2245	iemFpuUpdateFSWThenPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2246	/** Updates the FSW, FOP, FPUIP, FPUCS, FPUDP and FPUDS, and then pops the
2247	* stack. */
2248	#define IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2249	iemFpuUpdateFSWWithMemOpThenPop(pVCpu, a_u16FSW, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2250	/** Updates the FSW, FOP, FPUIP, and FPUCS, and then pops the stack twice. */
2251	#define IEM_MC_UPDATE_FSW_THEN_POP_POP(a_u16FSW, a_uFpuOpcode) \
2252	iemFpuUpdateFSWThenPopPop(pVCpu, a_u16FSW, a_uFpuOpcode)
2253
2254	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. */
2255	#define IEM_MC_FPU_STACK_UNDERFLOW(a_iStDst, a_uFpuOpcode) \
2256	iemFpuStackUnderflow(pVCpu, a_iStDst, a_uFpuOpcode)
2257	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2258	* stack. */
2259	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(a_iStDst, a_uFpuOpcode) \
2260	iemFpuStackUnderflowThenPop(pVCpu, a_iStDst, a_uFpuOpcode)
2261	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2262	* FPUDS. */
2263	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2264	iemFpuStackUnderflowWithMemOp(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2265	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS, FOP, FPUDP and
2266	* FPUDS. Pops stack. */
2267	#define IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2268	iemFpuStackUnderflowWithMemOpThenPop(pVCpu, a_iStDst, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2269	/** Raises a FPU stack underflow exception. Sets FPUIP, FPUCS and FOP. Pops
2270	* stack twice. */
2271	#define IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP(a_uFpuOpcode) \
2272	iemFpuStackUnderflowThenPopPop(pVCpu, a_uFpuOpcode)
2273	/** Raises a FPU stack underflow exception for an instruction pushing a result
2274	* value onto the stack. Sets FPUIP, FPUCS and FOP. */
2275	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW(a_uFpuOpcode) \
2276	iemFpuStackPushUnderflow(pVCpu, a_uFpuOpcode)
2277	/** Raises a FPU stack underflow exception for an instruction pushing a result
2278	* value onto the stack and replacing ST0. Sets FPUIP, FPUCS and FOP. */
2279	#define IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO(a_uFpuOpcode) \
2280	iemFpuStackPushUnderflowTwo(pVCpu, a_uFpuOpcode)
2281
2282	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2283	* FPUIP, FPUCS and FOP. */
2284	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW(a_uFpuOpcode) \
2285	iemFpuStackPushOverflow(pVCpu, a_uFpuOpcode)
2286	/** Raises a FPU stack overflow exception as part of a push attempt. Sets
2287	* FPUIP, FPUCS, FOP, FPUDP and FPUDS. */
2288	#define IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(a_iEffSeg, a_GCPtrEff, a_uFpuOpcode) \
2289	iemFpuStackPushOverflowWithMemOp(pVCpu, a_iEffSeg, a_GCPtrEff, a_uFpuOpcode)
2290	/** Prepares for using the FPU state.
2291	* Ensures that we can use the host FPU in the current context (RC+R0.
2292	* Ensures the guest FPU state in the CPUMCTX is up to date. */
2293	#define IEM_MC_PREPARE_FPU_USAGE() iemFpuPrepareUsage(pVCpu)
2294	/** Actualizes the guest FPU state so it can be accessed read-only fashion. */
2295	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ() iemFpuActualizeStateForRead(pVCpu)
2296	/** Actualizes the guest FPU state so it can be accessed and modified. */
2297	#define IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE() iemFpuActualizeStateForChange(pVCpu)
2298
2299	/** Stores SSE SIMD result updating MXCSR. */
2300	#define IEM_MC_STORE_SSE_RESULT(a_SseData, a_iXmmReg) \
2301	iemSseStoreResult(pVCpu, &a_SseData, a_iXmmReg)
2302	/** Updates MXCSR. */
2303	#define IEM_MC_SSE_UPDATE_MXCSR(a_fMxcsr) \
2304	iemSseUpdateMxcsr(pVCpu, a_fMxcsr)
2305
2306	/** Prepares for using the SSE state.
2307	* Ensures that we can use the host SSE/FPU in the current context (RC+R0.
2308	* Ensures the guest SSE state in the CPUMCTX is up to date. */
2309	#define IEM_MC_PREPARE_SSE_USAGE() iemFpuPrepareUsageSse(pVCpu)
2310	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2311	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ() iemFpuActualizeSseStateForRead(pVCpu)
2312	/** Actualizes the guest XMM0..15 and MXCSR register state for read-write access. */
2313	#define IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE() iemFpuActualizeSseStateForChange(pVCpu)
2314
2315	/** Prepares for using the AVX state.
2316	* Ensures that we can use the host AVX/FPU in the current context (RC+R0.
2317	* Ensures the guest AVX state in the CPUMCTX is up to date.
2318	* @note This will include the AVX512 state too when support for it is added
2319	* due to the zero extending feature of VEX instruction. */
2320	#define IEM_MC_PREPARE_AVX_USAGE() iemFpuPrepareUsageAvx(pVCpu)
2321	/** Actualizes the guest XMM0..15 and MXCSR register state for read-only access. */
2322	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ() iemFpuActualizeAvxStateForRead(pVCpu)
2323	/** Actualizes the guest YMM0..15 and MXCSR register state for read-write access. */
2324	#define IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE() iemFpuActualizeAvxStateForChange(pVCpu)
2325
2326	/**
2327	* Calls a MMX assembly implementation taking two visible arguments.
2328	*
2329	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2330	* @param a0 The first extra argument.
2331	* @param a1 The second extra argument.
2332	*/
2333	#define IEM_MC_CALL_MMX_AIMPL_2(a_pfnAImpl, a0, a1) \
2334	do { \
2335	IEM_MC_PREPARE_FPU_USAGE(); \
2336	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2337	} while (0)
2338
2339	/**
2340	* Calls a MMX assembly implementation taking three visible arguments.
2341	*
2342	* @param a_pfnAImpl Pointer to the assembly MMX routine.
2343	* @param a0 The first extra argument.
2344	* @param a1 The second extra argument.
2345	* @param a2 The third extra argument.
2346	*/
2347	#define IEM_MC_CALL_MMX_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2348	do { \
2349	IEM_MC_PREPARE_FPU_USAGE(); \
2350	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2351	} while (0)
2352
2353
2354	/**
2355	* Calls a SSE assembly implementation taking two visible arguments.
2356	*
2357	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2358	* @param a0 The first extra argument.
2359	* @param a1 The second extra argument.
2360	*/
2361	#define IEM_MC_CALL_SSE_AIMPL_2(a_pfnAImpl, a0, a1) \
2362	do { \
2363	IEM_MC_PREPARE_SSE_USAGE(); \
2364	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1)); \
2365	} while (0)
2366
2367	/**
2368	* Calls a SSE assembly implementation taking three visible arguments.
2369	*
2370	* @param a_pfnAImpl Pointer to the assembly SSE routine.
2371	* @param a0 The first extra argument.
2372	* @param a1 The second extra argument.
2373	* @param a2 The third extra argument.
2374	*/
2375	#define IEM_MC_CALL_SSE_AIMPL_3(a_pfnAImpl, a0, a1, a2) \
2376	do { \
2377	IEM_MC_PREPARE_SSE_USAGE(); \
2378	a_pfnAImpl(&pVCpu->cpum.GstCtx.XState.x87, (a0), (a1), (a2)); \
2379	} while (0)
2380
2381
2382	/** Declares implicit arguments for IEM_MC_CALL_AVX_AIMPL_2,
2383	* IEM_MC_CALL_AVX_AIMPL_3, IEM_MC_CALL_AVX_AIMPL_4, ...
2384	* @note IEMAllInstPython.py duplicates the expansion. */
2385	#define IEM_MC_IMPLICIT_AVX_AIMPL_ARGS() \
2386	IEM_MC_ARG_CONST(PX86XSAVEAREA, pXState, &pVCpu->cpum.GstCtx.XState, 0)
2387
2388	/**
2389	* Calls a AVX assembly implementation taking two visible arguments.
2390	*
2391	* There is one implicit zero'th argument, a pointer to the extended state.
2392	*
2393	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2394	* @param a1 The first extra argument.
2395	* @param a2 The second extra argument.
2396	*/
2397	#define IEM_MC_CALL_AVX_AIMPL_2(a_pfnAImpl, a1, a2) \
2398	do { \
2399	IEM_MC_PREPARE_AVX_USAGE(); \
2400	a_pfnAImpl(pXState, (a1), (a2)); \
2401	} while (0)
2402
2403	/**
2404	* Calls a AVX assembly implementation taking three visible arguments.
2405	*
2406	* There is one implicit zero'th argument, a pointer to the extended state.
2407	*
2408	* @param a_pfnAImpl Pointer to the assembly AVX routine.
2409	* @param a1 The first extra argument.
2410	* @param a2 The second extra argument.
2411	* @param a3 The third extra argument.
2412	*/
2413	#define IEM_MC_CALL_AVX_AIMPL_3(a_pfnAImpl, a1, a2, a3) \
2414	do { \
2415	IEM_MC_PREPARE_AVX_USAGE(); \
2416	a_pfnAImpl(pXState, (a1), (a2), (a3)); \
2417	} while (0)
2418
2419	/** @note Not for IOPL or IF testing. */
2420	#define IEM_MC_IF_EFL_BIT_SET(a_fBit) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) {
2421	/** @note Not for IOPL or IF testing. */
2422	#define IEM_MC_IF_EFL_BIT_NOT_SET(a_fBit) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit))) {
2423	/** @note Not for IOPL or IF testing. */
2424	#define IEM_MC_IF_EFL_ANY_BITS_SET(a_fBits) if (pVCpu->cpum.GstCtx.eflags.u & (a_fBits)) {
2425	/** @note Not for IOPL or IF testing. */
2426	#define IEM_MC_IF_EFL_NO_BITS_SET(a_fBits) if (!(pVCpu->cpum.GstCtx.eflags.u & (a_fBits))) {
2427	/** @note Not for IOPL or IF testing. */
2428	#define IEM_MC_IF_EFL_BITS_NE(a_fBit1, a_fBit2) \
2429	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2430	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2431	/** @note Not for IOPL or IF testing. */
2432	#define IEM_MC_IF_EFL_BITS_EQ(a_fBit1, a_fBit2) \
2433	if ( !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2434	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2435	/** @note Not for IOPL or IF testing. */
2436	#define IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(a_fBit, a_fBit1, a_fBit2) \
2437	if ( (pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2438	\|\| !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2439	!= !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2440	/** @note Not for IOPL or IF testing. */
2441	#define IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(a_fBit, a_fBit1, a_fBit2) \
2442	if ( !(pVCpu->cpum.GstCtx.eflags.u & (a_fBit)) \
2443	&& !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit1)) \
2444	== !!(pVCpu->cpum.GstCtx.eflags.u & (a_fBit2)) ) {
2445	#define IEM_MC_IF_CX_IS_NZ() if (pVCpu->cpum.GstCtx.cx != 0) {
2446	#define IEM_MC_IF_ECX_IS_NZ() if (pVCpu->cpum.GstCtx.ecx != 0) {
2447	#define IEM_MC_IF_RCX_IS_NZ() if (pVCpu->cpum.GstCtx.rcx != 0) {
2448	/** @note Not for IOPL or IF testing. */
2449	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2450	if ( pVCpu->cpum.GstCtx.cx != 0 \
2451	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2452	/** @note Not for IOPL or IF testing. */
2453	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2454	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2455	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2456	/** @note Not for IOPL or IF testing. */
2457	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(a_fBit) \
2458	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2459	&& (pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2460	/** @note Not for IOPL or IF testing. */
2461	#define IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2462	if ( pVCpu->cpum.GstCtx.cx != 0 \
2463	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2464	/** @note Not for IOPL or IF testing. */
2465	#define IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2466	if ( pVCpu->cpum.GstCtx.ecx != 0 \
2467	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2468	/** @note Not for IOPL or IF testing. */
2469	#define IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(a_fBit) \
2470	if ( pVCpu->cpum.GstCtx.rcx != 0 \
2471	&& !(pVCpu->cpum.GstCtx.eflags.u & a_fBit)) {
2472	#define IEM_MC_IF_LOCAL_IS_Z(a_Local) if ((a_Local) == 0) {
2473	#define IEM_MC_IF_GREG_BIT_SET(a_iGReg, a_iBitNo) if (iemGRegFetchU64(pVCpu, (a_iGReg)) & RT_BIT_64(a_iBitNo)) {
2474
2475	#define IEM_MC_REF_FPUREG(a_pr80Dst, a_iSt) \
2476	do { (a_pr80Dst) = &pVCpu->cpum.GstCtx.XState.x87.aRegs[a_iSt].r80; } while (0)
2477	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2478	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2479	#define IEM_MC_IF_FPUREG_NOT_EMPTY(a_iSt) \
2480	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) == VINF_SUCCESS) {
2481	#define IEM_MC_IF_FPUREG_IS_EMPTY(a_iSt) \
2482	if (iemFpuStRegNotEmpty(pVCpu, (a_iSt)) != VINF_SUCCESS) {
2483	#define IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(a_pr80Dst, a_iSt) \
2484	if (iemFpuStRegNotEmptyRef(pVCpu, (a_iSt), &(a_pr80Dst)) == VINF_SUCCESS) {
2485	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(a_pr80Dst0, a_iSt0, a_pr80Dst1, a_iSt1) \
2486	if (iemFpu2StRegsNotEmptyRef(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1), &(a_pr80Dst1)) == VINF_SUCCESS) {
2487	#define IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(a_pr80Dst0, a_iSt0, a_iSt1) \
2488	if (iemFpu2StRegsNotEmptyRefFirst(pVCpu, (a_iSt0), &(a_pr80Dst0), (a_iSt1)) == VINF_SUCCESS) {
2489	#define IEM_MC_IF_FCW_IM() \
2490	if (pVCpu->cpum.GstCtx.XState.x87.FCW & X86_FCW_IM) {
2491	#define IEM_MC_IF_MXCSR_XCPT_PENDING() \
2492	if (( ~((pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT) \
2493	& (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_XCPT_FLAGS)) != 0) {
2494
2495	#define IEM_MC_ELSE() } else {
2496	#define IEM_MC_ENDIF() } do {} while (0)
2497
2498	/** @} */
2499
2500	#endif /* !VMM_INCLUDED_SRC_include_IEMMc_h */
2501

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/include/IEMMc.h@ 101587

以其他格式下載: