IEMInline.h@ 101163

最後變更在這個檔案從101163是 101163,由 vboxsync 提交於 18 月前
VMM/IEM: Experimental recompilation of threaded blocks into native code on linux.amd64. bugref:10370
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 124.7 KB

行
1	/* $Id: IEMInline.h 101163 2023-09-18 20:44:24Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.alldomusa.eu.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	/* Deal with the cases that should be treated as VINF_SUCCESS first. */
49	if ( rcStrict == VINF_IEM_YIELD_PENDING_FF
50	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX /** @todo r=bird: Why do we need TWO status codes here? */
51	\|\| rcStrict == VINF_VMX_VMEXIT
52	#endif
53	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
54	\|\| rcStrict == VINF_SVM_VMEXIT
55	#endif
56	)
57	{
58	if (pVCpu->iem.s.rcPassUp == VINF_SUCCESS)
59	rcStrict = VINF_SUCCESS;
60	else
61	{
62	pVCpu->iem.s.cRetPassUpStatus++;
63	rcStrict = pVCpu->iem.s.rcPassUp;
64	}
65	}
66	else if (RT_SUCCESS(rcStrict))
67	{
68	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
69	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
70	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
71	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
72	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
73	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
74	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
75	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
76	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
77	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
78	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
79	\|\| rcStrict == VINF_EM_RAW_TO_R3
80	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
81	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
82	/* raw-mode / virt handlers only: */
83	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
84	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
85	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
86	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
87	\|\| rcStrict == VINF_SELM_SYNC_GDT
88	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
89	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
90	/* nested hw.virt codes: */
91	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
92	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
93	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
94	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
95	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
96	if (rcPassUp == VINF_SUCCESS)
97	pVCpu->iem.s.cRetInfStatuses++;
98	else if ( rcPassUp < VINF_EM_FIRST
99	\|\| rcPassUp > VINF_EM_LAST
100	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
101	{
102	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
103	pVCpu->iem.s.cRetPassUpStatus++;
104	rcStrict = rcPassUp;
105	}
106	else
107	{
108	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
109	pVCpu->iem.s.cRetInfStatuses++;
110	}
111	}
112	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
113	pVCpu->iem.s.cRetAspectNotImplemented++;
114	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
115	pVCpu->iem.s.cRetInstrNotImplemented++;
116	else
117	pVCpu->iem.s.cRetErrStatuses++;
118	}
119	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
120	{
121	pVCpu->iem.s.cRetPassUpStatus++;
122	rcStrict = pVCpu->iem.s.rcPassUp;
123	}
124
125	/* Just clear it here as well. */
126	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
127
128	return rcStrict;
129	}
130
131
132	/**
133	* Sets the pass up status.
134	*
135	* @returns VINF_SUCCESS.
136	* @param pVCpu The cross context virtual CPU structure of the
137	* calling thread.
138	* @param rcPassUp The pass up status. Must be informational.
139	* VINF_SUCCESS is not allowed.
140	*/
141	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp) RT_NOEXCEPT
142	{
143	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
144
145	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
146	if (rcOldPassUp == VINF_SUCCESS)
147	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
148	/* If both are EM scheduling codes, use EM priority rules. */
149	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
150	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
151	{
152	if (rcPassUp < rcOldPassUp)
153	{
154	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
155	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
156	}
157	else
158	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
159	}
160	/* Override EM scheduling with specific status code. */
161	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
162	{
163	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
164	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
165	}
166	/* Don't override specific status code, first come first served. */
167	else
168	LogEx(LOG_GROUP_IEM,("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
169	return VINF_SUCCESS;
170	}
171
172
173	/**
174	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag.
175	*
176	* Checks if CS, SS, DS and SS are all wide open flat 32-bit segments. This will
177	* reject expand down data segments and conforming code segments.
178	*
179	* ASSUMES that the CPU is in 32-bit mode.
180	*
181	* @note Will return zero when if any of the segment register state is marked
182	* external, this must be factored into assertions checking fExec
183	* consistency.
184	*
185	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
186	* @param pVCpu The cross context virtual CPU structure of the
187	* calling thread.
188	* @sa iemCalc32BitFlatIndicatorEsDs
189	*/
190	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicator(PVMCPUCC pVCpu) RT_NOEXCEPT
191	{
192	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
193	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
194	\| pVCpu->cpum.GstCtx.cs.Attr.u
195	\| pVCpu->cpum.GstCtx.ss.Attr.u
196	\| pVCpu->cpum.GstCtx.ds.Attr.u)
197	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
198	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
199	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
200	\| (pVCpu->cpum.GstCtx.cs.u32Limit + 1)
201	\| (pVCpu->cpum.GstCtx.ss.u32Limit + 1)
202	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
203	== 0
204	&& ( pVCpu->cpum.GstCtx.es.u64Base
205	\| pVCpu->cpum.GstCtx.cs.u64Base
206	\| pVCpu->cpum.GstCtx.ss.u64Base
207	\| pVCpu->cpum.GstCtx.ds.u64Base)
208	== 0
209	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
210	? IEM_F_MODE_X86_32BIT_FLAT : 0;
211	}
212
213
214	/**
215	* Calculates the IEM_F_MODE_X86_32BIT_FLAT flag, ASSUMING the CS and SS are
216	* flat already.
217	*
218	* This is used by sysenter.
219	*
220	* @note Will return zero when if any of the segment register state is marked
221	* external, this must be factored into assertions checking fExec
222	* consistency.
223	*
224	* @returns IEM_F_MODE_X86_32BIT_FLAT or zero.
225	* @param pVCpu The cross context virtual CPU structure of the
226	* calling thread.
227	* @sa iemCalc32BitFlatIndicator
228	*/
229	DECL_FORCE_INLINE(uint32_t) iemCalc32BitFlatIndicatorEsDs(PVMCPUCC pVCpu) RT_NOEXCEPT
230	{
231	AssertCompile(X86_SEL_TYPE_DOWN == X86_SEL_TYPE_CONF);
232	return ( ( pVCpu->cpum.GstCtx.es.Attr.u
233	\| pVCpu->cpum.GstCtx.ds.Attr.u)
234	& (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86DESCATTR_UNUSABLE))
235	== (X86_SEL_TYPE_ACCESSED \| X86DESCATTR_G \| X86DESCATTR_D \| X86DESCATTR_P)
236	&& ( (pVCpu->cpum.GstCtx.es.u32Limit + 1)
237	\| (pVCpu->cpum.GstCtx.ds.u32Limit + 1))
238	== 0
239	&& ( pVCpu->cpum.GstCtx.es.u64Base
240	\| pVCpu->cpum.GstCtx.ds.u64Base)
241	== 0
242	&& !(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ES \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_ES))
243	? IEM_F_MODE_X86_32BIT_FLAT : 0;
244	}
245
246
247	/**
248	* Calculates the IEM_F_MODE_XXX and CPL flags.
249	*
250	* @returns IEM_F_MODE_XXX
251	* @param pVCpu The cross context virtual CPU structure of the
252	* calling thread.
253	*/
254	DECL_FORCE_INLINE(uint32_t) iemCalcExecModeAndCplFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
255	{
256	/*
257	* We're duplicates code from CPUMGetGuestCPL and CPUMIsGuestIn64BitCodeEx
258	* here to try get this done as efficiently as possible.
259	*/
260	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_EFER \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS);
261
262	if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
263	{
264	if (!pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
265	{
266	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
267	uint32_t fExec = ((uint32_t)pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl << IEM_F_X86_CPL_SHIFT);
268	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
269	{
270	Assert(!pVCpu->cpum.GstCtx.cs.Attr.n.u1Long \|\| !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA));
271	fExec \|= IEM_F_MODE_X86_32BIT_PROT \| iemCalc32BitFlatIndicator(pVCpu);
272	}
273	else if ( pVCpu->cpum.GstCtx.cs.Attr.n.u1Long
274	&& (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA))
275	fExec \|= IEM_F_MODE_X86_64BIT;
276	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
277	fExec \|= IEM_F_MODE_X86_16BIT_PROT;
278	else
279	fExec \|= IEM_F_MODE_X86_16BIT_PROT_PRE_386;
280	return fExec;
281	}
282	return IEM_F_MODE_X86_16BIT_PROT_V86 \| (UINT32_C(3) << IEM_F_X86_CPL_SHIFT);
283	}
284
285	/* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
286	if (RT_LIKELY(!pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig))
287	{
288	if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
289	return IEM_F_MODE_X86_16BIT;
290	return IEM_F_MODE_X86_16BIT_PRE_386;
291	}
292
293	/* 32-bit unreal mode. */
294	return IEM_F_MODE_X86_32BIT \| iemCalc32BitFlatIndicator(pVCpu);
295	}
296
297
298	/**
299	* Calculates the AMD-V and VT-x related context flags.
300	*
301	* @returns 0 or a combination of IEM_F_X86_CTX_IN_GUEST, IEM_F_X86_CTX_SVM and
302	* IEM_F_X86_CTX_VMX.
303	* @param pVCpu The cross context virtual CPU structure of the
304	* calling thread.
305	*/
306	DECL_FORCE_INLINE(uint32_t) iemCalcExecHwVirtFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
307	{
308	/*
309	* This duplicates code from CPUMIsGuestVmxEnabled, CPUMIsGuestSvmEnabled
310	* and CPUMIsGuestInNestedHwvirtMode to some extent.
311	*/
312	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
313
314	AssertCompile(X86_CR4_VMXE != MSR_K6_EFER_SVME);
315	uint64_t const fTmp = (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VMXE)
316	\| (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SVME);
317	if (RT_LIKELY(!fTmp))
318	return 0; /* likely */
319
320	if (fTmp & X86_CR4_VMXE)
321	{
322	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_VMX);
323	if (pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode)
324	return IEM_F_X86_CTX_VMX \| IEM_F_X86_CTX_IN_GUEST;
325	return IEM_F_X86_CTX_VMX;
326	}
327
328	Assert(pVCpu->cpum.GstCtx.hwvirt.enmHwvirt == CPUMHWVIRT_SVM);
329	if (pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VMRUN)
330	return IEM_F_X86_CTX_SVM \| IEM_F_X86_CTX_IN_GUEST;
331	return IEM_F_X86_CTX_SVM;
332	}
333
334	#ifdef VBOX_INCLUDED_vmm_dbgf_h /* VM::dbgf.ro.cEnabledHwBreakpoints is only accessible if VBox/vmm/dbgf.h is included. */
335
336	/**
337	* Calculates IEM_F_BRK_PENDING_XXX (IEM_F_PENDING_BRK_MASK) flags.
338	*
339	* @returns IEM_F_BRK_PENDING_XXX or zero.
340	* @param pVCpu The cross context virtual CPU structure of the
341	* calling thread.
342	*/
343	DECL_FORCE_INLINE(uint32_t) iemCalcExecDbgFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
344	{
345	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
346
347	if (RT_LIKELY( !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
348	&& pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
349	return 0;
350	return iemCalcExecDbgFlagsSlow(pVCpu);
351	}
352
353	/**
354	* Calculates the the IEM_F_XXX flags.
355	*
356	* @returns IEM_F_XXX combination match the current CPU state.
357	* @param pVCpu The cross context virtual CPU structure of the
358	* calling thread.
359	*/
360	DECL_FORCE_INLINE(uint32_t) iemCalcExecFlags(PVMCPUCC pVCpu) RT_NOEXCEPT
361	{
362	return iemCalcExecModeAndCplFlags(pVCpu)
363	\| iemCalcExecHwVirtFlags(pVCpu)
364	/* SMM is not yet implemented */
365	\| iemCalcExecDbgFlags(pVCpu)
366	;
367	}
368
369
370	/**
371	* Re-calculates the MODE and CPL parts of IEMCPU::fExec.
372	*
373	* @param pVCpu The cross context virtual CPU structure of the
374	* calling thread.
375	*/
376	DECL_FORCE_INLINE(void) iemRecalcExecModeAndCplFlags(PVMCPUCC pVCpu)
377	{
378	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~(IEM_F_MODE_MASK \| IEM_F_X86_CPL_MASK))
379	\| iemCalcExecModeAndCplFlags(pVCpu);
380	}
381
382
383	/**
384	* Re-calculates the IEM_F_PENDING_BRK_MASK part of IEMCPU::fExec.
385	*
386	* @param pVCpu The cross context virtual CPU structure of the
387	* calling thread.
388	*/
389	DECL_FORCE_INLINE(void) iemRecalcExecDbgFlags(PVMCPUCC pVCpu)
390	{
391	pVCpu->iem.s.fExec = (pVCpu->iem.s.fExec & ~IEM_F_PENDING_BRK_MASK)
392	\| iemCalcExecDbgFlags(pVCpu);
393	}
394
395	#endif /* VBOX_INCLUDED_vmm_dbgf_h */
396
397
398	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
399
400	# if defined(VBOX_INCLUDED_vmm_dbgf_h) \|\| defined(DOXYGEN_RUNNING) /* dbgf.ro.cEnabledHwBreakpoints */
401
402	/**
403	* Initializes the execution state.
404	*
405	* @param pVCpu The cross context virtual CPU structure of the
406	* calling thread.
407	* @param fExecOpts Optional execution flags:
408	* - IEM_F_BYPASS_HANDLERS
409	* - IEM_F_X86_DISREGARD_LOCK
410	*
411	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
412	* side-effects in strict builds.
413	*/
414	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, uint32_t fExecOpts) RT_NOEXCEPT
415	{
416	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
417	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
418	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
419	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
420	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
421	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
422	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
423	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
424	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
425	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
426
427	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
428	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| fExecOpts;
429	pVCpu->iem.s.cActiveMappings = 0;
430	pVCpu->iem.s.iNextMapping = 0;
431
432	# ifdef VBOX_STRICT
433	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
434	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
435	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
436	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
437	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
438	pVCpu->iem.s.uRexReg = 127;
439	pVCpu->iem.s.uRexB = 127;
440	pVCpu->iem.s.offModRm = 127;
441	pVCpu->iem.s.uRexIndex = 127;
442	pVCpu->iem.s.iEffSeg = 127;
443	pVCpu->iem.s.idxPrefix = 127;
444	pVCpu->iem.s.uVex3rdReg = 127;
445	pVCpu->iem.s.uVexLength = 127;
446	pVCpu->iem.s.fEvexStuff = 127;
447	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
448	# ifdef IEM_WITH_CODE_TLB
449	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
450	pVCpu->iem.s.pbInstrBuf = NULL;
451	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
452	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
453	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
454	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
455	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
456	pVCpu->iem.s.offOpcode = 127;
457	# endif
458	# else
459	pVCpu->iem.s.offOpcode = 127;
460	pVCpu->iem.s.cbOpcode = 127;
461	# endif
462	# endif /* VBOX_STRICT */
463	}
464
465
466	# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
467	/**
468	* Performs a minimal reinitialization of the execution state.
469	*
470	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
471	* 'world-switch' types operations on the CPU. Currently only nested
472	* hardware-virtualization uses it.
473	*
474	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
475	* @param cbInstr The instruction length (for flushing).
476	*/
477	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
478	{
479	pVCpu->iem.s.fExec = iemCalcExecFlags(pVCpu) \| (pVCpu->iem.s.fExec & IEM_F_USER_OPTS);
480	iemOpcodeFlushHeavy(pVCpu, cbInstr);
481	}
482	# endif
483
484	# endif /* VBOX_INCLUDED_vmm_dbgf_h \|\| DOXYGEN_RUNNING */
485
486	/**
487	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
488	*
489	* @param pVCpu The cross context virtual CPU structure of the
490	* calling thread.
491	*/
492	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu) RT_NOEXCEPT
493	{
494	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
495	# ifdef VBOX_STRICT
496	# ifdef IEM_WITH_CODE_TLB
497	NOREF(pVCpu);
498	# else
499	pVCpu->iem.s.cbOpcode = 0;
500	# endif
501	# else
502	NOREF(pVCpu);
503	# endif
504	}
505
506
507	/**
508	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
509	*
510	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
511	*
512	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
514	* @param rcStrict The status code to fiddle.
515	*/
516	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
517	{
518	iemUninitExec(pVCpu);
519	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
520	}
521
522
523	/**
524	* Macro used by the IEMExec* method to check the given instruction length.
525	*
526	* Will return on failure!
527	*
528	* @param a_cbInstr The given instruction length.
529	* @param a_cbMin The minimum length.
530	*/
531	# define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
532	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
533	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
534
535
536	# ifndef IEM_WITH_SETJMP
537
538	/**
539	* Fetches the first opcode byte.
540	*
541	* @returns Strict VBox status code.
542	* @param pVCpu The cross context virtual CPU structure of the
543	* calling thread.
544	* @param pu8 Where to return the opcode byte.
545	*/
546	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetFirstU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
547	{
548	/*
549	* Check for hardware instruction breakpoints.
550	*/
551	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
552	{ /* likely */ }
553	else
554	{
555	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
556	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
557	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
558	{ /* likely */ }
559	else
560	{
561	pu8 = 0xff; / shut up gcc. sigh */
562	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
563	return iemRaiseDebugException(pVCpu);
564	return rcStrict;
565	}
566	}
567
568	/*
569	* Fetch the first opcode byte.
570	*/
571	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
572	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
573	{
574	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
575	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
576	return VINF_SUCCESS;
577	}
578	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
579	}
580
581	# else /* IEM_WITH_SETJMP */
582
583	/**
584	* Fetches the first opcode byte, longjmp on error.
585	*
586	* @returns The opcode byte.
587	* @param pVCpu The cross context virtual CPU structure of the calling thread.
588	*/
589	DECL_INLINE_THROW(uint8_t) iemOpcodeGetFirstU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
590	{
591	/*
592	* Check for hardware instruction breakpoints.
593	*/
594	if (RT_LIKELY(!(pVCpu->iem.s.fExec & IEM_F_PENDING_BRK_INSTR)))
595	{ /* likely */ }
596	else
597	{
598	VBOXSTRICTRC rcStrict = DBGFBpCheckInstruction(pVCpu->CTX_SUFF(pVM), pVCpu,
599	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
600	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
601	{ /* likely */ }
602	else
603	{
604	if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
605	rcStrict = iemRaiseDebugException(pVCpu);
606	IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
607	}
608	}
609
610	/*
611	* Fetch the first opcode byte.
612	*/
613	# ifdef IEM_WITH_CODE_TLB
614	uint8_t bRet;
615	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
616	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
617	if (RT_LIKELY( pbBuf != NULL
618	&& offBuf < pVCpu->iem.s.cbInstrBuf))
619	{
620	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
621	bRet = pbBuf[offBuf];
622	}
623	else
624	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
625	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
626	Assert(pVCpu->iem.s.offOpcode == 0);
627	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
628	# endif
629	return bRet;
630
631	# else /* !IEM_WITH_CODE_TLB */
632	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
633	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
634	{
635	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
636	return pVCpu->iem.s.abOpcode[offOpcode];
637	}
638	return iemOpcodeGetNextU8SlowJmp(pVCpu);
639	# endif
640	}
641
642	# endif /* IEM_WITH_SETJMP */
643
644	/**
645	* Fetches the first opcode byte, returns/throws automatically on failure.
646	*
647	* @param a_pu8 Where to return the opcode byte.
648	* @remark Implicitly references pVCpu.
649	*/
650	# ifndef IEM_WITH_SETJMP
651	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) \
652	do \
653	{ \
654	VBOXSTRICTRC rcStrict2 = iemOpcodeGetFirstU8(pVCpu, (a_pu8)); \
655	if (rcStrict2 == VINF_SUCCESS) \
656	{ /* likely */ } \
657	else \
658	return rcStrict2; \
659	} while (0)
660	# else
661	# define IEM_OPCODE_GET_FIRST_U8(a_pu8) (*(a_pu8) = iemOpcodeGetFirstU8Jmp(pVCpu))
662	# endif /* IEM_WITH_SETJMP */
663
664
665	# ifndef IEM_WITH_SETJMP
666
667	/**
668	* Fetches the next opcode byte.
669	*
670	* @returns Strict VBox status code.
671	* @param pVCpu The cross context virtual CPU structure of the
672	* calling thread.
673	* @param pu8 Where to return the opcode byte.
674	*/
675	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8) RT_NOEXCEPT
676	{
677	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
678	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
679	{
680	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
681	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
682	return VINF_SUCCESS;
683	}
684	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
685	}
686
687	# else /* IEM_WITH_SETJMP */
688
689	/**
690	* Fetches the next opcode byte, longjmp on error.
691	*
692	* @returns The opcode byte.
693	* @param pVCpu The cross context virtual CPU structure of the calling thread.
694	*/
695	DECL_INLINE_THROW(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
696	{
697	# ifdef IEM_WITH_CODE_TLB
698	uint8_t bRet;
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf < pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
705	bRet = pbBuf[offBuf];
706	}
707	else
708	bRet = iemOpcodeGetNextU8SlowJmp(pVCpu);
709	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
710	Assert(pVCpu->iem.s.offOpcode < sizeof(pVCpu->iem.s.abOpcode));
711	pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++] = bRet;
712	# endif
713	return bRet;
714
715	# else /* !IEM_WITH_CODE_TLB */
716	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
717	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
718	{
719	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
720	return pVCpu->iem.s.abOpcode[offOpcode];
721	}
722	return iemOpcodeGetNextU8SlowJmp(pVCpu);
723	# endif
724	}
725
726	# endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode byte, returns automatically on failure.
730	*
731	* @param a_pu8 Where to return the opcode byte.
732	* @remark Implicitly references pVCpu.
733	*/
734	# ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
739	if (rcStrict2 == VINF_SUCCESS) \
740	{ /* likely */ } \
741	else \
742	return rcStrict2; \
743	} while (0)
744	# else
745	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
746	# endif /* IEM_WITH_SETJMP */
747
748
749	# ifndef IEM_WITH_SETJMP
750	/**
751	* Fetches the next signed byte from the opcode stream.
752	*
753	* @returns Strict VBox status code.
754	* @param pVCpu The cross context virtual CPU structure of the calling thread.
755	* @param pi8 Where to return the signed byte.
756	*/
757	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8) RT_NOEXCEPT
758	{
759	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
760	}
761	# endif /* !IEM_WITH_SETJMP */
762
763
764	/**
765	* Fetches the next signed byte from the opcode stream, returning automatically
766	* on failure.
767	*
768	* @param a_pi8 Where to return the signed byte.
769	* @remark Implicitly references pVCpu.
770	*/
771	# ifndef IEM_WITH_SETJMP
772	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
773	do \
774	{ \
775	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
776	if (rcStrict2 != VINF_SUCCESS) \
777	return rcStrict2; \
778	} while (0)
779	# else /* IEM_WITH_SETJMP */
780	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
781
782	# endif /* IEM_WITH_SETJMP */
783
784
785	# ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next signed byte from the opcode stream, extending it to
788	* unsigned 16-bit.
789	*
790	* @returns Strict VBox status code.
791	* @param pVCpu The cross context virtual CPU structure of the calling thread.
792	* @param pu16 Where to return the unsigned word.
793	*/
794	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
795	{
796	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
797	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
798	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
799
800	*pu16 = (uint16_t)(int16_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
801	pVCpu->iem.s.offOpcode = offOpcode + 1;
802	return VINF_SUCCESS;
803	}
804	# endif /* !IEM_WITH_SETJMP */
805
806	/**
807	* Fetches the next signed byte from the opcode stream and sign-extending it to
808	* a word, returning automatically on failure.
809	*
810	* @param a_pu16 Where to return the word.
811	* @remark Implicitly references pVCpu.
812	*/
813	# ifndef IEM_WITH_SETJMP
814	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
815	do \
816	{ \
817	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
818	if (rcStrict2 != VINF_SUCCESS) \
819	return rcStrict2; \
820	} while (0)
821	# else
822	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (uint16_t)(int16_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
823	# endif
824
825	# ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed byte from the opcode stream, extending it to
828	* unsigned 32-bit.
829	*
830	* @returns Strict VBox status code.
831	* @param pVCpu The cross context virtual CPU structure of the calling thread.
832	* @param pu32 Where to return the unsigned dword.
833	*/
834	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
835	{
836	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
837	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
838	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
839
840	*pu32 = (uint32_t)(int32_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
841	pVCpu->iem.s.offOpcode = offOpcode + 1;
842	return VINF_SUCCESS;
843	}
844	# endif /* !IEM_WITH_SETJMP */
845
846	/**
847	* Fetches the next signed byte from the opcode stream and sign-extending it to
848	* a word, returning automatically on failure.
849	*
850	* @param a_pu32 Where to return the word.
851	* @remark Implicitly references pVCpu.
852	*/
853	# ifndef IEM_WITH_SETJMP
854	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
855	do \
856	{ \
857	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
858	if (rcStrict2 != VINF_SUCCESS) \
859	return rcStrict2; \
860	} while (0)
861	# else
862	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (uint32_t)(int32_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
863	# endif
864
865
866	# ifndef IEM_WITH_SETJMP
867	/**
868	* Fetches the next signed byte from the opcode stream, extending it to
869	* unsigned 64-bit.
870	*
871	* @returns Strict VBox status code.
872	* @param pVCpu The cross context virtual CPU structure of the calling thread.
873	* @param pu64 Where to return the unsigned qword.
874	*/
875	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
876	{
877	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
878	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
879	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
880
881	*pu64 = (uint64_t)(int64_t)(int8_t)pVCpu->iem.s.abOpcode[offOpcode];
882	pVCpu->iem.s.offOpcode = offOpcode + 1;
883	return VINF_SUCCESS;
884	}
885	# endif /* !IEM_WITH_SETJMP */
886
887	/**
888	* Fetches the next signed byte from the opcode stream and sign-extending it to
889	* a word, returning automatically on failure.
890	*
891	* @param a_pu64 Where to return the word.
892	* @remark Implicitly references pVCpu.
893	*/
894	# ifndef IEM_WITH_SETJMP
895	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
896	do \
897	{ \
898	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
899	if (rcStrict2 != VINF_SUCCESS) \
900	return rcStrict2; \
901	} while (0)
902	# else
903	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
904	# endif
905
906
907	# ifndef IEM_WITH_SETJMP
908
909	/**
910	* Fetches the next opcode word.
911	*
912	* @returns Strict VBox status code.
913	* @param pVCpu The cross context virtual CPU structure of the calling thread.
914	* @param pu16 Where to return the opcode word.
915	*/
916	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
917	{
918	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
919	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
920	{
921	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
922	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
923	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
924	# else
925	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
926	# endif
927	return VINF_SUCCESS;
928	}
929	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
930	}
931
932	# else /* IEM_WITH_SETJMP */
933
934	/**
935	* Fetches the next opcode word, longjmp on error.
936	*
937	* @returns The opcode word.
938	* @param pVCpu The cross context virtual CPU structure of the calling thread.
939	*/
940	DECL_INLINE_THROW(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
941	{
942	# ifdef IEM_WITH_CODE_TLB
943	uint16_t u16Ret;
944	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
945	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
946	if (RT_LIKELY( pbBuf != NULL
947	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
948	{
949	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
950	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
951	u16Ret = (uint16_t const )&pbBuf[offBuf];
952	# else
953	u16Ret = RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
954	# endif
955	}
956	else
957	u16Ret = iemOpcodeGetNextU16SlowJmp(pVCpu);
958
959	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
960	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
961	Assert(offOpcode + 1 < sizeof(pVCpu->iem.s.abOpcode));
962	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
963	(uint16_t )&pVCpu->iem.s.abOpcode[offOpcode] = u16Ret;
964	# else
965	pVCpu->iem.s.abOpcode[offOpcode] = RT_LO_U8(u16Ret);
966	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_HI_U8(u16Ret);
967	# endif
968	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)2;
969	# endif
970
971	return u16Ret;
972
973	# else /* !IEM_WITH_CODE_TLB */
974	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
975	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
976	{
977	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
978	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
979	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
980	# else
981	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
982	# endif
983	}
984	return iemOpcodeGetNextU16SlowJmp(pVCpu);
985	# endif /* !IEM_WITH_CODE_TLB */
986	}
987
988	# endif /* IEM_WITH_SETJMP */
989
990	/**
991	* Fetches the next opcode word, returns automatically on failure.
992	*
993	* @param a_pu16 Where to return the opcode word.
994	* @remark Implicitly references pVCpu.
995	*/
996	# ifndef IEM_WITH_SETJMP
997	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
998	do \
999	{ \
1000	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
1001	if (rcStrict2 != VINF_SUCCESS) \
1002	return rcStrict2; \
1003	} while (0)
1004	# else
1005	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
1006	# endif
1007
1008	# ifndef IEM_WITH_SETJMP
1009	/**
1010	* Fetches the next opcode word, zero extending it to a double word.
1011	*
1012	* @returns Strict VBox status code.
1013	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1014	* @param pu32 Where to return the opcode double word.
1015	*/
1016	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1017	{
1018	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1019	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1020	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
1021
1022	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1023	pVCpu->iem.s.offOpcode = offOpcode + 2;
1024	return VINF_SUCCESS;
1025	}
1026	# endif /* !IEM_WITH_SETJMP */
1027
1028	/**
1029	* Fetches the next opcode word and zero extends it to a double word, returns
1030	* automatically on failure.
1031	*
1032	* @param a_pu32 Where to return the opcode double word.
1033	* @remark Implicitly references pVCpu.
1034	*/
1035	# ifndef IEM_WITH_SETJMP
1036	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
1037	do \
1038	{ \
1039	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
1040	if (rcStrict2 != VINF_SUCCESS) \
1041	return rcStrict2; \
1042	} while (0)
1043	# else
1044	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
1045	# endif
1046
1047	# ifndef IEM_WITH_SETJMP
1048	/**
1049	* Fetches the next opcode word, zero extending it to a quad word.
1050	*
1051	* @returns Strict VBox status code.
1052	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1053	* @param pu64 Where to return the opcode quad word.
1054	*/
1055	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1056	{
1057	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1058	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
1059	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
1060
1061	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
1062	pVCpu->iem.s.offOpcode = offOpcode + 2;
1063	return VINF_SUCCESS;
1064	}
1065	# endif /* !IEM_WITH_SETJMP */
1066
1067	/**
1068	* Fetches the next opcode word and zero extends it to a quad word, returns
1069	* automatically on failure.
1070	*
1071	* @param a_pu64 Where to return the opcode quad word.
1072	* @remark Implicitly references pVCpu.
1073	*/
1074	# ifndef IEM_WITH_SETJMP
1075	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
1076	do \
1077	{ \
1078	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
1079	if (rcStrict2 != VINF_SUCCESS) \
1080	return rcStrict2; \
1081	} while (0)
1082	# else
1083	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
1084	# endif
1085
1086
1087	# ifndef IEM_WITH_SETJMP
1088	/**
1089	* Fetches the next signed word from the opcode stream.
1090	*
1091	* @returns Strict VBox status code.
1092	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1093	* @param pi16 Where to return the signed word.
1094	*/
1095	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16) RT_NOEXCEPT
1096	{
1097	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
1098	}
1099	# endif /* !IEM_WITH_SETJMP */
1100
1101
1102	/**
1103	* Fetches the next signed word from the opcode stream, returning automatically
1104	* on failure.
1105	*
1106	* @param a_pi16 Where to return the signed word.
1107	* @remark Implicitly references pVCpu.
1108	*/
1109	# ifndef IEM_WITH_SETJMP
1110	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
1111	do \
1112	{ \
1113	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
1114	if (rcStrict2 != VINF_SUCCESS) \
1115	return rcStrict2; \
1116	} while (0)
1117	# else
1118	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
1119	# endif
1120
1121	# ifndef IEM_WITH_SETJMP
1122
1123	/**
1124	* Fetches the next opcode dword.
1125	*
1126	* @returns Strict VBox status code.
1127	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1128	* @param pu32 Where to return the opcode double word.
1129	*/
1130	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
1131	{
1132	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1133	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1134	{
1135	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1136	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1137	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1138	# else
1139	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1140	pVCpu->iem.s.abOpcode[offOpcode + 1],
1141	pVCpu->iem.s.abOpcode[offOpcode + 2],
1142	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1143	# endif
1144	return VINF_SUCCESS;
1145	}
1146	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
1147	}
1148
1149	# else /* IEM_WITH_SETJMP */
1150
1151	/**
1152	* Fetches the next opcode dword, longjmp on error.
1153	*
1154	* @returns The opcode dword.
1155	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1156	*/
1157	DECL_INLINE_THROW(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1158	{
1159	# ifdef IEM_WITH_CODE_TLB
1160	uint32_t u32Ret;
1161	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1162	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1163	if (RT_LIKELY( pbBuf != NULL
1164	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
1165	{
1166	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
1167	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1168	u32Ret = (uint32_t const )&pbBuf[offBuf];
1169	# else
1170	u32Ret = RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
1171	pbBuf[offBuf + 1],
1172	pbBuf[offBuf + 2],
1173	pbBuf[offBuf + 3]);
1174	# endif
1175	}
1176	else
1177	u32Ret = iemOpcodeGetNextU32SlowJmp(pVCpu);
1178
1179	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1180	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1181	Assert(offOpcode + 3 < sizeof(pVCpu->iem.s.abOpcode));
1182	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1183	(uint32_t )&pVCpu->iem.s.abOpcode[offOpcode] = u32Ret;
1184	# else
1185	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u32Ret);
1186	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u32Ret);
1187	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u32Ret);
1188	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u32Ret);
1189	# endif
1190	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)4;
1191	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1192
1193	return u32Ret;
1194
1195	# else /* !IEM_WITH_CODE_TLB */
1196	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1197	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
1198	{
1199	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
1200	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1201	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1202	# else
1203	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1204	pVCpu->iem.s.abOpcode[offOpcode + 1],
1205	pVCpu->iem.s.abOpcode[offOpcode + 2],
1206	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1207	# endif
1208	}
1209	return iemOpcodeGetNextU32SlowJmp(pVCpu);
1210	# endif
1211	}
1212
1213	# endif /* IEM_WITH_SETJMP */
1214
1215	/**
1216	* Fetches the next opcode dword, returns automatically on failure.
1217	*
1218	* @param a_pu32 Where to return the opcode dword.
1219	* @remark Implicitly references pVCpu.
1220	*/
1221	# ifndef IEM_WITH_SETJMP
1222	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
1223	do \
1224	{ \
1225	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
1226	if (rcStrict2 != VINF_SUCCESS) \
1227	return rcStrict2; \
1228	} while (0)
1229	# else
1230	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
1231	# endif
1232
1233	# ifndef IEM_WITH_SETJMP
1234	/**
1235	* Fetches the next opcode dword, zero extending it to a quad word.
1236	*
1237	* @returns Strict VBox status code.
1238	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1239	* @param pu64 Where to return the opcode quad word.
1240	*/
1241	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1242	{
1243	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1244	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1245	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
1246
1247	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1248	pVCpu->iem.s.abOpcode[offOpcode + 1],
1249	pVCpu->iem.s.abOpcode[offOpcode + 2],
1250	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1251	pVCpu->iem.s.offOpcode = offOpcode + 4;
1252	return VINF_SUCCESS;
1253	}
1254	# endif /* !IEM_WITH_SETJMP */
1255
1256	/**
1257	* Fetches the next opcode dword and zero extends it to a quad word, returns
1258	* automatically on failure.
1259	*
1260	* @param a_pu64 Where to return the opcode quad word.
1261	* @remark Implicitly references pVCpu.
1262	*/
1263	# ifndef IEM_WITH_SETJMP
1264	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
1265	do \
1266	{ \
1267	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
1268	if (rcStrict2 != VINF_SUCCESS) \
1269	return rcStrict2; \
1270	} while (0)
1271	# else
1272	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
1273	# endif
1274
1275
1276	# ifndef IEM_WITH_SETJMP
1277	/**
1278	* Fetches the next signed double word from the opcode stream.
1279	*
1280	* @returns Strict VBox status code.
1281	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1282	* @param pi32 Where to return the signed double word.
1283	*/
1284	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32) RT_NOEXCEPT
1285	{
1286	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1287	}
1288	# endif
1289
1290	/**
1291	* Fetches the next signed double word from the opcode stream, returning
1292	* automatically on failure.
1293	*
1294	* @param a_pi32 Where to return the signed double word.
1295	* @remark Implicitly references pVCpu.
1296	*/
1297	# ifndef IEM_WITH_SETJMP
1298	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1299	do \
1300	{ \
1301	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1302	if (rcStrict2 != VINF_SUCCESS) \
1303	return rcStrict2; \
1304	} while (0)
1305	# else
1306	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1307	# endif
1308
1309	# ifndef IEM_WITH_SETJMP
1310	/**
1311	* Fetches the next opcode dword, sign extending it into a quad word.
1312	*
1313	* @returns Strict VBox status code.
1314	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1315	* @param pu64 Where to return the opcode quad word.
1316	*/
1317	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1318	{
1319	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1320	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1321	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1322
1323	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1324	pVCpu->iem.s.abOpcode[offOpcode + 1],
1325	pVCpu->iem.s.abOpcode[offOpcode + 2],
1326	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1327	*pu64 = (uint64_t)(int64_t)i32;
1328	pVCpu->iem.s.offOpcode = offOpcode + 4;
1329	return VINF_SUCCESS;
1330	}
1331	# endif /* !IEM_WITH_SETJMP */
1332
1333	/**
1334	* Fetches the next opcode double word and sign extends it to a quad word,
1335	* returns automatically on failure.
1336	*
1337	* @param a_pu64 Where to return the opcode quad word.
1338	* @remark Implicitly references pVCpu.
1339	*/
1340	# ifndef IEM_WITH_SETJMP
1341	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1342	do \
1343	{ \
1344	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1345	if (rcStrict2 != VINF_SUCCESS) \
1346	return rcStrict2; \
1347	} while (0)
1348	# else
1349	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (uint64_t)(int64_t)(int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1350	# endif
1351
1352	# ifndef IEM_WITH_SETJMP
1353
1354	/**
1355	* Fetches the next opcode qword.
1356	*
1357	* @returns Strict VBox status code.
1358	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1359	* @param pu64 Where to return the opcode qword.
1360	*/
1361	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
1362	{
1363	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1364	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1365	{
1366	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1367	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1368	# else
1369	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1370	pVCpu->iem.s.abOpcode[offOpcode + 1],
1371	pVCpu->iem.s.abOpcode[offOpcode + 2],
1372	pVCpu->iem.s.abOpcode[offOpcode + 3],
1373	pVCpu->iem.s.abOpcode[offOpcode + 4],
1374	pVCpu->iem.s.abOpcode[offOpcode + 5],
1375	pVCpu->iem.s.abOpcode[offOpcode + 6],
1376	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1377	# endif
1378	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1379	return VINF_SUCCESS;
1380	}
1381	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1382	}
1383
1384	# else /* IEM_WITH_SETJMP */
1385
1386	/**
1387	* Fetches the next opcode qword, longjmp on error.
1388	*
1389	* @returns The opcode qword.
1390	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1391	*/
1392	DECL_INLINE_THROW(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
1393	{
1394	# ifdef IEM_WITH_CODE_TLB
1395	uint64_t u64Ret;
1396	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1397	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1398	if (RT_LIKELY( pbBuf != NULL
1399	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1400	{
1401	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1402	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1403	u64Ret = (uint64_t const )&pbBuf[offBuf];
1404	# else
1405	u64Ret = RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1406	pbBuf[offBuf + 1],
1407	pbBuf[offBuf + 2],
1408	pbBuf[offBuf + 3],
1409	pbBuf[offBuf + 4],
1410	pbBuf[offBuf + 5],
1411	pbBuf[offBuf + 6],
1412	pbBuf[offBuf + 7]);
1413	# endif
1414	}
1415	else
1416	u64Ret = iemOpcodeGetNextU64SlowJmp(pVCpu);
1417
1418	# ifdef IEM_WITH_CODE_TLB_AND_OPCODE_BUF
1419	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1420	Assert(offOpcode + 7 < sizeof(pVCpu->iem.s.abOpcode));
1421	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1422	(uint64_t )&pVCpu->iem.s.abOpcode[offOpcode] = u64Ret;
1423	# else
1424	pVCpu->iem.s.abOpcode[offOpcode] = RT_BYTE1(u64Ret);
1425	pVCpu->iem.s.abOpcode[offOpcode + 1] = RT_BYTE2(u64Ret);
1426	pVCpu->iem.s.abOpcode[offOpcode + 2] = RT_BYTE3(u64Ret);
1427	pVCpu->iem.s.abOpcode[offOpcode + 3] = RT_BYTE4(u64Ret);
1428	pVCpu->iem.s.abOpcode[offOpcode + 4] = RT_BYTE5(u64Ret);
1429	pVCpu->iem.s.abOpcode[offOpcode + 5] = RT_BYTE6(u64Ret);
1430	pVCpu->iem.s.abOpcode[offOpcode + 6] = RT_BYTE7(u64Ret);
1431	pVCpu->iem.s.abOpcode[offOpcode + 7] = RT_BYTE8(u64Ret);
1432	# endif
1433	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + (uint8_t)8;
1434	# endif /* IEM_WITH_CODE_TLB_AND_OPCODE_BUF */
1435
1436	return u64Ret;
1437
1438	# else /* !IEM_WITH_CODE_TLB */
1439	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1440	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1441	{
1442	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1443	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1444	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1445	# else
1446	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1447	pVCpu->iem.s.abOpcode[offOpcode + 1],
1448	pVCpu->iem.s.abOpcode[offOpcode + 2],
1449	pVCpu->iem.s.abOpcode[offOpcode + 3],
1450	pVCpu->iem.s.abOpcode[offOpcode + 4],
1451	pVCpu->iem.s.abOpcode[offOpcode + 5],
1452	pVCpu->iem.s.abOpcode[offOpcode + 6],
1453	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1454	# endif
1455	}
1456	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1457	# endif /* !IEM_WITH_CODE_TLB */
1458	}
1459
1460	# endif /* IEM_WITH_SETJMP */
1461
1462	/**
1463	* Fetches the next opcode quad word, returns automatically on failure.
1464	*
1465	* @param a_pu64 Where to return the opcode quad word.
1466	* @remark Implicitly references pVCpu.
1467	*/
1468	# ifndef IEM_WITH_SETJMP
1469	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1470	do \
1471	{ \
1472	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1473	if (rcStrict2 != VINF_SUCCESS) \
1474	return rcStrict2; \
1475	} while (0)
1476	# else
1477	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1478	# endif
1479
1480	/**
1481	* For fetching the opcode bytes for an ModR/M effective address, but throw
1482	* away the result.
1483	*
1484	* This is used when decoding undefined opcodes and such where we want to avoid
1485	* unnecessary MC blocks.
1486	*
1487	* @note The recompiler code overrides this one so iemOpHlpCalcRmEffAddrJmpEx is
1488	* used instead. At least for now...
1489	*/
1490	# ifndef IEM_WITH_SETJMP
1491	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1492	RTGCPTR GCPtrEff; \
1493	VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff); \
1494	if (rcStrict != VINF_SUCCESS) \
1495	return rcStrict; \
1496	} while (0)
1497	# else
1498	# define IEM_OPCODE_SKIP_RM_EFF_ADDR_BYTES(a_bRm) do { \
1499	(void)iemOpHlpCalcRmEffAddrJmp(pVCpu, bRm, 0); \
1500	} while (0)
1501	# endif
1502
1503	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1504
1505
1506	/** @name Misc Worker Functions.
1507	* @{
1508	*/
1509
1510	/**
1511	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1512	* not (kind of obsolete now).
1513	*
1514	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1515	*/
1516	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1517
1518	/**
1519	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1520	*
1521	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1522	* @param a_fEfl The new EFLAGS.
1523	*/
1524	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1525
1526
1527	/**
1528	* Loads a NULL data selector into a selector register, both the hidden and
1529	* visible parts, in protected mode.
1530	*
1531	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1532	* @param pSReg Pointer to the segment register.
1533	* @param uRpl The RPL.
1534	*/
1535	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl) RT_NOEXCEPT
1536	{
1537	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1538	* data selector in protected mode. */
1539	pSReg->Sel = uRpl;
1540	pSReg->ValidSel = uRpl;
1541	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1542	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1543	{
1544	/* VT-x (Intel 3960x) observed doing something like this. */
1545	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (IEM_GET_CPL(pVCpu) << X86DESCATTR_DPL_SHIFT);
1546	pSReg->u32Limit = UINT32_MAX;
1547	pSReg->u64Base = 0;
1548	}
1549	else
1550	{
1551	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1552	pSReg->u32Limit = 0;
1553	pSReg->u64Base = 0;
1554	}
1555	}
1556
1557	/** @} */
1558
1559
1560	/*
1561	*
1562	* Helpers routines.
1563	* Helpers routines.
1564	* Helpers routines.
1565	*
1566	*/
1567
1568	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1569
1570	/**
1571	* Recalculates the effective operand size.
1572	*
1573	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1574	*/
1575	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu) RT_NOEXCEPT
1576	{
1577	switch (IEM_GET_CPU_MODE(pVCpu))
1578	{
1579	case IEMMODE_16BIT:
1580	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1581	break;
1582	case IEMMODE_32BIT:
1583	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1584	break;
1585	case IEMMODE_64BIT:
1586	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1587	{
1588	case 0:
1589	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1590	break;
1591	case IEM_OP_PRF_SIZE_OP:
1592	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1593	break;
1594	case IEM_OP_PRF_SIZE_REX_W:
1595	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1596	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1597	break;
1598	}
1599	break;
1600	default:
1601	AssertFailed();
1602	}
1603	}
1604
1605
1606	/**
1607	* Sets the default operand size to 64-bit and recalculates the effective
1608	* operand size.
1609	*
1610	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1611	*/
1612	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu) RT_NOEXCEPT
1613	{
1614	Assert(IEM_IS_64BIT_CODE(pVCpu));
1615	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1616	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1617	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1618	else
1619	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1620	}
1621
1622
1623	/**
1624	* Sets the default operand size to 64-bit and recalculates the effective
1625	* operand size, with intel ignoring any operand size prefix (AMD respects it).
1626	*
1627	* This is for the relative jumps.
1628	*
1629	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1630	*/
1631	DECLINLINE(void) iemRecalEffOpSize64DefaultAndIntelIgnoresOpSizePrefix(PVMCPUCC pVCpu) RT_NOEXCEPT
1632	{
1633	Assert(IEM_IS_64BIT_CODE(pVCpu));
1634	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1635	if ( (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP
1636	\|\| pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
1637	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1638	else
1639	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1640	}
1641
1642	#endif /* !IEM_WITH_OPAQUE_DECODER_STATE */
1643
1644
1645
1646	/** @name Register Access.
1647	* @{
1648	*/
1649
1650	/**
1651	* Gets a reference (pointer) to the specified hidden segment register.
1652	*
1653	* @returns Hidden register reference.
1654	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1655	* @param iSegReg The segment register.
1656	*/
1657	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1658	{
1659	Assert(iSegReg < X86_SREG_COUNT);
1660	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1661	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1662
1663	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1664	return pSReg;
1665	}
1666
1667
1668	/**
1669	* Ensures that the given hidden segment register is up to date.
1670	*
1671	* @returns Hidden register reference.
1672	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1673	* @param pSReg The segment register.
1674	*/
1675	DECL_FORCE_INLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg) RT_NOEXCEPT
1676	{
1677	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1678	NOREF(pVCpu);
1679	return pSReg;
1680	}
1681
1682
1683	/**
1684	* Gets a reference (pointer) to the specified segment register (the selector
1685	* value).
1686	*
1687	* @returns Pointer to the selector variable.
1688	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1689	* @param iSegReg The segment register.
1690	*/
1691	DECL_FORCE_INLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1692	{
1693	Assert(iSegReg < X86_SREG_COUNT);
1694	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1695	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1696	}
1697
1698
1699	/**
1700	* Fetches the selector value of a segment register.
1701	*
1702	* @returns The selector value.
1703	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1704	* @param iSegReg The segment register.
1705	*/
1706	DECL_FORCE_INLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1707	{
1708	Assert(iSegReg < X86_SREG_COUNT);
1709	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1710	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1711	}
1712
1713
1714	/**
1715	* Fetches the base address value of a segment register.
1716	*
1717	* @returns The selector value.
1718	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1719	* @param iSegReg The segment register.
1720	*/
1721	DECL_FORCE_INLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1722	{
1723	Assert(iSegReg < X86_SREG_COUNT);
1724	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1725	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1726	}
1727
1728
1729	/**
1730	* Gets a reference (pointer) to the specified general purpose register.
1731	*
1732	* @returns Register reference.
1733	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1734	* @param iReg The general purpose register.
1735	*/
1736	DECL_FORCE_INLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1737	{
1738	Assert(iReg < 16);
1739	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1740	}
1741
1742
1743	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1744	/**
1745	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1746	*
1747	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1748	*
1749	* @returns Register reference.
1750	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1751	* @param iReg The register.
1752	*/
1753	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1754	{
1755	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1756	{
1757	Assert(iReg < 16);
1758	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1759	}
1760	/* high 8-bit register. */
1761	Assert(iReg < 8);
1762	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1763	}
1764	#endif
1765
1766
1767	/**
1768	* Gets a reference (pointer) to the specified 8-bit general purpose register,
1769	* alternative version with extended (20) register index.
1770	*
1771	* @returns Register reference.
1772	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1773	* @param iRegEx The register. The 16 first are regular ones,
1774	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1775	*/
1776	DECL_FORCE_INLINE(uint8_t *) iemGRegRefU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1777	{
1778	/** @todo This could be done by double indexing on little endian hosts:
1779	* return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 15].ab[iRegEx >> 4]; */
1780	if (iRegEx < 16)
1781	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx].u8;
1782
1783	/* high 8-bit register. */
1784	Assert(iRegEx < 20);
1785	return &pVCpu->cpum.GstCtx.aGRegs[iRegEx & 3].bHi;
1786	}
1787
1788
1789	/**
1790	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1791	*
1792	* @returns Register reference.
1793	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1794	* @param iReg The register.
1795	*/
1796	DECL_FORCE_INLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1797	{
1798	Assert(iReg < 16);
1799	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1800	}
1801
1802
1803	/**
1804	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1805	*
1806	* @returns Register reference.
1807	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1808	* @param iReg The register.
1809	*/
1810	DECL_FORCE_INLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1811	{
1812	Assert(iReg < 16);
1813	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1814	}
1815
1816
1817	/**
1818	* Gets a reference (pointer) to the specified signed 32-bit general purpose register.
1819	*
1820	* @returns Register reference.
1821	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1822	* @param iReg The register.
1823	*/
1824	DECL_FORCE_INLINE(int32_t *) iemGRegRefI32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1825	{
1826	Assert(iReg < 16);
1827	return (int32_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1828	}
1829
1830
1831	/**
1832	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1833	*
1834	* @returns Register reference.
1835	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1836	* @param iReg The register.
1837	*/
1838	DECL_FORCE_INLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1839	{
1840	Assert(iReg < 64);
1841	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1842	}
1843
1844
1845	/**
1846	* Gets a reference (pointer) to the specified signed 64-bit general purpose register.
1847	*
1848	* @returns Register reference.
1849	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1850	* @param iReg The register.
1851	*/
1852	DECL_FORCE_INLINE(int64_t *) iemGRegRefI64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1853	{
1854	Assert(iReg < 16);
1855	return (int64_t *)&pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1856	}
1857
1858
1859	/**
1860	* Gets a reference (pointer) to the specified segment register's base address.
1861	*
1862	* @returns Segment register base address reference.
1863	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1864	* @param iSegReg The segment selector.
1865	*/
1866	DECL_FORCE_INLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg) RT_NOEXCEPT
1867	{
1868	Assert(iSegReg < X86_SREG_COUNT);
1869	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1870	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1871	}
1872
1873
1874	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
1875	/**
1876	* Fetches the value of a 8-bit general purpose register.
1877	*
1878	* @returns The register value.
1879	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1880	* @param iReg The register.
1881	*/
1882	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1883	{
1884	return *iemGRegRefU8(pVCpu, iReg);
1885	}
1886	#endif
1887
1888
1889	/**
1890	* Fetches the value of a 8-bit general purpose register, alternative version
1891	* with extended (20) register index.
1892
1893	* @returns The register value.
1894	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1895	* @param iRegEx The register. The 16 first are regular ones,
1896	* whereas 16 thru 19 maps to AH, CH, DH and BH.
1897	*/
1898	DECL_FORCE_INLINE(uint8_t) iemGRegFetchU8Ex(PVMCPUCC pVCpu, uint8_t iRegEx) RT_NOEXCEPT
1899	{
1900	return *iemGRegRefU8Ex(pVCpu, iRegEx);
1901	}
1902
1903
1904	/**
1905	* Fetches the value of a 16-bit general purpose register.
1906	*
1907	* @returns The register value.
1908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1909	* @param iReg The register.
1910	*/
1911	DECL_FORCE_INLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1912	{
1913	Assert(iReg < 16);
1914	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1915	}
1916
1917
1918	/**
1919	* Fetches the value of a 32-bit general purpose register.
1920	*
1921	* @returns The register value.
1922	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1923	* @param iReg The register.
1924	*/
1925	DECL_FORCE_INLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1926	{
1927	Assert(iReg < 16);
1928	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1929	}
1930
1931
1932	/**
1933	* Fetches the value of a 64-bit general purpose register.
1934	*
1935	* @returns The register value.
1936	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1937	* @param iReg The register.
1938	*/
1939	DECL_FORCE_INLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg) RT_NOEXCEPT
1940	{
1941	Assert(iReg < 16);
1942	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1943	}
1944
1945
1946	/**
1947	* Stores a 16-bit value to a general purpose register.
1948	*
1949	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1950	* @param iReg The register.
1951	* @param uValue The value to store.
1952	*/
1953	DECL_FORCE_INLINE(void) iemGRegStoreU16(PVMCPUCC pVCpu, uint8_t iReg, uint16_t uValue) RT_NOEXCEPT
1954	{
1955	Assert(iReg < 16);
1956	pVCpu->cpum.GstCtx.aGRegs[iReg].u16 = uValue;
1957	}
1958
1959
1960	/**
1961	* Stores a 32-bit value to a general purpose register, implicitly clearing high
1962	* values.
1963	*
1964	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1965	* @param iReg The register.
1966	* @param uValue The value to store.
1967	*/
1968	DECL_FORCE_INLINE(void) iemGRegStoreU32(PVMCPUCC pVCpu, uint8_t iReg, uint32_t uValue) RT_NOEXCEPT
1969	{
1970	Assert(iReg < 16);
1971	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1972	}
1973
1974
1975	/**
1976	* Stores a 64-bit value to a general purpose register.
1977	*
1978	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1979	* @param iReg The register.
1980	* @param uValue The value to store.
1981	*/
1982	DECL_FORCE_INLINE(void) iemGRegStoreU64(PVMCPUCC pVCpu, uint8_t iReg, uint64_t uValue) RT_NOEXCEPT
1983	{
1984	Assert(iReg < 16);
1985	pVCpu->cpum.GstCtx.aGRegs[iReg].u64 = uValue;
1986	}
1987
1988
1989	/**
1990	* Get the address of the top of the stack.
1991	*
1992	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1993	*/
1994	DECL_FORCE_INLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu) RT_NOEXCEPT
1995	{
1996	if (IEM_IS_64BIT_CODE(pVCpu))
1997	return pVCpu->cpum.GstCtx.rsp;
1998	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1999	return pVCpu->cpum.GstCtx.esp;
2000	return pVCpu->cpum.GstCtx.sp;
2001	}
2002
2003
2004	/**
2005	* Updates the RIP/EIP/IP to point to the next instruction.
2006	*
2007	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2008	* @param cbInstr The number of bytes to add.
2009	*/
2010	DECL_FORCE_INLINE(void) iemRegAddToRip(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2011	{
2012	/*
2013	* Advance RIP.
2014	*
2015	* When we're targetting 8086/8, 80186/8 or 80286 mode the updates are 16-bit,
2016	* while in all other modes except LM64 the updates are 32-bit. This means
2017	* we need to watch for both 32-bit and 16-bit "carry" situations, i.e.
2018	* 4GB and 64KB rollovers, and decide whether anything needs masking.
2019	*
2020	* See PC wrap around tests in bs3-cpu-weird-1.
2021	*/
2022	uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
2023	uint64_t const uRipNext = uRipPrev + cbInstr;
2024	if (RT_LIKELY( !((uRipNext ^ uRipPrev) & (RT_BIT_64(32) \| RT_BIT_64(16)))
2025	\|\| IEM_IS_64BIT_CODE(pVCpu)))
2026	pVCpu->cpum.GstCtx.rip = uRipNext;
2027	else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
2028	pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
2029	else
2030	pVCpu->cpum.GstCtx.rip = (uint16_t)uRipNext;
2031	}
2032
2033
2034	/**
2035	* Called by iemRegAddToRipAndFinishingClearingRF and others when any of the
2036	* following EFLAGS bits are set:
2037	* - X86_EFL_RF - clear it.
2038	* - CPUMCTX_INHIBIT_SHADOW (_SS/_STI) - clear them.
2039	* - X86_EFL_TF - generate single step \#DB trap.
2040	* - CPUMCTX_DBG_HIT_DR0/1/2/3 - generate \#DB trap (data or I/O, not
2041	* instruction).
2042	*
2043	* According to @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events},
2044	* a \#DB due to TF (single stepping) or a DRx non-instruction breakpoint
2045	* takes priority over both NMIs and hardware interrupts. So, neither is
2046	* considered here. (The RESET, \#MC, SMI, INIT, STOPCLK and FLUSH events are
2047	* either unsupported will be triggered on-top of any \#DB raised here.)
2048	*
2049	* The RF flag only needs to be cleared here as it only suppresses instruction
2050	* breakpoints which are not raised here (happens synchronously during
2051	* instruction fetching).
2052	*
2053	* The CPUMCTX_INHIBIT_SHADOW_SS flag will be cleared by this function, so its
2054	* status has no bearing on whether \#DB exceptions are raised.
2055	*
2056	* @note This must NOT be called by the two instructions setting the
2057	* CPUMCTX_INHIBIT_SHADOW_SS flag.
2058	*
2059	* @see @sdmv3{077,200,Table 6-2,Priority Among Concurrent Events}
2060	* @see @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching
2061	* Stacks}
2062	*/
2063	static VBOXSTRICTRC iemFinishInstructionWithFlagsSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2064	{
2065	/*
2066	* Normally we're just here to clear RF and/or interrupt shadow bits.
2067	*/
2068	if (RT_LIKELY((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) == 0))
2069	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW);
2070	else
2071	{
2072	/*
2073	* Raise a #DB or/and DBGF event.
2074	*/
2075	VBOXSTRICTRC rcStrict;
2076	if (pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_TF \| CPUMCTX_DBG_HIT_DRX_MASK))
2077	{
2078	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2079	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2080	if (pVCpu->cpum.GstCtx.eflags.uBoth & X86_EFL_TF)
2081	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS;
2082	pVCpu->cpum.GstCtx.dr[6] \|= (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2083	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64\n",
2084	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2085	pVCpu->cpum.GstCtx.rflags.uBoth));
2086
2087	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK);
2088	rcStrict = iemRaiseDebugException(pVCpu);
2089
2090	/* A DBGF event/breakpoint trumps the iemRaiseDebugException informational status code. */
2091	if ((pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK) && RT_FAILURE(rcStrict))
2092	{
2093	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2094	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2095	}
2096	}
2097	else
2098	{
2099	Assert(pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_MASK);
2100	rcStrict = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_DBGF_BP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_EVENT;
2101	LogFlowFunc(("dbgf at %04X:%016llX: %Rrc\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
2102	}
2103	pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_DBG_DBGF_MASK;
2104	return rcStrict;
2105	}
2106	return VINF_SUCCESS;
2107	}
2108
2109
2110	/**
2111	* Clears the RF and CPUMCTX_INHIBIT_SHADOW, triggering \#DB if pending.
2112	*
2113	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2114	*/
2115	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2116	{
2117	/*
2118	* We assume that most of the time nothing actually needs doing here.
2119	*/
2120	AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
2121	if (RT_LIKELY(!( pVCpu->cpum.GstCtx.eflags.uBoth
2122	& (X86_EFL_TF \| X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK)) ))
2123	return VINF_SUCCESS;
2124	return iemFinishInstructionWithFlagsSet(pVCpu);
2125	}
2126
2127
2128	/**
2129	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF
2130	* and CPUMCTX_INHIBIT_SHADOW.
2131	*
2132	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2133	* @param cbInstr The number of bytes to add.
2134	*/
2135	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2136	{
2137	iemRegAddToRip(pVCpu, cbInstr);
2138	return iemRegFinishClearingRF(pVCpu);
2139	}
2140
2141
2142	/**
2143	* Updates the RIP to point to the next instruction and clears EFLAGS.RF
2144	* and CPUMCTX_INHIBIT_SHADOW.
2145	*
2146	* Only called from 64-bit code.
2147	*
2148	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2149	* @param cbInstr The number of bytes to add.
2150	*/
2151	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToRip64AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2152	{
2153	pVCpu->cpum.GstCtx.rip = pVCpu->cpum.GstCtx.rip + cbInstr;
2154	return iemRegFinishClearingRF(pVCpu);
2155	}
2156
2157
2158	/**
2159	* Updates the EIP to point to the next instruction and clears EFLAGS.RF and
2160	* CPUMCTX_INHIBIT_SHADOW.
2161	*
2162	* This is never from 64-bit code.
2163	*
2164	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2165	* @param cbInstr The number of bytes to add.
2166	*/
2167	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToEip32AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2168	{
2169	pVCpu->cpum.GstCtx.rip = (uint32_t)(pVCpu->cpum.GstCtx.eip + cbInstr);
2170	return iemRegFinishClearingRF(pVCpu);
2171	}
2172
2173
2174	/**
2175	* Updates the IP to point to the next instruction and clears EFLAGS.RF and
2176	* CPUMCTX_INHIBIT_SHADOW.
2177	*
2178	* This is only ever used from 16-bit code on a pre-386 CPU.
2179	*
2180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2181	* @param cbInstr The number of bytes to add.
2182	*/
2183	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegAddToIp16AndFinishingClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
2184	{
2185	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr);
2186	return iemRegFinishClearingRF(pVCpu);
2187	}
2188
2189
2190	/**
2191	* Adds a 8-bit signed jump offset to RIP from 64-bit code.
2192	*
2193	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2194	* segment limit.
2195	*
2196	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2197	* @param cbInstr Instruction size.
2198	* @param offNextInstr The offset of the next instruction.
2199	* @param enmEffOpSize Effective operand size.
2200	*/
2201	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2202	IEMMODE enmEffOpSize) RT_NOEXCEPT
2203	{
2204	Assert(IEM_IS_64BIT_CODE(pVCpu));
2205	Assert(enmEffOpSize == IEMMODE_64BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2206
2207	uint64_t uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2208	if (enmEffOpSize == IEMMODE_16BIT)
2209	uNewRip &= UINT16_MAX;
2210
2211	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2212	pVCpu->cpum.GstCtx.rip = uNewRip;
2213	else
2214	return iemRaiseGeneralProtectionFault0(pVCpu);
2215
2216	#ifndef IEM_WITH_CODE_TLB
2217	iemOpcodeFlushLight(pVCpu, cbInstr);
2218	#endif
2219
2220	/*
2221	* Clear RF and finish the instruction (maybe raise #DB).
2222	*/
2223	return iemRegFinishClearingRF(pVCpu);
2224	}
2225
2226
2227	/**
2228	* Adds a 8-bit signed jump offset to EIP, on 386 or later from 16-bit or 32-bit
2229	* code (never 64-bit).
2230	*
2231	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2232	* segment limit.
2233	*
2234	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2235	* @param cbInstr Instruction size.
2236	* @param offNextInstr The offset of the next instruction.
2237	* @param enmEffOpSize Effective operand size.
2238	*/
2239	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
2240	IEMMODE enmEffOpSize) RT_NOEXCEPT
2241	{
2242	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2243	Assert(enmEffOpSize == IEMMODE_32BIT \|\| enmEffOpSize == IEMMODE_16BIT);
2244
2245	uint32_t uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
2246	if (enmEffOpSize == IEMMODE_16BIT)
2247	uNewEip &= UINT16_MAX;
2248	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2249	pVCpu->cpum.GstCtx.rip = uNewEip;
2250	else
2251	return iemRaiseGeneralProtectionFault0(pVCpu);
2252
2253	#ifndef IEM_WITH_CODE_TLB
2254	iemOpcodeFlushLight(pVCpu, cbInstr);
2255	#endif
2256
2257	/*
2258	* Clear RF and finish the instruction (maybe raise #DB).
2259	*/
2260	return iemRegFinishClearingRF(pVCpu);
2261	}
2262
2263
2264	/**
2265	* Adds a 8-bit signed jump offset to IP, on a pre-386 CPU.
2266	*
2267	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2268	* segment limit.
2269	*
2270	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2271	* @param cbInstr Instruction size.
2272	* @param offNextInstr The offset of the next instruction.
2273	*/
2274	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegIp16RelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2275	int8_t offNextInstr) RT_NOEXCEPT
2276	{
2277	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2278
2279	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
2280	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2281	pVCpu->cpum.GstCtx.rip = uNewIp;
2282	else
2283	return iemRaiseGeneralProtectionFault0(pVCpu);
2284
2285	#ifndef IEM_WITH_CODE_TLB
2286	iemOpcodeFlushLight(pVCpu, cbInstr);
2287	#endif
2288
2289	/*
2290	* Clear RF and finish the instruction (maybe raise #DB).
2291	*/
2292	return iemRegFinishClearingRF(pVCpu);
2293	}
2294
2295
2296	/**
2297	* Adds a 16-bit signed jump offset to RIP from 64-bit code.
2298	*
2299	* @returns Strict VBox status code.
2300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2301	* @param cbInstr Instruction size.
2302	* @param offNextInstr The offset of the next instruction.
2303	*/
2304	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2305	int16_t offNextInstr) RT_NOEXCEPT
2306	{
2307	Assert(IEM_IS_64BIT_CODE(pVCpu));
2308
2309	pVCpu->cpum.GstCtx.rip = (uint16_t)(pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr);
2310
2311	#ifndef IEM_WITH_CODE_TLB
2312	iemOpcodeFlushLight(pVCpu, cbInstr);
2313	#endif
2314
2315	/*
2316	* Clear RF and finish the instruction (maybe raise #DB).
2317	*/
2318	return iemRegFinishClearingRF(pVCpu);
2319	}
2320
2321
2322	/**
2323	* Adds a 16-bit signed jump offset to EIP from 16-bit or 32-bit code.
2324	*
2325	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2326	* segment limit.
2327	*
2328	* @returns Strict VBox status code.
2329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2330	* @param cbInstr Instruction size.
2331	* @param offNextInstr The offset of the next instruction.
2332	*
2333	* @note This is also used by 16-bit code in pre-386 mode, as the code is
2334	* identical.
2335	*/
2336	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2337	int16_t offNextInstr) RT_NOEXCEPT
2338	{
2339	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2340
2341	uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
2342	if (RT_LIKELY(uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit))
2343	pVCpu->cpum.GstCtx.rip = uNewIp;
2344	else
2345	return iemRaiseGeneralProtectionFault0(pVCpu);
2346
2347	#ifndef IEM_WITH_CODE_TLB
2348	iemOpcodeFlushLight(pVCpu, cbInstr);
2349	#endif
2350
2351	/*
2352	* Clear RF and finish the instruction (maybe raise #DB).
2353	*/
2354	return iemRegFinishClearingRF(pVCpu);
2355	}
2356
2357
2358	/**
2359	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2360	*
2361	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2362	* segment limit.
2363	*
2364	* We ASSUME that the effective operand size is 64-bit here, as 16-bit is the
2365	* only alternative for relative jumps in 64-bit code and that is already
2366	* handled in the decoder stage.
2367	*
2368	* @returns Strict VBox status code.
2369	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2370	* @param cbInstr Instruction size.
2371	* @param offNextInstr The offset of the next instruction.
2372	*/
2373	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegRip64RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2374	int32_t offNextInstr) RT_NOEXCEPT
2375	{
2376	Assert(IEM_IS_64BIT_CODE(pVCpu));
2377
2378	uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
2379	if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
2380	pVCpu->cpum.GstCtx.rip = uNewRip;
2381	else
2382	return iemRaiseGeneralProtectionFault0(pVCpu);
2383
2384	#ifndef IEM_WITH_CODE_TLB
2385	iemOpcodeFlushLight(pVCpu, cbInstr);
2386	#endif
2387
2388	/*
2389	* Clear RF and finish the instruction (maybe raise #DB).
2390	*/
2391	return iemRegFinishClearingRF(pVCpu);
2392	}
2393
2394
2395	/**
2396	* Adds a 32-bit signed jump offset to RIP from 64-bit code.
2397	*
2398	* May raise a \#GP(0) if the new RIP is non-canonical or outside the code
2399	* segment limit.
2400	*
2401	* We ASSUME that the effective operand size is 32-bit here, as 16-bit is the
2402	* only alternative for relative jumps in 32-bit code and that is already
2403	* handled in the decoder stage.
2404	*
2405	* @returns Strict VBox status code.
2406	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2407	* @param cbInstr Instruction size.
2408	* @param offNextInstr The offset of the next instruction.
2409	*/
2410	DECL_FORCE_INLINE(VBOXSTRICTRC) iemRegEip32RelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr,
2411	int32_t offNextInstr) RT_NOEXCEPT
2412	{
2413	Assert(!IEM_IS_64BIT_CODE(pVCpu));
2414	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
2415
2416	uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
2417	if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
2418	pVCpu->cpum.GstCtx.rip = uNewEip;
2419	else
2420	return iemRaiseGeneralProtectionFault0(pVCpu);
2421
2422	#ifndef IEM_WITH_CODE_TLB
2423	iemOpcodeFlushLight(pVCpu, cbInstr);
2424	#endif
2425
2426	/*
2427	* Clear RF and finish the instruction (maybe raise #DB).
2428	*/
2429	return iemRegFinishClearingRF(pVCpu);
2430	}
2431
2432
2433	/**
2434	* Extended version of iemFinishInstructionWithFlagsSet that goes with
2435	* iemRegAddToRipAndFinishingClearingRfEx.
2436	*
2437	* See iemFinishInstructionWithFlagsSet() for details.
2438	*/
2439	static VBOXSTRICTRC iemFinishInstructionWithTfSet(PVMCPUCC pVCpu) RT_NOEXCEPT
2440	{
2441	/*
2442	* Raise a #DB.
2443	*/
2444	IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
2445	pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
2446	pVCpu->cpum.GstCtx.dr[6] \|= X86_DR6_BS
2447	\| (pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_DBG_HIT_DRX_MASK) >> CPUMCTX_DBG_HIT_DRX_SHIFT;
2448	/** @todo Do we set all pending \#DB events, or just one? */
2449	LogFlowFunc(("Guest #DB fired at %04X:%016llX: DR6=%08X, RFLAGS=%16RX64 (popf)\n",
2450	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, (unsigned)pVCpu->cpum.GstCtx.dr[6],
2451	pVCpu->cpum.GstCtx.rflags.uBoth));
2452	pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF \| CPUMCTX_INHIBIT_SHADOW \| CPUMCTX_DBG_HIT_DRX_MASK \| CPUMCTX_DBG_DBGF_MASK);
2453	return iemRaiseDebugException(pVCpu);
2454	}
2455
2456
2457	/**
2458	* Extended version of iemRegAddToRipAndFinishingClearingRF for use by POPF and
2459	* others potentially updating EFLAGS.TF.
2460	*
2461	* The single step event must be generated using the TF value at the start of
2462	* the instruction, not the new value set by it.
2463	*
2464	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2465	* @param cbInstr The number of bytes to add.
2466	* @param fEflOld The EFLAGS at the start of the instruction
2467	* execution.
2468	*/
2469	DECLINLINE(VBOXSTRICTRC) iemRegAddToRipAndFinishingClearingRfEx(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t fEflOld) RT_NOEXCEPT
2470	{
2471	iemRegAddToRip(pVCpu, cbInstr);
2472	if (!(fEflOld & X86_EFL_TF))
2473	return iemRegFinishClearingRF(pVCpu);
2474	return iemFinishInstructionWithTfSet(pVCpu);
2475	}
2476
2477
2478	#ifndef IEM_WITH_OPAQUE_DECODER_STATE
2479	/**
2480	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
2481	*
2482	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2483	*/
2484	DECLINLINE(VBOXSTRICTRC) iemRegUpdateRipAndFinishClearingRF(PVMCPUCC pVCpu) RT_NOEXCEPT
2485	{
2486	return iemRegAddToRipAndFinishingClearingRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
2487	}
2488	#endif
2489
2490
2491	/**
2492	* Adds to the stack pointer.
2493	*
2494	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2495	* @param cbToAdd The number of bytes to add (8-bit!).
2496	*/
2497	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd) RT_NOEXCEPT
2498	{
2499	if (IEM_IS_64BIT_CODE(pVCpu))
2500	pVCpu->cpum.GstCtx.rsp += cbToAdd;
2501	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2502	pVCpu->cpum.GstCtx.esp += cbToAdd;
2503	else
2504	pVCpu->cpum.GstCtx.sp += cbToAdd;
2505	}
2506
2507
2508	/**
2509	* Subtracts from the stack pointer.
2510	*
2511	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2512	* @param cbToSub The number of bytes to subtract (8-bit!).
2513	*/
2514	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub) RT_NOEXCEPT
2515	{
2516	if (IEM_IS_64BIT_CODE(pVCpu))
2517	pVCpu->cpum.GstCtx.rsp -= cbToSub;
2518	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2519	pVCpu->cpum.GstCtx.esp -= cbToSub;
2520	else
2521	pVCpu->cpum.GstCtx.sp -= cbToSub;
2522	}
2523
2524
2525	/**
2526	* Adds to the temporary stack pointer.
2527	*
2528	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2529	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2530	* @param cbToAdd The number of bytes to add (16-bit).
2531	*/
2532	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd) RT_NOEXCEPT
2533	{
2534	if (IEM_IS_64BIT_CODE(pVCpu))
2535	pTmpRsp->u += cbToAdd;
2536	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2537	pTmpRsp->DWords.dw0 += cbToAdd;
2538	else
2539	pTmpRsp->Words.w0 += cbToAdd;
2540	}
2541
2542
2543	/**
2544	* Subtracts from the temporary stack pointer.
2545	*
2546	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2547	* @param pTmpRsp The temporary SP/ESP/RSP to update.
2548	* @param cbToSub The number of bytes to subtract.
2549	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
2550	* expecting that.
2551	*/
2552	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub) RT_NOEXCEPT
2553	{
2554	if (IEM_IS_64BIT_CODE(pVCpu))
2555	pTmpRsp->u -= cbToSub;
2556	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2557	pTmpRsp->DWords.dw0 -= cbToSub;
2558	else
2559	pTmpRsp->Words.w0 -= cbToSub;
2560	}
2561
2562
2563	/**
2564	* Calculates the effective stack address for a push of the specified size as
2565	* well as the new RSP value (upper bits may be masked).
2566	*
2567	* @returns Effective stack addressf for the push.
2568	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2569	* @param cbItem The size of the stack item to pop.
2570	* @param puNewRsp Where to return the new RSP value.
2571	*/
2572	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2573	{
2574	RTUINT64U uTmpRsp;
2575	RTGCPTR GCPtrTop;
2576	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2577
2578	if (IEM_IS_64BIT_CODE(pVCpu))
2579	GCPtrTop = uTmpRsp.u -= cbItem;
2580	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2581	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
2582	else
2583	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
2584	*puNewRsp = uTmpRsp.u;
2585	return GCPtrTop;
2586	}
2587
2588
2589	/**
2590	* Gets the current stack pointer and calculates the value after a pop of the
2591	* specified size.
2592	*
2593	* @returns Current stack pointer.
2594	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2595	* @param cbItem The size of the stack item to pop.
2596	* @param puNewRsp Where to return the new RSP value.
2597	*/
2598	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp) RT_NOEXCEPT
2599	{
2600	RTUINT64U uTmpRsp;
2601	RTGCPTR GCPtrTop;
2602	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
2603
2604	if (IEM_IS_64BIT_CODE(pVCpu))
2605	{
2606	GCPtrTop = uTmpRsp.u;
2607	uTmpRsp.u += cbItem;
2608	}
2609	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2610	{
2611	GCPtrTop = uTmpRsp.DWords.dw0;
2612	uTmpRsp.DWords.dw0 += cbItem;
2613	}
2614	else
2615	{
2616	GCPtrTop = uTmpRsp.Words.w0;
2617	uTmpRsp.Words.w0 += cbItem;
2618	}
2619	*puNewRsp = uTmpRsp.u;
2620	return GCPtrTop;
2621	}
2622
2623
2624	/**
2625	* Calculates the effective stack address for a push of the specified size as
2626	* well as the new temporary RSP value (upper bits may be masked).
2627	*
2628	* @returns Effective stack addressf for the push.
2629	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2630	* @param pTmpRsp The temporary stack pointer. This is updated.
2631	* @param cbItem The size of the stack item to pop.
2632	*/
2633	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2634	{
2635	RTGCPTR GCPtrTop;
2636
2637	if (IEM_IS_64BIT_CODE(pVCpu))
2638	GCPtrTop = pTmpRsp->u -= cbItem;
2639	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2640	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
2641	else
2642	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
2643	return GCPtrTop;
2644	}
2645
2646
2647	/**
2648	* Gets the effective stack address for a pop of the specified size and
2649	* calculates and updates the temporary RSP.
2650	*
2651	* @returns Current stack pointer.
2652	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2653	* @param pTmpRsp The temporary stack pointer. This is updated.
2654	* @param cbItem The size of the stack item to pop.
2655	*/
2656	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem) RT_NOEXCEPT
2657	{
2658	RTGCPTR GCPtrTop;
2659	if (IEM_IS_64BIT_CODE(pVCpu))
2660	{
2661	GCPtrTop = pTmpRsp->u;
2662	pTmpRsp->u += cbItem;
2663	}
2664	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
2665	{
2666	GCPtrTop = pTmpRsp->DWords.dw0;
2667	pTmpRsp->DWords.dw0 += cbItem;
2668	}
2669	else
2670	{
2671	GCPtrTop = pTmpRsp->Words.w0;
2672	pTmpRsp->Words.w0 += cbItem;
2673	}
2674	return GCPtrTop;
2675	}
2676
2677	/** @} */
2678
2679
2680	/** @name FPU access and helpers.
2681	*
2682	* @{
2683	*/
2684
2685
2686	/**
2687	* Hook for preparing to use the host FPU.
2688	*
2689	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2690	*
2691	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2692	*/
2693	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu) RT_NOEXCEPT
2694	{
2695	#ifdef IN_RING3
2696	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2697	#else
2698	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
2699	#endif
2700	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2701	}
2702
2703
2704	/**
2705	* Hook for preparing to use the host FPU for SSE.
2706	*
2707	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2708	*
2709	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2710	*/
2711	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu) RT_NOEXCEPT
2712	{
2713	iemFpuPrepareUsage(pVCpu);
2714	}
2715
2716
2717	/**
2718	* Hook for preparing to use the host FPU for AVX.
2719	*
2720	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2721	*
2722	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2723	*/
2724	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu) RT_NOEXCEPT
2725	{
2726	iemFpuPrepareUsage(pVCpu);
2727	}
2728
2729
2730	/**
2731	* Hook for actualizing the guest FPU state before the interpreter reads it.
2732	*
2733	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2734	*
2735	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2736	*/
2737	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2738	{
2739	#ifdef IN_RING3
2740	NOREF(pVCpu);
2741	#else
2742	CPUMRZFpuStateActualizeForRead(pVCpu);
2743	#endif
2744	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2745	}
2746
2747
2748	/**
2749	* Hook for actualizing the guest FPU state before the interpreter changes it.
2750	*
2751	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2752	*
2753	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2754	*/
2755	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2756	{
2757	#ifdef IN_RING3
2758	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2759	#else
2760	CPUMRZFpuStateActualizeForChange(pVCpu);
2761	#endif
2762	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2763	}
2764
2765
2766	/**
2767	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
2768	* only.
2769	*
2770	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2771	*
2772	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2773	*/
2774	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2775	{
2776	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2777	NOREF(pVCpu);
2778	#else
2779	CPUMRZFpuStateActualizeSseForRead(pVCpu);
2780	#endif
2781	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2782	}
2783
2784
2785	/**
2786	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
2787	* read+write.
2788	*
2789	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2790	*
2791	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2792	*/
2793	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2794	{
2795	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
2796	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2797	#else
2798	CPUMRZFpuStateActualizeForChange(pVCpu);
2799	#endif
2800	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2801
2802	/* Make sure any changes are loaded the next time around. */
2803	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
2804	}
2805
2806
2807	/**
2808	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
2809	* only.
2810	*
2811	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2812	*
2813	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2814	*/
2815	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu) RT_NOEXCEPT
2816	{
2817	#ifdef IN_RING3
2818	NOREF(pVCpu);
2819	#else
2820	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
2821	#endif
2822	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2823	}
2824
2825
2826	/**
2827	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
2828	* read+write.
2829	*
2830	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
2831	*
2832	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2833	*/
2834	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu) RT_NOEXCEPT
2835	{
2836	#ifdef IN_RING3
2837	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
2838	#else
2839	CPUMRZFpuStateActualizeForChange(pVCpu);
2840	#endif
2841	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
2842
2843	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
2844	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
2845	}
2846
2847
2848	/**
2849	* Stores a QNaN value into a FPU register.
2850	*
2851	* @param pReg Pointer to the register.
2852	*/
2853	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg) RT_NOEXCEPT
2854	{
2855	pReg->au32[0] = UINT32_C(0x00000000);
2856	pReg->au32[1] = UINT32_C(0xc0000000);
2857	pReg->au16[4] = UINT16_C(0xffff);
2858	}
2859
2860
2861	/**
2862	* Updates the FOP, FPU.CS and FPUIP registers, extended version.
2863	*
2864	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2865	* @param pFpuCtx The FPU context.
2866	* @param uFpuOpcode The FPU opcode value (see IEMCPU::uFpuOpcode).
2867	*/
2868	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorkerEx(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint16_t uFpuOpcode) RT_NOEXCEPT
2869	{
2870	Assert(uFpuOpcode != UINT16_MAX);
2871	pFpuCtx->FOP = uFpuOpcode;
2872	/** @todo x87.CS and FPUIP needs to be kept seperately. */
2873	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2874	{
2875	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2876	* happens in real mode here based on the fnsave and fnstenv images. */
2877	pFpuCtx->CS = 0;
2878	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2879	}
2880	else if (!IEM_IS_LONG_MODE(pVCpu))
2881	{
2882	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2883	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2884	}
2885	else
2886	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2887	}
2888
2889
2890	/**
2891	* Marks the specified stack register as free (for FFREE).
2892	*
2893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2894	* @param iStReg The register to free.
2895	*/
2896	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2897	{
2898	Assert(iStReg < 8);
2899	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2900	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2901	pFpuCtx->FTW &= ~RT_BIT(iReg);
2902	}
2903
2904
2905	/**
2906	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2907	*
2908	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2909	*/
2910	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2911	{
2912	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2913	uint16_t uFsw = pFpuCtx->FSW;
2914	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2915	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2916	uFsw &= ~X86_FSW_TOP_MASK;
2917	uFsw \|= uTop;
2918	pFpuCtx->FSW = uFsw;
2919	}
2920
2921
2922	/**
2923	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2924	*
2925	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2926	*/
2927	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu) RT_NOEXCEPT
2928	{
2929	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2930	uint16_t uFsw = pFpuCtx->FSW;
2931	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2932	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2933	uFsw &= ~X86_FSW_TOP_MASK;
2934	uFsw \|= uTop;
2935	pFpuCtx->FSW = uFsw;
2936	}
2937
2938
2939
2940
2941	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
2942	{
2943	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2944	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2945	if (pFpuCtx->FTW & RT_BIT(iReg))
2946	return VINF_SUCCESS;
2947	return VERR_NOT_FOUND;
2948	}
2949
2950
2951	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef) RT_NOEXCEPT
2952	{
2953	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2954	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2955	if (pFpuCtx->FTW & RT_BIT(iReg))
2956	{
2957	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2958	return VINF_SUCCESS;
2959	}
2960	return VERR_NOT_FOUND;
2961	}
2962
2963
2964	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2965	uint8_t iStReg1, PCRTFLOAT80U *ppRef1) RT_NOEXCEPT
2966	{
2967	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2968	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2969	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2970	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2971	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2972	{
2973	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2974	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2975	return VINF_SUCCESS;
2976	}
2977	return VERR_NOT_FOUND;
2978	}
2979
2980
2981	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1) RT_NOEXCEPT
2982	{
2983	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2984	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2985	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2986	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2987	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2988	{
2989	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2990	return VINF_SUCCESS;
2991	}
2992	return VERR_NOT_FOUND;
2993	}
2994
2995
2996	/**
2997	* Rotates the stack registers when setting new TOS.
2998	*
2999	* @param pFpuCtx The FPU context.
3000	* @param iNewTop New TOS value.
3001	* @remarks We only do this to speed up fxsave/fxrstor which
3002	* arrange the FP registers in stack order.
3003	* MUST be done before writing the new TOS (FSW).
3004	*/
3005	DECLINLINE(void) iemFpuRotateStackSetTop(PX86FXSTATE pFpuCtx, uint16_t iNewTop) RT_NOEXCEPT
3006	{
3007	uint16_t iOldTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3008	RTFLOAT80U ar80Temp[8];
3009
3010	if (iOldTop == iNewTop)
3011	return;
3012
3013	/* Unscrew the stack and get it into 'native' order. */
3014	ar80Temp[0] = pFpuCtx->aRegs[(8 - iOldTop + 0) & X86_FSW_TOP_SMASK].r80;
3015	ar80Temp[1] = pFpuCtx->aRegs[(8 - iOldTop + 1) & X86_FSW_TOP_SMASK].r80;
3016	ar80Temp[2] = pFpuCtx->aRegs[(8 - iOldTop + 2) & X86_FSW_TOP_SMASK].r80;
3017	ar80Temp[3] = pFpuCtx->aRegs[(8 - iOldTop + 3) & X86_FSW_TOP_SMASK].r80;
3018	ar80Temp[4] = pFpuCtx->aRegs[(8 - iOldTop + 4) & X86_FSW_TOP_SMASK].r80;
3019	ar80Temp[5] = pFpuCtx->aRegs[(8 - iOldTop + 5) & X86_FSW_TOP_SMASK].r80;
3020	ar80Temp[6] = pFpuCtx->aRegs[(8 - iOldTop + 6) & X86_FSW_TOP_SMASK].r80;
3021	ar80Temp[7] = pFpuCtx->aRegs[(8 - iOldTop + 7) & X86_FSW_TOP_SMASK].r80;
3022
3023	/* Now rotate the stack to the new position. */
3024	pFpuCtx->aRegs[0].r80 = ar80Temp[(iNewTop + 0) & X86_FSW_TOP_SMASK];
3025	pFpuCtx->aRegs[1].r80 = ar80Temp[(iNewTop + 1) & X86_FSW_TOP_SMASK];
3026	pFpuCtx->aRegs[2].r80 = ar80Temp[(iNewTop + 2) & X86_FSW_TOP_SMASK];
3027	pFpuCtx->aRegs[3].r80 = ar80Temp[(iNewTop + 3) & X86_FSW_TOP_SMASK];
3028	pFpuCtx->aRegs[4].r80 = ar80Temp[(iNewTop + 4) & X86_FSW_TOP_SMASK];
3029	pFpuCtx->aRegs[5].r80 = ar80Temp[(iNewTop + 5) & X86_FSW_TOP_SMASK];
3030	pFpuCtx->aRegs[6].r80 = ar80Temp[(iNewTop + 6) & X86_FSW_TOP_SMASK];
3031	pFpuCtx->aRegs[7].r80 = ar80Temp[(iNewTop + 7) & X86_FSW_TOP_SMASK];
3032	}
3033
3034
3035	/**
3036	* Updates the FPU exception status after FCW is changed.
3037	*
3038	* @param pFpuCtx The FPU context.
3039	*/
3040	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
3041	{
3042	uint16_t u16Fsw = pFpuCtx->FSW;
3043	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
3044	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
3045	else
3046	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
3047	pFpuCtx->FSW = u16Fsw;
3048	}
3049
3050
3051	/**
3052	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
3053	*
3054	* @returns The full FTW.
3055	* @param pFpuCtx The FPU context.
3056	*/
3057	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx) RT_NOEXCEPT
3058	{
3059	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
3060	uint16_t u16Ftw = 0;
3061	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
3062	for (unsigned iSt = 0; iSt < 8; iSt++)
3063	{
3064	unsigned const iReg = (iSt + iTop) & 7;
3065	if (!(u8Ftw & RT_BIT(iReg)))
3066	u16Ftw \|= 3 << (iReg * 2); /* empty */
3067	else
3068	{
3069	uint16_t uTag;
3070	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
3071	if (pr80Reg->s.uExponent == 0x7fff)
3072	uTag = 2; /* Exponent is all 1's => Special. */
3073	else if (pr80Reg->s.uExponent == 0x0000)
3074	{
3075	if (pr80Reg->s.uMantissa == 0x0000)
3076	uTag = 1; /* All bits are zero => Zero. */
3077	else
3078	uTag = 2; /* Must be special. */
3079	}
3080	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
3081	uTag = 0; /* Valid. */
3082	else
3083	uTag = 2; /* Must be special. */
3084
3085	u16Ftw \|= uTag << (iReg * 2);
3086	}
3087	}
3088
3089	return u16Ftw;
3090	}
3091
3092
3093	/**
3094	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
3095	*
3096	* @returns The compressed FTW.
3097	* @param u16FullFtw The full FTW to convert.
3098	*/
3099	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw) RT_NOEXCEPT
3100	{
3101	uint8_t u8Ftw = 0;
3102	for (unsigned i = 0; i < 8; i++)
3103	{
3104	if ((u16FullFtw & 3) != 3 /empty/)
3105	u8Ftw \|= RT_BIT(i);
3106	u16FullFtw >>= 2;
3107	}
3108
3109	return u8Ftw;
3110	}
3111
3112	/** @} */
3113
3114
3115	/** @name Memory access.
3116	*
3117	* @{
3118	*/
3119
3120
3121	/**
3122	* Checks whether alignment checks are enabled or not.
3123	*
3124	* @returns true if enabled, false if not.
3125	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3126	*/
3127	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu) RT_NOEXCEPT
3128	{
3129	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3130	return IEM_GET_CPL(pVCpu) == 3
3131	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
3132	}
3133
3134	/**
3135	* Checks if the given segment can be written to, raise the appropriate
3136	* exception if not.
3137	*
3138	* @returns VBox strict status code.
3139	*
3140	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3141	* @param pHid Pointer to the hidden register.
3142	* @param iSegReg The register number.
3143	* @param pu64BaseAddr Where to return the base address to use for the
3144	* segment. (In 64-bit code it may differ from the
3145	* base in the hidden segment.)
3146	*/
3147	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3148	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3149	{
3150	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3151
3152	if (IEM_IS_64BIT_CODE(pVCpu))
3153	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3154	else
3155	{
3156	if (!pHid->Attr.n.u1Present)
3157	{
3158	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3159	AssertRelease(uSel == 0);
3160	LogEx(LOG_GROUP_IEM,("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3161	return iemRaiseGeneralProtectionFault0(pVCpu);
3162	}
3163
3164	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
3165	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
3166	&& !IEM_IS_64BIT_CODE(pVCpu) )
3167	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3168	*pu64BaseAddr = pHid->u64Base;
3169	}
3170	return VINF_SUCCESS;
3171	}
3172
3173
3174	/**
3175	* Checks if the given segment can be read from, raise the appropriate
3176	* exception if not.
3177	*
3178	* @returns VBox strict status code.
3179	*
3180	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3181	* @param pHid Pointer to the hidden register.
3182	* @param iSegReg The register number.
3183	* @param pu64BaseAddr Where to return the base address to use for the
3184	* segment. (In 64-bit code it may differ from the
3185	* base in the hidden segment.)
3186	*/
3187	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid,
3188	uint8_t iSegReg, uint64_t *pu64BaseAddr) RT_NOEXCEPT
3189	{
3190	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3191
3192	if (IEM_IS_64BIT_CODE(pVCpu))
3193	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
3194	else
3195	{
3196	if (!pHid->Attr.n.u1Present)
3197	{
3198	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
3199	AssertRelease(uSel == 0);
3200	LogEx(LOG_GROUP_IEM,("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
3201	return iemRaiseGeneralProtectionFault0(pVCpu);
3202	}
3203
3204	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
3205	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3206	*pu64BaseAddr = pHid->u64Base;
3207	}
3208	return VINF_SUCCESS;
3209	}
3210
3211
3212	/**
3213	* Maps a physical page.
3214	*
3215	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
3216	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3217	* @param GCPhysMem The physical address.
3218	* @param fAccess The intended access.
3219	* @param ppvMem Where to return the mapping address.
3220	* @param pLock The PGM lock.
3221	*/
3222	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3223	void **ppvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3224	{
3225	#ifdef IEM_LOG_MEMORY_WRITES
3226	if (fAccess & IEM_ACCESS_TYPE_WRITE)
3227	return VERR_PGM_PHYS_TLB_CATCH_ALL;
3228	#endif
3229
3230	/** @todo This API may require some improving later. A private deal with PGM
3231	* regarding locking and unlocking needs to be struct. A couple of TLBs
3232	* living in PGM, but with publicly accessible inlined access methods
3233	* could perhaps be an even better solution. */
3234	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
3235	GCPhysMem,
3236	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
3237	RT_BOOL(pVCpu->iem.s.fExec & IEM_F_BYPASS_HANDLERS),
3238	ppvMem,
3239	pLock);
3240	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
3241	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
3242
3243	return rc;
3244	}
3245
3246
3247	/**
3248	* Unmap a page previously mapped by iemMemPageMap.
3249	*
3250	* @param pVCpu The cross context virtual CPU structure of the calling thread.
3251	* @param GCPhysMem The physical address.
3252	* @param fAccess The intended access.
3253	* @param pvMem What iemMemPageMap returned.
3254	* @param pLock The PGM lock.
3255	*/
3256	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess,
3257	const void *pvMem, PPGMPAGEMAPLOCK pLock) RT_NOEXCEPT
3258	{
3259	NOREF(pVCpu);
3260	NOREF(GCPhysMem);
3261	NOREF(fAccess);
3262	NOREF(pvMem);
3263	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
3264	}
3265
3266	#ifdef IEM_WITH_SETJMP
3267
3268	/** @todo slim this down */
3269	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg,
3270	size_t cbMem, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3271	{
3272	Assert(cbMem >= 1);
3273	Assert(iSegReg < X86_SREG_COUNT);
3274
3275	/*
3276	* 64-bit mode is simpler.
3277	*/
3278	if (IEM_IS_64BIT_CODE(pVCpu))
3279	{
3280	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
3281	{
3282	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3283	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3284	GCPtrMem += pSel->u64Base;
3285	}
3286
3287	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3288	return GCPtrMem;
3289	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3290	}
3291	/*
3292	* 16-bit and 32-bit segmentation.
3293	*/
3294	else if (iSegReg != UINT8_MAX)
3295	{
3296	/** @todo Does this apply to segments with 4G-1 limit? */
3297	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3298	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
3299	{
3300	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3301	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
3302	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3303	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
3304	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
3305	\| X86_SEL_TYPE_CODE))
3306	{
3307	case X86DESCATTR_P: /* readonly data, expand up */
3308	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
3309	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
3310	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
3311	/* expand up */
3312	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
3313	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3314	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
3315	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
3316	break;
3317
3318	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
3319	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
3320	/* expand down */
3321	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
3322	&& ( pSel->Attr.n.u1DefBig
3323	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
3324	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3325	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
3326	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
3327	break;
3328
3329	default:
3330	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
3331	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3332	break;
3333	}
3334	}
3335	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
3336	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
3337	}
3338	/*
3339	* 32-bit flat address.
3340	*/
3341	else
3342	return GCPtrMem;
3343	}
3344
3345
3346	/** @todo slim this down */
3347	DECL_INLINE_THROW(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem,
3348	RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
3349	{
3350	Assert(cbMem >= 1);
3351	Assert(iSegReg < X86_SREG_COUNT);
3352
3353	/*
3354	* 64-bit mode is simpler.
3355	*/
3356	if (IEM_IS_64BIT_CODE(pVCpu))
3357	{
3358	if (iSegReg >= X86_SREG_FS)
3359	{
3360	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3361	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3362	GCPtrMem += pSel->u64Base;
3363	}
3364
3365	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
3366	return GCPtrMem;
3367	}
3368	/*
3369	* 16-bit and 32-bit segmentation.
3370	*/
3371	else
3372	{
3373	Assert(GCPtrMem <= UINT32_MAX);
3374	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
3375	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
3376	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
3377	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
3378	if ( fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE) /* data, expand up */
3379	/** @todo explore exactly how the CS stuff works in real mode. See also
3380	* http://www.rcollins.org/Productivity/DescriptorCache.html and
3381	* http://www.rcollins.org/ddj/Aug98/Aug98.html for some insight. */
3382	\|\| (iSegReg == X86_SREG_CS && IEM_IS_REAL_OR_V86_MODE(pVCpu)) ) /* Ignored for CS. / /* @todo testcase! */
3383	{
3384	/* expand up */
3385	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3386	if (RT_LIKELY( GCPtrLast32 <= pSel->u32Limit
3387	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3388	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3389	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3390	}
3391	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
3392	{
3393	/* expand down - the uppger boundary is defined by the B bit, not G. */
3394	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
3395	if (RT_LIKELY( (uint32_t)GCPtrMem >= pSel->u32Limit
3396	&& (pSel->Attr.n.u1DefBig \|\| GCPtrLast32 <= UINT32_C(0xffff))
3397	&& GCPtrLast32 >= (uint32_t)GCPtrMem))
3398	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
3399	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3400	}
3401	else
3402	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
3403	}
3404	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
3405	}
3406
3407	#endif /* IEM_WITH_SETJMP */
3408
3409	/**
3410	* Fakes a long mode stack selector for SS = 0.
3411	*
3412	* @param pDescSs Where to return the fake stack descriptor.
3413	* @param uDpl The DPL we want.
3414	*/
3415	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl) RT_NOEXCEPT
3416	{
3417	pDescSs->Long.au64[0] = 0;
3418	pDescSs->Long.au64[1] = 0;
3419	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
3420	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
3421	pDescSs->Long.Gen.u2Dpl = uDpl;
3422	pDescSs->Long.Gen.u1Present = 1;
3423	pDescSs->Long.Gen.u1Long = 1;
3424	}
3425
3426
3427	/*
3428	* Unmap helpers.
3429	*/
3430
3431	#ifdef IEM_WITH_SETJMP
3432
3433	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRwJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3434	{
3435	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3436	if (RT_LIKELY(bMapInfo == 0))
3437	return;
3438	# endif
3439	iemMemCommitAndUnmapRwSafeJmp(pVCpu, pvMem, bMapInfo);
3440	}
3441
3442
3443	DECL_INLINE_THROW(void) iemMemCommitAndUnmapWoJmp(PVMCPUCC pVCpu, void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3444	{
3445	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3446	if (RT_LIKELY(bMapInfo == 0))
3447	return;
3448	# endif
3449	iemMemCommitAndUnmapWoSafeJmp(pVCpu, pvMem, bMapInfo);
3450	}
3451
3452
3453	DECL_INLINE_THROW(void) iemMemCommitAndUnmapRoJmp(PVMCPUCC pVCpu, const void *pvMem, uint8_t bMapInfo) IEM_NOEXCEPT_MAY_LONGJMP
3454	{
3455	# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
3456	if (RT_LIKELY(bMapInfo == 0))
3457	return;
3458	# endif
3459	iemMemCommitAndUnmapRoSafeJmp(pVCpu, pvMem, bMapInfo);
3460	}
3461
3462	#endif /* IEM_WITH_SETJMP */
3463
3464
3465	/*
3466	* Instantiate R/W inline templates.
3467	*/
3468
3469	/** @def TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3470	* Used to check if an unaligned access is if within the page and won't
3471	* trigger an \#AC.
3472	*
3473	* This can be used to deal with misaligned accesses on platforms that are
3474	* senstive to such if desires.
3475	*/
3476	AssertCompile(X86_CR0_AM == X86_EFL_AC);
3477	AssertCompile(((3U + 1U) << 16) == X86_CR0_AM);
3478	#if 1
3479	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) \
3480	( ((a_GCPtrEff) & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(a_TmplMemType) \
3481	&& !( (uint32_t)(a_pVCpu)->cpum.GstCtx.cr0 \
3482	& (a_pVCpu)->cpum.GstCtx.eflags.u \
3483	& ((IEM_GET_CPL((a_pVCpu)) + 1U) << 16) /* IEM_GET_CPL(a_pVCpu) == 3 ? X86_CR0_AM : 0 */ \
3484	& X86_CR0_AM) )
3485	#else
3486	# define TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK(a_pVCpu, a_GCPtrEff, a_TmplMemType) 0
3487	#endif
3488
3489	#define TMPL_MEM_TYPE uint8_t
3490	#define TMPL_MEM_TYPE_ALIGN 0
3491	#define TMPL_MEM_TYPE_SIZE 1
3492	#define TMPL_MEM_FN_SUFF U8
3493	#define TMPL_MEM_FMT_TYPE "%#04x"
3494	#define TMPL_MEM_FMT_DESC "byte"
3495	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3496
3497	#define TMPL_MEM_WITH_STACK
3498
3499	#define TMPL_MEM_TYPE uint16_t
3500	#define TMPL_MEM_TYPE_ALIGN 1
3501	#define TMPL_MEM_TYPE_SIZE 2
3502	#define TMPL_MEM_FN_SUFF U16
3503	#define TMPL_MEM_FMT_TYPE "%#06x"
3504	#define TMPL_MEM_FMT_DESC "word"
3505	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3506
3507	#define TMPL_WITH_PUSH_SREG
3508	#define TMPL_MEM_TYPE uint32_t
3509	#define TMPL_MEM_TYPE_ALIGN 3
3510	#define TMPL_MEM_TYPE_SIZE 4
3511	#define TMPL_MEM_FN_SUFF U32
3512	#define TMPL_MEM_FMT_TYPE "%#010x"
3513	#define TMPL_MEM_FMT_DESC "dword"
3514	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3515	#undef TMPL_WITH_PUSH_SREG
3516
3517	#define TMPL_MEM_TYPE uint64_t
3518	#define TMPL_MEM_TYPE_ALIGN 7
3519	#define TMPL_MEM_TYPE_SIZE 8
3520	#define TMPL_MEM_FN_SUFF U64
3521	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3522	#define TMPL_MEM_FMT_DESC "qword"
3523	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3524
3525	#undef TMPL_MEM_WITH_STACK
3526
3527	#define TMPL_MEM_NO_STORE
3528	#define TMPL_MEM_NO_MAPPING
3529	#define TMPL_MEM_TYPE uint64_t
3530	#define TMPL_MEM_TYPE_ALIGN 15
3531	#define TMPL_MEM_TYPE_SIZE 8
3532	#define TMPL_MEM_FN_SUFF U64AlignedU128
3533	#define TMPL_MEM_FMT_TYPE "%#018RX64"
3534	#define TMPL_MEM_FMT_DESC "qword"
3535	#include "../VMMAll/IEMAllMemRWTmplInline.cpp.h"
3536
3537	#undef TMPL_MEM_CHECK_UNALIGNED_WITHIN_PAGE_OK
3538
3539	/** @} */
3540
3541
3542	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3543
3544	/**
3545	* Gets CR0 fixed-0 bits in VMX operation.
3546	*
3547	* We do this rather than fetching what we report to the guest (in
3548	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
3549	* values regardless of whether unrestricted-guest feature is available on the CPU.
3550	*
3551	* @returns CR0 fixed-0 bits.
3552	* @param pVCpu The cross context virtual CPU structure.
3553	* @param fVmxNonRootMode Whether the CR0 fixed-0 bits for VMX non-root mode
3554	* must be returned. When @c false, the CR0 fixed-0
3555	* bits for VMX root mode is returned.
3556	*
3557	*/
3558	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu, bool fVmxNonRootMode) RT_NOEXCEPT
3559	{
3560	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
3561
3562	PCVMXMSRS pMsrs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs;
3563	if ( fVmxNonRootMode
3564	&& (pMsrs->ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST))
3565	return VMX_V_CR0_FIXED0_UX;
3566	return VMX_V_CR0_FIXED0;
3567	}
3568
3569
3570	# ifdef XAPIC_OFF_END /* Requires VBox/apic.h to be included before IEMInline.h. */
3571	/**
3572	* Sets virtual-APIC write emulation as pending.
3573	*
3574	* @param pVCpu The cross context virtual CPU structure.
3575	* @param offApic The offset in the virtual-APIC page that was written.
3576	*/
3577	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
3578	{
3579	Assert(offApic < XAPIC_OFF_END + 4);
3580
3581	/*
3582	* Record the currently updated APIC offset, as we need this later for figuring
3583	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
3584	* as for supplying the exit qualification when causing an APIC-write VM-exit.
3585	*/
3586	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
3587
3588	/*
3589	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
3590	* virtualization or APIC-write emulation).
3591	*/
3592	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
3593	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
3594	}
3595	# endif /* XAPIC_OFF_END */
3596
3597	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
3598
3599	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 101163

以其他格式下載: