IEMInline.h@ 96478

最後變更在這個檔案從96478是 96407,由 vboxsync 提交於 3 年前
scm copyright and license note update
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 84.9 KB

行
1	/* $Id: IEMInline.h 96407 2022-08-22 17:43:14Z vboxsync $ */
2	/** @file
3	* IEM - Interpreted Execution Manager - Inlined Functions.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.alldomusa.eu.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28	#ifndef VMM_INCLUDED_SRC_include_IEMInline_h
29	#define VMM_INCLUDED_SRC_include_IEMInline_h
30	#ifndef RT_WITHOUT_PRAGMA_ONCE
31	# pragma once
32	#endif
33
34
35
36	/**
37	* Makes status code addjustments (pass up from I/O and access handler)
38	* as well as maintaining statistics.
39	*
40	* @returns Strict VBox status code to pass up.
41	* @param pVCpu The cross context virtual CPU structure of the calling thread.
42	* @param rcStrict The status from executing an instruction.
43	*/
44	DECL_FORCE_INLINE(VBOXSTRICTRC) iemExecStatusCodeFiddling(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
45	{
46	if (rcStrict != VINF_SUCCESS)
47	{
48	if (RT_SUCCESS(rcStrict))
49	{
50	AssertMsg( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
51	\|\| rcStrict == VINF_IOM_R3_IOPORT_READ
52	\|\| rcStrict == VINF_IOM_R3_IOPORT_WRITE
53	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
54	\|\| rcStrict == VINF_IOM_R3_MMIO_READ
55	\|\| rcStrict == VINF_IOM_R3_MMIO_READ_WRITE
56	\|\| rcStrict == VINF_IOM_R3_MMIO_WRITE
57	\|\| rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE
58	\|\| rcStrict == VINF_CPUM_R3_MSR_READ
59	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE
60	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
61	\|\| rcStrict == VINF_EM_RAW_TO_R3
62	\|\| rcStrict == VINF_EM_TRIPLE_FAULT
63	\|\| rcStrict == VINF_GIM_R3_HYPERCALL
64	/* raw-mode / virt handlers only: */
65	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT
66	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT
67	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT
68	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT
69	\|\| rcStrict == VINF_SELM_SYNC_GDT
70	\|\| rcStrict == VINF_CSAM_PENDING_ACTION
71	\|\| rcStrict == VINF_PATM_CHECK_PATCH_PAGE
72	/* nested hw.virt codes: */
73	\|\| rcStrict == VINF_VMX_VMEXIT
74	\|\| rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE
75	\|\| rcStrict == VINF_VMX_MODIFIES_BEHAVIOR
76	\|\| rcStrict == VINF_SVM_VMEXIT
77	, ("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
78	/** @todo adjust for VINF_EM_RAW_EMULATE_INSTR. */
79	int32_t const rcPassUp = pVCpu->iem.s.rcPassUp;
80	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
81	if ( rcStrict == VINF_VMX_VMEXIT
82	&& rcPassUp == VINF_SUCCESS)
83	rcStrict = VINF_SUCCESS;
84	else
85	#endif
86	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
87	if ( rcStrict == VINF_SVM_VMEXIT
88	&& rcPassUp == VINF_SUCCESS)
89	rcStrict = VINF_SUCCESS;
90	else
91	#endif
92	if (rcPassUp == VINF_SUCCESS)
93	pVCpu->iem.s.cRetInfStatuses++;
94	else if ( rcPassUp < VINF_EM_FIRST
95	\|\| rcPassUp > VINF_EM_LAST
96	\|\| rcPassUp < VBOXSTRICTRC_VAL(rcStrict))
97	{
98	Log(("IEM: rcPassUp=%Rrc! rcStrict=%Rrc\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
99	pVCpu->iem.s.cRetPassUpStatus++;
100	rcStrict = rcPassUp;
101	}
102	else
103	{
104	Log(("IEM: rcPassUp=%Rrc rcStrict=%Rrc!\n", rcPassUp, VBOXSTRICTRC_VAL(rcStrict)));
105	pVCpu->iem.s.cRetInfStatuses++;
106	}
107	}
108	else if (rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
109	pVCpu->iem.s.cRetAspectNotImplemented++;
110	else if (rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
111	pVCpu->iem.s.cRetInstrNotImplemented++;
112	else
113	pVCpu->iem.s.cRetErrStatuses++;
114	}
115	else if (pVCpu->iem.s.rcPassUp != VINF_SUCCESS)
116	{
117	pVCpu->iem.s.cRetPassUpStatus++;
118	rcStrict = pVCpu->iem.s.rcPassUp;
119	}
120
121	return rcStrict;
122	}
123
124
125	/**
126	* Sets the pass up status.
127	*
128	* @returns VINF_SUCCESS.
129	* @param pVCpu The cross context virtual CPU structure of the
130	* calling thread.
131	* @param rcPassUp The pass up status. Must be informational.
132	* VINF_SUCCESS is not allowed.
133	*/
134	DECLINLINE(int) iemSetPassUpStatus(PVMCPUCC pVCpu, VBOXSTRICTRC rcPassUp)
135	{
136	AssertRC(VBOXSTRICTRC_VAL(rcPassUp)); Assert(rcPassUp != VINF_SUCCESS);
137
138	int32_t const rcOldPassUp = pVCpu->iem.s.rcPassUp;
139	if (rcOldPassUp == VINF_SUCCESS)
140	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
141	/* If both are EM scheduling codes, use EM priority rules. */
142	else if ( rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST
143	&& rcPassUp >= VINF_EM_FIRST && rcPassUp <= VINF_EM_LAST)
144	{
145	if (rcPassUp < rcOldPassUp)
146	{
147	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
148	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
149	}
150	else
151	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
152	}
153	/* Override EM scheduling with specific status code. */
154	else if (rcOldPassUp >= VINF_EM_FIRST && rcOldPassUp <= VINF_EM_LAST)
155	{
156	Log(("IEM: rcPassUp=%Rrc! rcOldPassUp=%Rrc\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
157	pVCpu->iem.s.rcPassUp = VBOXSTRICTRC_VAL(rcPassUp);
158	}
159	/* Don't override specific status code, first come first served. */
160	else
161	Log(("IEM: rcPassUp=%Rrc rcOldPassUp=%Rrc!\n", VBOXSTRICTRC_VAL(rcPassUp), rcOldPassUp));
162	return VINF_SUCCESS;
163	}
164
165
166	/**
167	* Calculates the CPU mode.
168	*
169	* This is mainly for updating IEMCPU::enmCpuMode.
170	*
171	* @returns CPU mode.
172	* @param pVCpu The cross context virtual CPU structure of the
173	* calling thread.
174	*/
175	DECLINLINE(IEMMODE) iemCalcCpuMode(PVMCPUCC pVCpu)
176	{
177	if (CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx))
178	return IEMMODE_64BIT;
179	if (pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig) /** @todo check if this is correct... */
180	return IEMMODE_32BIT;
181	return IEMMODE_16BIT;
182	}
183
184
185	/**
186	* Initializes the execution state.
187	*
188	* @param pVCpu The cross context virtual CPU structure of the
189	* calling thread.
190	* @param fBypassHandlers Whether to bypass access handlers.
191	*
192	* @remarks Callers of this must call iemUninitExec() to undo potentially fatal
193	* side-effects in strict builds.
194	*/
195	DECLINLINE(void) iemInitExec(PVMCPUCC pVCpu, bool fBypassHandlers)
196	{
197	IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
198	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
199	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
200	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
201	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
202	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
203	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
204	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
205	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
206	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
207
208	pVCpu->iem.s.uCpl = CPUMGetGuestCPL(pVCpu);
209	pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
210	#ifdef VBOX_STRICT
211	pVCpu->iem.s.enmDefAddrMode = (IEMMODE)0xfe;
212	pVCpu->iem.s.enmEffAddrMode = (IEMMODE)0xfe;
213	pVCpu->iem.s.enmDefOpSize = (IEMMODE)0xfe;
214	pVCpu->iem.s.enmEffOpSize = (IEMMODE)0xfe;
215	pVCpu->iem.s.fPrefixes = 0xfeedbeef;
216	pVCpu->iem.s.uRexReg = 127;
217	pVCpu->iem.s.uRexB = 127;
218	pVCpu->iem.s.offModRm = 127;
219	pVCpu->iem.s.uRexIndex = 127;
220	pVCpu->iem.s.iEffSeg = 127;
221	pVCpu->iem.s.idxPrefix = 127;
222	pVCpu->iem.s.uVex3rdReg = 127;
223	pVCpu->iem.s.uVexLength = 127;
224	pVCpu->iem.s.fEvexStuff = 127;
225	pVCpu->iem.s.uFpuOpcode = UINT16_MAX;
226	# ifdef IEM_WITH_CODE_TLB
227	pVCpu->iem.s.offInstrNextByte = UINT16_MAX;
228	pVCpu->iem.s.pbInstrBuf = NULL;
229	pVCpu->iem.s.cbInstrBuf = UINT16_MAX;
230	pVCpu->iem.s.cbInstrBufTotal = UINT16_MAX;
231	pVCpu->iem.s.offCurInstrStart = INT16_MAX;
232	pVCpu->iem.s.uInstrBufPc = UINT64_C(0xc0ffc0ffcff0c0ff);
233	# else
234	pVCpu->iem.s.offOpcode = 127;
235	pVCpu->iem.s.cbOpcode = 127;
236	# endif
237	#endif
238
239	pVCpu->iem.s.cActiveMappings = 0;
240	pVCpu->iem.s.iNextMapping = 0;
241	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
242	pVCpu->iem.s.fBypassHandlers = fBypassHandlers;
243	#if 0
244	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
245	if ( CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)
246	&& CPUMIsGuestVmxProcCtls2Set(pVCpu, &pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
247	{
248	PCVMXVVMCS pVmcs = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
249	Assert(pVmcs);
250	RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
251	if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
252	{
253	int rc = PGMHandlerPhysicalRegister(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess, GCPhysApicAccess + X86_PAGE_4K_SIZE - 1,
254	pVCpu->iem.s.hVmxApicAccessPage, NIL_RTR3PTR /* pvUserR3 */,
255	NIL_RTR0PTR /* pvUserR0 /, NIL_RTRCPTR / pvUserRC /, NULL / pszDesc */);
256	AssertRC(rc);
257	}
258	}
259	#endif
260	#endif
261	}
262
263	#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) \|\| defined(VBOX_WITH_NESTED_HWVIRT_VMX)
264	/**
265	* Performs a minimal reinitialization of the execution state.
266	*
267	* This is intended to be used by VM-exits, SMM, LOADALL and other similar
268	* 'world-switch' types operations on the CPU. Currently only nested
269	* hardware-virtualization uses it.
270	*
271	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
272	*/
273	DECLINLINE(void) iemReInitExec(PVMCPUCC pVCpu)
274	{
275	IEMMODE const enmMode = iemCalcCpuMode(pVCpu);
276	uint8_t const uCpl = CPUMGetGuestCPL(pVCpu);
277
278	pVCpu->iem.s.uCpl = uCpl;
279	pVCpu->iem.s.enmCpuMode = enmMode;
280	pVCpu->iem.s.enmDefAddrMode = enmMode; /** @todo check if this is correct... */
281	pVCpu->iem.s.enmEffAddrMode = enmMode;
282	if (enmMode != IEMMODE_64BIT)
283	{
284	pVCpu->iem.s.enmDefOpSize = enmMode; /** @todo check if this is correct... */
285	pVCpu->iem.s.enmEffOpSize = enmMode;
286	}
287	else
288	{
289	pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
290	pVCpu->iem.s.enmEffOpSize = enmMode;
291	}
292	pVCpu->iem.s.iEffSeg = X86_SREG_DS;
293	#ifndef IEM_WITH_CODE_TLB
294	/** @todo Shouldn't we be doing this in IEMTlbInvalidateAll()? */
295	pVCpu->iem.s.offOpcode = 0;
296	pVCpu->iem.s.cbOpcode = 0;
297	#endif
298	pVCpu->iem.s.rcPassUp = VINF_SUCCESS;
299	}
300	#endif
301
302	/**
303	* Counterpart to #iemInitExec that undoes evil strict-build stuff.
304	*
305	* @param pVCpu The cross context virtual CPU structure of the
306	* calling thread.
307	*/
308	DECLINLINE(void) iemUninitExec(PVMCPUCC pVCpu)
309	{
310	/* Note! do not touch fInPatchCode here! (see iemUninitExecAndFiddleStatusAndMaybeReenter) */
311	#ifdef VBOX_STRICT
312	# ifdef IEM_WITH_CODE_TLB
313	NOREF(pVCpu);
314	# else
315	pVCpu->iem.s.cbOpcode = 0;
316	# endif
317	#else
318	NOREF(pVCpu);
319	#endif
320	}
321
322
323	/**
324	* Calls iemUninitExec, iemExecStatusCodeFiddling and iemRCRawMaybeReenter.
325	*
326	* Only calling iemRCRawMaybeReenter in raw-mode, obviously.
327	*
328	* @returns Fiddled strict vbox status code, ready to return to non-IEM caller.
329	* @param pVCpu The cross context virtual CPU structure of the calling thread.
330	* @param rcStrict The status code to fiddle.
331	*/
332	DECLINLINE(VBOXSTRICTRC) iemUninitExecAndFiddleStatusAndMaybeReenter(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
333	{
334	iemUninitExec(pVCpu);
335	return iemExecStatusCodeFiddling(pVCpu, rcStrict);
336	}
337
338
339	/**
340	* Macro used by the IEMExec* method to check the given instruction length.
341	*
342	* Will return on failure!
343	*
344	* @param a_cbInstr The given instruction length.
345	* @param a_cbMin The minimum length.
346	*/
347	#define IEMEXEC_ASSERT_INSTR_LEN_RETURN(a_cbInstr, a_cbMin) \
348	AssertMsgReturn((unsigned)(a_cbInstr) - (unsigned)(a_cbMin) <= (unsigned)15 - (unsigned)(a_cbMin), \
349	("cbInstr=%u cbMin=%u\n", (a_cbInstr), (a_cbMin)), VERR_IEM_INVALID_INSTR_LENGTH)
350
351
352
353	#ifndef IEM_WITH_SETJMP
354
355	/**
356	* Fetches the next opcode byte.
357	*
358	* @returns Strict VBox status code.
359	* @param pVCpu The cross context virtual CPU structure of the
360	* calling thread.
361	* @param pu8 Where to return the opcode byte.
362	*/
363	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU8(PVMCPUCC pVCpu, uint8_t *pu8)
364	{
365	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
366	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
367	{
368	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
369	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
370	return VINF_SUCCESS;
371	}
372	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
373	}
374
375	#else /* IEM_WITH_SETJMP */
376
377	/**
378	* Fetches the next opcode byte, longjmp on error.
379	*
380	* @returns The opcode byte.
381	* @param pVCpu The cross context virtual CPU structure of the calling thread.
382	*/
383	DECLINLINE(uint8_t) iemOpcodeGetNextU8Jmp(PVMCPUCC pVCpu)
384	{
385	# ifdef IEM_WITH_CODE_TLB
386	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
387	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
388	if (RT_LIKELY( pbBuf != NULL
389	&& offBuf < pVCpu->iem.s.cbInstrBuf))
390	{
391	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
392	return pbBuf[offBuf];
393	}
394	# else
395	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
396	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
397	{
398	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
399	return pVCpu->iem.s.abOpcode[offOpcode];
400	}
401	# endif
402	return iemOpcodeGetNextU8SlowJmp(pVCpu);
403	}
404
405	#endif /* IEM_WITH_SETJMP */
406
407	/**
408	* Fetches the next opcode byte, returns automatically on failure.
409	*
410	* @param a_pu8 Where to return the opcode byte.
411	* @remark Implicitly references pVCpu.
412	*/
413	#ifndef IEM_WITH_SETJMP
414	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) \
415	do \
416	{ \
417	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU8(pVCpu, (a_pu8)); \
418	if (rcStrict2 == VINF_SUCCESS) \
419	{ /* likely */ } \
420	else \
421	return rcStrict2; \
422	} while (0)
423	#else
424	# define IEM_OPCODE_GET_NEXT_U8(a_pu8) (*(a_pu8) = iemOpcodeGetNextU8Jmp(pVCpu))
425	#endif /* IEM_WITH_SETJMP */
426
427
428	#ifndef IEM_WITH_SETJMP
429	/**
430	* Fetches the next signed byte from the opcode stream.
431	*
432	* @returns Strict VBox status code.
433	* @param pVCpu The cross context virtual CPU structure of the calling thread.
434	* @param pi8 Where to return the signed byte.
435	*/
436	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8(PVMCPUCC pVCpu, int8_t *pi8)
437	{
438	return iemOpcodeGetNextU8(pVCpu, (uint8_t *)pi8);
439	}
440	#endif /* !IEM_WITH_SETJMP */
441
442
443	/**
444	* Fetches the next signed byte from the opcode stream, returning automatically
445	* on failure.
446	*
447	* @param a_pi8 Where to return the signed byte.
448	* @remark Implicitly references pVCpu.
449	*/
450	#ifndef IEM_WITH_SETJMP
451	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) \
452	do \
453	{ \
454	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8(pVCpu, (a_pi8)); \
455	if (rcStrict2 != VINF_SUCCESS) \
456	return rcStrict2; \
457	} while (0)
458	#else /* IEM_WITH_SETJMP */
459	# define IEM_OPCODE_GET_NEXT_S8(a_pi8) (*(a_pi8) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
460
461	#endif /* IEM_WITH_SETJMP */
462
463
464	#ifndef IEM_WITH_SETJMP
465	/**
466	* Fetches the next signed byte from the opcode stream, extending it to
467	* unsigned 16-bit.
468	*
469	* @returns Strict VBox status code.
470	* @param pVCpu The cross context virtual CPU structure of the calling thread.
471	* @param pu16 Where to return the unsigned word.
472	*/
473	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU16(PVMCPUCC pVCpu, uint16_t *pu16)
474	{
475	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
476	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
477	return iemOpcodeGetNextS8SxU16Slow(pVCpu, pu16);
478
479	*pu16 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
480	pVCpu->iem.s.offOpcode = offOpcode + 1;
481	return VINF_SUCCESS;
482	}
483	#endif /* !IEM_WITH_SETJMP */
484
485	/**
486	* Fetches the next signed byte from the opcode stream and sign-extending it to
487	* a word, returning automatically on failure.
488	*
489	* @param a_pu16 Where to return the word.
490	* @remark Implicitly references pVCpu.
491	*/
492	#ifndef IEM_WITH_SETJMP
493	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) \
494	do \
495	{ \
496	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU16(pVCpu, (a_pu16)); \
497	if (rcStrict2 != VINF_SUCCESS) \
498	return rcStrict2; \
499	} while (0)
500	#else
501	# define IEM_OPCODE_GET_NEXT_S8_SX_U16(a_pu16) (*(a_pu16) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
502	#endif
503
504	#ifndef IEM_WITH_SETJMP
505	/**
506	* Fetches the next signed byte from the opcode stream, extending it to
507	* unsigned 32-bit.
508	*
509	* @returns Strict VBox status code.
510	* @param pVCpu The cross context virtual CPU structure of the calling thread.
511	* @param pu32 Where to return the unsigned dword.
512	*/
513	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU32(PVMCPUCC pVCpu, uint32_t *pu32)
514	{
515	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
516	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
517	return iemOpcodeGetNextS8SxU32Slow(pVCpu, pu32);
518
519	*pu32 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
520	pVCpu->iem.s.offOpcode = offOpcode + 1;
521	return VINF_SUCCESS;
522	}
523	#endif /* !IEM_WITH_SETJMP */
524
525	/**
526	* Fetches the next signed byte from the opcode stream and sign-extending it to
527	* a word, returning automatically on failure.
528	*
529	* @param a_pu32 Where to return the word.
530	* @remark Implicitly references pVCpu.
531	*/
532	#ifndef IEM_WITH_SETJMP
533	#define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) \
534	do \
535	{ \
536	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU32(pVCpu, (a_pu32)); \
537	if (rcStrict2 != VINF_SUCCESS) \
538	return rcStrict2; \
539	} while (0)
540	#else
541	# define IEM_OPCODE_GET_NEXT_S8_SX_U32(a_pu32) (*(a_pu32) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
542	#endif
543
544
545	#ifndef IEM_WITH_SETJMP
546	/**
547	* Fetches the next signed byte from the opcode stream, extending it to
548	* unsigned 64-bit.
549	*
550	* @returns Strict VBox status code.
551	* @param pVCpu The cross context virtual CPU structure of the calling thread.
552	* @param pu64 Where to return the unsigned qword.
553	*/
554	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS8SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
555	{
556	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
557	if (RT_UNLIKELY(offOpcode >= pVCpu->iem.s.cbOpcode))
558	return iemOpcodeGetNextS8SxU64Slow(pVCpu, pu64);
559
560	*pu64 = (int8_t)pVCpu->iem.s.abOpcode[offOpcode];
561	pVCpu->iem.s.offOpcode = offOpcode + 1;
562	return VINF_SUCCESS;
563	}
564	#endif /* !IEM_WITH_SETJMP */
565
566	/**
567	* Fetches the next signed byte from the opcode stream and sign-extending it to
568	* a word, returning automatically on failure.
569	*
570	* @param a_pu64 Where to return the word.
571	* @remark Implicitly references pVCpu.
572	*/
573	#ifndef IEM_WITH_SETJMP
574	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) \
575	do \
576	{ \
577	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS8SxU64(pVCpu, (a_pu64)); \
578	if (rcStrict2 != VINF_SUCCESS) \
579	return rcStrict2; \
580	} while (0)
581	#else
582	# define IEM_OPCODE_GET_NEXT_S8_SX_U64(a_pu64) (*(a_pu64) = (int8_t)iemOpcodeGetNextU8Jmp(pVCpu))
583	#endif
584
585
586	#ifndef IEM_WITH_SETJMP
587	/**
588	* Fetches the next opcode byte.
589	*
590	* @returns Strict VBox status code.
591	* @param pVCpu The cross context virtual CPU structure of the
592	* calling thread.
593	* @param pu8 Where to return the opcode byte.
594	*/
595	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextRm(PVMCPUCC pVCpu, uint8_t *pu8)
596	{
597	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
598	pVCpu->iem.s.offModRm = offOpcode;
599	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
600	{
601	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
602	*pu8 = pVCpu->iem.s.abOpcode[offOpcode];
603	return VINF_SUCCESS;
604	}
605	return iemOpcodeGetNextU8Slow(pVCpu, pu8);
606	}
607	#else /* IEM_WITH_SETJMP */
608	/**
609	* Fetches the next opcode byte, longjmp on error.
610	*
611	* @returns The opcode byte.
612	* @param pVCpu The cross context virtual CPU structure of the calling thread.
613	*/
614	DECLINLINE(uint8_t) iemOpcodeGetNextRmJmp(PVMCPUCC pVCpu)
615	{
616	# ifdef IEM_WITH_CODE_TLB
617	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
618	pVCpu->iem.s.offModRm = offBuf;
619	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
620	if (RT_LIKELY( pbBuf != NULL
621	&& offBuf < pVCpu->iem.s.cbInstrBuf))
622	{
623	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 1;
624	return pbBuf[offBuf];
625	}
626	# else
627	uintptr_t offOpcode = pVCpu->iem.s.offOpcode;
628	pVCpu->iem.s.offModRm = offOpcode;
629	if (RT_LIKELY((uint8_t)offOpcode < pVCpu->iem.s.cbOpcode))
630	{
631	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 1;
632	return pVCpu->iem.s.abOpcode[offOpcode];
633	}
634	# endif
635	return iemOpcodeGetNextU8SlowJmp(pVCpu);
636	}
637	#endif /* IEM_WITH_SETJMP */
638
639	/**
640	* Fetches the next opcode byte, which is a ModR/M byte, returns automatically
641	* on failure.
642	*
643	* Will note down the position of the ModR/M byte for VT-x exits.
644	*
645	* @param a_pbRm Where to return the RM opcode byte.
646	* @remark Implicitly references pVCpu.
647	*/
648	#ifndef IEM_WITH_SETJMP
649	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) \
650	do \
651	{ \
652	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextRm(pVCpu, (a_pbRm)); \
653	if (rcStrict2 == VINF_SUCCESS) \
654	{ /* likely */ } \
655	else \
656	return rcStrict2; \
657	} while (0)
658	#else
659	# define IEM_OPCODE_GET_NEXT_RM(a_pbRm) (*(a_pbRm) = iemOpcodeGetNextRmJmp(pVCpu))
660	#endif /* IEM_WITH_SETJMP */
661
662
663	#ifndef IEM_WITH_SETJMP
664
665	/**
666	* Fetches the next opcode word.
667	*
668	* @returns Strict VBox status code.
669	* @param pVCpu The cross context virtual CPU structure of the calling thread.
670	* @param pu16 Where to return the opcode word.
671	*/
672	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16(PVMCPUCC pVCpu, uint16_t *pu16)
673	{
674	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
675	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
676	{
677	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
678	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
679	pu16 = (uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
680	# else
681	*pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
682	# endif
683	return VINF_SUCCESS;
684	}
685	return iemOpcodeGetNextU16Slow(pVCpu, pu16);
686	}
687
688	#else /* IEM_WITH_SETJMP */
689
690	/**
691	* Fetches the next opcode word, longjmp on error.
692	*
693	* @returns The opcode word.
694	* @param pVCpu The cross context virtual CPU structure of the calling thread.
695	*/
696	DECLINLINE(uint16_t) iemOpcodeGetNextU16Jmp(PVMCPUCC pVCpu)
697	{
698	# ifdef IEM_WITH_CODE_TLB
699	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
700	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
701	if (RT_LIKELY( pbBuf != NULL
702	&& offBuf + 2 <= pVCpu->iem.s.cbInstrBuf))
703	{
704	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 2;
705	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
706	return (uint16_t const )&pbBuf[offBuf];
707	# else
708	return RT_MAKE_U16(pbBuf[offBuf], pbBuf[offBuf + 1]);
709	# endif
710	}
711	# else
712	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
713	if (RT_LIKELY((uint8_t)offOpcode + 2 <= pVCpu->iem.s.cbOpcode))
714	{
715	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 2;
716	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
717	return (uint16_t const )&pVCpu->iem.s.abOpcode[offOpcode];
718	# else
719	return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
720	# endif
721	}
722	# endif
723	return iemOpcodeGetNextU16SlowJmp(pVCpu);
724	}
725
726	#endif /* IEM_WITH_SETJMP */
727
728	/**
729	* Fetches the next opcode word, returns automatically on failure.
730	*
731	* @param a_pu16 Where to return the opcode word.
732	* @remark Implicitly references pVCpu.
733	*/
734	#ifndef IEM_WITH_SETJMP
735	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) \
736	do \
737	{ \
738	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16(pVCpu, (a_pu16)); \
739	if (rcStrict2 != VINF_SUCCESS) \
740	return rcStrict2; \
741	} while (0)
742	#else
743	# define IEM_OPCODE_GET_NEXT_U16(a_pu16) (*(a_pu16) = iemOpcodeGetNextU16Jmp(pVCpu))
744	#endif
745
746	#ifndef IEM_WITH_SETJMP
747	/**
748	* Fetches the next opcode word, zero extending it to a double word.
749	*
750	* @returns Strict VBox status code.
751	* @param pVCpu The cross context virtual CPU structure of the calling thread.
752	* @param pu32 Where to return the opcode double word.
753	*/
754	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU32(PVMCPUCC pVCpu, uint32_t *pu32)
755	{
756	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
757	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
758	return iemOpcodeGetNextU16ZxU32Slow(pVCpu, pu32);
759
760	*pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
761	pVCpu->iem.s.offOpcode = offOpcode + 2;
762	return VINF_SUCCESS;
763	}
764	#endif /* !IEM_WITH_SETJMP */
765
766	/**
767	* Fetches the next opcode word and zero extends it to a double word, returns
768	* automatically on failure.
769	*
770	* @param a_pu32 Where to return the opcode double word.
771	* @remark Implicitly references pVCpu.
772	*/
773	#ifndef IEM_WITH_SETJMP
774	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) \
775	do \
776	{ \
777	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU32(pVCpu, (a_pu32)); \
778	if (rcStrict2 != VINF_SUCCESS) \
779	return rcStrict2; \
780	} while (0)
781	#else
782	# define IEM_OPCODE_GET_NEXT_U16_ZX_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU16Jmp(pVCpu))
783	#endif
784
785	#ifndef IEM_WITH_SETJMP
786	/**
787	* Fetches the next opcode word, zero extending it to a quad word.
788	*
789	* @returns Strict VBox status code.
790	* @param pVCpu The cross context virtual CPU structure of the calling thread.
791	* @param pu64 Where to return the opcode quad word.
792	*/
793	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU16ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
794	{
795	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
796	if (RT_UNLIKELY(offOpcode + 2 > pVCpu->iem.s.cbOpcode))
797	return iemOpcodeGetNextU16ZxU64Slow(pVCpu, pu64);
798
799	*pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
800	pVCpu->iem.s.offOpcode = offOpcode + 2;
801	return VINF_SUCCESS;
802	}
803	#endif /* !IEM_WITH_SETJMP */
804
805	/**
806	* Fetches the next opcode word and zero extends it to a quad word, returns
807	* automatically on failure.
808	*
809	* @param a_pu64 Where to return the opcode quad word.
810	* @remark Implicitly references pVCpu.
811	*/
812	#ifndef IEM_WITH_SETJMP
813	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) \
814	do \
815	{ \
816	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU16ZxU64(pVCpu, (a_pu64)); \
817	if (rcStrict2 != VINF_SUCCESS) \
818	return rcStrict2; \
819	} while (0)
820	#else
821	# define IEM_OPCODE_GET_NEXT_U16_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU16Jmp(pVCpu))
822	#endif
823
824
825	#ifndef IEM_WITH_SETJMP
826	/**
827	* Fetches the next signed word from the opcode stream.
828	*
829	* @returns Strict VBox status code.
830	* @param pVCpu The cross context virtual CPU structure of the calling thread.
831	* @param pi16 Where to return the signed word.
832	*/
833	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS16(PVMCPUCC pVCpu, int16_t *pi16)
834	{
835	return iemOpcodeGetNextU16(pVCpu, (uint16_t *)pi16);
836	}
837	#endif /* !IEM_WITH_SETJMP */
838
839
840	/**
841	* Fetches the next signed word from the opcode stream, returning automatically
842	* on failure.
843	*
844	* @param a_pi16 Where to return the signed word.
845	* @remark Implicitly references pVCpu.
846	*/
847	#ifndef IEM_WITH_SETJMP
848	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) \
849	do \
850	{ \
851	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS16(pVCpu, (a_pi16)); \
852	if (rcStrict2 != VINF_SUCCESS) \
853	return rcStrict2; \
854	} while (0)
855	#else
856	# define IEM_OPCODE_GET_NEXT_S16(a_pi16) (*(a_pi16) = (int16_t)iemOpcodeGetNextU16Jmp(pVCpu))
857	#endif
858
859	#ifndef IEM_WITH_SETJMP
860
861	/**
862	* Fetches the next opcode dword.
863	*
864	* @returns Strict VBox status code.
865	* @param pVCpu The cross context virtual CPU structure of the calling thread.
866	* @param pu32 Where to return the opcode double word.
867	*/
868	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32(PVMCPUCC pVCpu, uint32_t *pu32)
869	{
870	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
871	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
872	{
873	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
874	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
875	pu32 = (uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
876	# else
877	*pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
878	pVCpu->iem.s.abOpcode[offOpcode + 1],
879	pVCpu->iem.s.abOpcode[offOpcode + 2],
880	pVCpu->iem.s.abOpcode[offOpcode + 3]);
881	# endif
882	return VINF_SUCCESS;
883	}
884	return iemOpcodeGetNextU32Slow(pVCpu, pu32);
885	}
886
887	#else /* IEM_WITH_SETJMP */
888
889	/**
890	* Fetches the next opcode dword, longjmp on error.
891	*
892	* @returns The opcode dword.
893	* @param pVCpu The cross context virtual CPU structure of the calling thread.
894	*/
895	DECLINLINE(uint32_t) iemOpcodeGetNextU32Jmp(PVMCPUCC pVCpu)
896	{
897	# ifdef IEM_WITH_CODE_TLB
898	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
899	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
900	if (RT_LIKELY( pbBuf != NULL
901	&& offBuf + 4 <= pVCpu->iem.s.cbInstrBuf))
902	{
903	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 4;
904	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
905	return (uint32_t const )&pbBuf[offBuf];
906	# else
907	return RT_MAKE_U32_FROM_U8(pbBuf[offBuf],
908	pbBuf[offBuf + 1],
909	pbBuf[offBuf + 2],
910	pbBuf[offBuf + 3]);
911	# endif
912	}
913	# else
914	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
915	if (RT_LIKELY((uint8_t)offOpcode + 4 <= pVCpu->iem.s.cbOpcode))
916	{
917	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 4;
918	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
919	return (uint32_t const )&pVCpu->iem.s.abOpcode[offOpcode];
920	# else
921	return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
922	pVCpu->iem.s.abOpcode[offOpcode + 1],
923	pVCpu->iem.s.abOpcode[offOpcode + 2],
924	pVCpu->iem.s.abOpcode[offOpcode + 3]);
925	# endif
926	}
927	# endif
928	return iemOpcodeGetNextU32SlowJmp(pVCpu);
929	}
930
931	#endif /* IEM_WITH_SETJMP */
932
933	/**
934	* Fetches the next opcode dword, returns automatically on failure.
935	*
936	* @param a_pu32 Where to return the opcode dword.
937	* @remark Implicitly references pVCpu.
938	*/
939	#ifndef IEM_WITH_SETJMP
940	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) \
941	do \
942	{ \
943	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32(pVCpu, (a_pu32)); \
944	if (rcStrict2 != VINF_SUCCESS) \
945	return rcStrict2; \
946	} while (0)
947	#else
948	# define IEM_OPCODE_GET_NEXT_U32(a_pu32) (*(a_pu32) = iemOpcodeGetNextU32Jmp(pVCpu))
949	#endif
950
951	#ifndef IEM_WITH_SETJMP
952	/**
953	* Fetches the next opcode dword, zero extending it to a quad word.
954	*
955	* @returns Strict VBox status code.
956	* @param pVCpu The cross context virtual CPU structure of the calling thread.
957	* @param pu64 Where to return the opcode quad word.
958	*/
959	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU32ZxU64(PVMCPUCC pVCpu, uint64_t *pu64)
960	{
961	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
962	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
963	return iemOpcodeGetNextU32ZxU64Slow(pVCpu, pu64);
964
965	*pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
966	pVCpu->iem.s.abOpcode[offOpcode + 1],
967	pVCpu->iem.s.abOpcode[offOpcode + 2],
968	pVCpu->iem.s.abOpcode[offOpcode + 3]);
969	pVCpu->iem.s.offOpcode = offOpcode + 4;
970	return VINF_SUCCESS;
971	}
972	#endif /* !IEM_WITH_SETJMP */
973
974	/**
975	* Fetches the next opcode dword and zero extends it to a quad word, returns
976	* automatically on failure.
977	*
978	* @param a_pu64 Where to return the opcode quad word.
979	* @remark Implicitly references pVCpu.
980	*/
981	#ifndef IEM_WITH_SETJMP
982	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) \
983	do \
984	{ \
985	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU32ZxU64(pVCpu, (a_pu64)); \
986	if (rcStrict2 != VINF_SUCCESS) \
987	return rcStrict2; \
988	} while (0)
989	#else
990	# define IEM_OPCODE_GET_NEXT_U32_ZX_U64(a_pu64) (*(a_pu64) = iemOpcodeGetNextU32Jmp(pVCpu))
991	#endif
992
993
994	#ifndef IEM_WITH_SETJMP
995	/**
996	* Fetches the next signed double word from the opcode stream.
997	*
998	* @returns Strict VBox status code.
999	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1000	* @param pi32 Where to return the signed double word.
1001	*/
1002	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32(PVMCPUCC pVCpu, int32_t *pi32)
1003	{
1004	return iemOpcodeGetNextU32(pVCpu, (uint32_t *)pi32);
1005	}
1006	#endif
1007
1008	/**
1009	* Fetches the next signed double word from the opcode stream, returning
1010	* automatically on failure.
1011	*
1012	* @param a_pi32 Where to return the signed double word.
1013	* @remark Implicitly references pVCpu.
1014	*/
1015	#ifndef IEM_WITH_SETJMP
1016	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) \
1017	do \
1018	{ \
1019	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32(pVCpu, (a_pi32)); \
1020	if (rcStrict2 != VINF_SUCCESS) \
1021	return rcStrict2; \
1022	} while (0)
1023	#else
1024	# define IEM_OPCODE_GET_NEXT_S32(a_pi32) (*(a_pi32) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1025	#endif
1026
1027	#ifndef IEM_WITH_SETJMP
1028	/**
1029	* Fetches the next opcode dword, sign extending it into a quad word.
1030	*
1031	* @returns Strict VBox status code.
1032	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1033	* @param pu64 Where to return the opcode quad word.
1034	*/
1035	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64)
1036	{
1037	uint8_t const offOpcode = pVCpu->iem.s.offOpcode;
1038	if (RT_UNLIKELY(offOpcode + 4 > pVCpu->iem.s.cbOpcode))
1039	return iemOpcodeGetNextS32SxU64Slow(pVCpu, pu64);
1040
1041	int32_t i32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1042	pVCpu->iem.s.abOpcode[offOpcode + 1],
1043	pVCpu->iem.s.abOpcode[offOpcode + 2],
1044	pVCpu->iem.s.abOpcode[offOpcode + 3]);
1045	*pu64 = i32;
1046	pVCpu->iem.s.offOpcode = offOpcode + 4;
1047	return VINF_SUCCESS;
1048	}
1049	#endif /* !IEM_WITH_SETJMP */
1050
1051	/**
1052	* Fetches the next opcode double word and sign extends it to a quad word,
1053	* returns automatically on failure.
1054	*
1055	* @param a_pu64 Where to return the opcode quad word.
1056	* @remark Implicitly references pVCpu.
1057	*/
1058	#ifndef IEM_WITH_SETJMP
1059	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) \
1060	do \
1061	{ \
1062	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextS32SxU64(pVCpu, (a_pu64)); \
1063	if (rcStrict2 != VINF_SUCCESS) \
1064	return rcStrict2; \
1065	} while (0)
1066	#else
1067	# define IEM_OPCODE_GET_NEXT_S32_SX_U64(a_pu64) (*(a_pu64) = (int32_t)iemOpcodeGetNextU32Jmp(pVCpu))
1068	#endif
1069
1070	#ifndef IEM_WITH_SETJMP
1071
1072	/**
1073	* Fetches the next opcode qword.
1074	*
1075	* @returns Strict VBox status code.
1076	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1077	* @param pu64 Where to return the opcode qword.
1078	*/
1079	DECLINLINE(VBOXSTRICTRC) iemOpcodeGetNextU64(PVMCPUCC pVCpu, uint64_t *pu64)
1080	{
1081	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1082	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1083	{
1084	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1085	pu64 = (uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
1086	# else
1087	*pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1088	pVCpu->iem.s.abOpcode[offOpcode + 1],
1089	pVCpu->iem.s.abOpcode[offOpcode + 2],
1090	pVCpu->iem.s.abOpcode[offOpcode + 3],
1091	pVCpu->iem.s.abOpcode[offOpcode + 4],
1092	pVCpu->iem.s.abOpcode[offOpcode + 5],
1093	pVCpu->iem.s.abOpcode[offOpcode + 6],
1094	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1095	# endif
1096	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1097	return VINF_SUCCESS;
1098	}
1099	return iemOpcodeGetNextU64Slow(pVCpu, pu64);
1100	}
1101
1102	#else /* IEM_WITH_SETJMP */
1103
1104	/**
1105	* Fetches the next opcode qword, longjmp on error.
1106	*
1107	* @returns The opcode qword.
1108	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1109	*/
1110	DECLINLINE(uint64_t) iemOpcodeGetNextU64Jmp(PVMCPUCC pVCpu)
1111	{
1112	# ifdef IEM_WITH_CODE_TLB
1113	uintptr_t offBuf = pVCpu->iem.s.offInstrNextByte;
1114	uint8_t const *pbBuf = pVCpu->iem.s.pbInstrBuf;
1115	if (RT_LIKELY( pbBuf != NULL
1116	&& offBuf + 8 <= pVCpu->iem.s.cbInstrBuf))
1117	{
1118	pVCpu->iem.s.offInstrNextByte = (uint32_t)offBuf + 8;
1119	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1120	return (uint64_t const )&pbBuf[offBuf];
1121	# else
1122	return RT_MAKE_U64_FROM_U8(pbBuf[offBuf],
1123	pbBuf[offBuf + 1],
1124	pbBuf[offBuf + 2],
1125	pbBuf[offBuf + 3],
1126	pbBuf[offBuf + 4],
1127	pbBuf[offBuf + 5],
1128	pbBuf[offBuf + 6],
1129	pbBuf[offBuf + 7]);
1130	# endif
1131	}
1132	# else
1133	uintptr_t const offOpcode = pVCpu->iem.s.offOpcode;
1134	if (RT_LIKELY((uint8_t)offOpcode + 8 <= pVCpu->iem.s.cbOpcode))
1135	{
1136	pVCpu->iem.s.offOpcode = (uint8_t)offOpcode + 8;
1137	# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
1138	return (uint64_t const )&pVCpu->iem.s.abOpcode[offOpcode];
1139	# else
1140	return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
1141	pVCpu->iem.s.abOpcode[offOpcode + 1],
1142	pVCpu->iem.s.abOpcode[offOpcode + 2],
1143	pVCpu->iem.s.abOpcode[offOpcode + 3],
1144	pVCpu->iem.s.abOpcode[offOpcode + 4],
1145	pVCpu->iem.s.abOpcode[offOpcode + 5],
1146	pVCpu->iem.s.abOpcode[offOpcode + 6],
1147	pVCpu->iem.s.abOpcode[offOpcode + 7]);
1148	# endif
1149	}
1150	# endif
1151	return iemOpcodeGetNextU64SlowJmp(pVCpu);
1152	}
1153
1154	#endif /* IEM_WITH_SETJMP */
1155
1156	/**
1157	* Fetches the next opcode quad word, returns automatically on failure.
1158	*
1159	* @param a_pu64 Where to return the opcode quad word.
1160	* @remark Implicitly references pVCpu.
1161	*/
1162	#ifndef IEM_WITH_SETJMP
1163	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) \
1164	do \
1165	{ \
1166	VBOXSTRICTRC rcStrict2 = iemOpcodeGetNextU64(pVCpu, (a_pu64)); \
1167	if (rcStrict2 != VINF_SUCCESS) \
1168	return rcStrict2; \
1169	} while (0)
1170	#else
1171	# define IEM_OPCODE_GET_NEXT_U64(a_pu64) ( *(a_pu64) = iemOpcodeGetNextU64Jmp(pVCpu) )
1172	#endif
1173
1174
1175	/** @name Misc Worker Functions.
1176	* @{
1177	*/
1178
1179	/**
1180	* Gets the correct EFLAGS regardless of whether PATM stores parts of them or
1181	* not (kind of obsolete now).
1182	*
1183	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1184	*/
1185	#define IEMMISC_GET_EFL(a_pVCpu) ( (a_pVCpu)->cpum.GstCtx.eflags.u )
1186
1187	/**
1188	* Updates the EFLAGS in the correct manner wrt. PATM (kind of obsolete).
1189	*
1190	* @param a_pVCpu The cross context virtual CPU structure of the calling thread.
1191	* @param a_fEfl The new EFLAGS.
1192	*/
1193	#define IEMMISC_SET_EFL(a_pVCpu, a_fEfl) do { (a_pVCpu)->cpum.GstCtx.eflags.u = (a_fEfl); } while (0)
1194
1195
1196	/**
1197	* Loads a NULL data selector into a selector register, both the hidden and
1198	* visible parts, in protected mode.
1199	*
1200	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1201	* @param pSReg Pointer to the segment register.
1202	* @param uRpl The RPL.
1203	*/
1204	DECLINLINE(void) iemHlpLoadNullDataSelectorProt(PVMCPUCC pVCpu, PCPUMSELREG pSReg, RTSEL uRpl)
1205	{
1206	/** @todo Testcase: write a testcase checking what happends when loading a NULL
1207	* data selector in protected mode. */
1208	pSReg->Sel = uRpl;
1209	pSReg->ValidSel = uRpl;
1210	pSReg->fFlags = CPUMSELREG_FLAGS_VALID;
1211	if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
1212	{
1213	/* VT-x (Intel 3960x) observed doing something like this. */
1214	pSReg->Attr.u = X86DESCATTR_UNUSABLE \| X86DESCATTR_G \| X86DESCATTR_D \| (pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT);
1215	pSReg->u32Limit = UINT32_MAX;
1216	pSReg->u64Base = 0;
1217	}
1218	else
1219	{
1220	pSReg->Attr.u = X86DESCATTR_UNUSABLE;
1221	pSReg->u32Limit = 0;
1222	pSReg->u64Base = 0;
1223	}
1224	}
1225
1226	/** @} */
1227
1228
1229	/*
1230	*
1231	* Helpers routines.
1232	* Helpers routines.
1233	* Helpers routines.
1234	*
1235	*/
1236
1237	/**
1238	* Recalculates the effective operand size.
1239	*
1240	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1241	*/
1242	DECLINLINE(void) iemRecalEffOpSize(PVMCPUCC pVCpu)
1243	{
1244	switch (pVCpu->iem.s.enmCpuMode)
1245	{
1246	case IEMMODE_16BIT:
1247	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_32BIT : IEMMODE_16BIT;
1248	break;
1249	case IEMMODE_32BIT:
1250	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP ? IEMMODE_16BIT : IEMMODE_32BIT;
1251	break;
1252	case IEMMODE_64BIT:
1253	switch (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP))
1254	{
1255	case 0:
1256	pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize;
1257	break;
1258	case IEM_OP_PRF_SIZE_OP:
1259	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1260	break;
1261	case IEM_OP_PRF_SIZE_REX_W:
1262	case IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP:
1263	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1264	break;
1265	}
1266	break;
1267	default:
1268	AssertFailed();
1269	}
1270	}
1271
1272
1273	/**
1274	* Sets the default operand size to 64-bit and recalculates the effective
1275	* operand size.
1276	*
1277	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1278	*/
1279	DECLINLINE(void) iemRecalEffOpSize64Default(PVMCPUCC pVCpu)
1280	{
1281	Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
1282	pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
1283	if ((pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_SIZE_REX_W \| IEM_OP_PRF_SIZE_OP)) != IEM_OP_PRF_SIZE_OP)
1284	pVCpu->iem.s.enmEffOpSize = IEMMODE_64BIT;
1285	else
1286	pVCpu->iem.s.enmEffOpSize = IEMMODE_16BIT;
1287	}
1288
1289
1290
1291
1292	/** @name Register Access.
1293	* @{
1294	*/
1295
1296	/**
1297	* Gets a reference (pointer) to the specified hidden segment register.
1298	*
1299	* @returns Hidden register reference.
1300	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1301	* @param iSegReg The segment register.
1302	*/
1303	DECLINLINE(PCPUMSELREG) iemSRegGetHid(PVMCPUCC pVCpu, uint8_t iSegReg)
1304	{
1305	Assert(iSegReg < X86_SREG_COUNT);
1306	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1307	PCPUMSELREG pSReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
1308
1309	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1310	return pSReg;
1311	}
1312
1313
1314	/**
1315	* Ensures that the given hidden segment register is up to date.
1316	*
1317	* @returns Hidden register reference.
1318	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1319	* @param pSReg The segment register.
1320	*/
1321	DECLINLINE(PCPUMSELREG) iemSRegUpdateHid(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
1322	{
1323	Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
1324	NOREF(pVCpu);
1325	return pSReg;
1326	}
1327
1328
1329	/**
1330	* Gets a reference (pointer) to the specified segment register (the selector
1331	* value).
1332	*
1333	* @returns Pointer to the selector variable.
1334	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1335	* @param iSegReg The segment register.
1336	*/
1337	DECLINLINE(uint16_t *) iemSRegRef(PVMCPUCC pVCpu, uint8_t iSegReg)
1338	{
1339	Assert(iSegReg < X86_SREG_COUNT);
1340	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1341	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1342	}
1343
1344
1345	/**
1346	* Fetches the selector value of a segment register.
1347	*
1348	* @returns The selector value.
1349	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1350	* @param iSegReg The segment register.
1351	*/
1352	DECLINLINE(uint16_t) iemSRegFetchU16(PVMCPUCC pVCpu, uint8_t iSegReg)
1353	{
1354	Assert(iSegReg < X86_SREG_COUNT);
1355	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1356	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].Sel;
1357	}
1358
1359
1360	/**
1361	* Fetches the base address value of a segment register.
1362	*
1363	* @returns The selector value.
1364	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1365	* @param iSegReg The segment register.
1366	*/
1367	DECLINLINE(uint64_t) iemSRegBaseFetchU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1368	{
1369	Assert(iSegReg < X86_SREG_COUNT);
1370	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1371	return pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1372	}
1373
1374
1375	/**
1376	* Gets a reference (pointer) to the specified general purpose register.
1377	*
1378	* @returns Register reference.
1379	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1380	* @param iReg The general purpose register.
1381	*/
1382	DECLINLINE(void *) iemGRegRef(PVMCPUCC pVCpu, uint8_t iReg)
1383	{
1384	Assert(iReg < 16);
1385	return &pVCpu->cpum.GstCtx.aGRegs[iReg];
1386	}
1387
1388
1389	/**
1390	* Gets a reference (pointer) to the specified 8-bit general purpose register.
1391	*
1392	* Because of AH, CH, DH and BH we cannot use iemGRegRef directly here.
1393	*
1394	* @returns Register reference.
1395	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1396	* @param iReg The register.
1397	*/
1398	DECLINLINE(uint8_t *) iemGRegRefU8(PVMCPUCC pVCpu, uint8_t iReg)
1399	{
1400	if (iReg < 4 \|\| (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX))
1401	{
1402	Assert(iReg < 16);
1403	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u8;
1404	}
1405	/* high 8-bit register. */
1406	Assert(iReg < 8);
1407	return &pVCpu->cpum.GstCtx.aGRegs[iReg & 3].bHi;
1408	}
1409
1410
1411	/**
1412	* Gets a reference (pointer) to the specified 16-bit general purpose register.
1413	*
1414	* @returns Register reference.
1415	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1416	* @param iReg The register.
1417	*/
1418	DECLINLINE(uint16_t *) iemGRegRefU16(PVMCPUCC pVCpu, uint8_t iReg)
1419	{
1420	Assert(iReg < 16);
1421	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1422	}
1423
1424
1425	/**
1426	* Gets a reference (pointer) to the specified 32-bit general purpose register.
1427	*
1428	* @returns Register reference.
1429	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1430	* @param iReg The register.
1431	*/
1432	DECLINLINE(uint32_t *) iemGRegRefU32(PVMCPUCC pVCpu, uint8_t iReg)
1433	{
1434	Assert(iReg < 16);
1435	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1436	}
1437
1438
1439	/**
1440	* Gets a reference (pointer) to the specified 64-bit general purpose register.
1441	*
1442	* @returns Register reference.
1443	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1444	* @param iReg The register.
1445	*/
1446	DECLINLINE(uint64_t *) iemGRegRefU64(PVMCPUCC pVCpu, uint8_t iReg)
1447	{
1448	Assert(iReg < 64);
1449	return &pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1450	}
1451
1452
1453	/**
1454	* Gets a reference (pointer) to the specified segment register's base address.
1455	*
1456	* @returns Segment register base address reference.
1457	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1458	* @param iSegReg The segment selector.
1459	*/
1460	DECLINLINE(uint64_t *) iemSRegBaseRefU64(PVMCPUCC pVCpu, uint8_t iSegReg)
1461	{
1462	Assert(iSegReg < X86_SREG_COUNT);
1463	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
1464	return &pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
1465	}
1466
1467
1468	/**
1469	* Fetches the value of a 8-bit general purpose register.
1470	*
1471	* @returns The register value.
1472	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1473	* @param iReg The register.
1474	*/
1475	DECLINLINE(uint8_t) iemGRegFetchU8(PVMCPUCC pVCpu, uint8_t iReg)
1476	{
1477	return *iemGRegRefU8(pVCpu, iReg);
1478	}
1479
1480
1481	/**
1482	* Fetches the value of a 16-bit general purpose register.
1483	*
1484	* @returns The register value.
1485	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1486	* @param iReg The register.
1487	*/
1488	DECLINLINE(uint16_t) iemGRegFetchU16(PVMCPUCC pVCpu, uint8_t iReg)
1489	{
1490	Assert(iReg < 16);
1491	return pVCpu->cpum.GstCtx.aGRegs[iReg].u16;
1492	}
1493
1494
1495	/**
1496	* Fetches the value of a 32-bit general purpose register.
1497	*
1498	* @returns The register value.
1499	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1500	* @param iReg The register.
1501	*/
1502	DECLINLINE(uint32_t) iemGRegFetchU32(PVMCPUCC pVCpu, uint8_t iReg)
1503	{
1504	Assert(iReg < 16);
1505	return pVCpu->cpum.GstCtx.aGRegs[iReg].u32;
1506	}
1507
1508
1509	/**
1510	* Fetches the value of a 64-bit general purpose register.
1511	*
1512	* @returns The register value.
1513	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1514	* @param iReg The register.
1515	*/
1516	DECLINLINE(uint64_t) iemGRegFetchU64(PVMCPUCC pVCpu, uint8_t iReg)
1517	{
1518	Assert(iReg < 16);
1519	return pVCpu->cpum.GstCtx.aGRegs[iReg].u64;
1520	}
1521
1522
1523	/**
1524	* Get the address of the top of the stack.
1525	*
1526	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1527	*/
1528	DECLINLINE(RTGCPTR) iemRegGetEffRsp(PCVMCPU pVCpu)
1529	{
1530	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1531	return pVCpu->cpum.GstCtx.rsp;
1532	if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1533	return pVCpu->cpum.GstCtx.esp;
1534	return pVCpu->cpum.GstCtx.sp;
1535	}
1536
1537
1538	/**
1539	* Updates the RIP/EIP/IP to point to the next instruction.
1540	*
1541	* This function leaves the EFLAGS.RF flag alone.
1542	*
1543	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1544	* @param cbInstr The number of bytes to add.
1545	*/
1546	DECLINLINE(void) iemRegAddToRipKeepRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1547	{
1548	switch (pVCpu->iem.s.enmCpuMode)
1549	{
1550	case IEMMODE_16BIT:
1551	Assert(pVCpu->cpum.GstCtx.rip <= UINT16_MAX);
1552	pVCpu->cpum.GstCtx.eip += cbInstr;
1553	pVCpu->cpum.GstCtx.eip &= UINT32_C(0xffff);
1554	break;
1555
1556	case IEMMODE_32BIT:
1557	pVCpu->cpum.GstCtx.eip += cbInstr;
1558	Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
1559	break;
1560
1561	case IEMMODE_64BIT:
1562	pVCpu->cpum.GstCtx.rip += cbInstr;
1563	break;
1564	default: AssertFailed();
1565	}
1566	}
1567
1568
1569	#if 0
1570	/**
1571	* Updates the RIP/EIP/IP to point to the next instruction.
1572	*
1573	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1574	*/
1575	DECLINLINE(void) iemRegUpdateRipKeepRF(PVMCPUCC pVCpu)
1576	{
1577	return iemRegAddToRipKeepRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1578	}
1579	#endif
1580
1581
1582
1583	/**
1584	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1585	*
1586	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1587	* @param cbInstr The number of bytes to add.
1588	*/
1589	DECLINLINE(void) iemRegAddToRipAndClearRF(PVMCPUCC pVCpu, uint8_t cbInstr)
1590	{
1591	pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
1592
1593	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
1594	#if ARCH_BITS >= 64
1595	static uint64_t const s_aRipMasks[] = { UINT64_C(0xffffffff), UINT64_C(0xffffffff), UINT64_MAX };
1596	Assert(pVCpu->cpum.GstCtx.rip <= s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode]);
1597	pVCpu->cpum.GstCtx.rip = (pVCpu->cpum.GstCtx.rip + cbInstr) & s_aRipMasks[(unsigned)pVCpu->iem.s.enmCpuMode];
1598	#else
1599	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1600	pVCpu->cpum.GstCtx.rip += cbInstr;
1601	else
1602	pVCpu->cpum.GstCtx.eip += cbInstr;
1603	#endif
1604	}
1605
1606
1607	/**
1608	* Updates the RIP/EIP/IP to point to the next instruction and clears EFLAGS.RF.
1609	*
1610	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1611	*/
1612	DECLINLINE(void) iemRegUpdateRipAndClearRF(PVMCPUCC pVCpu)
1613	{
1614	return iemRegAddToRipAndClearRF(pVCpu, IEM_GET_INSTR_LEN(pVCpu));
1615	}
1616
1617
1618	/**
1619	* Adds to the stack pointer.
1620	*
1621	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1622	* @param cbToAdd The number of bytes to add (8-bit!).
1623	*/
1624	DECLINLINE(void) iemRegAddToRsp(PVMCPUCC pVCpu, uint8_t cbToAdd)
1625	{
1626	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1627	pVCpu->cpum.GstCtx.rsp += cbToAdd;
1628	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1629	pVCpu->cpum.GstCtx.esp += cbToAdd;
1630	else
1631	pVCpu->cpum.GstCtx.sp += cbToAdd;
1632	}
1633
1634
1635	/**
1636	* Subtracts from the stack pointer.
1637	*
1638	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1639	* @param cbToSub The number of bytes to subtract (8-bit!).
1640	*/
1641	DECLINLINE(void) iemRegSubFromRsp(PVMCPUCC pVCpu, uint8_t cbToSub)
1642	{
1643	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1644	pVCpu->cpum.GstCtx.rsp -= cbToSub;
1645	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1646	pVCpu->cpum.GstCtx.esp -= cbToSub;
1647	else
1648	pVCpu->cpum.GstCtx.sp -= cbToSub;
1649	}
1650
1651
1652	/**
1653	* Adds to the temporary stack pointer.
1654	*
1655	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1656	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1657	* @param cbToAdd The number of bytes to add (16-bit).
1658	*/
1659	DECLINLINE(void) iemRegAddToRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToAdd)
1660	{
1661	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1662	pTmpRsp->u += cbToAdd;
1663	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1664	pTmpRsp->DWords.dw0 += cbToAdd;
1665	else
1666	pTmpRsp->Words.w0 += cbToAdd;
1667	}
1668
1669
1670	/**
1671	* Subtracts from the temporary stack pointer.
1672	*
1673	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1674	* @param pTmpRsp The temporary SP/ESP/RSP to update.
1675	* @param cbToSub The number of bytes to subtract.
1676	* @remarks The @a cbToSub argument MUST be 16-bit, iemCImpl_enter is
1677	* expecting that.
1678	*/
1679	DECLINLINE(void) iemRegSubFromRspEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint16_t cbToSub)
1680	{
1681	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1682	pTmpRsp->u -= cbToSub;
1683	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1684	pTmpRsp->DWords.dw0 -= cbToSub;
1685	else
1686	pTmpRsp->Words.w0 -= cbToSub;
1687	}
1688
1689
1690	/**
1691	* Calculates the effective stack address for a push of the specified size as
1692	* well as the new RSP value (upper bits may be masked).
1693	*
1694	* @returns Effective stack addressf for the push.
1695	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1696	* @param cbItem The size of the stack item to pop.
1697	* @param puNewRsp Where to return the new RSP value.
1698	*/
1699	DECLINLINE(RTGCPTR) iemRegGetRspForPush(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1700	{
1701	RTUINT64U uTmpRsp;
1702	RTGCPTR GCPtrTop;
1703	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1704
1705	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1706	GCPtrTop = uTmpRsp.u -= cbItem;
1707	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1708	GCPtrTop = uTmpRsp.DWords.dw0 -= cbItem;
1709	else
1710	GCPtrTop = uTmpRsp.Words.w0 -= cbItem;
1711	*puNewRsp = uTmpRsp.u;
1712	return GCPtrTop;
1713	}
1714
1715
1716	/**
1717	* Gets the current stack pointer and calculates the value after a pop of the
1718	* specified size.
1719	*
1720	* @returns Current stack pointer.
1721	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1722	* @param cbItem The size of the stack item to pop.
1723	* @param puNewRsp Where to return the new RSP value.
1724	*/
1725	DECLINLINE(RTGCPTR) iemRegGetRspForPop(PCVMCPU pVCpu, uint8_t cbItem, uint64_t *puNewRsp)
1726	{
1727	RTUINT64U uTmpRsp;
1728	RTGCPTR GCPtrTop;
1729	uTmpRsp.u = pVCpu->cpum.GstCtx.rsp;
1730
1731	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1732	{
1733	GCPtrTop = uTmpRsp.u;
1734	uTmpRsp.u += cbItem;
1735	}
1736	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1737	{
1738	GCPtrTop = uTmpRsp.DWords.dw0;
1739	uTmpRsp.DWords.dw0 += cbItem;
1740	}
1741	else
1742	{
1743	GCPtrTop = uTmpRsp.Words.w0;
1744	uTmpRsp.Words.w0 += cbItem;
1745	}
1746	*puNewRsp = uTmpRsp.u;
1747	return GCPtrTop;
1748	}
1749
1750
1751	/**
1752	* Calculates the effective stack address for a push of the specified size as
1753	* well as the new temporary RSP value (upper bits may be masked).
1754	*
1755	* @returns Effective stack addressf for the push.
1756	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1757	* @param pTmpRsp The temporary stack pointer. This is updated.
1758	* @param cbItem The size of the stack item to pop.
1759	*/
1760	DECLINLINE(RTGCPTR) iemRegGetRspForPushEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1761	{
1762	RTGCPTR GCPtrTop;
1763
1764	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1765	GCPtrTop = pTmpRsp->u -= cbItem;
1766	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1767	GCPtrTop = pTmpRsp->DWords.dw0 -= cbItem;
1768	else
1769	GCPtrTop = pTmpRsp->Words.w0 -= cbItem;
1770	return GCPtrTop;
1771	}
1772
1773
1774	/**
1775	* Gets the effective stack address for a pop of the specified size and
1776	* calculates and updates the temporary RSP.
1777	*
1778	* @returns Current stack pointer.
1779	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1780	* @param pTmpRsp The temporary stack pointer. This is updated.
1781	* @param cbItem The size of the stack item to pop.
1782	*/
1783	DECLINLINE(RTGCPTR) iemRegGetRspForPopEx(PCVMCPU pVCpu, PRTUINT64U pTmpRsp, uint8_t cbItem)
1784	{
1785	RTGCPTR GCPtrTop;
1786	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
1787	{
1788	GCPtrTop = pTmpRsp->u;
1789	pTmpRsp->u += cbItem;
1790	}
1791	else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
1792	{
1793	GCPtrTop = pTmpRsp->DWords.dw0;
1794	pTmpRsp->DWords.dw0 += cbItem;
1795	}
1796	else
1797	{
1798	GCPtrTop = pTmpRsp->Words.w0;
1799	pTmpRsp->Words.w0 += cbItem;
1800	}
1801	return GCPtrTop;
1802	}
1803
1804	/** @} */
1805
1806
1807	/** @name FPU access and helpers.
1808	*
1809	* @{
1810	*/
1811
1812
1813	/**
1814	* Hook for preparing to use the host FPU.
1815	*
1816	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1817	*
1818	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1819	*/
1820	DECLINLINE(void) iemFpuPrepareUsage(PVMCPUCC pVCpu)
1821	{
1822	#ifdef IN_RING3
1823	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1824	#else
1825	CPUMRZFpuStatePrepareHostCpuForUse(pVCpu);
1826	#endif
1827	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1828	}
1829
1830
1831	/**
1832	* Hook for preparing to use the host FPU for SSE.
1833	*
1834	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1835	*
1836	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1837	*/
1838	DECLINLINE(void) iemFpuPrepareUsageSse(PVMCPUCC pVCpu)
1839	{
1840	iemFpuPrepareUsage(pVCpu);
1841	}
1842
1843
1844	/**
1845	* Hook for preparing to use the host FPU for AVX.
1846	*
1847	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1848	*
1849	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1850	*/
1851	DECLINLINE(void) iemFpuPrepareUsageAvx(PVMCPUCC pVCpu)
1852	{
1853	iemFpuPrepareUsage(pVCpu);
1854	}
1855
1856
1857	/**
1858	* Hook for actualizing the guest FPU state before the interpreter reads it.
1859	*
1860	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1861	*
1862	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1863	*/
1864	DECLINLINE(void) iemFpuActualizeStateForRead(PVMCPUCC pVCpu)
1865	{
1866	#ifdef IN_RING3
1867	NOREF(pVCpu);
1868	#else
1869	CPUMRZFpuStateActualizeForRead(pVCpu);
1870	#endif
1871	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1872	}
1873
1874
1875	/**
1876	* Hook for actualizing the guest FPU state before the interpreter changes it.
1877	*
1878	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1879	*
1880	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1881	*/
1882	DECLINLINE(void) iemFpuActualizeStateForChange(PVMCPUCC pVCpu)
1883	{
1884	#ifdef IN_RING3
1885	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1886	#else
1887	CPUMRZFpuStateActualizeForChange(pVCpu);
1888	#endif
1889	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1890	}
1891
1892
1893	/**
1894	* Hook for actualizing the guest XMM0..15 and MXCSR register state for read
1895	* only.
1896	*
1897	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1898	*
1899	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1900	*/
1901	DECLINLINE(void) iemFpuActualizeSseStateForRead(PVMCPUCC pVCpu)
1902	{
1903	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1904	NOREF(pVCpu);
1905	#else
1906	CPUMRZFpuStateActualizeSseForRead(pVCpu);
1907	#endif
1908	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1909	}
1910
1911
1912	/**
1913	* Hook for actualizing the guest XMM0..15 and MXCSR register state for
1914	* read+write.
1915	*
1916	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1917	*
1918	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1919	*/
1920	DECLINLINE(void) iemFpuActualizeSseStateForChange(PVMCPUCC pVCpu)
1921	{
1922	#if defined(IN_RING3) \|\| defined(VBOX_WITH_KERNEL_USING_XMM)
1923	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1924	#else
1925	CPUMRZFpuStateActualizeForChange(pVCpu);
1926	#endif
1927	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1928
1929	/* Make sure any changes are loaded the next time around. */
1930	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_SSE;
1931	}
1932
1933
1934	/**
1935	* Hook for actualizing the guest YMM0..15 and MXCSR register state for read
1936	* only.
1937	*
1938	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1939	*
1940	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1941	*/
1942	DECLINLINE(void) iemFpuActualizeAvxStateForRead(PVMCPUCC pVCpu)
1943	{
1944	#ifdef IN_RING3
1945	NOREF(pVCpu);
1946	#else
1947	CPUMRZFpuStateActualizeAvxForRead(pVCpu);
1948	#endif
1949	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1950	}
1951
1952
1953	/**
1954	* Hook for actualizing the guest YMM0..15 and MXCSR register state for
1955	* read+write.
1956	*
1957	* This is necessary in ring-0 and raw-mode context (nop in ring-3).
1958	*
1959	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1960	*/
1961	DECLINLINE(void) iemFpuActualizeAvxStateForChange(PVMCPUCC pVCpu)
1962	{
1963	#ifdef IN_RING3
1964	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
1965	#else
1966	CPUMRZFpuStateActualizeForChange(pVCpu);
1967	#endif
1968	IEM_CTX_IMPORT_NORET(pVCpu, CPUMCTX_EXTRN_X87 \| CPUMCTX_EXTRN_SSE_AVX \| CPUMCTX_EXTRN_OTHER_XSAVE \| CPUMCTX_EXTRN_XCRx);
1969
1970	/* Just assume we're going to make changes to the SSE and YMM_HI parts. */
1971	pVCpu->cpum.GstCtx.XState.Hdr.bmXState \|= XSAVE_C_YMM \| XSAVE_C_SSE;
1972	}
1973
1974
1975	/**
1976	* Stores a QNaN value into a FPU register.
1977	*
1978	* @param pReg Pointer to the register.
1979	*/
1980	DECLINLINE(void) iemFpuStoreQNan(PRTFLOAT80U pReg)
1981	{
1982	pReg->au32[0] = UINT32_C(0x00000000);
1983	pReg->au32[1] = UINT32_C(0xc0000000);
1984	pReg->au16[4] = UINT16_C(0xffff);
1985	}
1986
1987
1988	/**
1989	* Updates the FOP, FPU.CS and FPUIP registers.
1990	*
1991	* @param pVCpu The cross context virtual CPU structure of the calling thread.
1992	* @param pFpuCtx The FPU context.
1993	*/
1994	DECLINLINE(void) iemFpuUpdateOpcodeAndIpWorker(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx)
1995	{
1996	Assert(pVCpu->iem.s.uFpuOpcode != UINT16_MAX);
1997	pFpuCtx->FOP = pVCpu->iem.s.uFpuOpcode;
1998	/** @todo x87.CS and FPUIP needs to be kept seperately. */
1999	if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
2000	{
2001	/** @todo Testcase: making assumptions about how FPUIP and FPUDP are handled
2002	* happens in real mode here based on the fnsave and fnstenv images. */
2003	pFpuCtx->CS = 0;
2004	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.eip \| ((uint32_t)pVCpu->cpum.GstCtx.cs.Sel << 4);
2005	}
2006	else if (!IEM_IS_LONG_MODE(pVCpu))
2007	{
2008	pFpuCtx->CS = pVCpu->cpum.GstCtx.cs.Sel;
2009	pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2010	}
2011	else
2012	(uint64_t )&pFpuCtx->FPUIP = pVCpu->cpum.GstCtx.rip;
2013	}
2014
2015
2016
2017
2018
2019	/**
2020	* Marks the specified stack register as free (for FFREE).
2021	*
2022	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2023	* @param iStReg The register to free.
2024	*/
2025	DECLINLINE(void) iemFpuStackFree(PVMCPUCC pVCpu, uint8_t iStReg)
2026	{
2027	Assert(iStReg < 8);
2028	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2029	uint8_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2030	pFpuCtx->FTW &= ~RT_BIT(iReg);
2031	}
2032
2033
2034	/**
2035	* Increments FSW.TOP, i.e. pops an item off the stack without freeing it.
2036	*
2037	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2038	*/
2039	DECLINLINE(void) iemFpuStackIncTop(PVMCPUCC pVCpu)
2040	{
2041	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2042	uint16_t uFsw = pFpuCtx->FSW;
2043	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2044	uTop = (uTop + (1 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2045	uFsw &= ~X86_FSW_TOP_MASK;
2046	uFsw \|= uTop;
2047	pFpuCtx->FSW = uFsw;
2048	}
2049
2050
2051	/**
2052	* Decrements FSW.TOP, i.e. push an item off the stack without storing anything.
2053	*
2054	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2055	*/
2056	DECLINLINE(void) iemFpuStackDecTop(PVMCPUCC pVCpu)
2057	{
2058	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2059	uint16_t uFsw = pFpuCtx->FSW;
2060	uint16_t uTop = uFsw & X86_FSW_TOP_MASK;
2061	uTop = (uTop + (7 << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
2062	uFsw &= ~X86_FSW_TOP_MASK;
2063	uFsw \|= uTop;
2064	pFpuCtx->FSW = uFsw;
2065	}
2066
2067
2068
2069
2070	DECLINLINE(int) iemFpuStRegNotEmpty(PVMCPUCC pVCpu, uint8_t iStReg)
2071	{
2072	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2073	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2074	if (pFpuCtx->FTW & RT_BIT(iReg))
2075	return VINF_SUCCESS;
2076	return VERR_NOT_FOUND;
2077	}
2078
2079
2080	DECLINLINE(int) iemFpuStRegNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg, PCRTFLOAT80U *ppRef)
2081	{
2082	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2083	uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
2084	if (pFpuCtx->FTW & RT_BIT(iReg))
2085	{
2086	*ppRef = &pFpuCtx->aRegs[iStReg].r80;
2087	return VINF_SUCCESS;
2088	}
2089	return VERR_NOT_FOUND;
2090	}
2091
2092
2093	DECLINLINE(int) iemFpu2StRegsNotEmptyRef(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0,
2094	uint8_t iStReg1, PCRTFLOAT80U *ppRef1)
2095	{
2096	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2097	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2098	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2099	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2100	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2101	{
2102	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2103	*ppRef1 = &pFpuCtx->aRegs[iStReg1].r80;
2104	return VINF_SUCCESS;
2105	}
2106	return VERR_NOT_FOUND;
2107	}
2108
2109
2110	DECLINLINE(int) iemFpu2StRegsNotEmptyRefFirst(PVMCPUCC pVCpu, uint8_t iStReg0, PCRTFLOAT80U *ppRef0, uint8_t iStReg1)
2111	{
2112	PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
2113	uint16_t iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2114	uint16_t iReg0 = (iTop + iStReg0) & X86_FSW_TOP_SMASK;
2115	uint16_t iReg1 = (iTop + iStReg1) & X86_FSW_TOP_SMASK;
2116	if ((pFpuCtx->FTW & (RT_BIT(iReg0) \| RT_BIT(iReg1))) == (RT_BIT(iReg0) \| RT_BIT(iReg1)))
2117	{
2118	*ppRef0 = &pFpuCtx->aRegs[iStReg0].r80;
2119	return VINF_SUCCESS;
2120	}
2121	return VERR_NOT_FOUND;
2122	}
2123
2124
2125	/**
2126	* Updates the FPU exception status after FCW is changed.
2127	*
2128	* @param pFpuCtx The FPU context.
2129	*/
2130	DECLINLINE(void) iemFpuRecalcExceptionStatus(PX86FXSTATE pFpuCtx)
2131	{
2132	uint16_t u16Fsw = pFpuCtx->FSW;
2133	if ((u16Fsw & X86_FSW_XCPT_MASK) & ~(pFpuCtx->FCW & X86_FCW_XCPT_MASK))
2134	u16Fsw \|= X86_FSW_ES \| X86_FSW_B;
2135	else
2136	u16Fsw &= ~(X86_FSW_ES \| X86_FSW_B);
2137	pFpuCtx->FSW = u16Fsw;
2138	}
2139
2140
2141	/**
2142	* Calculates the full FTW (FPU tag word) for use in FNSTENV and FNSAVE.
2143	*
2144	* @returns The full FTW.
2145	* @param pFpuCtx The FPU context.
2146	*/
2147	DECLINLINE(uint16_t) iemFpuCalcFullFtw(PCX86FXSTATE pFpuCtx)
2148	{
2149	uint8_t const u8Ftw = (uint8_t)pFpuCtx->FTW;
2150	uint16_t u16Ftw = 0;
2151	unsigned const iTop = X86_FSW_TOP_GET(pFpuCtx->FSW);
2152	for (unsigned iSt = 0; iSt < 8; iSt++)
2153	{
2154	unsigned const iReg = (iSt + iTop) & 7;
2155	if (!(u8Ftw & RT_BIT(iReg)))
2156	u16Ftw \|= 3 << (iReg * 2); /* empty */
2157	else
2158	{
2159	uint16_t uTag;
2160	PCRTFLOAT80U const pr80Reg = &pFpuCtx->aRegs[iSt].r80;
2161	if (pr80Reg->s.uExponent == 0x7fff)
2162	uTag = 2; /* Exponent is all 1's => Special. */
2163	else if (pr80Reg->s.uExponent == 0x0000)
2164	{
2165	if (pr80Reg->s.uMantissa == 0x0000)
2166	uTag = 1; /* All bits are zero => Zero. */
2167	else
2168	uTag = 2; /* Must be special. */
2169	}
2170	else if (pr80Reg->s.uMantissa & RT_BIT_64(63)) /* The J bit. */
2171	uTag = 0; /* Valid. */
2172	else
2173	uTag = 2; /* Must be special. */
2174
2175	u16Ftw \|= uTag << (iReg * 2); /* empty */
2176	}
2177	}
2178
2179	return u16Ftw;
2180	}
2181
2182
2183	/**
2184	* Converts a full FTW to a compressed one (for use in FLDENV and FRSTOR).
2185	*
2186	* @returns The compressed FTW.
2187	* @param u16FullFtw The full FTW to convert.
2188	*/
2189	DECLINLINE(uint16_t) iemFpuCompressFtw(uint16_t u16FullFtw)
2190	{
2191	uint8_t u8Ftw = 0;
2192	for (unsigned i = 0; i < 8; i++)
2193	{
2194	if ((u16FullFtw & 3) != 3 /empty/)
2195	u8Ftw \|= RT_BIT(i);
2196	u16FullFtw >>= 2;
2197	}
2198
2199	return u8Ftw;
2200	}
2201
2202	/** @} */
2203
2204
2205	/** @name Memory access.
2206	*
2207	* @{
2208	*/
2209
2210
2211	/**
2212	* Checks whether alignment checks are enabled or not.
2213	*
2214	* @returns true if enabled, false if not.
2215	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2216	*/
2217	DECLINLINE(bool) iemMemAreAlignmentChecksEnabled(PVMCPUCC pVCpu)
2218	{
2219	AssertCompile(X86_CR0_AM == X86_EFL_AC);
2220	return pVCpu->iem.s.uCpl == 3
2221	&& (((uint32_t)pVCpu->cpum.GstCtx.cr0 & pVCpu->cpum.GstCtx.eflags.u) & X86_CR0_AM);
2222	}
2223
2224	/**
2225	* Checks if the given segment can be written to, raise the appropriate
2226	* exception if not.
2227	*
2228	* @returns VBox strict status code.
2229	*
2230	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2231	* @param pHid Pointer to the hidden register.
2232	* @param iSegReg The register number.
2233	* @param pu64BaseAddr Where to return the base address to use for the
2234	* segment. (In 64-bit code it may differ from the
2235	* base in the hidden segment.)
2236	*/
2237	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckWriteAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2238	{
2239	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2240
2241	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2242	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2243	else
2244	{
2245	if (!pHid->Attr.n.u1Present)
2246	{
2247	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2248	AssertRelease(uSel == 0);
2249	Log(("iemMemSegCheckWriteAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2250	return iemRaiseGeneralProtectionFault0(pVCpu);
2251	}
2252
2253	if ( ( (pHid->Attr.n.u4Type & X86_SEL_TYPE_CODE)
2254	\|\| !(pHid->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
2255	&& pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT )
2256	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2257	*pu64BaseAddr = pHid->u64Base;
2258	}
2259	return VINF_SUCCESS;
2260	}
2261
2262
2263	/**
2264	* Checks if the given segment can be read from, raise the appropriate
2265	* exception if not.
2266	*
2267	* @returns VBox strict status code.
2268	*
2269	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2270	* @param pHid Pointer to the hidden register.
2271	* @param iSegReg The register number.
2272	* @param pu64BaseAddr Where to return the base address to use for the
2273	* segment. (In 64-bit code it may differ from the
2274	* base in the hidden segment.)
2275	*/
2276	DECLINLINE(VBOXSTRICTRC) iemMemSegCheckReadAccessEx(PVMCPUCC pVCpu, PCCPUMSELREGHID pHid, uint8_t iSegReg, uint64_t *pu64BaseAddr)
2277	{
2278	IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2279
2280	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2281	*pu64BaseAddr = iSegReg < X86_SREG_FS ? 0 : pHid->u64Base;
2282	else
2283	{
2284	if (!pHid->Attr.n.u1Present)
2285	{
2286	uint16_t uSel = iemSRegFetchU16(pVCpu, iSegReg);
2287	AssertRelease(uSel == 0);
2288	Log(("iemMemSegCheckReadAccessEx: %#x (index %u) - bad selector -> #GP\n", uSel, iSegReg));
2289	return iemRaiseGeneralProtectionFault0(pVCpu);
2290	}
2291
2292	if ((pHid->Attr.n.u4Type & (X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
2293	return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2294	*pu64BaseAddr = pHid->u64Base;
2295	}
2296	return VINF_SUCCESS;
2297	}
2298
2299
2300	/**
2301	* Maps a physical page.
2302	*
2303	* @returns VBox status code (see PGMR3PhysTlbGCPhys2Ptr).
2304	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2305	* @param GCPhysMem The physical address.
2306	* @param fAccess The intended access.
2307	* @param ppvMem Where to return the mapping address.
2308	* @param pLock The PGM lock.
2309	*/
2310	DECLINLINE(int) iemMemPageMap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, void **ppvMem, PPGMPAGEMAPLOCK pLock)
2311	{
2312	#ifdef IEM_LOG_MEMORY_WRITES
2313	if (fAccess & IEM_ACCESS_TYPE_WRITE)
2314	return VERR_PGM_PHYS_TLB_CATCH_ALL;
2315	#endif
2316
2317	/** @todo This API may require some improving later. A private deal with PGM
2318	* regarding locking and unlocking needs to be struct. A couple of TLBs
2319	* living in PGM, but with publicly accessible inlined access methods
2320	* could perhaps be an even better solution. */
2321	int rc = PGMPhysIemGCPhys2Ptr(pVCpu->CTX_SUFF(pVM), pVCpu,
2322	GCPhysMem,
2323	RT_BOOL(fAccess & IEM_ACCESS_TYPE_WRITE),
2324	pVCpu->iem.s.fBypassHandlers,
2325	ppvMem,
2326	pLock);
2327	/Log(("PGMPhysIemGCPhys2Ptr %Rrc pLock=%.Rhxs\n", rc, sizeof(pLock), pLock));/
2328	AssertMsg(rc == VINF_SUCCESS \|\| RT_FAILURE_NP(rc), ("%Rrc\n", rc));
2329
2330	return rc;
2331	}
2332
2333
2334	/**
2335	* Unmap a page previously mapped by iemMemPageMap.
2336	*
2337	* @param pVCpu The cross context virtual CPU structure of the calling thread.
2338	* @param GCPhysMem The physical address.
2339	* @param fAccess The intended access.
2340	* @param pvMem What iemMemPageMap returned.
2341	* @param pLock The PGM lock.
2342	*/
2343	DECLINLINE(void) iemMemPageUnmap(PVMCPUCC pVCpu, RTGCPHYS GCPhysMem, uint32_t fAccess, const void *pvMem, PPGMPAGEMAPLOCK pLock)
2344	{
2345	NOREF(pVCpu);
2346	NOREF(GCPhysMem);
2347	NOREF(fAccess);
2348	NOREF(pvMem);
2349	PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), pLock);
2350	}
2351
2352	#ifdef IEM_WITH_SETJMP
2353
2354	/** @todo slim this down */
2355	DECLINLINE(RTGCPTR) iemMemApplySegmentToReadJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2356	{
2357	Assert(cbMem >= 1);
2358	Assert(iSegReg < X86_SREG_COUNT);
2359
2360	/*
2361	* 64-bit mode is simpler.
2362	*/
2363	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2364	{
2365	if (iSegReg >= X86_SREG_FS && iSegReg != UINT8_MAX)
2366	{
2367	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2368	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2369	GCPtrMem += pSel->u64Base;
2370	}
2371
2372	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2373	return GCPtrMem;
2374	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2375	}
2376	/*
2377	* 16-bit and 32-bit segmentation.
2378	*/
2379	else if (iSegReg != UINT8_MAX)
2380	{
2381	/** @todo Does this apply to segments with 4G-1 limit? */
2382	uint32_t const GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem - 1;
2383	if (RT_LIKELY(GCPtrLast32 >= (uint32_t)GCPtrMem))
2384	{
2385	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2386	PCPUMSELREGHID const pSel = iemSRegGetHid(pVCpu, iSegReg);
2387	switch (pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2388	\| X86_SEL_TYPE_READ \| X86_SEL_TYPE_WRITE /* same as read */
2389	\| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_CONF /* same as down */
2390	\| X86_SEL_TYPE_CODE))
2391	{
2392	case X86DESCATTR_P: /* readonly data, expand up */
2393	case X86DESCATTR_P \| X86_SEL_TYPE_WRITE: /* writable data, expand up */
2394	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ: /* code, read-only */
2395	case X86DESCATTR_P \| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_READ \| X86_SEL_TYPE_CONF: /* conforming code, read-only */
2396	/* expand up */
2397	if (RT_LIKELY(GCPtrLast32 <= pSel->u32Limit))
2398	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2399	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x vs %#x\n",
2400	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit));
2401	break;
2402
2403	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN: /* readonly data, expand down */
2404	case X86DESCATTR_P \| X86_SEL_TYPE_DOWN \| X86_SEL_TYPE_WRITE: /* writable data, expand down */
2405	/* expand down */
2406	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2407	&& ( pSel->Attr.n.u1DefBig
2408	\|\| GCPtrLast32 <= UINT32_C(0xffff)) ))
2409	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2410	Log10(("iemMemApplySegmentToReadJmp: expand down out of bounds %#x..%#x vs %#x..%#x\n",
2411	(uint32_t)GCPtrMem, GCPtrLast32, pSel->u32Limit, pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT16_MAX));
2412	break;
2413
2414	default:
2415	Log10(("iemMemApplySegmentToReadJmp: bad selector %#x\n", pSel->Attr.u));
2416	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2417	break;
2418	}
2419	}
2420	Log10(("iemMemApplySegmentToReadJmp: out of bounds %#x..%#x\n",(uint32_t)GCPtrMem, GCPtrLast32));
2421	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_R);
2422	}
2423	/*
2424	* 32-bit flat address.
2425	*/
2426	else
2427	return GCPtrMem;
2428	}
2429
2430
2431	/** @todo slim this down */
2432	DECLINLINE(RTGCPTR) iemMemApplySegmentToWriteJmp(PVMCPUCC pVCpu, uint8_t iSegReg, size_t cbMem, RTGCPTR GCPtrMem)
2433	{
2434	Assert(cbMem >= 1);
2435	Assert(iSegReg < X86_SREG_COUNT);
2436
2437	/*
2438	* 64-bit mode is simpler.
2439	*/
2440	if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
2441	{
2442	if (iSegReg >= X86_SREG_FS)
2443	{
2444	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2445	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2446	GCPtrMem += pSel->u64Base;
2447	}
2448
2449	if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
2450	return GCPtrMem;
2451	}
2452	/*
2453	* 16-bit and 32-bit segmentation.
2454	*/
2455	else
2456	{
2457	IEM_CTX_IMPORT_JMP(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
2458	PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
2459	uint32_t const fRelevantAttrs = pSel->Attr.u & ( X86DESCATTR_P \| X86DESCATTR_UNUSABLE
2460	\| X86_SEL_TYPE_CODE \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN);
2461	if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE)) /* data, expand up */
2462	{
2463	/* expand up */
2464	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2465	if (RT_LIKELY( GCPtrLast32 > pSel->u32Limit
2466	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2467	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2468	}
2469	else if (fRelevantAttrs == (X86DESCATTR_P \| X86_SEL_TYPE_WRITE \| X86_SEL_TYPE_DOWN)) /* data, expand up */
2470	{
2471	/* expand down */
2472	uint32_t GCPtrLast32 = (uint32_t)GCPtrMem + (uint32_t)cbMem;
2473	if (RT_LIKELY( (uint32_t)GCPtrMem > pSel->u32Limit
2474	&& GCPtrLast32 <= (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
2475	&& GCPtrLast32 > (uint32_t)GCPtrMem))
2476	return (uint32_t)GCPtrMem + (uint32_t)pSel->u64Base;
2477	}
2478	else
2479	iemRaiseSelectorInvalidAccessJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2480	iemRaiseSelectorBoundsJmp(pVCpu, iSegReg, IEM_ACCESS_DATA_W);
2481	}
2482	iemRaiseGeneralProtectionFault0Jmp(pVCpu);
2483	}
2484
2485	#endif /* IEM_WITH_SETJMP */
2486
2487	/**
2488	* Fakes a long mode stack selector for SS = 0.
2489	*
2490	* @param pDescSs Where to return the fake stack descriptor.
2491	* @param uDpl The DPL we want.
2492	*/
2493	DECLINLINE(void) iemMemFakeStackSelDesc(PIEMSELDESC pDescSs, uint32_t uDpl)
2494	{
2495	pDescSs->Long.au64[0] = 0;
2496	pDescSs->Long.au64[1] = 0;
2497	pDescSs->Long.Gen.u4Type = X86_SEL_TYPE_RW_ACC;
2498	pDescSs->Long.Gen.u1DescType = 1; /* 1 = code / data, 0 = system. */
2499	pDescSs->Long.Gen.u2Dpl = uDpl;
2500	pDescSs->Long.Gen.u1Present = 1;
2501	pDescSs->Long.Gen.u1Long = 1;
2502	}
2503
2504	/** @} */
2505
2506
2507	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
2508
2509	/**
2510	* Gets CR0 fixed-0 bits in VMX non-root mode.
2511	*
2512	* We do this rather than fetching what we report to the guest (in
2513	* IA32_VMX_CR0_FIXED0 MSR) because real hardware (and so do we) report the same
2514	* values regardless of whether unrestricted-guest feature is available on the CPU.
2515	*
2516	* @returns CR0 fixed-0 bits.
2517	* @param pVCpu The cross context virtual CPU structure.
2518	*/
2519	DECLINLINE(uint64_t) iemVmxGetCr0Fixed0(PCVMCPUCC pVCpu)
2520	{
2521	Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
2522	Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
2523
2524	static uint64_t const s_auCr0Fixed0[2] = { VMX_V_CR0_FIXED0, VMX_V_CR0_FIXED0_UX };
2525	PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
2526	uint8_t const fUnrestrictedGuest = !!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
2527	uint64_t const uCr0Fixed0 = s_auCr0Fixed0[fUnrestrictedGuest];
2528	Assert(!(uCr0Fixed0 & (X86_CR0_NW \| X86_CR0_CD)));
2529	return uCr0Fixed0;
2530	}
2531
2532
2533	/**
2534	* Sets virtual-APIC write emulation as pending.
2535	*
2536	* @param pVCpu The cross context virtual CPU structure.
2537	* @param offApic The offset in the virtual-APIC page that was written.
2538	*/
2539	DECLINLINE(void) iemVmxVirtApicSetPendingWrite(PVMCPUCC pVCpu, uint16_t offApic)
2540	{
2541	Assert(offApic < XAPIC_OFF_END + 4);
2542
2543	/*
2544	* Record the currently updated APIC offset, as we need this later for figuring
2545	* out whether to perform TPR, EOI or self-IPI virtualization as well as well
2546	* as for supplying the exit qualification when causing an APIC-write VM-exit.
2547	*/
2548	pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = offApic;
2549
2550	/*
2551	* Flag that we need to perform virtual-APIC write emulation (TPR/PPR/EOI/Self-IPI
2552	* virtualization or APIC-write emulation).
2553	*/
2554	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
2555	VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
2556	}
2557
2558	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
2559
2560	#endif /* !VMM_INCLUDED_SRC_include_IEMInline_h */

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/VMM/include/IEMInline.h@ 96478

以其他格式下載: