VirtualBox

source: vbox/trunk/src/VBox/VMM/include/IEMN8veRecompiler.h@ 101484

最後變更 在這個檔案從101484是 101484,由 vboxsync 提交於 17 月 前

VMM/IEM: Basic register allocator sketches that incorporates simple skipping of guest register value loads. Sketched out varable and argument managmenet. Start telling GDB our jitted code to help with backtraces. ++ bugref:10371
  • 屬性 svn:eol-style 設為 native
  • 屬性 svn:keywords 設為 Author Date Id Revision
檔案大小: 47.5 KB
行 
1/* $Id: IEMN8veRecompiler.h 101484 2023-10-18 01:32:17Z vboxsync $ */
2/** @file
3 * IEM - Interpreted Execution Manager - Native Recompiler Internals.
4 */
5
6/*
7 * Copyright (C) 2011-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.alldomusa.eu.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28#ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
29#define VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h
30#ifndef RT_WITHOUT_PRAGMA_ONCE
31# pragma once
32#endif
33
34
35/** @defgroup grp_iem_n8ve_re Native Recompiler Internals.
36 * @ingroup grp_iem_int
37 * @{
38 */
39
40/** @name Stack Frame Layout
41 *
42 * @{ */
43/** The size of the area for stack variables and spills and stuff.
44 * @note This limit is duplicated in the python script(s). */
45#define IEMNATIVE_FRAME_VAR_SIZE 0xc0
46#ifdef RT_ARCH_AMD64
47/** Number of stack arguments slots for calls made from the frame. */
48# define IEMNATIVE_FRAME_STACK_ARG_COUNT 4
49/** An stack alignment adjustment (between non-volatile register pushes and
50 * the stack variable area, so the latter better aligned). */
51# define IEMNATIVE_FRAME_ALIGN_SIZE 8
52/** Number of any shadow arguments (spill area) for calls we make. */
53# ifdef RT_OS_WINDOWS
54# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 4
55# else
56# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
57# endif
58
59/** Frame pointer (RBP) relative offset of the last push. */
60# ifdef RT_OS_WINDOWS
61# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
62# else
63# define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)
64# endif
65/** Frame pointer (RBP) relative offset of the stack variable area (the lowest
66 * address for it). */
67# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
68/** Frame pointer (RBP) relative offset of the first stack argument for calls. */
69# define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)
70/** Frame pointer (RBP) relative offset of the second stack argument for calls. */
71# define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)
72/** Frame pointer (RBP) relative offset of the third stack argument for calls. */
73# define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)
74/** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */
75# define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)
76
77# ifdef RT_OS_WINDOWS
78/** Frame pointer (RBP) relative offset of the first incoming shadow argument. */
79# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)
80/** Frame pointer (RBP) relative offset of the second incoming shadow argument. */
81# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)
82/** Frame pointer (RBP) relative offset of the third incoming shadow argument. */
83# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)
84/** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */
85# define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)
86# endif
87
88#elif RT_ARCH_ARM64
89/** No stack argument slots, enough got 8 registers for arguments. */
90# define IEMNATIVE_FRAME_STACK_ARG_COUNT 0
91/** There are no argument spill area. */
92# define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0
93
94/** Number of saved registers at the top of our stack frame.
95 * This includes the return address and old frame pointer, so x19 thru x30. */
96# define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)
97/** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */
98# define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)
99
100/** Frame pointer (BP) relative offset of the last push. */
101# define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)
102
103/** Frame pointer (BP) relative offset of the stack variable area (the lowest
104 * address for it). */
105# define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)
106
107#else
108# error "port me"
109#endif
110/** @} */
111
112
113/** @name Fixed Register Allocation(s)
114 * @{ */
115/** @def IEMNATIVE_REG_FIXED_PVMCPU
116 * The number of the register holding the pVCpu pointer. */
117/** @def IEMNATIVE_REG_FIXED_PCPUMCTX
118 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.
119 * @note This not available on AMD64, only ARM64. */
120/** @def IEMNATIVE_REG_FIXED_TMP0
121 * Dedicated temporary register.
122 * @todo replace this by a register allocator and content tracker. */
123/** @def IEMNATIVE_REG_FIXED_MASK
124 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS
125 * architecture. */
126#ifdef RT_ARCH_AMD64
127# define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX
128# define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11
129# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
130 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \
131 | RT_BIT_32(X86_GREG_xSP) \
132 | RT_BIT_32(X86_GREG_xBP) )
133
134#elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
135# define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28
136# define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27
137# define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15
138# define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \
139 | RT_BIT_32(ARMV8_A64_REG_LR) \
140 | RT_BIT_32(ARMV8_A64_REG_BP) \
141 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \
142 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \
143 | RT_BIT_32(ARMV8_A64_REG_X18) \
144 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )
145
146#else
147# error "port me"
148#endif
149/** @} */
150
151/** @name Call related registers.
152 * @{ */
153/** @def IEMNATIVE_CALL_RET_GREG
154 * The return value register. */
155/** @def IEMNATIVE_CALL_ARG_GREG_COUNT
156 * Number of arguments in registers. */
157/** @def IEMNATIVE_CALL_ARG0_GREG
158 * The general purpose register carrying argument \#0. */
159/** @def IEMNATIVE_CALL_ARG1_GREG
160 * The general purpose register carrying argument \#1. */
161/** @def IEMNATIVE_CALL_ARG2_GREG
162 * The general purpose register carrying argument \#2. */
163/** @def IEMNATIVE_CALL_ARG3_GREG
164 * The general purpose register carrying argument \#3. */
165/** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK
166 * Mask of registers the callee will not save and may trash. */
167#ifdef RT_ARCH_AMD64
168# define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX
169
170# ifdef RT_OS_WINDOWS
171# define IEMNATIVE_CALL_ARG_GREG_COUNT 4
172# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX
173# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX
174# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8
175# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9
176# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
177 | RT_BIT_32(X86_GREG_xCX) \
178 | RT_BIT_32(X86_GREG_xDX) \
179 | RT_BIT_32(X86_GREG_x8) \
180 | RT_BIT_32(X86_GREG_x9) \
181 | RT_BIT_32(X86_GREG_x10) \
182 | RT_BIT_32(X86_GREG_x11) )
183# else
184# define IEMNATIVE_CALL_ARG_GREG_COUNT 6
185# define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI
186# define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI
187# define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX
188# define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX
189# define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8
190# define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9
191# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \
192 | RT_BIT_32(X86_GREG_xCX) \
193 | RT_BIT_32(X86_GREG_xDX) \
194 | RT_BIT_32(X86_GREG_xDI) \
195 | RT_BIT_32(X86_GREG_xSI) \
196 | RT_BIT_32(X86_GREG_x8) \
197 | RT_BIT_32(X86_GREG_x9) \
198 | RT_BIT_32(X86_GREG_x10) \
199 | RT_BIT_32(X86_GREG_x11) )
200# endif
201
202#elif defined(RT_ARCH_ARM64)
203# define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0
204# define IEMNATIVE_CALL_ARG_GREG_COUNT 8
205# define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0
206# define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1
207# define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2
208# define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3
209# define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4
210# define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5
211# define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6
212# define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7
213# define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \
214 | RT_BIT_32(ARMV8_A64_REG_X1) \
215 | RT_BIT_32(ARMV8_A64_REG_X2) \
216 | RT_BIT_32(ARMV8_A64_REG_X3) \
217 | RT_BIT_32(ARMV8_A64_REG_X4) \
218 | RT_BIT_32(ARMV8_A64_REG_X5) \
219 | RT_BIT_32(ARMV8_A64_REG_X6) \
220 | RT_BIT_32(ARMV8_A64_REG_X7) \
221 | RT_BIT_32(ARMV8_A64_REG_X8) \
222 | RT_BIT_32(ARMV8_A64_REG_X9) \
223 | RT_BIT_32(ARMV8_A64_REG_X10) \
224 | RT_BIT_32(ARMV8_A64_REG_X11) \
225 | RT_BIT_32(ARMV8_A64_REG_X12) \
226 | RT_BIT_32(ARMV8_A64_REG_X13) \
227 | RT_BIT_32(ARMV8_A64_REG_X14) \
228 | RT_BIT_32(ARMV8_A64_REG_X15) \
229 | RT_BIT_32(ARMV8_A64_REG_X16) \
230 | RT_BIT_32(ARMV8_A64_REG_X17) )
231
232#endif
233
234/** @} */
235
236
237/** @def IEMNATIVE_HST_GREG_COUNT
238 * Number of host general purpose registers we tracker. */
239/** @def IEMNATIVE_HST_GREG_MASK
240 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to
241 * inverted register masks and such to get down to a correct set of regs. */
242#ifdef RT_ARCH_AMD64
243# define IEMNATIVE_HST_GREG_COUNT 16
244# define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)
245
246#elif defined(RT_ARCH_ARM64)
247# define IEMNATIVE_HST_GREG_COUNT 32
248# define IEMNATIVE_HST_GREG_MASK UINT32_MAX
249#else
250# error "Port me!"
251#endif
252
253
254/** Native code generator label types. */
255typedef enum
256{
257 kIemNativeLabelType_Invalid = 0,
258 kIemNativeLabelType_Return,
259 kIemNativeLabelType_NonZeroRetOrPassUp,
260 kIemNativeLabelType_End
261} IEMNATIVELABELTYPE;
262
263/** Native code generator label definition. */
264typedef struct IEMNATIVELABEL
265{
266 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in
267 * the epilog. */
268 uint32_t off;
269 /** The type of label (IEMNATIVELABELTYPE). */
270 uint16_t enmType;
271 /** Additional label data, type specific. */
272 uint16_t uData;
273} IEMNATIVELABEL;
274/** Pointer to a label. */
275typedef IEMNATIVELABEL *PIEMNATIVELABEL;
276
277
278/** Native code generator fixup types. */
279typedef enum
280{
281 kIemNativeFixupType_Invalid = 0,
282#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
283 /** AMD64 fixup: PC relative 32-bit with addend in bData. */
284 kIemNativeFixupType_Rel32,
285#elif defined(RT_ARCH_ARM64)
286 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ). */
287 kIemNativeFixupType_RelImm19At5,
288#endif
289 kIemNativeFixupType_End
290} IEMNATIVEFIXUPTYPE;
291
292/** Native code generator fixup. */
293typedef struct IEMNATIVEFIXUP
294{
295 /** Code offset of the fixup location. */
296 uint32_t off;
297 /** The IEMNATIVELABEL this is a fixup for. */
298 uint16_t idxLabel;
299 /** The fixup type (IEMNATIVEFIXUPTYPE). */
300 uint8_t enmType;
301 /** Addend or other data. */
302 int8_t offAddend;
303} IEMNATIVEFIXUP;
304/** Pointer to a native code generator fixup. */
305typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;
306
307
308/**
309 * Guest registers that can be shadowed in GPRs.
310 */
311typedef enum IEMNATIVEGSTREG : uint8_t
312{
313 kIemNativeGstReg_GprFirst = 0,
314 kIemNativeGstReg_GprLast = 15,
315 kIemNativeGstReg_Pc,
316 kIemNativeGstReg_Rflags,
317 /* gap: 18..23 */
318 kIemNativeGstReg_SegSelFirst = 24,
319 kIemNativeGstReg_SegSelLast = 29,
320 kIemNativeGstReg_SegBaseFirst = 30,
321 kIemNativeGstReg_SegBaseLast = 35,
322 kIemNativeGstReg_SegLimitFirst = 36,
323 kIemNativeGstReg_SegLimitLast = 41,
324 kIemNativeGstReg_End
325} IEMNATIVEGSTREG;
326
327/**
328 * Guest registers (classes) that can be referenced.
329 */
330typedef enum IEMNATIVEGSTREGREF : uint8_t
331{
332 kIemNativeGstRegRef_Invalid = 0,
333 kIemNativeGstRegRef_Gpr,
334 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/
335 kIemNativeGstRegRef_EFlags,
336 kIemNativeGstRegRef_MxCsr,
337 kIemNativeGstRegRef_FpuReg,
338 kIemNativeGstRegRef_MReg,
339 kIemNativeGstRegRef_XReg,
340 kIemNativeGstRegRef_YReg,
341 kIemNativeGstRegRef_End
342} IEMNATIVEGSTREGREF;
343
344
345/** Variable kinds. */
346typedef enum IEMNATIVEVARKIND : uint8_t
347{
348 /** Customary invalid zero value. */
349 kIemNativeVarKind_Invalid = 0,
350 /** This is either in a register or on the stack. */
351 kIemNativeVarKind_Stack,
352 /** Immediate value - loaded into register when needed, or can live on the
353 * stack if referenced (in theory). */
354 kIemNativeVarKind_Immediate,
355 /** Variable reference - loaded into register when needed, never stack. */
356 kIemNativeVarKind_VarRef,
357 /** Guest register reference - loaded into register when needed, never stack. */
358 kIemNativeVarKind_GstRegRef,
359 /** End of valid values. */
360 kIemNativeVarKind_End
361} IEMNATIVEVARKIND;
362
363
364/** Variable or argument. */
365typedef struct IEMNATIVEVAR
366{
367 /** The kind of variable. */
368 IEMNATIVEVARKIND enmKind;
369 /** The variable size in bytes. */
370 uint8_t cbVar;
371 /** The first stack slot (uint64_t), except for immediate and references
372 * where it usually is UINT8_MAX. */
373 uint8_t idxStackSlot;
374 /** The host register allocated for the variable, UINT8_MAX if not. */
375 uint8_t idxReg;
376 /** The argument number if argument, UINT8_MAX if regular variable. */
377 uint8_t uArgNo;
378 /** If referenced, the index of the variable referencing this one, otherwise
379 * UINT8_MAX. A referenced variable must only be placed on the stack and
380 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */
381 uint8_t idxReferrerVar;
382 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */
383 IEMNATIVEGSTREG enmGstReg;
384 uint8_t bAlign;
385
386 union
387 {
388 /** kIemNativeVarKind_Immediate: The immediate value. */
389 uint64_t uValue;
390 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */
391 uint8_t idxRefVar;
392 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */
393 struct
394 {
395 /** The class of register. */
396 IEMNATIVEGSTREGREF enmClass;
397 /** Index within the class. */
398 uint8_t idx;
399 } GstRegRef;
400 } u;
401} IEMNATIVEVAR;
402
403/** What is being kept in a host register. */
404typedef enum IEMNATIVEWHAT : uint8_t
405{
406 /** The traditional invalid zero value. */
407 kIemNativeWhat_Invalid = 0,
408 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */
409 kIemNativeWhat_Var,
410 /** Temporary register, this is typically freed when a MC completes. */
411 kIemNativeWhat_Tmp,
412 /** Call argument w/o a variable mapping. This is free (via
413 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */
414 kIemNativeWhat_Arg,
415 /** Return status code.
416 * @todo not sure if we need this... */
417 kIemNativeWhat_rc,
418 /** The fixed pVCpu (PVMCPUCC) register.
419 * @todo consider offsetting this on amd64 to use negative offsets to access
420 * more members using 8-byte disp. */
421 kIemNativeWhat_pVCpuFixed,
422 /** The fixed pCtx (PCPUMCTX) register.
423 * @todo consider offsetting this on amd64 to use negative offsets to access
424 * more members using 8-byte disp. */
425 kIemNativeWhat_pCtxFixed,
426 /** Fixed temporary register. */
427 kIemNativeWhat_FixedTmp,
428 /** Register reserved by the CPU or OS architecture. */
429 kIemNativeWhat_FixedReserved,
430 /** End of valid values. */
431 kIemNativeWhat_End
432} IEMNATIVEWHAT;
433
434/**
435 * Host general register entry.
436 *
437 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.
438 *
439 * @todo Track immediate values in host registers similarlly to how we track the
440 * guest register shadow copies. For it to be real helpful, though,
441 * we probably need to know which will be reused and put them into
442 * non-volatile registers, otherwise it's going to be more or less
443 * restricted to an instruction or two.
444 */
445typedef struct IEMNATIVEHSTREG
446{
447 /** Set of guest registers this one shadows.
448 *
449 * Using a bitmap here so we can designate the same host register as a copy
450 * for more than one guest register. This is expected to be useful in
451 * situations where one value is copied to several registers in a sequence.
452 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'
453 * sequence we'd want to let this register follow to be a copy of and there
454 * will always be places where we'd be picking the wrong one.
455 */
456 uint64_t fGstRegShadows;
457 /** What is being kept in this register. */
458 IEMNATIVEWHAT enmWhat;
459 /** Variable index if holding a variable, otherwise UINT8_MAX. */
460 uint8_t idxVar;
461 /** Alignment padding. */
462 uint8_t abAlign[6];
463} IEMNATIVEHSTREG;
464
465
466/**
467 * Native recompiler state.
468 */
469typedef struct IEMRECOMPILERSTATE
470{
471 /** Size of the buffer that pbNativeRecompileBufR3 points to in
472 * IEMNATIVEINSTR units. */
473 uint32_t cInstrBufAlloc;
474 uint32_t uPadding; /* We don't keep track of this here... */
475 /** Fixed temporary code buffer for native recompilation. */
476 PIEMNATIVEINSTR pInstrBuf;
477
478 /** Actual number of labels in paLabels. */
479 uint32_t cLabels;
480 /** Max number of entries allowed in paLabels before reallocating it. */
481 uint32_t cLabelsAlloc;
482 /** Labels defined while recompiling (referenced by fixups). */
483 PIEMNATIVELABEL paLabels;
484
485 /** Actual number of fixups paFixups. */
486 uint32_t cFixups;
487 /** Max number of entries allowed in paFixups before reallocating it. */
488 uint32_t cFixupsAlloc;
489 /** Buffer used by the recompiler for recording fixups when generating code. */
490 PIEMNATIVEFIXUP paFixups;
491
492 /** The translation block being recompiled. */
493 PCIEMTB pTbOrg;
494
495 /** Allocation bitmap fro aHstRegs. */
496 uint32_t bmHstRegs;
497
498 /** Bitmap marking which host register contains guest register shadow copies.
499 * This is used during register allocation to try preserve copies. */
500 uint32_t bmHstRegsWithGstShadow;
501 /** Bitmap marking valid entries in aidxGstRegShadows. */
502 uint64_t bmGstRegShadows;
503
504 /** Allocation bitmap for aVars. */
505 uint32_t bmVars;
506 uint32_t u32Align;
507 union
508 {
509 /** Index of variable arguments, UINT8_MAX if not valid. */
510 uint8_t aidxArgVars[8];
511 /** For more efficient resetting. */
512 uint64_t u64ArgVars;
513 };
514
515 /** Host register allocation tracking. */
516 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];
517 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).
518 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.
519 * (A shadow copy of a guest register can only be held in a one host register,
520 * there are no duplicate copies or ambiguities like that). */
521 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];
522 /** Variables and arguments. */
523 IEMNATIVEVAR aVars[16];
524} IEMRECOMPILERSTATE;
525/** Pointer to a native recompiler state. */
526typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;
527
528
529/**
530 * Native recompiler worker for a threaded function.
531 *
532 * @returns New code buffer offset, UINT32_MAX in case of failure.
533 * @param pReNative The native recompiler state.
534 * @param off The current code buffer offset.
535 * @param pCallEntry The threaded call entry.
536 *
537 * @note This is not allowed to throw anything atm.
538 */
539typedef DECLCALLBACKTYPE(uint32_t, FNIEMNATIVERECOMPFUNC,(PIEMRECOMPILERSTATE pReNative, uint32_t off,
540 PCIEMTHRDEDCALLENTRY pCallEntry));
541/** Pointer to a native recompiler worker for a threaded function. */
542typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;
543
544/** Defines a native recompiler worker for a threaded function. */
545#define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \
546 DECLCALLBACK(uint32_t) a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)
547/** Prototypes a native recompiler function for a threaded function. */
548#define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name
549
550
551DECLHIDDEN(uint32_t) iemNativeMakeLabel(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,
552 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0) RT_NOEXCEPT;
553DECLHIDDEN(bool) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,
554 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0) RT_NOEXCEPT;
555DECLHIDDEN(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off,
556 uint32_t cInstrReq) RT_NOEXCEPT;
557
558DECLHIDDEN(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
559 bool fPreferVolatile = true) RT_NOEXCEPT;
560DECLHIDDEN(uint8_t) iemNativeRegAllocTmpForGuest(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,
561 IEMNATIVEGSTREG enmGstReg) RT_NOEXCEPT;
562DECLHIDDEN(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar) RT_NOEXCEPT;
563DECLHIDDEN(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs) RT_NOEXCEPT;
564DECLHIDDEN(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
565DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
566DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;
567DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;
568
569DECLHIDDEN(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off,
570 uint8_t idxInstr) RT_NOEXCEPT;
571
572
573/**
574 * Ensures that there is sufficient space in the instruction output buffer.
575 *
576 * This will reallocate the buffer if needed and allowed.
577 *
578 * @returns Pointer to the instruction output buffer on success, NULL on
579 * failure.
580 * @param pReNative The native recompile state.
581 * @param off Current instruction offset. Works safely for UINT32_MAX
582 * as well.
583 * @param cInstrReq Number of instruction about to be added. It's okay to
584 * overestimate this a bit.
585 */
586DECL_FORCE_INLINE(PIEMNATIVEINSTR) iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)
587{
588 if (RT_LIKELY(off + (uint64_t)cInstrReq <= pReNative->cInstrBufAlloc))
589 return pReNative->pInstrBuf;
590 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);
591}
592
593
594/**
595 * Emit a simple marker instruction to more easily tell where something starts
596 * in the disassembly.
597 */
598DECLINLINE(uint32_t) iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off)
599{
600#ifdef RT_ARCH_AMD64
601 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
602 AssertReturn(pbCodeBuf, UINT32_MAX);
603 /* nop */
604 pbCodeBuf[off++] = 0x90;
605
606#elif RT_ARCH_ARM64
607 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
608 AssertReturn(pu32CodeBuf, UINT32_MAX);
609 /* nop */
610 pu32CodeBuf[off++] = 0xd503201f;
611
612#else
613# error "port me"
614#endif
615 return off;
616}
617
618
619/**
620 * Emits setting a GPR to zero.
621 */
622DECLINLINE(uint32_t) iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)
623{
624#ifdef RT_ARCH_AMD64
625 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
626 AssertReturn(pbCodeBuf, UINT32_MAX);
627 /* xor gpr32, gpr32 */
628 if (iGpr >= 8)
629 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;
630 pbCodeBuf[off++] = 0x33;
631 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);
632
633#elif RT_ARCH_ARM64
634 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
635 AssertReturn(pu32CodeBuf, UINT32_MAX);
636 /* mov gpr, #0x0 */
637 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;
638
639#else
640# error "port me"
641#endif
642 RT_NOREF(pReNative);
643 return off;
644}
645
646
647/**
648 * Emits loading a constant into a 64-bit GPR
649 */
650DECLINLINE(uint32_t) iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)
651{
652 if (!uImm64)
653 return iemNativeEmitGprZero(pReNative, off, iGpr);
654
655#ifdef RT_ARCH_AMD64
656 if (uImm64 <= UINT32_MAX)
657 {
658 /* mov gpr, imm32 */
659 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);
660 AssertReturn(pbCodeBuf, UINT32_MAX);
661 if (iGpr >= 8)
662 pbCodeBuf[off++] = X86_OP_REX_B;
663 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
664 pbCodeBuf[off++] = RT_BYTE1(uImm64);
665 pbCodeBuf[off++] = RT_BYTE2(uImm64);
666 pbCodeBuf[off++] = RT_BYTE3(uImm64);
667 pbCodeBuf[off++] = RT_BYTE4(uImm64);
668 }
669 else
670 {
671 /* mov gpr, imm64 */
672 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);
673 AssertReturn(pbCodeBuf, UINT32_MAX);
674 if (iGpr < 8)
675 pbCodeBuf[off++] = X86_OP_REX_W;
676 else
677 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
678 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);
679 pbCodeBuf[off++] = RT_BYTE1(uImm64);
680 pbCodeBuf[off++] = RT_BYTE2(uImm64);
681 pbCodeBuf[off++] = RT_BYTE3(uImm64);
682 pbCodeBuf[off++] = RT_BYTE4(uImm64);
683 pbCodeBuf[off++] = RT_BYTE5(uImm64);
684 pbCodeBuf[off++] = RT_BYTE6(uImm64);
685 pbCodeBuf[off++] = RT_BYTE7(uImm64);
686 pbCodeBuf[off++] = RT_BYTE8(uImm64);
687 }
688
689#elif RT_ARCH_ARM64
690 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
691 AssertReturn(pu32CodeBuf, UINT32_MAX);
692
693 /*
694 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and
695 * supply remaining bits using 'movk grp, imm16, lsl #x'.
696 *
697 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,
698 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff
699 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it
700 * after the first non-zero immediate component so we switch to movk for
701 * the remainder.
702 */
703 uint32_t fMovK = 0;
704 /* mov gpr, imm16 */
705 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);
706 if (uImmPart)
707 {
708 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;
709 fMovK |= RT_BIT_32(29);
710 }
711 /* mov[k] gpr, imm16, lsl #16 */
712 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);
713 if (uImmPart)
714 {
715 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;
716 fMovK |= RT_BIT_32(29);
717 }
718 /* mov[k] gpr, imm16, lsl #32 */
719 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);
720 if (uImmPart)
721 {
722 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;
723 fMovK |= RT_BIT_32(29);
724 }
725 /* mov[k] gpr, imm16, lsl #48 */
726 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);
727 if (uImmPart)
728 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;
729
730 /** @todo there is an inverted mask variant we might want to explore if it
731 * reduces the number of instructions... */
732 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?
733 * clang 12.x does that, only to use the 'x' version for the
734 * addressing in the following ldr). */
735
736#else
737# error "port me"
738#endif
739 return off;
740}
741
742
743#ifdef RT_ARCH_AMD64
744/**
745 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
746 */
747DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)
748{
749 if (offVCpu < 128)
750 {
751 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
752 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;
753 }
754 else
755 {
756 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);
757 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);
758 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);
759 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);
760 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);
761 }
762 return off;
763}
764#elif RT_ARCH_ARM64
765/**
766 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.
767 */
768DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,
769 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)
770{
771 /*
772 * There are a couple of ldr variants that takes an immediate offset, so
773 * try use those if we can, otherwise we have to use the temporary register
774 * help with the addressing.
775 */
776 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))
777 {
778 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */
779 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
780 AssertReturn(pu32CodeBuf, UINT32_MAX);
781 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGrp, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);
782 }
783 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))
784 {
785 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
786 AssertReturn(pu32CodeBuf, UINT32_MAX);
787 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGrp, IEMNATIVE_REG_FIXED_PCPUMCTX,
788 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);
789 }
790 else
791 {
792 /* The offset is too large, so we must load it into a register and use
793 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */
794 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */
795 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);
796
797 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
798 AssertReturn(pu32CodeBuf, UINT32_MAX);
799 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGpr, IEMNATIVE_REG_FIXED_PVMCPU, IEMNATIVE_REG_FIXED_TMP);
800 }
801 return off;
802}
803#endif
804
805
806/**
807 * Emits a 64-bit GPR load of a VCpu value.
808 */
809DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
810{
811#ifdef RT_ARCH_AMD64
812 /* mov reg64, mem64 */
813 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
814 AssertReturn(pbCodeBuf, UINT32_MAX);
815 if (iGpr < 8)
816 pbCodeBuf[off++] = X86_OP_REX_W;
817 else
818 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
819 pbCodeBuf[off++] = 0x8b;
820 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
821
822#elif RT_ARCH_ARM64
823 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));
824
825#else
826# error "port me"
827#endif
828 return off;
829}
830
831
832/**
833 * Emits a 32-bit GPR load of a VCpu value.
834 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
835 */
836DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
837{
838#ifdef RT_ARCH_AMD64
839 /* mov reg32, mem32 */
840 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
841 AssertReturn(pbCodeBuf, UINT32_MAX);
842 if (iGpr >= 8)
843 pbCodeBuf[off++] = X86_OP_REX_R;
844 pbCodeBuf[off++] = 0x8b;
845 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
846
847#elif RT_ARCH_ARM64
848 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));
849
850#else
851# error "port me"
852#endif
853 return off;
854}
855
856
857/**
858 * Emits a 16-bit GPR load of a VCpu value.
859 * @note Bits 16 thru 63 in the GPR will be zero after the operation.
860 */
861DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
862{
863#ifdef RT_ARCH_AMD64
864 /* movzx reg32, mem16 */
865 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
866 AssertReturn(pbCodeBuf, UINT32_MAX);
867 if (iGpr >= 8)
868 pbCodeBuf[off++] = X86_OP_REX_R;
869 pbCodeBuf[off++] = 0x0f;
870 pbCodeBuf[off++] = 0xb7;
871 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
872
873#elif RT_ARCH_ARM64
874 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));
875
876#else
877# error "port me"
878#endif
879 return off;
880}
881
882
883/**
884 * Emits a 8-bit GPR load of a VCpu value.
885 * @note Bits 8 thru 63 in the GPR will be zero after the operation.
886 */
887DECLINLINE(uint32_t) iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
888{
889#ifdef RT_ARCH_AMD64
890 /* movzx reg32, mem8 */
891 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
892 AssertReturn(pbCodeBuf, UINT32_MAX);
893 if (iGpr >= 8)
894 pbCodeBuf[off++] = X86_OP_REX_R;
895 pbCodeBuf[off++] = 0x0f;
896 pbCodeBuf[off++] = 0xb6;
897 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
898
899#elif RT_ARCH_ARM64
900 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));
901
902#else
903# error "port me"
904#endif
905 return off;
906}
907
908
909/**
910 * Emits a store of a GPR value to a 64-bit VCpu field.
911 */
912DECLINLINE(uint32_t) iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
913{
914#ifdef RT_ARCH_AMD64
915 /* mov mem64, reg64 */
916 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
917 AssertReturn(pbCodeBuf, UINT32_MAX);
918 if (iGpr < 8)
919 pbCodeBuf[off++] = X86_OP_REX_W;
920 else
921 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
922 pbCodeBuf[off++] = 0x89;
923 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);
924
925#elif RT_ARCH_ARM64
926 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));
927
928#else
929# error "port me"
930#endif
931 return off;
932}
933
934
935/**
936 * Emits a store of a GPR value to a 32-bit VCpu field.
937 */
938DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
939{
940#ifdef RT_ARCH_AMD64
941 /* mov mem32, reg32 */
942 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
943 AssertReturn(pbCodeBuf, UINT32_MAX);
944 if (iGpr >= 8)
945 pbCodeBuf[off++] = X86_OP_REX_R;
946 pbCodeBuf[off++] = 0x89;
947 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
948
949#elif RT_ARCH_ARM64
950 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));
951
952#else
953# error "port me"
954#endif
955 return off;
956}
957
958
959/**
960 * Emits a store of a GPR value to a 16-bit VCpu field.
961 */
962DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
963{
964#ifdef RT_ARCH_AMD64
965 /* mov mem16, reg16 */
966 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);
967 AssertReturn(pbCodeBuf, UINT32_MAX);
968 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;
969 if (iGpr >= 8)
970 pbCodeBuf[off++] = X86_OP_REX_R;
971 pbCodeBuf[off++] = 0x89;
972 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
973
974#elif RT_ARCH_ARM64
975 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));
976
977#else
978# error "port me"
979#endif
980 return off;
981}
982
983
984/**
985 * Emits a store of a GPR value to a 8-bit VCpu field.
986 */
987DECLINLINE(uint32_t) iemNativeEmitStoreGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)
988{
989#ifdef RT_ARCH_AMD64
990 /* mov mem8, reg8 */
991 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
992 AssertReturn(pbCodeBuf, UINT32_MAX);
993 if (iGpr >= 8)
994 pbCodeBuf[off++] = X86_OP_REX_R;
995 pbCodeBuf[off++] = 0x88;
996 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);
997
998#elif RT_ARCH_ARM64
999 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));
1000
1001#else
1002# error "port me"
1003#endif
1004 return off;
1005}
1006
1007
1008/**
1009 * Emits a gprdst = gprsrc load.
1010 */
1011DECLINLINE(uint32_t) iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)
1012{
1013#ifdef RT_ARCH_AMD64
1014 /* mov gprdst, gprsrc */
1015 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);
1016 AssertReturn(pbCodeBuf, UINT32_MAX);
1017 if ((iGprDst | iGprSrc) >= 8)
1018 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B
1019 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B
1020 : X86_OP_REX_W | X86_OP_REX_R;
1021 else
1022 pbCodeBuf[off++] = X86_OP_REX_W;
1023 pbCodeBuf[off++] = 0x8b;
1024 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);
1025
1026#elif RT_ARCH_ARM64
1027 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1028 AssertReturn(pu32CodeBuf, UINT32_MAX);
1029 /* mov dst, src; alias for: orr dst, xzr, src */
1030 pu32CodeBuf[off++] = UINT32_C(0xaa000000) | ((uint32_t)iGprSrc << 16) | ((uint32_t)ARMV8_A64_REG_XZR << 5) | iGprDst;
1031
1032#else
1033# error "port me"
1034#endif
1035 return off;
1036}
1037
1038#ifdef RT_ARCH_AMD64
1039/**
1040 * Common bit of iemNativeEmitLoadGprByBp and friends.
1041 */
1042DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp)
1043{
1044 if (offDisp < 128 && offDisp >= -128)
1045 {
1046 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);
1047 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;
1048 }
1049 else
1050 {
1051 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);
1052 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1053 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1054 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1055 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1056 }
1057 return off;
1058}
1059#endif
1060
1061
1062#ifdef RT_ARCH_AMD64
1063/**
1064 * Emits a 64-bit GRP load instruction with an BP relative source address.
1065 */
1066DECLINLINE(uint32_t) iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1067{
1068 /* mov gprdst, qword [rbp + offDisp] */
1069 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1070 AssertReturn(pbCodeBuf, UINT32_MAX);
1071 if (iGprDst < 8)
1072 pbCodeBuf[off++] = X86_OP_REX_W;
1073 else
1074 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1075 pbCodeBuf[off++] = 0x8b;
1076 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1077}
1078#endif
1079
1080
1081#ifdef RT_ARCH_AMD64
1082/**
1083 * Emits a 32-bit GRP load instruction with an BP relative source address.
1084 */
1085DECLINLINE(uint32_t) iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1086{
1087 /* mov gprdst, dword [rbp + offDisp] */
1088 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1089 AssertReturn(pbCodeBuf, UINT32_MAX);
1090 if (iGprDst >= 8)
1091 pbCodeBuf[off++] = X86_OP_REX_R;
1092 pbCodeBuf[off++] = 0x8b;
1093 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1094}
1095#endif
1096
1097
1098#ifdef RT_ARCH_AMD64
1099/**
1100 * Emits a load effective address to a GRP with an BP relative source address.
1101 */
1102DECLINLINE(uint32_t) iemNativeEmitLeaGrpByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)
1103{
1104 /* lea gprdst, [rbp + offDisp] */
1105 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1106 AssertReturn(pbCodeBuf, UINT32_MAX);
1107 if (iGprDst < 8)
1108 pbCodeBuf[off++] = X86_OP_REX_W;
1109 else
1110 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1111 pbCodeBuf[off++] = 0x8d;
1112 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp);
1113}
1114#endif
1115
1116
1117/**
1118 * Emits a 64-bit GPR store with an BP relative destination address.
1119 *
1120 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1121 */
1122DECLINLINE(uint32_t) iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)
1123{
1124#ifdef RT_ARCH_AMD64
1125 /* mov qword [rbp + offDisp], gprdst */
1126 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1127 AssertReturn(pbCodeBuf, UINT32_MAX);
1128 if (iGprSrc < 8)
1129 pbCodeBuf[off++] = X86_OP_REX_W;
1130 else
1131 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;
1132 pbCodeBuf[off++] = 0x89;
1133 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp);
1134
1135#elif defined(RT_ARCH_ARM64)
1136 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))
1137 {
1138 /* str w/ unsigned imm12 (scaled) */
1139 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1140 AssertReturn(pu32CodeBuf, UINT32_MAX);
1141 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,
1142 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);
1143 }
1144 else if (offDisp >= -256 && offDisp <= 256)
1145 {
1146 /* stur w/ signed imm9 (unscaled) */
1147 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1148 AssertReturn(pu32CodeBuf, UINT32_MAX);
1149 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);
1150 }
1151 else
1152 {
1153 /* Use temporary indexing register. */
1154 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);
1155 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1156 AssertReturn(pu32CodeBuf, UINT32_MAX);
1157 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,
1158 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);
1159 }
1160 return off;
1161
1162#else
1163# error "Port me!"
1164#endif
1165}
1166
1167
1168/**
1169 * Emits a 64-bit immediate store with an BP relative destination address.
1170 *
1171 * @note May trash IEMNATIVE_REG_FIXED_TMP0.
1172 */
1173DECLINLINE(uint32_t) iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)
1174{
1175#ifdef RT_ARCH_AMD64
1176 if ((int64_t)uImm64 == (int32_t)uImm64)
1177 {
1178 /* mov qword [rbp + offDisp], imm32 - sign extended */
1179 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);
1180 AssertReturn(pbCodeBuf, UINT32_MAX);
1181
1182 pbCodeBuf[off++] = X86_OP_REX_W;
1183 pbCodeBuf[off++] = 0xc7;
1184 if (offDisp < 128 && offDisp >= -128)
1185 {
1186 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);
1187 pbCodeBuf[off++] = (uint8_t)offDisp;
1188 }
1189 else
1190 {
1191 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);
1192 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);
1193 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);
1194 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);
1195 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);
1196 }
1197 pbCodeBuf[off++] = RT_BYTE1(uImm64);
1198 pbCodeBuf[off++] = RT_BYTE2(uImm64);
1199 pbCodeBuf[off++] = RT_BYTE3(uImm64);
1200 pbCodeBuf[off++] = RT_BYTE4(uImm64);
1201 return off;
1202 }
1203#endif
1204
1205 /* Load tmp0, imm64; Store tmp to bp+disp. */
1206 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);
1207 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);
1208}
1209
1210
1211#ifdef RT_ARCH_AMD64
1212/**
1213 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.
1214 */
1215DECLINLINE(uint32_t) iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)
1216{
1217 /* sub gprdst, imm8/imm32 */
1218 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);
1219 AssertReturn(pbCodeBuf, UINT32_MAX);
1220 if (iGprDst < 8)
1221 pbCodeBuf[off++] = X86_OP_REX_W;
1222 else
1223 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;
1224 if (iSubtrahend < 128 && iSubtrahend >= -128)
1225 {
1226 pbCodeBuf[off++] = 0x83;
1227 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1228 pbCodeBuf[off++] = (uint8_t)iSubtrahend;
1229 }
1230 else
1231 {
1232 pbCodeBuf[off++] = 0x81;
1233 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);
1234 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);
1235 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);
1236 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);
1237 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);
1238 }
1239 return off;
1240}
1241#endif
1242
1243
1244/**
1245 * Emits a 32-bit GPR additions with a 8-bit signed immediate.
1246 * @note Bits 32 thru 63 in the GPR will be zero after the operation.
1247 */
1248DECLINLINE(uint32_t ) iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)
1249{
1250#if defined(RT_ARCH_AMD64)
1251 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);
1252 AssertReturn(pbCodeBuf, UINT32_MAX);
1253 if (iGprDst >= 8)
1254 pbCodeBuf[off++] = X86_OP_REX_B;
1255 pbCodeBuf[off++] = 0x83;
1256 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);
1257 pbCodeBuf[off++] = (uint8_t)iImm8;
1258
1259#elif defined(RT_ARCH_ARM64)
1260 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);
1261 AssertReturn(pu32CodeBuf, UINT32_MAX);
1262 if (iImm8 >= 0)
1263 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);
1264 else
1265 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);
1266
1267#else
1268# error "Port me"
1269#endif
1270 return off;
1271}
1272
1273/** @} */
1274
1275#endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler_h */
1276
注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette