translate.c@ 37675

最後變更在這個檔案從37675是 37675,由 vboxsync 提交於 14 年前
rem: Synced with v0.12.5.
屬性 svn:eol-style 設為 `native`
檔案大小: 270.7 KB

行
1	/*
2	* i386 translation
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	/*
21	* Oracle LGPL Disclaimer: For the avoidance of doubt, except that if any license choice
22	* other than GPL or LGPL is available it will apply instead, Oracle elects to use only
23	* the Lesser General Public License version 2.1 (LGPLv2) at this time for any software where
24	* a choice of LGPL license versions is made available with the language indicating
25	* that LGPLv2 or any later version may be used, or where a choice of which version
26	* of the LGPL is applied is otherwise unspecified.
27	*/
28
29	#include <stdarg.h>
30	#include <stdlib.h>
31	#include <stdio.h>
32	#include <string.h>
33	#ifndef VBOX
34	#include <inttypes.h>
35	#include <signal.h>
36	#endif /* !VBOX */
37
38	#include "cpu.h"
39	#include "exec-all.h"
40	#include "disas.h"
41	#include "tcg-op.h"
42
43	#include "helper.h"
44	#define GEN_HELPER 1
45	#include "helper.h"
46
47	#define PREFIX_REPZ 0x01
48	#define PREFIX_REPNZ 0x02
49	#define PREFIX_LOCK 0x04
50	#define PREFIX_DATA 0x08
51	#define PREFIX_ADR 0x10
52
53	#ifdef TARGET_X86_64
54	#define X86_64_ONLY(x) x
55	#define X86_64_DEF(...) __VA_ARGS__
56	#define CODE64(s) ((s)->code64)
57	#define REX_X(s) ((s)->rex_x)
58	#define REX_B(s) ((s)->rex_b)
59	/* XXX: gcc generates push/pop in some opcodes, so we cannot use them */
60	#if 1
61	#define BUGGY_64(x) NULL
62	#endif
63	#else
64	#define X86_64_ONLY(x) NULL
65	#define X86_64_DEF(...)
66	#define CODE64(s) 0
67	#define REX_X(s) 0
68	#define REX_B(s) 0
69	#endif
70
71	//#define MACRO_TEST 1
72
73	/* global register indexes */
74	static TCGv_ptr cpu_env;
75	static TCGv cpu_A0, cpu_cc_src, cpu_cc_dst, cpu_cc_tmp;
76	static TCGv_i32 cpu_cc_op;
77	static TCGv cpu_regs[CPU_NB_REGS];
78	/* local temps */
79	static TCGv cpu_T[2], cpu_T3;
80	/* local register indexes (only used inside old micro ops) */
81	static TCGv cpu_tmp0, cpu_tmp4;
82	static TCGv_ptr cpu_ptr0, cpu_ptr1;
83	static TCGv_i32 cpu_tmp2_i32, cpu_tmp3_i32;
84	static TCGv_i64 cpu_tmp1_i64;
85	static TCGv cpu_tmp5;
86
87	#include "gen-icount.h"
88
89	#ifdef TARGET_X86_64
90	static int x86_64_hregs;
91	#endif
92
93	#ifdef VBOX
94
95	/* Special/override code readers to hide patched code. */
96
97	uint8_t ldub_code_raw(target_ulong pc)
98	{
99	uint8_t b;
100
101	if (!remR3GetOpcode(cpu_single_env, pc, &b))
102	b = ldub_code(pc);
103	return b;
104	}
105	#define ldub_code(a) ldub_code_raw(a)
106
107	uint16_t lduw_code_raw(target_ulong pc)
108	{
109	return (ldub_code(pc+1) << 8) \| ldub_code(pc);
110	}
111	#define lduw_code(a) lduw_code_raw(a)
112
113
114	uint32_t ldl_code_raw(target_ulong pc)
115	{
116	return (ldub_code(pc+3) << 24) \| (ldub_code(pc+2) << 16) \| (ldub_code(pc+1) << 8) \| ldub_code(pc);
117	}
118	#define ldl_code(a) ldl_code_raw(a)
119
120	#endif /* VBOX */
121
122	typedef struct DisasContext {
123	/* current insn context */
124	int override; /* -1 if no override */
125	int prefix;
126	int aflag, dflag;
127	target_ulong pc; /* pc = eip + cs_base */
128	int is_jmp; /* 1 = means jump (stop translation), 2 means CPU
129	static state change (stop translation) */
130	/* current block context */
131	target_ulong cs_base; /* base of CS segment */
132	int pe; /* protected mode */
133	int code32; /* 32 bit code segment */
134	#ifdef TARGET_X86_64
135	int lma; /* long mode active */
136	int code64; /* 64 bit code segment */
137	int rex_x, rex_b;
138	#endif
139	int ss32; /* 32 bit stack segment */
140	int cc_op; /* current CC operation */
141	int addseg; /* non zero if either DS/ES/SS have a non zero base */
142	int f_st; /* currently unused */
143	int vm86; /* vm86 mode */
144	#ifdef VBOX
145	int vme; /* CR4.VME */
146	int pvi; /* CR4.PVI */
147	int record_call; /* record calls for CSAM or not? */
148	#endif
149	int cpl;
150	int iopl;
151	int tf; /* TF cpu flag */
152	int singlestep_enabled; /* "hardware" single step enabled */
153	int jmp_opt; /* use direct block chaining for direct jumps */
154	int mem_index; /* select memory access functions */
155	uint64_t flags; /* all execution flags */
156	struct TranslationBlock *tb;
157	int popl_esp_hack; /* for correct popl with esp base handling */
158	int rip_offset; /* only used in x86_64, but left for simplicity */
159	int cpuid_features;
160	int cpuid_ext_features;
161	int cpuid_ext2_features;
162	int cpuid_ext3_features;
163	} DisasContext;
164
165	static void gen_eob(DisasContext *s);
166	static void gen_jmp(DisasContext *s, target_ulong eip);
167	static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num);
168
169	#ifdef VBOX
170	static void gen_check_external_event(void);
171	#endif
172
173	/* i386 arith/logic operations */
174	enum {
175	OP_ADDL,
176	OP_ORL,
177	OP_ADCL,
178	OP_SBBL,
179	OP_ANDL,
180	OP_SUBL,
181	OP_XORL,
182	OP_CMPL,
183	};
184
185	/* i386 shift ops */
186	enum {
187	OP_ROL,
188	OP_ROR,
189	OP_RCL,
190	OP_RCR,
191	OP_SHL,
192	OP_SHR,
193	OP_SHL1, /* undocumented */
194	OP_SAR = 7,
195	};
196
197	enum {
198	JCC_O,
199	JCC_B,
200	JCC_Z,
201	JCC_BE,
202	JCC_S,
203	JCC_P,
204	JCC_L,
205	JCC_LE,
206	};
207
208	/* operand size */
209	enum {
210	OT_BYTE = 0,
211	OT_WORD,
212	OT_LONG,
213	OT_QUAD,
214	};
215
216	enum {
217	/* I386 int registers */
218	OR_EAX, /* MUST be even numbered */
219	OR_ECX,
220	OR_EDX,
221	OR_EBX,
222	OR_ESP,
223	OR_EBP,
224	OR_ESI,
225	OR_EDI,
226
227	OR_TMP0 = 16, /* temporary operand register */
228	OR_TMP1,
229	OR_A0, /* temporary register used when doing address evaluation */
230	};
231
232	static inline void gen_op_movl_T0_0(void)
233	{
234	tcg_gen_movi_tl(cpu_T[0], 0);
235	}
236
237	static inline void gen_op_movl_T0_im(int32_t val)
238	{
239	tcg_gen_movi_tl(cpu_T[0], val);
240	}
241
242	static inline void gen_op_movl_T0_imu(uint32_t val)
243	{
244	tcg_gen_movi_tl(cpu_T[0], val);
245	}
246
247	static inline void gen_op_movl_T1_im(int32_t val)
248	{
249	tcg_gen_movi_tl(cpu_T[1], val);
250	}
251
252	static inline void gen_op_movl_T1_imu(uint32_t val)
253	{
254	tcg_gen_movi_tl(cpu_T[1], val);
255	}
256
257	static inline void gen_op_movl_A0_im(uint32_t val)
258	{
259	tcg_gen_movi_tl(cpu_A0, val);
260	}
261
262	#ifdef TARGET_X86_64
263	static inline void gen_op_movq_A0_im(int64_t val)
264	{
265	tcg_gen_movi_tl(cpu_A0, val);
266	}
267	#endif
268
269	static inline void gen_movtl_T0_im(target_ulong val)
270	{
271	tcg_gen_movi_tl(cpu_T[0], val);
272	}
273
274	static inline void gen_movtl_T1_im(target_ulong val)
275	{
276	tcg_gen_movi_tl(cpu_T[1], val);
277	}
278
279	static inline void gen_op_andl_T0_ffff(void)
280	{
281	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff);
282	}
283
284	static inline void gen_op_andl_T0_im(uint32_t val)
285	{
286	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], val);
287	}
288
289	static inline void gen_op_movl_T0_T1(void)
290	{
291	tcg_gen_mov_tl(cpu_T[0], cpu_T[1]);
292	}
293
294	static inline void gen_op_andl_A0_ffff(void)
295	{
296	tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffff);
297	}
298
299	#ifdef TARGET_X86_64
300
301	#define NB_OP_SIZES 4
302
303	#else /* !TARGET_X86_64 */
304
305	#define NB_OP_SIZES 3
306
307	#endif /* !TARGET_X86_64 */
308
309	#if defined(HOST_WORDS_BIGENDIAN)
310	#define REG_B_OFFSET (sizeof(target_ulong) - 1)
311	#define REG_H_OFFSET (sizeof(target_ulong) - 2)
312	#define REG_W_OFFSET (sizeof(target_ulong) - 2)
313	#define REG_L_OFFSET (sizeof(target_ulong) - 4)
314	#define REG_LH_OFFSET (sizeof(target_ulong) - 8)
315	#else
316	#define REG_B_OFFSET 0
317	#define REG_H_OFFSET 1
318	#define REG_W_OFFSET 0
319	#define REG_L_OFFSET 0
320	#define REG_LH_OFFSET 4
321	#endif
322
323	static inline void gen_op_mov_reg_v(int ot, int reg, TCGv t0)
324	{
325	TCGv tmp;
326
327	switch(ot) {
328	case OT_BYTE:
329	tmp = tcg_temp_new();
330	tcg_gen_ext8u_tl(tmp, t0);
331	if (reg < 4 X86_64_DEF( \|\| reg >= 8 \|\| x86_64_hregs)) {
332	tcg_gen_andi_tl(cpu_regs[reg], cpu_regs[reg], ~0xff);
333	tcg_gen_or_tl(cpu_regs[reg], cpu_regs[reg], tmp);
334	} else {
335	tcg_gen_shli_tl(tmp, tmp, 8);
336	tcg_gen_andi_tl(cpu_regs[reg - 4], cpu_regs[reg - 4], ~0xff00);
337	tcg_gen_or_tl(cpu_regs[reg - 4], cpu_regs[reg - 4], tmp);
338	}
339	tcg_temp_free(tmp);
340	break;
341	case OT_WORD:
342	tmp = tcg_temp_new();
343	tcg_gen_ext16u_tl(tmp, t0);
344	tcg_gen_andi_tl(cpu_regs[reg], cpu_regs[reg], ~0xffff);
345	tcg_gen_or_tl(cpu_regs[reg], cpu_regs[reg], tmp);
346	tcg_temp_free(tmp);
347	break;
348	default: /* XXX this shouldn't be reached; abort? */
349	case OT_LONG:
350	/* For x86_64, this sets the higher half of register to zero.
351	For i386, this is equivalent to a mov. */
352	tcg_gen_ext32u_tl(cpu_regs[reg], t0);
353	break;
354	#ifdef TARGET_X86_64
355	case OT_QUAD:
356	tcg_gen_mov_tl(cpu_regs[reg], t0);
357	break;
358	#endif
359	}
360	}
361
362	static inline void gen_op_mov_reg_T0(int ot, int reg)
363	{
364	gen_op_mov_reg_v(ot, reg, cpu_T[0]);
365	}
366
367	static inline void gen_op_mov_reg_T1(int ot, int reg)
368	{
369	gen_op_mov_reg_v(ot, reg, cpu_T[1]);
370	}
371
372	static inline void gen_op_mov_reg_A0(int size, int reg)
373	{
374	TCGv tmp;
375
376	switch(size) {
377	case 0:
378	tmp = tcg_temp_new();
379	tcg_gen_ext16u_tl(tmp, cpu_A0);
380	tcg_gen_andi_tl(cpu_regs[reg], cpu_regs[reg], ~0xffff);
381	tcg_gen_or_tl(cpu_regs[reg], cpu_regs[reg], tmp);
382	tcg_temp_free(tmp);
383	break;
384	default: /* XXX this shouldn't be reached; abort? */
385	case 1:
386	/* For x86_64, this sets the higher half of register to zero.
387	For i386, this is equivalent to a mov. */
388	tcg_gen_ext32u_tl(cpu_regs[reg], cpu_A0);
389	break;
390	#ifdef TARGET_X86_64
391	case 2:
392	tcg_gen_mov_tl(cpu_regs[reg], cpu_A0);
393	break;
394	#endif
395	}
396	}
397
398	static inline void gen_op_mov_v_reg(int ot, TCGv t0, int reg)
399	{
400	switch(ot) {
401	case OT_BYTE:
402	if (reg < 4 X86_64_DEF( \|\| reg >= 8 \|\| x86_64_hregs)) {
403	goto std_case;
404	} else {
405	tcg_gen_shri_tl(t0, cpu_regs[reg - 4], 8);
406	tcg_gen_ext8u_tl(t0, t0);
407	}
408	break;
409	default:
410	std_case:
411	tcg_gen_mov_tl(t0, cpu_regs[reg]);
412	break;
413	}
414	}
415
416	static inline void gen_op_mov_TN_reg(int ot, int t_index, int reg)
417	{
418	gen_op_mov_v_reg(ot, cpu_T[t_index], reg);
419	}
420
421	static inline void gen_op_movl_A0_reg(int reg)
422	{
423	tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
424	}
425
426	static inline void gen_op_addl_A0_im(int32_t val)
427	{
428	tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
429	#ifdef TARGET_X86_64
430	tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
431	#endif
432	}
433
434	#ifdef TARGET_X86_64
435	static inline void gen_op_addq_A0_im(int64_t val)
436	{
437	tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
438	}
439	#endif
440
441	static void gen_add_A0_im(DisasContext *s, int val)
442	{
443	#ifdef TARGET_X86_64
444	if (CODE64(s))
445	gen_op_addq_A0_im(val);
446	else
447	#endif
448	gen_op_addl_A0_im(val);
449	}
450
451	static inline void gen_op_addl_T0_T1(void)
452	{
453	tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
454	}
455
456	static inline void gen_op_jmp_T0(void)
457	{
458	tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUState, eip));
459	}
460
461	static inline void gen_op_add_reg_im(int size, int reg, int32_t val)
462	{
463	switch(size) {
464	case 0:
465	tcg_gen_addi_tl(cpu_tmp0, cpu_regs[reg], val);
466	tcg_gen_ext16u_tl(cpu_tmp0, cpu_tmp0);
467	tcg_gen_andi_tl(cpu_regs[reg], cpu_regs[reg], ~0xffff);
468	tcg_gen_or_tl(cpu_regs[reg], cpu_regs[reg], cpu_tmp0);
469	break;
470	case 1:
471	tcg_gen_addi_tl(cpu_tmp0, cpu_regs[reg], val);
472	/* For x86_64, this sets the higher half of register to zero.
473	For i386, this is equivalent to a nop. */
474	tcg_gen_ext32u_tl(cpu_tmp0, cpu_tmp0);
475	tcg_gen_mov_tl(cpu_regs[reg], cpu_tmp0);
476	break;
477	#ifdef TARGET_X86_64
478	case 2:
479	tcg_gen_addi_tl(cpu_regs[reg], cpu_regs[reg], val);
480	break;
481	#endif
482	}
483	}
484
485	static inline void gen_op_add_reg_T0(int size, int reg)
486	{
487	switch(size) {
488	case 0:
489	tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T[0]);
490	tcg_gen_ext16u_tl(cpu_tmp0, cpu_tmp0);
491	tcg_gen_andi_tl(cpu_regs[reg], cpu_regs[reg], ~0xffff);
492	tcg_gen_or_tl(cpu_regs[reg], cpu_regs[reg], cpu_tmp0);
493	break;
494	case 1:
495	tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T[0]);
496	/* For x86_64, this sets the higher half of register to zero.
497	For i386, this is equivalent to a nop. */
498	tcg_gen_ext32u_tl(cpu_tmp0, cpu_tmp0);
499	tcg_gen_mov_tl(cpu_regs[reg], cpu_tmp0);
500	break;
501	#ifdef TARGET_X86_64
502	case 2:
503	tcg_gen_add_tl(cpu_regs[reg], cpu_regs[reg], cpu_T[0]);
504	break;
505	#endif
506	}
507	}
508
509	static inline void gen_op_set_cc_op(int32_t val)
510	{
511	tcg_gen_movi_i32(cpu_cc_op, val);
512	}
513
514	static inline void gen_op_addl_A0_reg_sN(int shift, int reg)
515	{
516	tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
517	if (shift != 0)
518	tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
519	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
520	/* For x86_64, this sets the higher half of register to zero.
521	For i386, this is equivalent to a nop. */
522	tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
523	}
524
525	#ifdef VBOX
526	DECLINLINE(void) gen_op_seg_check(int reg, bool keepA0)
527	{
528	/* It seems segments doesn't get out of sync - if they do in fact - enable below code. */
529	# ifdef FORCE_SEGMENT_SYNC
530	# if 1
531	TCGv t0;
532
533	/* Considering poor quality of TCG optimizer - better call directly */
534	t0 = tcg_temp_local_new(TCG_TYPE_TL);
535	tcg_gen_movi_tl(t0, reg);
536	tcg_gen_helper_0_1(helper_sync_seg, t0);
537	tcg_temp_free(t0);
538	# else
539	/* Our segments could be outdated, thus check for newselector field to see if update really needed */
540	int skip_label;
541	TCGv t0, a0;
542
543	/* For other segments this check is waste of time, and also TCG is unable to cope with this code,
544	for data/stack segments, as expects alive cpu_T[0] */
545	if (reg != R_GS)
546	return;
547
548	if (keepA0)
549	{
550	/* we need to store old cpu_A0 */
551	a0 = tcg_temp_local_new(TCG_TYPE_TL);
552	tcg_gen_mov_tl(a0, cpu_A0);
553	}
554
555	skip_label = gen_new_label();
556	t0 = tcg_temp_local_new(TCG_TYPE_TL);
557
558	tcg_gen_ld32u_tl(t0, cpu_env, offsetof(CPUState, segs[reg].newselector) + REG_L_OFFSET);
559	tcg_gen_brcondi_i32(TCG_COND_EQ, t0, 0, skip_label);
560	tcg_gen_ld32u_tl(t0, cpu_env, offsetof(CPUState, eflags) + REG_L_OFFSET);
561	tcg_gen_andi_tl(t0, t0, VM_MASK);
562	tcg_gen_brcondi_i32(TCG_COND_NE, t0, 0, skip_label);
563	tcg_gen_movi_tl(t0, reg);
564
565	tcg_gen_helper_0_1(helper_sync_seg, t0);
566
567	tcg_temp_free(t0);
568
569	gen_set_label(skip_label);
570	if (keepA0)
571	{
572	tcg_gen_mov_tl(cpu_A0, a0);
573	tcg_temp_free(a0);
574	}
575	# endif /* 0 */
576	# endif /* FORCE_SEGMENT_SYNC */
577	}
578	#endif /* VBOX */
579
580	static inline void gen_op_movl_A0_seg(int reg)
581	{
582	#ifdef VBOX
583	gen_op_seg_check(reg, false);
584	#endif
585	tcg_gen_ld32u_tl(cpu_A0, cpu_env, offsetof(CPUState, segs[reg].base) + REG_L_OFFSET);
586	}
587
588	static inline void gen_op_addl_A0_seg(int reg)
589	{
590	#ifdef VBOX
591	gen_op_seg_check(reg, true);
592	#endif
593	tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUState, segs[reg].base));
594	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
595	#ifdef TARGET_X86_64
596	tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
597	#endif
598	}
599
600	#ifdef TARGET_X86_64
601	static inline void gen_op_movq_A0_seg(int reg)
602	{
603	#ifdef VBOX
604	gen_op_seg_check(reg, false);
605	#endif
606	tcg_gen_ld_tl(cpu_A0, cpu_env, offsetof(CPUState, segs[reg].base));
607	}
608
609	static inline void gen_op_addq_A0_seg(int reg)
610	{
611	#ifdef VBOX
612	gen_op_seg_check(reg, true);
613	#endif
614	tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUState, segs[reg].base));
615	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
616	}
617
618	static inline void gen_op_movq_A0_reg(int reg)
619	{
620	tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
621	}
622
623	static inline void gen_op_addq_A0_reg_sN(int shift, int reg)
624	{
625	tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
626	if (shift != 0)
627	tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
628	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
629	}
630	#endif
631
632	static inline void gen_op_lds_T0_A0(int idx)
633	{
634	int mem_index = (idx >> 2) - 1;
635	switch(idx & 3) {
636	case 0:
637	tcg_gen_qemu_ld8s(cpu_T[0], cpu_A0, mem_index);
638	break;
639	case 1:
640	tcg_gen_qemu_ld16s(cpu_T[0], cpu_A0, mem_index);
641	break;
642	default:
643	case 2:
644	tcg_gen_qemu_ld32s(cpu_T[0], cpu_A0, mem_index);
645	break;
646	}
647	}
648
649	static inline void gen_op_ld_v(int idx, TCGv t0, TCGv a0)
650	{
651	int mem_index = (idx >> 2) - 1;
652	switch(idx & 3) {
653	case 0:
654	tcg_gen_qemu_ld8u(t0, a0, mem_index);
655	break;
656	case 1:
657	tcg_gen_qemu_ld16u(t0, a0, mem_index);
658	break;
659	case 2:
660	tcg_gen_qemu_ld32u(t0, a0, mem_index);
661	break;
662	default:
663	case 3:
664	/* Should never happen on 32-bit targets. */
665	#ifdef TARGET_X86_64
666	tcg_gen_qemu_ld64(t0, a0, mem_index);
667	#endif
668	break;
669	}
670	}
671
672	/* XXX: always use ldu or lds */
673	static inline void gen_op_ld_T0_A0(int idx)
674	{
675	gen_op_ld_v(idx, cpu_T[0], cpu_A0);
676	}
677
678	static inline void gen_op_ldu_T0_A0(int idx)
679	{
680	gen_op_ld_v(idx, cpu_T[0], cpu_A0);
681	}
682
683	static inline void gen_op_ld_T1_A0(int idx)
684	{
685	gen_op_ld_v(idx, cpu_T[1], cpu_A0);
686	}
687
688	static inline void gen_op_st_v(int idx, TCGv t0, TCGv a0)
689	{
690	int mem_index = (idx >> 2) - 1;
691	switch(idx & 3) {
692	case 0:
693	tcg_gen_qemu_st8(t0, a0, mem_index);
694	break;
695	case 1:
696	tcg_gen_qemu_st16(t0, a0, mem_index);
697	break;
698	case 2:
699	tcg_gen_qemu_st32(t0, a0, mem_index);
700	break;
701	default:
702	case 3:
703	/* Should never happen on 32-bit targets. */
704	#ifdef TARGET_X86_64
705	tcg_gen_qemu_st64(t0, a0, mem_index);
706	#endif
707	break;
708	}
709	}
710
711	static inline void gen_op_st_T0_A0(int idx)
712	{
713	gen_op_st_v(idx, cpu_T[0], cpu_A0);
714	}
715
716	static inline void gen_op_st_T1_A0(int idx)
717	{
718	gen_op_st_v(idx, cpu_T[1], cpu_A0);
719	}
720
721	#ifdef VBOX
722
723	static void gen_check_external_event(void)
724	{
725	# if 1
726	/** @todo: once TCG codegen improves, we may want to use version
727	from else version */
728	gen_helper_check_external_event();
729	# else
730	int skip_label;
731	TCGv t0;
732
733	skip_label = gen_new_label();
734	t0 = tcg_temp_local_new(TCG_TYPE_TL);
735	/* t0 = cpu_tmp0; */
736
737	tcg_gen_ld32u_tl(t0, cpu_env, offsetof(CPUState, interrupt_request));
738	/* Keep in sync with helper_check_external_event() */
739	tcg_gen_andi_tl(t0, t0,
740	CPU_INTERRUPT_EXTERNAL_EXIT
741	\| CPU_INTERRUPT_EXTERNAL_TIMER
742	\| CPU_INTERRUPT_EXTERNAL_DMA
743	\| CPU_INTERRUPT_EXTERNAL_HARD);
744	/** @todo: predict branch as taken */
745	tcg_gen_brcondi_i32(TCG_COND_EQ, t0, 0, skip_label);
746	tcg_temp_free(t0);
747
748	gen_helper_check_external_event();
749
750	gen_set_label(skip_label);
751	# endif
752	}
753
754	# if 0 /* unused code? */
755	static void gen_check_external_event2()
756	{
757	gen_helper_check_external_event();
758	}
759	# endif
760
761	#endif /* VBOX */
762
763	static inline void gen_jmp_im(target_ulong pc)
764	{
765	tcg_gen_movi_tl(cpu_tmp0, pc);
766	tcg_gen_st_tl(cpu_tmp0, cpu_env, offsetof(CPUState, eip));
767	}
768
769	#ifdef VBOX
770	DECLINLINE(void) gen_update_eip(target_ulong pc)
771	{
772	gen_jmp_im(pc);
773	# ifdef VBOX_DUMP_STATE
774	gen_helper_dump_state();
775	# endif
776	}
777	#endif /* VBOX */
778
779	static inline void gen_string_movl_A0_ESI(DisasContext *s)
780	{
781	int override;
782
783	override = s->override;
784	#ifdef TARGET_X86_64
785	if (s->aflag == 2) {
786	if (override >= 0) {
787	gen_op_movq_A0_seg(override);
788	gen_op_addq_A0_reg_sN(0, R_ESI);
789	} else {
790	gen_op_movq_A0_reg(R_ESI);
791	}
792	} else
793	#endif
794	if (s->aflag) {
795	/* 32 bit address */
796	if (s->addseg && override < 0)
797	override = R_DS;
798	if (override >= 0) {
799	gen_op_movl_A0_seg(override);
800	gen_op_addl_A0_reg_sN(0, R_ESI);
801	} else {
802	gen_op_movl_A0_reg(R_ESI);
803	}
804	} else {
805	/* 16 address, always override */
806	if (override < 0)
807	override = R_DS;
808	gen_op_movl_A0_reg(R_ESI);
809	gen_op_andl_A0_ffff();
810	gen_op_addl_A0_seg(override);
811	}
812	}
813
814	static inline void gen_string_movl_A0_EDI(DisasContext *s)
815	{
816	#ifdef TARGET_X86_64
817	if (s->aflag == 2) {
818	gen_op_movq_A0_reg(R_EDI);
819	} else
820	#endif
821	if (s->aflag) {
822	if (s->addseg) {
823	gen_op_movl_A0_seg(R_ES);
824	gen_op_addl_A0_reg_sN(0, R_EDI);
825	} else {
826	gen_op_movl_A0_reg(R_EDI);
827	}
828	} else {
829	gen_op_movl_A0_reg(R_EDI);
830	gen_op_andl_A0_ffff();
831	gen_op_addl_A0_seg(R_ES);
832	}
833	}
834
835	static inline void gen_op_movl_T0_Dshift(int ot)
836	{
837	tcg_gen_ld32s_tl(cpu_T[0], cpu_env, offsetof(CPUState, df));
838	tcg_gen_shli_tl(cpu_T[0], cpu_T[0], ot);
839	};
840
841	static void gen_extu(int ot, TCGv reg)
842	{
843	switch(ot) {
844	case OT_BYTE:
845	tcg_gen_ext8u_tl(reg, reg);
846	break;
847	case OT_WORD:
848	tcg_gen_ext16u_tl(reg, reg);
849	break;
850	case OT_LONG:
851	tcg_gen_ext32u_tl(reg, reg);
852	break;
853	default:
854	break;
855	}
856	}
857
858	static void gen_exts(int ot, TCGv reg)
859	{
860	switch(ot) {
861	case OT_BYTE:
862	tcg_gen_ext8s_tl(reg, reg);
863	break;
864	case OT_WORD:
865	tcg_gen_ext16s_tl(reg, reg);
866	break;
867	case OT_LONG:
868	tcg_gen_ext32s_tl(reg, reg);
869	break;
870	default:
871	break;
872	}
873	}
874
875	static inline void gen_op_jnz_ecx(int size, int label1)
876	{
877	tcg_gen_mov_tl(cpu_tmp0, cpu_regs[R_ECX]);
878	gen_extu(size + 1, cpu_tmp0);
879	tcg_gen_brcondi_tl(TCG_COND_NE, cpu_tmp0, 0, label1);
880	}
881
882	static inline void gen_op_jz_ecx(int size, int label1)
883	{
884	tcg_gen_mov_tl(cpu_tmp0, cpu_regs[R_ECX]);
885	gen_extu(size + 1, cpu_tmp0);
886	tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_tmp0, 0, label1);
887	}
888
889	static void gen_helper_in_func(int ot, TCGv v, TCGv_i32 n)
890	{
891	switch (ot) {
892	case 0: gen_helper_inb(v, n); break;
893	case 1: gen_helper_inw(v, n); break;
894	case 2: gen_helper_inl(v, n); break;
895	}
896
897	}
898
899	static void gen_helper_out_func(int ot, TCGv_i32 v, TCGv_i32 n)
900	{
901	switch (ot) {
902	case 0: gen_helper_outb(v, n); break;
903	case 1: gen_helper_outw(v, n); break;
904	case 2: gen_helper_outl(v, n); break;
905	}
906
907	}
908
909	static void gen_check_io(DisasContext *s, int ot, target_ulong cur_eip,
910	uint32_t svm_flags)
911	{
912	int state_saved;
913	target_ulong next_eip;
914
915	state_saved = 0;
916	if (s->pe && (s->cpl > s->iopl \|\| s->vm86)) {
917	if (s->cc_op != CC_OP_DYNAMIC)
918	gen_op_set_cc_op(s->cc_op);
919	gen_jmp_im(cur_eip);
920	state_saved = 1;
921	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
922	switch (ot) {
923	case 0: gen_helper_check_iob(cpu_tmp2_i32); break;
924	case 1: gen_helper_check_iow(cpu_tmp2_i32); break;
925	case 2: gen_helper_check_iol(cpu_tmp2_i32); break;
926	}
927	}
928	if(s->flags & HF_SVMI_MASK) {
929	if (!state_saved) {
930	if (s->cc_op != CC_OP_DYNAMIC)
931	gen_op_set_cc_op(s->cc_op);
932	gen_jmp_im(cur_eip);
933	state_saved = 1;
934	}
935	svm_flags \|= (1 << (4 + ot));
936	next_eip = s->pc - s->cs_base;
937	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
938	gen_helper_svm_check_io(cpu_tmp2_i32, tcg_const_i32(svm_flags),
939	tcg_const_i32(next_eip - cur_eip));
940	}
941	}
942
943	static inline void gen_movs(DisasContext *s, int ot)
944	{
945	gen_string_movl_A0_ESI(s);
946	gen_op_ld_T0_A0(ot + s->mem_index);
947	gen_string_movl_A0_EDI(s);
948	gen_op_st_T0_A0(ot + s->mem_index);
949	gen_op_movl_T0_Dshift(ot);
950	gen_op_add_reg_T0(s->aflag, R_ESI);
951	gen_op_add_reg_T0(s->aflag, R_EDI);
952	}
953
954	static inline void gen_update_cc_op(DisasContext *s)
955	{
956	if (s->cc_op != CC_OP_DYNAMIC) {
957	gen_op_set_cc_op(s->cc_op);
958	s->cc_op = CC_OP_DYNAMIC;
959	}
960	}
961
962	static void gen_op_update1_cc(void)
963	{
964	tcg_gen_discard_tl(cpu_cc_src);
965	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
966	}
967
968	static void gen_op_update2_cc(void)
969	{
970	tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
971	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
972	}
973
974	static inline void gen_op_cmpl_T0_T1_cc(void)
975	{
976	tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
977	tcg_gen_sub_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
978	}
979
980	static inline void gen_op_testl_T0_T1_cc(void)
981	{
982	tcg_gen_discard_tl(cpu_cc_src);
983	tcg_gen_and_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
984	}
985
986	static void gen_op_update_neg_cc(void)
987	{
988	tcg_gen_neg_tl(cpu_cc_src, cpu_T[0]);
989	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
990	}
991
992	/* compute eflags.C to reg */
993	static void gen_compute_eflags_c(TCGv reg)
994	{
995	gen_helper_cc_compute_c(cpu_tmp2_i32, cpu_cc_op);
996	tcg_gen_extu_i32_tl(reg, cpu_tmp2_i32);
997	}
998
999	/* compute all eflags to cc_src */
1000	static void gen_compute_eflags(TCGv reg)
1001	{
1002	gen_helper_cc_compute_all(cpu_tmp2_i32, cpu_cc_op);
1003	tcg_gen_extu_i32_tl(reg, cpu_tmp2_i32);
1004	}
1005
1006	static inline void gen_setcc_slow_T0(DisasContext *s, int jcc_op)
1007	{
1008	if (s->cc_op != CC_OP_DYNAMIC)
1009	gen_op_set_cc_op(s->cc_op);
1010	switch(jcc_op) {
1011	case JCC_O:
1012	gen_compute_eflags(cpu_T[0]);
1013	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 11);
1014	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1015	break;
1016	case JCC_B:
1017	gen_compute_eflags_c(cpu_T[0]);
1018	break;
1019	case JCC_Z:
1020	gen_compute_eflags(cpu_T[0]);
1021	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 6);
1022	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1023	break;
1024	case JCC_BE:
1025	gen_compute_eflags(cpu_tmp0);
1026	tcg_gen_shri_tl(cpu_T[0], cpu_tmp0, 6);
1027	tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
1028	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1029	break;
1030	case JCC_S:
1031	gen_compute_eflags(cpu_T[0]);
1032	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 7);
1033	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1034	break;
1035	case JCC_P:
1036	gen_compute_eflags(cpu_T[0]);
1037	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 2);
1038	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1039	break;
1040	case JCC_L:
1041	gen_compute_eflags(cpu_tmp0);
1042	tcg_gen_shri_tl(cpu_T[0], cpu_tmp0, 11); /* CC_O */
1043	tcg_gen_shri_tl(cpu_tmp0, cpu_tmp0, 7); /* CC_S */
1044	tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
1045	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1046	break;
1047	default:
1048	case JCC_LE:
1049	gen_compute_eflags(cpu_tmp0);
1050	tcg_gen_shri_tl(cpu_T[0], cpu_tmp0, 11); /* CC_O */
1051	tcg_gen_shri_tl(cpu_tmp4, cpu_tmp0, 7); /* CC_S */
1052	tcg_gen_shri_tl(cpu_tmp0, cpu_tmp0, 6); /* CC_Z */
1053	tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
1054	tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
1055	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 1);
1056	break;
1057	}
1058	}
1059
1060	/* return true if setcc_slow is not needed (WARNING: must be kept in
1061	sync with gen_jcc1) */
1062	static int is_fast_jcc_case(DisasContext *s, int b)
1063	{
1064	int jcc_op;
1065	jcc_op = (b >> 1) & 7;
1066	switch(s->cc_op) {
1067	/* we optimize the cmp/jcc case */
1068	case CC_OP_SUBB:
1069	case CC_OP_SUBW:
1070	case CC_OP_SUBL:
1071	case CC_OP_SUBQ:
1072	if (jcc_op == JCC_O \|\| jcc_op == JCC_P)
1073	goto slow_jcc;
1074	break;
1075
1076	/* some jumps are easy to compute */
1077	case CC_OP_ADDB:
1078	case CC_OP_ADDW:
1079	case CC_OP_ADDL:
1080	case CC_OP_ADDQ:
1081
1082	case CC_OP_LOGICB:
1083	case CC_OP_LOGICW:
1084	case CC_OP_LOGICL:
1085	case CC_OP_LOGICQ:
1086
1087	case CC_OP_INCB:
1088	case CC_OP_INCW:
1089	case CC_OP_INCL:
1090	case CC_OP_INCQ:
1091
1092	case CC_OP_DECB:
1093	case CC_OP_DECW:
1094	case CC_OP_DECL:
1095	case CC_OP_DECQ:
1096
1097	case CC_OP_SHLB:
1098	case CC_OP_SHLW:
1099	case CC_OP_SHLL:
1100	case CC_OP_SHLQ:
1101	if (jcc_op != JCC_Z && jcc_op != JCC_S)
1102	goto slow_jcc;
1103	break;
1104	default:
1105	slow_jcc:
1106	return 0;
1107	}
1108	return 1;
1109	}
1110
1111	/* generate a conditional jump to label 'l1' according to jump opcode
1112	value 'b'. In the fast case, T0 is guaranted not to be used. */
1113	static inline void gen_jcc1(DisasContext *s, int cc_op, int b, int l1)
1114	{
1115	int inv, jcc_op, size, cond;
1116	TCGv t0;
1117
1118	inv = b & 1;
1119	jcc_op = (b >> 1) & 7;
1120
1121	switch(cc_op) {
1122	/* we optimize the cmp/jcc case */
1123	case CC_OP_SUBB:
1124	case CC_OP_SUBW:
1125	case CC_OP_SUBL:
1126	case CC_OP_SUBQ:
1127
1128	size = cc_op - CC_OP_SUBB;
1129	switch(jcc_op) {
1130	case JCC_Z:
1131	fast_jcc_z:
1132	switch(size) {
1133	case 0:
1134	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0xff);
1135	t0 = cpu_tmp0;
1136	break;
1137	case 1:
1138	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0xffff);
1139	t0 = cpu_tmp0;
1140	break;
1141	#ifdef TARGET_X86_64
1142	case 2:
1143	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0xffffffff);
1144	t0 = cpu_tmp0;
1145	break;
1146	#endif
1147	default:
1148	t0 = cpu_cc_dst;
1149	break;
1150	}
1151	tcg_gen_brcondi_tl(inv ? TCG_COND_NE : TCG_COND_EQ, t0, 0, l1);
1152	break;
1153	case JCC_S:
1154	fast_jcc_s:
1155	switch(size) {
1156	case 0:
1157	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0x80);
1158	tcg_gen_brcondi_tl(inv ? TCG_COND_EQ : TCG_COND_NE, cpu_tmp0,
1159	0, l1);
1160	break;
1161	case 1:
1162	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0x8000);
1163	tcg_gen_brcondi_tl(inv ? TCG_COND_EQ : TCG_COND_NE, cpu_tmp0,
1164	0, l1);
1165	break;
1166	#ifdef TARGET_X86_64
1167	case 2:
1168	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_dst, 0x80000000);
1169	tcg_gen_brcondi_tl(inv ? TCG_COND_EQ : TCG_COND_NE, cpu_tmp0,
1170	0, l1);
1171	break;
1172	#endif
1173	default:
1174	tcg_gen_brcondi_tl(inv ? TCG_COND_GE : TCG_COND_LT, cpu_cc_dst,
1175	0, l1);
1176	break;
1177	}
1178	break;
1179
1180	case JCC_B:
1181	cond = inv ? TCG_COND_GEU : TCG_COND_LTU;
1182	goto fast_jcc_b;
1183	case JCC_BE:
1184	cond = inv ? TCG_COND_GTU : TCG_COND_LEU;
1185	fast_jcc_b:
1186	tcg_gen_add_tl(cpu_tmp4, cpu_cc_dst, cpu_cc_src);
1187	switch(size) {
1188	case 0:
1189	t0 = cpu_tmp0;
1190	tcg_gen_andi_tl(cpu_tmp4, cpu_tmp4, 0xff);
1191	tcg_gen_andi_tl(t0, cpu_cc_src, 0xff);
1192	break;
1193	case 1:
1194	t0 = cpu_tmp0;
1195	tcg_gen_andi_tl(cpu_tmp4, cpu_tmp4, 0xffff);
1196	tcg_gen_andi_tl(t0, cpu_cc_src, 0xffff);
1197	break;
1198	#ifdef TARGET_X86_64
1199	case 2:
1200	t0 = cpu_tmp0;
1201	tcg_gen_andi_tl(cpu_tmp4, cpu_tmp4, 0xffffffff);
1202	tcg_gen_andi_tl(t0, cpu_cc_src, 0xffffffff);
1203	break;
1204	#endif
1205	default:
1206	t0 = cpu_cc_src;
1207	break;
1208	}
1209	tcg_gen_brcond_tl(cond, cpu_tmp4, t0, l1);
1210	break;
1211
1212	case JCC_L:
1213	cond = inv ? TCG_COND_GE : TCG_COND_LT;
1214	goto fast_jcc_l;
1215	case JCC_LE:
1216	cond = inv ? TCG_COND_GT : TCG_COND_LE;
1217	fast_jcc_l:
1218	tcg_gen_add_tl(cpu_tmp4, cpu_cc_dst, cpu_cc_src);
1219	switch(size) {
1220	case 0:
1221	t0 = cpu_tmp0;
1222	tcg_gen_ext8s_tl(cpu_tmp4, cpu_tmp4);
1223	tcg_gen_ext8s_tl(t0, cpu_cc_src);
1224	break;
1225	case 1:
1226	t0 = cpu_tmp0;
1227	tcg_gen_ext16s_tl(cpu_tmp4, cpu_tmp4);
1228	tcg_gen_ext16s_tl(t0, cpu_cc_src);
1229	break;
1230	#ifdef TARGET_X86_64
1231	case 2:
1232	t0 = cpu_tmp0;
1233	tcg_gen_ext32s_tl(cpu_tmp4, cpu_tmp4);
1234	tcg_gen_ext32s_tl(t0, cpu_cc_src);
1235	break;
1236	#endif
1237	default:
1238	t0 = cpu_cc_src;
1239	break;
1240	}
1241	tcg_gen_brcond_tl(cond, cpu_tmp4, t0, l1);
1242	break;
1243
1244	default:
1245	goto slow_jcc;
1246	}
1247	break;
1248
1249	/* some jumps are easy to compute */
1250	case CC_OP_ADDB:
1251	case CC_OP_ADDW:
1252	case CC_OP_ADDL:
1253	case CC_OP_ADDQ:
1254
1255	case CC_OP_ADCB:
1256	case CC_OP_ADCW:
1257	case CC_OP_ADCL:
1258	case CC_OP_ADCQ:
1259
1260	case CC_OP_SBBB:
1261	case CC_OP_SBBW:
1262	case CC_OP_SBBL:
1263	case CC_OP_SBBQ:
1264
1265	case CC_OP_LOGICB:
1266	case CC_OP_LOGICW:
1267	case CC_OP_LOGICL:
1268	case CC_OP_LOGICQ:
1269
1270	case CC_OP_INCB:
1271	case CC_OP_INCW:
1272	case CC_OP_INCL:
1273	case CC_OP_INCQ:
1274
1275	case CC_OP_DECB:
1276	case CC_OP_DECW:
1277	case CC_OP_DECL:
1278	case CC_OP_DECQ:
1279
1280	case CC_OP_SHLB:
1281	case CC_OP_SHLW:
1282	case CC_OP_SHLL:
1283	case CC_OP_SHLQ:
1284
1285	case CC_OP_SARB:
1286	case CC_OP_SARW:
1287	case CC_OP_SARL:
1288	case CC_OP_SARQ:
1289	switch(jcc_op) {
1290	case JCC_Z:
1291	size = (cc_op - CC_OP_ADDB) & 3;
1292	goto fast_jcc_z;
1293	case JCC_S:
1294	size = (cc_op - CC_OP_ADDB) & 3;
1295	goto fast_jcc_s;
1296	default:
1297	goto slow_jcc;
1298	}
1299	break;
1300	default:
1301	slow_jcc:
1302	gen_setcc_slow_T0(s, jcc_op);
1303	tcg_gen_brcondi_tl(inv ? TCG_COND_EQ : TCG_COND_NE,
1304	cpu_T[0], 0, l1);
1305	break;
1306	}
1307	}
1308
1309	/* XXX: does not work with gdbstub "ice" single step - not a
1310	serious problem */
1311	static int gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
1312	{
1313	int l1, l2;
1314
1315	l1 = gen_new_label();
1316	l2 = gen_new_label();
1317	gen_op_jnz_ecx(s->aflag, l1);
1318	gen_set_label(l2);
1319	gen_jmp_tb(s, next_eip, 1);
1320	gen_set_label(l1);
1321	return l2;
1322	}
1323
1324	static inline void gen_stos(DisasContext *s, int ot)
1325	{
1326	gen_op_mov_TN_reg(OT_LONG, 0, R_EAX);
1327	gen_string_movl_A0_EDI(s);
1328	gen_op_st_T0_A0(ot + s->mem_index);
1329	gen_op_movl_T0_Dshift(ot);
1330	gen_op_add_reg_T0(s->aflag, R_EDI);
1331	}
1332
1333	static inline void gen_lods(DisasContext *s, int ot)
1334	{
1335	gen_string_movl_A0_ESI(s);
1336	gen_op_ld_T0_A0(ot + s->mem_index);
1337	gen_op_mov_reg_T0(ot, R_EAX);
1338	gen_op_movl_T0_Dshift(ot);
1339	gen_op_add_reg_T0(s->aflag, R_ESI);
1340	}
1341
1342	static inline void gen_scas(DisasContext *s, int ot)
1343	{
1344	gen_op_mov_TN_reg(OT_LONG, 0, R_EAX);
1345	gen_string_movl_A0_EDI(s);
1346	gen_op_ld_T1_A0(ot + s->mem_index);
1347	gen_op_cmpl_T0_T1_cc();
1348	gen_op_movl_T0_Dshift(ot);
1349	gen_op_add_reg_T0(s->aflag, R_EDI);
1350	}
1351
1352	static inline void gen_cmps(DisasContext *s, int ot)
1353	{
1354	gen_string_movl_A0_ESI(s);
1355	gen_op_ld_T0_A0(ot + s->mem_index);
1356	gen_string_movl_A0_EDI(s);
1357	gen_op_ld_T1_A0(ot + s->mem_index);
1358	gen_op_cmpl_T0_T1_cc();
1359	gen_op_movl_T0_Dshift(ot);
1360	gen_op_add_reg_T0(s->aflag, R_ESI);
1361	gen_op_add_reg_T0(s->aflag, R_EDI);
1362	}
1363
1364	static inline void gen_ins(DisasContext *s, int ot)
1365	{
1366	if (use_icount)
1367	gen_io_start();
1368	gen_string_movl_A0_EDI(s);
1369	/* Note: we must do this dummy write first to be restartable in
1370	case of page fault. */
1371	gen_op_movl_T0_0();
1372	gen_op_st_T0_A0(ot + s->mem_index);
1373	gen_op_mov_TN_reg(OT_WORD, 1, R_EDX);
1374	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[1]);
1375	tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
1376	gen_helper_in_func(ot, cpu_T[0], cpu_tmp2_i32);
1377	gen_op_st_T0_A0(ot + s->mem_index);
1378	gen_op_movl_T0_Dshift(ot);
1379	gen_op_add_reg_T0(s->aflag, R_EDI);
1380	if (use_icount)
1381	gen_io_end();
1382	}
1383
1384	static inline void gen_outs(DisasContext *s, int ot)
1385	{
1386	if (use_icount)
1387	gen_io_start();
1388	gen_string_movl_A0_ESI(s);
1389	gen_op_ld_T0_A0(ot + s->mem_index);
1390
1391	gen_op_mov_TN_reg(OT_WORD, 1, R_EDX);
1392	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[1]);
1393	tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
1394	tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[0]);
1395	gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
1396
1397	gen_op_movl_T0_Dshift(ot);
1398	gen_op_add_reg_T0(s->aflag, R_ESI);
1399	if (use_icount)
1400	gen_io_end();
1401	}
1402
1403	/* same method as Valgrind : we generate jumps to current or next
1404	instruction */
1405	#define GEN_REPZ(op) \
1406	static inline void gen_repz_ ## op(DisasContext *s, int ot, \
1407	target_ulong cur_eip, target_ulong next_eip) \
1408	{ \
1409	int l2;\
1410	gen_update_cc_op(s); \
1411	l2 = gen_jz_ecx_string(s, next_eip); \
1412	gen_ ## op(s, ot); \
1413	gen_op_add_reg_im(s->aflag, R_ECX, -1); \
1414	/* a loop would cause two single step exceptions if ECX = 1 \
1415	before rep string_insn */ \
1416	if (!s->jmp_opt) \
1417	gen_op_jz_ecx(s->aflag, l2); \
1418	gen_jmp(s, cur_eip); \
1419	}
1420
1421	#define GEN_REPZ2(op) \
1422	static inline void gen_repz_ ## op(DisasContext *s, int ot, \
1423	target_ulong cur_eip, \
1424	target_ulong next_eip, \
1425	int nz) \
1426	{ \
1427	int l2;\
1428	gen_update_cc_op(s); \
1429	l2 = gen_jz_ecx_string(s, next_eip); \
1430	gen_ ## op(s, ot); \
1431	gen_op_add_reg_im(s->aflag, R_ECX, -1); \
1432	gen_op_set_cc_op(CC_OP_SUBB + ot); \
1433	gen_jcc1(s, CC_OP_SUBB + ot, (JCC_Z << 1) \| (nz ^ 1), l2); \
1434	if (!s->jmp_opt) \
1435	gen_op_jz_ecx(s->aflag, l2); \
1436	gen_jmp(s, cur_eip); \
1437	}
1438
1439	GEN_REPZ(movs)
1440	GEN_REPZ(stos)
1441	GEN_REPZ(lods)
1442	GEN_REPZ(ins)
1443	GEN_REPZ(outs)
1444	GEN_REPZ2(scas)
1445	GEN_REPZ2(cmps)
1446
1447	static void gen_helper_fp_arith_ST0_FT0(int op)
1448	{
1449	switch (op) {
1450	case 0: gen_helper_fadd_ST0_FT0(); break;
1451	case 1: gen_helper_fmul_ST0_FT0(); break;
1452	case 2: gen_helper_fcom_ST0_FT0(); break;
1453	case 3: gen_helper_fcom_ST0_FT0(); break;
1454	case 4: gen_helper_fsub_ST0_FT0(); break;
1455	case 5: gen_helper_fsubr_ST0_FT0(); break;
1456	case 6: gen_helper_fdiv_ST0_FT0(); break;
1457	case 7: gen_helper_fdivr_ST0_FT0(); break;
1458	}
1459	}
1460
1461	/* NOTE the exception in "r" op ordering */
1462	static void gen_helper_fp_arith_STN_ST0(int op, int opreg)
1463	{
1464	TCGv_i32 tmp = tcg_const_i32(opreg);
1465	switch (op) {
1466	case 0: gen_helper_fadd_STN_ST0(tmp); break;
1467	case 1: gen_helper_fmul_STN_ST0(tmp); break;
1468	case 4: gen_helper_fsubr_STN_ST0(tmp); break;
1469	case 5: gen_helper_fsub_STN_ST0(tmp); break;
1470	case 6: gen_helper_fdivr_STN_ST0(tmp); break;
1471	case 7: gen_helper_fdiv_STN_ST0(tmp); break;
1472	}
1473	}
1474
1475	/* if d == OR_TMP0, it means memory operand (address in A0) */
1476	static void gen_op(DisasContext *s1, int op, int ot, int d)
1477	{
1478	if (d != OR_TMP0) {
1479	gen_op_mov_TN_reg(ot, 0, d);
1480	} else {
1481	gen_op_ld_T0_A0(ot + s1->mem_index);
1482	}
1483	switch(op) {
1484	case OP_ADCL:
1485	if (s1->cc_op != CC_OP_DYNAMIC)
1486	gen_op_set_cc_op(s1->cc_op);
1487	gen_compute_eflags_c(cpu_tmp4);
1488	tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1489	tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
1490	if (d != OR_TMP0)
1491	gen_op_mov_reg_T0(ot, d);
1492	else
1493	gen_op_st_T0_A0(ot + s1->mem_index);
1494	tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
1495	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
1496	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp4);
1497	tcg_gen_shli_i32(cpu_tmp2_i32, cpu_tmp2_i32, 2);
1498	tcg_gen_addi_i32(cpu_cc_op, cpu_tmp2_i32, CC_OP_ADDB + ot);
1499	s1->cc_op = CC_OP_DYNAMIC;
1500	break;
1501	case OP_SBBL:
1502	if (s1->cc_op != CC_OP_DYNAMIC)
1503	gen_op_set_cc_op(s1->cc_op);
1504	gen_compute_eflags_c(cpu_tmp4);
1505	tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1506	tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
1507	if (d != OR_TMP0)
1508	gen_op_mov_reg_T0(ot, d);
1509	else
1510	gen_op_st_T0_A0(ot + s1->mem_index);
1511	tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
1512	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
1513	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp4);
1514	tcg_gen_shli_i32(cpu_tmp2_i32, cpu_tmp2_i32, 2);
1515	tcg_gen_addi_i32(cpu_cc_op, cpu_tmp2_i32, CC_OP_SUBB + ot);
1516	s1->cc_op = CC_OP_DYNAMIC;
1517	break;
1518	case OP_ADDL:
1519	gen_op_addl_T0_T1();
1520	if (d != OR_TMP0)
1521	gen_op_mov_reg_T0(ot, d);
1522	else
1523	gen_op_st_T0_A0(ot + s1->mem_index);
1524	gen_op_update2_cc();
1525	s1->cc_op = CC_OP_ADDB + ot;
1526	break;
1527	case OP_SUBL:
1528	tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1529	if (d != OR_TMP0)
1530	gen_op_mov_reg_T0(ot, d);
1531	else
1532	gen_op_st_T0_A0(ot + s1->mem_index);
1533	gen_op_update2_cc();
1534	s1->cc_op = CC_OP_SUBB + ot;
1535	break;
1536	default:
1537	case OP_ANDL:
1538	tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1539	if (d != OR_TMP0)
1540	gen_op_mov_reg_T0(ot, d);
1541	else
1542	gen_op_st_T0_A0(ot + s1->mem_index);
1543	gen_op_update1_cc();
1544	s1->cc_op = CC_OP_LOGICB + ot;
1545	break;
1546	case OP_ORL:
1547	tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1548	if (d != OR_TMP0)
1549	gen_op_mov_reg_T0(ot, d);
1550	else
1551	gen_op_st_T0_A0(ot + s1->mem_index);
1552	gen_op_update1_cc();
1553	s1->cc_op = CC_OP_LOGICB + ot;
1554	break;
1555	case OP_XORL:
1556	tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1557	if (d != OR_TMP0)
1558	gen_op_mov_reg_T0(ot, d);
1559	else
1560	gen_op_st_T0_A0(ot + s1->mem_index);
1561	gen_op_update1_cc();
1562	s1->cc_op = CC_OP_LOGICB + ot;
1563	break;
1564	case OP_CMPL:
1565	gen_op_cmpl_T0_T1_cc();
1566	s1->cc_op = CC_OP_SUBB + ot;
1567	break;
1568	}
1569	}
1570
1571	/* if d == OR_TMP0, it means memory operand (address in A0) */
1572	static void gen_inc(DisasContext *s1, int ot, int d, int c)
1573	{
1574	if (d != OR_TMP0)
1575	gen_op_mov_TN_reg(ot, 0, d);
1576	else
1577	gen_op_ld_T0_A0(ot + s1->mem_index);
1578	if (s1->cc_op != CC_OP_DYNAMIC)
1579	gen_op_set_cc_op(s1->cc_op);
1580	if (c > 0) {
1581	tcg_gen_addi_tl(cpu_T[0], cpu_T[0], 1);
1582	s1->cc_op = CC_OP_INCB + ot;
1583	} else {
1584	tcg_gen_addi_tl(cpu_T[0], cpu_T[0], -1);
1585	s1->cc_op = CC_OP_DECB + ot;
1586	}
1587	if (d != OR_TMP0)
1588	gen_op_mov_reg_T0(ot, d);
1589	else
1590	gen_op_st_T0_A0(ot + s1->mem_index);
1591	gen_compute_eflags_c(cpu_cc_src);
1592	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
1593	}
1594
1595	static void gen_shift_rm_T1(DisasContext *s, int ot, int op1,
1596	int is_right, int is_arith)
1597	{
1598	target_ulong mask;
1599	int shift_label;
1600	TCGv t0, t1;
1601
1602	if (ot == OT_QUAD)
1603	mask = 0x3f;
1604	else
1605	mask = 0x1f;
1606
1607	/* load */
1608	if (op1 == OR_TMP0)
1609	gen_op_ld_T0_A0(ot + s->mem_index);
1610	else
1611	gen_op_mov_TN_reg(ot, 0, op1);
1612
1613	tcg_gen_andi_tl(cpu_T[1], cpu_T[1], mask);
1614
1615	tcg_gen_addi_tl(cpu_tmp5, cpu_T[1], -1);
1616
1617	if (is_right) {
1618	if (is_arith) {
1619	gen_exts(ot, cpu_T[0]);
1620	tcg_gen_sar_tl(cpu_T3, cpu_T[0], cpu_tmp5);
1621	tcg_gen_sar_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1622	} else {
1623	gen_extu(ot, cpu_T[0]);
1624	tcg_gen_shr_tl(cpu_T3, cpu_T[0], cpu_tmp5);
1625	tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1626	}
1627	} else {
1628	tcg_gen_shl_tl(cpu_T3, cpu_T[0], cpu_tmp5);
1629	tcg_gen_shl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
1630	}
1631
1632	/* store */
1633	if (op1 == OR_TMP0)
1634	gen_op_st_T0_A0(ot + s->mem_index);
1635	else
1636	gen_op_mov_reg_T0(ot, op1);
1637
1638	/* update eflags if non zero shift */
1639	if (s->cc_op != CC_OP_DYNAMIC)
1640	gen_op_set_cc_op(s->cc_op);
1641
1642	/* XXX: inefficient */
1643	t0 = tcg_temp_local_new();
1644	t1 = tcg_temp_local_new();
1645
1646	tcg_gen_mov_tl(t0, cpu_T[0]);
1647	tcg_gen_mov_tl(t1, cpu_T3);
1648
1649	shift_label = gen_new_label();
1650	tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_T[1], 0, shift_label);
1651
1652	tcg_gen_mov_tl(cpu_cc_src, t1);
1653	tcg_gen_mov_tl(cpu_cc_dst, t0);
1654	if (is_right)
1655	tcg_gen_movi_i32(cpu_cc_op, CC_OP_SARB + ot);
1656	else
1657	tcg_gen_movi_i32(cpu_cc_op, CC_OP_SHLB + ot);
1658
1659	gen_set_label(shift_label);
1660	s->cc_op = CC_OP_DYNAMIC; /* cannot predict flags after */
1661
1662	tcg_temp_free(t0);
1663	tcg_temp_free(t1);
1664	}
1665
1666	static void gen_shift_rm_im(DisasContext *s, int ot, int op1, int op2,
1667	int is_right, int is_arith)
1668	{
1669	int mask;
1670
1671	if (ot == OT_QUAD)
1672	mask = 0x3f;
1673	else
1674	mask = 0x1f;
1675
1676	/* load */
1677	if (op1 == OR_TMP0)
1678	gen_op_ld_T0_A0(ot + s->mem_index);
1679	else
1680	gen_op_mov_TN_reg(ot, 0, op1);
1681
1682	op2 &= mask;
1683	if (op2 != 0) {
1684	if (is_right) {
1685	if (is_arith) {
1686	gen_exts(ot, cpu_T[0]);
1687	tcg_gen_sari_tl(cpu_tmp4, cpu_T[0], op2 - 1);
1688	tcg_gen_sari_tl(cpu_T[0], cpu_T[0], op2);
1689	} else {
1690	gen_extu(ot, cpu_T[0]);
1691	tcg_gen_shri_tl(cpu_tmp4, cpu_T[0], op2 - 1);
1692	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], op2);
1693	}
1694	} else {
1695	tcg_gen_shli_tl(cpu_tmp4, cpu_T[0], op2 - 1);
1696	tcg_gen_shli_tl(cpu_T[0], cpu_T[0], op2);
1697	}
1698	}
1699
1700	/* store */
1701	if (op1 == OR_TMP0)
1702	gen_op_st_T0_A0(ot + s->mem_index);
1703	else
1704	gen_op_mov_reg_T0(ot, op1);
1705
1706	/* update eflags if non zero shift */
1707	if (op2 != 0) {
1708	tcg_gen_mov_tl(cpu_cc_src, cpu_tmp4);
1709	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
1710	if (is_right)
1711	s->cc_op = CC_OP_SARB + ot;
1712	else
1713	s->cc_op = CC_OP_SHLB + ot;
1714	}
1715	}
1716
1717	static inline void tcg_gen_lshift(TCGv ret, TCGv arg1, target_long arg2)
1718	{
1719	if (arg2 >= 0)
1720	tcg_gen_shli_tl(ret, arg1, arg2);
1721	else
1722	tcg_gen_shri_tl(ret, arg1, -arg2);
1723	}
1724
1725	static void gen_rot_rm_T1(DisasContext *s, int ot, int op1,
1726	int is_right)
1727	{
1728	target_ulong mask;
1729	int label1, label2, data_bits;
1730	TCGv t0, t1, t2, a0;
1731
1732	/* XXX: inefficient, but we must use local temps */
1733	t0 = tcg_temp_local_new();
1734	t1 = tcg_temp_local_new();
1735	t2 = tcg_temp_local_new();
1736	a0 = tcg_temp_local_new();
1737
1738	if (ot == OT_QUAD)
1739	mask = 0x3f;
1740	else
1741	mask = 0x1f;
1742
1743	/* load */
1744	if (op1 == OR_TMP0) {
1745	tcg_gen_mov_tl(a0, cpu_A0);
1746	gen_op_ld_v(ot + s->mem_index, t0, a0);
1747	} else {
1748	gen_op_mov_v_reg(ot, t0, op1);
1749	}
1750
1751	tcg_gen_mov_tl(t1, cpu_T[1]);
1752
1753	tcg_gen_andi_tl(t1, t1, mask);
1754
1755	/* Must test zero case to avoid using undefined behaviour in TCG
1756	shifts. */
1757	label1 = gen_new_label();
1758	tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, label1);
1759
1760	if (ot <= OT_WORD)
1761	tcg_gen_andi_tl(cpu_tmp0, t1, (1 << (3 + ot)) - 1);
1762	else
1763	tcg_gen_mov_tl(cpu_tmp0, t1);
1764
1765	gen_extu(ot, t0);
1766	tcg_gen_mov_tl(t2, t0);
1767
1768	data_bits = 8 << ot;
1769	/* XXX: rely on behaviour of shifts when operand 2 overflows (XXX:
1770	fix TCG definition) */
1771	if (is_right) {
1772	tcg_gen_shr_tl(cpu_tmp4, t0, cpu_tmp0);
1773	tcg_gen_subfi_tl(cpu_tmp0, data_bits, cpu_tmp0);
1774	tcg_gen_shl_tl(t0, t0, cpu_tmp0);
1775	} else {
1776	tcg_gen_shl_tl(cpu_tmp4, t0, cpu_tmp0);
1777	tcg_gen_subfi_tl(cpu_tmp0, data_bits, cpu_tmp0);
1778	tcg_gen_shr_tl(t0, t0, cpu_tmp0);
1779	}
1780	tcg_gen_or_tl(t0, t0, cpu_tmp4);
1781
1782	gen_set_label(label1);
1783	/* store */
1784	if (op1 == OR_TMP0) {
1785	gen_op_st_v(ot + s->mem_index, t0, a0);
1786	} else {
1787	gen_op_mov_reg_v(ot, op1, t0);
1788	}
1789
1790	/* update eflags */
1791	if (s->cc_op != CC_OP_DYNAMIC)
1792	gen_op_set_cc_op(s->cc_op);
1793
1794	label2 = gen_new_label();
1795	tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, label2);
1796
1797	gen_compute_eflags(cpu_cc_src);
1798	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~(CC_O \| CC_C));
1799	tcg_gen_xor_tl(cpu_tmp0, t2, t0);
1800	tcg_gen_lshift(cpu_tmp0, cpu_tmp0, 11 - (data_bits - 1));
1801	tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, CC_O);
1802	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_tmp0);
1803	if (is_right) {
1804	tcg_gen_shri_tl(t0, t0, data_bits - 1);
1805	}
1806	tcg_gen_andi_tl(t0, t0, CC_C);
1807	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, t0);
1808
1809	tcg_gen_discard_tl(cpu_cc_dst);
1810	tcg_gen_movi_i32(cpu_cc_op, CC_OP_EFLAGS);
1811
1812	gen_set_label(label2);
1813	s->cc_op = CC_OP_DYNAMIC; /* cannot predict flags after */
1814
1815	tcg_temp_free(t0);
1816	tcg_temp_free(t1);
1817	tcg_temp_free(t2);
1818	tcg_temp_free(a0);
1819	}
1820
1821	static void gen_rot_rm_im(DisasContext *s, int ot, int op1, int op2,
1822	int is_right)
1823	{
1824	int mask;
1825	int data_bits;
1826	TCGv t0, t1, a0;
1827
1828	/* XXX: inefficient, but we must use local temps */
1829	t0 = tcg_temp_local_new();
1830	t1 = tcg_temp_local_new();
1831	a0 = tcg_temp_local_new();
1832
1833	if (ot == OT_QUAD)
1834	mask = 0x3f;
1835	else
1836	mask = 0x1f;
1837
1838	/* load */
1839	if (op1 == OR_TMP0) {
1840	tcg_gen_mov_tl(a0, cpu_A0);
1841	gen_op_ld_v(ot + s->mem_index, t0, a0);
1842	} else {
1843	gen_op_mov_v_reg(ot, t0, op1);
1844	}
1845
1846	gen_extu(ot, t0);
1847	tcg_gen_mov_tl(t1, t0);
1848
1849	op2 &= mask;
1850	data_bits = 8 << ot;
1851	if (op2 != 0) {
1852	int shift = op2 & ((1 << (3 + ot)) - 1);
1853	if (is_right) {
1854	tcg_gen_shri_tl(cpu_tmp4, t0, shift);
1855	tcg_gen_shli_tl(t0, t0, data_bits - shift);
1856	}
1857	else {
1858	tcg_gen_shli_tl(cpu_tmp4, t0, shift);
1859	tcg_gen_shri_tl(t0, t0, data_bits - shift);
1860	}
1861	tcg_gen_or_tl(t0, t0, cpu_tmp4);
1862	}
1863
1864	/* store */
1865	if (op1 == OR_TMP0) {
1866	gen_op_st_v(ot + s->mem_index, t0, a0);
1867	} else {
1868	gen_op_mov_reg_v(ot, op1, t0);
1869	}
1870
1871	if (op2 != 0) {
1872	/* update eflags */
1873	if (s->cc_op != CC_OP_DYNAMIC)
1874	gen_op_set_cc_op(s->cc_op);
1875
1876	gen_compute_eflags(cpu_cc_src);
1877	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~(CC_O \| CC_C));
1878	tcg_gen_xor_tl(cpu_tmp0, t1, t0);
1879	tcg_gen_lshift(cpu_tmp0, cpu_tmp0, 11 - (data_bits - 1));
1880	tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, CC_O);
1881	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_tmp0);
1882	if (is_right) {
1883	tcg_gen_shri_tl(t0, t0, data_bits - 1);
1884	}
1885	tcg_gen_andi_tl(t0, t0, CC_C);
1886	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, t0);
1887
1888	tcg_gen_discard_tl(cpu_cc_dst);
1889	tcg_gen_movi_i32(cpu_cc_op, CC_OP_EFLAGS);
1890	s->cc_op = CC_OP_EFLAGS;
1891	}
1892
1893	tcg_temp_free(t0);
1894	tcg_temp_free(t1);
1895	tcg_temp_free(a0);
1896	}
1897
1898	/* XXX: add faster immediate = 1 case */
1899	static void gen_rotc_rm_T1(DisasContext *s, int ot, int op1,
1900	int is_right)
1901	{
1902	int label1;
1903
1904	if (s->cc_op != CC_OP_DYNAMIC)
1905	gen_op_set_cc_op(s->cc_op);
1906
1907	/* load */
1908	if (op1 == OR_TMP0)
1909	gen_op_ld_T0_A0(ot + s->mem_index);
1910	else
1911	gen_op_mov_TN_reg(ot, 0, op1);
1912
1913	if (is_right) {
1914	switch (ot) {
1915	case 0: gen_helper_rcrb(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1916	case 1: gen_helper_rcrw(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1917	case 2: gen_helper_rcrl(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1918	#ifdef TARGET_X86_64
1919	case 3: gen_helper_rcrq(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1920	#endif
1921	}
1922	} else {
1923	switch (ot) {
1924	case 0: gen_helper_rclb(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1925	case 1: gen_helper_rclw(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1926	case 2: gen_helper_rcll(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1927	#ifdef TARGET_X86_64
1928	case 3: gen_helper_rclq(cpu_T[0], cpu_T[0], cpu_T[1]); break;
1929	#endif
1930	}
1931	}
1932	/* store */
1933	if (op1 == OR_TMP0)
1934	gen_op_st_T0_A0(ot + s->mem_index);
1935	else
1936	gen_op_mov_reg_T0(ot, op1);
1937
1938	/* update eflags */
1939	label1 = gen_new_label();
1940	tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_cc_tmp, -1, label1);
1941
1942	tcg_gen_mov_tl(cpu_cc_src, cpu_cc_tmp);
1943	tcg_gen_discard_tl(cpu_cc_dst);
1944	tcg_gen_movi_i32(cpu_cc_op, CC_OP_EFLAGS);
1945
1946	gen_set_label(label1);
1947	s->cc_op = CC_OP_DYNAMIC; /* cannot predict flags after */
1948	}
1949
1950	/* XXX: add faster immediate case */
1951	static void gen_shiftd_rm_T1_T3(DisasContext *s, int ot, int op1,
1952	int is_right)
1953	{
1954	int label1, label2, data_bits;
1955	target_ulong mask;
1956	TCGv t0, t1, t2, a0;
1957
1958	t0 = tcg_temp_local_new();
1959	t1 = tcg_temp_local_new();
1960	t2 = tcg_temp_local_new();
1961	a0 = tcg_temp_local_new();
1962
1963	if (ot == OT_QUAD)
1964	mask = 0x3f;
1965	else
1966	mask = 0x1f;
1967
1968	/* load */
1969	if (op1 == OR_TMP0) {
1970	tcg_gen_mov_tl(a0, cpu_A0);
1971	gen_op_ld_v(ot + s->mem_index, t0, a0);
1972	} else {
1973	gen_op_mov_v_reg(ot, t0, op1);
1974	}
1975
1976	tcg_gen_andi_tl(cpu_T3, cpu_T3, mask);
1977
1978	tcg_gen_mov_tl(t1, cpu_T[1]);
1979	tcg_gen_mov_tl(t2, cpu_T3);
1980
1981	/* Must test zero case to avoid using undefined behaviour in TCG
1982	shifts. */
1983	label1 = gen_new_label();
1984	tcg_gen_brcondi_tl(TCG_COND_EQ, t2, 0, label1);
1985
1986	tcg_gen_addi_tl(cpu_tmp5, t2, -1);
1987	if (ot == OT_WORD) {
1988	/* Note: we implement the Intel behaviour for shift count > 16 */
1989	if (is_right) {
1990	tcg_gen_andi_tl(t0, t0, 0xffff);
1991	tcg_gen_shli_tl(cpu_tmp0, t1, 16);
1992	tcg_gen_or_tl(t0, t0, cpu_tmp0);
1993	tcg_gen_ext32u_tl(t0, t0);
1994
1995	tcg_gen_shr_tl(cpu_tmp4, t0, cpu_tmp5);
1996
1997	/* only needed if count > 16, but a test would complicate */
1998	tcg_gen_subfi_tl(cpu_tmp5, 32, t2);
1999	tcg_gen_shl_tl(cpu_tmp0, t0, cpu_tmp5);
2000
2001	tcg_gen_shr_tl(t0, t0, t2);
2002
2003	tcg_gen_or_tl(t0, t0, cpu_tmp0);
2004	} else {
2005	/* XXX: not optimal */
2006	tcg_gen_andi_tl(t0, t0, 0xffff);
2007	tcg_gen_shli_tl(t1, t1, 16);
2008	tcg_gen_or_tl(t1, t1, t0);
2009	tcg_gen_ext32u_tl(t1, t1);
2010
2011	tcg_gen_shl_tl(cpu_tmp4, t0, cpu_tmp5);
2012	tcg_gen_subfi_tl(cpu_tmp0, 32, cpu_tmp5);
2013	tcg_gen_shr_tl(cpu_tmp5, t1, cpu_tmp0);
2014	tcg_gen_or_tl(cpu_tmp4, cpu_tmp4, cpu_tmp5);
2015
2016	tcg_gen_shl_tl(t0, t0, t2);
2017	tcg_gen_subfi_tl(cpu_tmp5, 32, t2);
2018	tcg_gen_shr_tl(t1, t1, cpu_tmp5);
2019	tcg_gen_or_tl(t0, t0, t1);
2020	}
2021	} else {
2022	data_bits = 8 << ot;
2023	if (is_right) {
2024	if (ot == OT_LONG)
2025	tcg_gen_ext32u_tl(t0, t0);
2026
2027	tcg_gen_shr_tl(cpu_tmp4, t0, cpu_tmp5);
2028
2029	tcg_gen_shr_tl(t0, t0, t2);
2030	tcg_gen_subfi_tl(cpu_tmp5, data_bits, t2);
2031	tcg_gen_shl_tl(t1, t1, cpu_tmp5);
2032	tcg_gen_or_tl(t0, t0, t1);
2033
2034	} else {
2035	if (ot == OT_LONG)
2036	tcg_gen_ext32u_tl(t1, t1);
2037
2038	tcg_gen_shl_tl(cpu_tmp4, t0, cpu_tmp5);
2039
2040	tcg_gen_shl_tl(t0, t0, t2);
2041	tcg_gen_subfi_tl(cpu_tmp5, data_bits, t2);
2042	tcg_gen_shr_tl(t1, t1, cpu_tmp5);
2043	tcg_gen_or_tl(t0, t0, t1);
2044	}
2045	}
2046	tcg_gen_mov_tl(t1, cpu_tmp4);
2047
2048	gen_set_label(label1);
2049	/* store */
2050	if (op1 == OR_TMP0) {
2051	gen_op_st_v(ot + s->mem_index, t0, a0);
2052	} else {
2053	gen_op_mov_reg_v(ot, op1, t0);
2054	}
2055
2056	/* update eflags */
2057	if (s->cc_op != CC_OP_DYNAMIC)
2058	gen_op_set_cc_op(s->cc_op);
2059
2060	label2 = gen_new_label();
2061	tcg_gen_brcondi_tl(TCG_COND_EQ, t2, 0, label2);
2062
2063	tcg_gen_mov_tl(cpu_cc_src, t1);
2064	tcg_gen_mov_tl(cpu_cc_dst, t0);
2065	if (is_right) {
2066	tcg_gen_movi_i32(cpu_cc_op, CC_OP_SARB + ot);
2067	} else {
2068	tcg_gen_movi_i32(cpu_cc_op, CC_OP_SHLB + ot);
2069	}
2070	gen_set_label(label2);
2071	s->cc_op = CC_OP_DYNAMIC; /* cannot predict flags after */
2072
2073	tcg_temp_free(t0);
2074	tcg_temp_free(t1);
2075	tcg_temp_free(t2);
2076	tcg_temp_free(a0);
2077	}
2078
2079	static void gen_shift(DisasContext *s1, int op, int ot, int d, int s)
2080	{
2081	if (s != OR_TMP1)
2082	gen_op_mov_TN_reg(ot, 1, s);
2083	switch(op) {
2084	case OP_ROL:
2085	gen_rot_rm_T1(s1, ot, d, 0);
2086	break;
2087	case OP_ROR:
2088	gen_rot_rm_T1(s1, ot, d, 1);
2089	break;
2090	case OP_SHL:
2091	case OP_SHL1:
2092	gen_shift_rm_T1(s1, ot, d, 0, 0);
2093	break;
2094	case OP_SHR:
2095	gen_shift_rm_T1(s1, ot, d, 1, 0);
2096	break;
2097	case OP_SAR:
2098	gen_shift_rm_T1(s1, ot, d, 1, 1);
2099	break;
2100	case OP_RCL:
2101	gen_rotc_rm_T1(s1, ot, d, 0);
2102	break;
2103	case OP_RCR:
2104	gen_rotc_rm_T1(s1, ot, d, 1);
2105	break;
2106	}
2107	}
2108
2109	static void gen_shifti(DisasContext *s1, int op, int ot, int d, int c)
2110	{
2111	switch(op) {
2112	case OP_ROL:
2113	gen_rot_rm_im(s1, ot, d, c, 0);
2114	break;
2115	case OP_ROR:
2116	gen_rot_rm_im(s1, ot, d, c, 1);
2117	break;
2118	case OP_SHL:
2119	case OP_SHL1:
2120	gen_shift_rm_im(s1, ot, d, c, 0, 0);
2121	break;
2122	case OP_SHR:
2123	gen_shift_rm_im(s1, ot, d, c, 1, 0);
2124	break;
2125	case OP_SAR:
2126	gen_shift_rm_im(s1, ot, d, c, 1, 1);
2127	break;
2128	default:
2129	/* currently not optimized */
2130	gen_op_movl_T1_im(c);
2131	gen_shift(s1, op, ot, d, OR_TMP1);
2132	break;
2133	}
2134	}
2135
2136	static void gen_lea_modrm(DisasContext s, int modrm, int reg_ptr, int *offset_ptr)
2137	{
2138	target_long disp;
2139	int havesib;
2140	int base;
2141	int index;
2142	int scale;
2143	int opreg;
2144	int mod, rm, code, override, must_add_seg;
2145
2146	override = s->override;
2147	must_add_seg = s->addseg;
2148	if (override >= 0)
2149	must_add_seg = 1;
2150	mod = (modrm >> 6) & 3;
2151	rm = modrm & 7;
2152
2153	if (s->aflag) {
2154
2155	havesib = 0;
2156	base = rm;
2157	index = 0;
2158	scale = 0;
2159
2160	if (base == 4) {
2161	havesib = 1;
2162	code = ldub_code(s->pc++);
2163	scale = (code >> 6) & 3;
2164	index = ((code >> 3) & 7) \| REX_X(s);
2165	base = (code & 7);
2166	}
2167	base \|= REX_B(s);
2168
2169	switch (mod) {
2170	case 0:
2171	if ((base & 7) == 5) {
2172	base = -1;
2173	disp = (int32_t)ldl_code(s->pc);
2174	s->pc += 4;
2175	if (CODE64(s) && !havesib) {
2176	disp += s->pc + s->rip_offset;
2177	}
2178	} else {
2179	disp = 0;
2180	}
2181	break;
2182	case 1:
2183	disp = (int8_t)ldub_code(s->pc++);
2184	break;
2185	default:
2186	case 2:
2187	#ifdef VBOX
2188	disp = (int32_t)ldl_code(s->pc);
2189	#else
2190	disp = ldl_code(s->pc);
2191	#endif
2192	s->pc += 4;
2193	break;
2194	}
2195
2196	if (base >= 0) {
2197	/* for correct popl handling with esp */
2198	if (base == 4 && s->popl_esp_hack)
2199	disp += s->popl_esp_hack;
2200	#ifdef TARGET_X86_64
2201	if (s->aflag == 2) {
2202	gen_op_movq_A0_reg(base);
2203	if (disp != 0) {
2204	gen_op_addq_A0_im(disp);
2205	}
2206	} else
2207	#endif
2208	{
2209	gen_op_movl_A0_reg(base);
2210	if (disp != 0)
2211	gen_op_addl_A0_im(disp);
2212	}
2213	} else {
2214	#ifdef TARGET_X86_64
2215	if (s->aflag == 2) {
2216	gen_op_movq_A0_im(disp);
2217	} else
2218	#endif
2219	{
2220	gen_op_movl_A0_im(disp);
2221	}
2222	}
2223	/* index == 4 means no index */
2224	if (havesib && (index != 4)) {
2225	#ifdef TARGET_X86_64
2226	if (s->aflag == 2) {
2227	gen_op_addq_A0_reg_sN(scale, index);
2228	} else
2229	#endif
2230	{
2231	gen_op_addl_A0_reg_sN(scale, index);
2232	}
2233	}
2234	if (must_add_seg) {
2235	if (override < 0) {
2236	if (base == R_EBP \|\| base == R_ESP)
2237	override = R_SS;
2238	else
2239	override = R_DS;
2240	}
2241	#ifdef TARGET_X86_64
2242	if (s->aflag == 2) {
2243	gen_op_addq_A0_seg(override);
2244	} else
2245	#endif
2246	{
2247	gen_op_addl_A0_seg(override);
2248	}
2249	}
2250	} else {
2251	switch (mod) {
2252	case 0:
2253	if (rm == 6) {
2254	disp = lduw_code(s->pc);
2255	s->pc += 2;
2256	gen_op_movl_A0_im(disp);
2257	rm = 0; /* avoid SS override */
2258	goto no_rm;
2259	} else {
2260	disp = 0;
2261	}
2262	break;
2263	case 1:
2264	disp = (int8_t)ldub_code(s->pc++);
2265	break;
2266	default:
2267	case 2:
2268	disp = lduw_code(s->pc);
2269	s->pc += 2;
2270	break;
2271	}
2272	switch(rm) {
2273	case 0:
2274	gen_op_movl_A0_reg(R_EBX);
2275	gen_op_addl_A0_reg_sN(0, R_ESI);
2276	break;
2277	case 1:
2278	gen_op_movl_A0_reg(R_EBX);
2279	gen_op_addl_A0_reg_sN(0, R_EDI);
2280	break;
2281	case 2:
2282	gen_op_movl_A0_reg(R_EBP);
2283	gen_op_addl_A0_reg_sN(0, R_ESI);
2284	break;
2285	case 3:
2286	gen_op_movl_A0_reg(R_EBP);
2287	gen_op_addl_A0_reg_sN(0, R_EDI);
2288	break;
2289	case 4:
2290	gen_op_movl_A0_reg(R_ESI);
2291	break;
2292	case 5:
2293	gen_op_movl_A0_reg(R_EDI);
2294	break;
2295	case 6:
2296	gen_op_movl_A0_reg(R_EBP);
2297	break;
2298	default:
2299	case 7:
2300	gen_op_movl_A0_reg(R_EBX);
2301	break;
2302	}
2303	if (disp != 0)
2304	gen_op_addl_A0_im(disp);
2305	gen_op_andl_A0_ffff();
2306	no_rm:
2307	if (must_add_seg) {
2308	if (override < 0) {
2309	if (rm == 2 \|\| rm == 3 \|\| rm == 6)
2310	override = R_SS;
2311	else
2312	override = R_DS;
2313	}
2314	gen_op_addl_A0_seg(override);
2315	}
2316	}
2317
2318	opreg = OR_A0;
2319	disp = 0;
2320	*reg_ptr = opreg;
2321	*offset_ptr = disp;
2322	}
2323
2324	static void gen_nop_modrm(DisasContext *s, int modrm)
2325	{
2326	int mod, rm, base, code;
2327
2328	mod = (modrm >> 6) & 3;
2329	if (mod == 3)
2330	return;
2331	rm = modrm & 7;
2332
2333	if (s->aflag) {
2334
2335	base = rm;
2336
2337	if (base == 4) {
2338	code = ldub_code(s->pc++);
2339	base = (code & 7);
2340	}
2341
2342	switch (mod) {
2343	case 0:
2344	if (base == 5) {
2345	s->pc += 4;
2346	}
2347	break;
2348	case 1:
2349	s->pc++;
2350	break;
2351	default:
2352	case 2:
2353	s->pc += 4;
2354	break;
2355	}
2356	} else {
2357	switch (mod) {
2358	case 0:
2359	if (rm == 6) {
2360	s->pc += 2;
2361	}
2362	break;
2363	case 1:
2364	s->pc++;
2365	break;
2366	default:
2367	case 2:
2368	s->pc += 2;
2369	break;
2370	}
2371	}
2372	}
2373
2374	/* used for LEA and MOV AX, mem */
2375	static void gen_add_A0_ds_seg(DisasContext *s)
2376	{
2377	int override, must_add_seg;
2378	must_add_seg = s->addseg;
2379	override = R_DS;
2380	if (s->override >= 0) {
2381	override = s->override;
2382	must_add_seg = 1;
2383	} else {
2384	override = R_DS;
2385	}
2386	if (must_add_seg) {
2387	#ifdef TARGET_X86_64
2388	if (CODE64(s)) {
2389	gen_op_addq_A0_seg(override);
2390	} else
2391	#endif
2392	{
2393	gen_op_addl_A0_seg(override);
2394	}
2395	}
2396	}
2397
2398	/* generate modrm memory load or store of 'reg'. TMP0 is used if reg ==
2399	OR_TMP0 */
2400	static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_store)
2401	{
2402	int mod, rm, opreg, disp;
2403
2404	mod = (modrm >> 6) & 3;
2405	rm = (modrm & 7) \| REX_B(s);
2406	if (mod == 3) {
2407	if (is_store) {
2408	if (reg != OR_TMP0)
2409	gen_op_mov_TN_reg(ot, 0, reg);
2410	gen_op_mov_reg_T0(ot, rm);
2411	} else {
2412	gen_op_mov_TN_reg(ot, 0, rm);
2413	if (reg != OR_TMP0)
2414	gen_op_mov_reg_T0(ot, reg);
2415	}
2416	} else {
2417	gen_lea_modrm(s, modrm, &opreg, &disp);
2418	if (is_store) {
2419	if (reg != OR_TMP0)
2420	gen_op_mov_TN_reg(ot, 0, reg);
2421	gen_op_st_T0_A0(ot + s->mem_index);
2422	} else {
2423	gen_op_ld_T0_A0(ot + s->mem_index);
2424	if (reg != OR_TMP0)
2425	gen_op_mov_reg_T0(ot, reg);
2426	}
2427	}
2428	}
2429
2430	static inline uint32_t insn_get(DisasContext *s, int ot)
2431	{
2432	uint32_t ret;
2433
2434	switch(ot) {
2435	case OT_BYTE:
2436	ret = ldub_code(s->pc);
2437	s->pc++;
2438	break;
2439	case OT_WORD:
2440	ret = lduw_code(s->pc);
2441	s->pc += 2;
2442	break;
2443	default:
2444	case OT_LONG:
2445	ret = ldl_code(s->pc);
2446	s->pc += 4;
2447	break;
2448	}
2449	return ret;
2450	}
2451
2452	static inline int insn_const_size(unsigned int ot)
2453	{
2454	if (ot <= OT_LONG)
2455	return 1 << ot;
2456	else
2457	return 4;
2458	}
2459
2460	static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip)
2461	{
2462	TranslationBlock *tb;
2463	target_ulong pc;
2464
2465	pc = s->cs_base + eip;
2466	tb = s->tb;
2467	/* NOTE: we handle the case where the TB spans two pages here */
2468	if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) \|\|
2469	(pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK)) {
2470	#ifdef VBOX
2471	gen_check_external_event();
2472	#endif /* VBOX */
2473	/* jump to same page: we can use a direct jump */
2474	tcg_gen_goto_tb(tb_num);
2475	gen_jmp_im(eip);
2476	tcg_gen_exit_tb((long)tb + tb_num);
2477	} else {
2478	/* jump to another page: currently not optimized */
2479	gen_jmp_im(eip);
2480	gen_eob(s);
2481	}
2482	}
2483
2484	static inline void gen_jcc(DisasContext *s, int b,
2485	target_ulong val, target_ulong next_eip)
2486	{
2487	int l1, l2, cc_op;
2488
2489	cc_op = s->cc_op;
2490	if (s->cc_op != CC_OP_DYNAMIC) {
2491	gen_op_set_cc_op(s->cc_op);
2492	s->cc_op = CC_OP_DYNAMIC;
2493	}
2494	if (s->jmp_opt) {
2495	l1 = gen_new_label();
2496	gen_jcc1(s, cc_op, b, l1);
2497
2498	gen_goto_tb(s, 0, next_eip);
2499
2500	gen_set_label(l1);
2501	gen_goto_tb(s, 1, val);
2502	s->is_jmp = 3;
2503	} else {
2504
2505	l1 = gen_new_label();
2506	l2 = gen_new_label();
2507	gen_jcc1(s, cc_op, b, l1);
2508
2509	gen_jmp_im(next_eip);
2510	tcg_gen_br(l2);
2511
2512	gen_set_label(l1);
2513	gen_jmp_im(val);
2514	gen_set_label(l2);
2515	gen_eob(s);
2516	}
2517	}
2518
2519	static void gen_setcc(DisasContext *s, int b)
2520	{
2521	int inv, jcc_op, l1;
2522	TCGv t0;
2523
2524	if (is_fast_jcc_case(s, b)) {
2525	/* nominal case: we use a jump */
2526	/* XXX: make it faster by adding new instructions in TCG */
2527	t0 = tcg_temp_local_new();
2528	tcg_gen_movi_tl(t0, 0);
2529	l1 = gen_new_label();
2530	gen_jcc1(s, s->cc_op, b ^ 1, l1);
2531	tcg_gen_movi_tl(t0, 1);
2532	gen_set_label(l1);
2533	tcg_gen_mov_tl(cpu_T[0], t0);
2534	tcg_temp_free(t0);
2535	} else {
2536	/* slow case: it is more efficient not to generate a jump,
2537	although it is questionnable whether this optimization is
2538	worth to */
2539	inv = b & 1;
2540	jcc_op = (b >> 1) & 7;
2541	gen_setcc_slow_T0(s, jcc_op);
2542	if (inv) {
2543	tcg_gen_xori_tl(cpu_T[0], cpu_T[0], 1);
2544	}
2545	}
2546	}
2547
2548	static inline void gen_op_movl_T0_seg(int seg_reg)
2549	{
2550	tcg_gen_ld32u_tl(cpu_T[0], cpu_env,
2551	offsetof(CPUX86State,segs[seg_reg].selector));
2552	}
2553
2554	static inline void gen_op_movl_seg_T0_vm(int seg_reg)
2555	{
2556	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff);
2557	tcg_gen_st32_tl(cpu_T[0], cpu_env,
2558	offsetof(CPUX86State,segs[seg_reg].selector));
2559	tcg_gen_shli_tl(cpu_T[0], cpu_T[0], 4);
2560	tcg_gen_st_tl(cpu_T[0], cpu_env,
2561	offsetof(CPUX86State,segs[seg_reg].base));
2562	#ifdef VBOX
2563	int flags = DESC_P_MASK \| DESC_S_MASK \| DESC_W_MASK;
2564	if (seg_reg == R_CS)
2565	flags \|= DESC_CS_MASK;
2566	gen_op_movl_T0_im(flags);
2567	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,segs[seg_reg].flags));
2568
2569	/* Set the limit to 0xffff. */
2570	gen_op_movl_T0_im(0xffff);
2571	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,segs[seg_reg].limit));
2572	#endif
2573	}
2574
2575	/* move T0 to seg_reg and compute if the CPU state may change. Never
2576	call this function with seg_reg == R_CS */
2577	static void gen_movl_seg_T0(DisasContext *s, int seg_reg, target_ulong cur_eip)
2578	{
2579	if (s->pe && !s->vm86) {
2580	/* XXX: optimize by finding processor state dynamically */
2581	if (s->cc_op != CC_OP_DYNAMIC)
2582	gen_op_set_cc_op(s->cc_op);
2583	gen_jmp_im(cur_eip);
2584	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
2585	gen_helper_load_seg(tcg_const_i32(seg_reg), cpu_tmp2_i32);
2586	/* abort translation because the addseg value may change or
2587	because ss32 may change. For R_SS, translation must always
2588	stop as a special handling must be done to disable hardware
2589	interrupts for the next instruction */
2590	if (seg_reg == R_SS \|\| (s->code32 && seg_reg < R_FS))
2591	s->is_jmp = 3;
2592	} else {
2593	gen_op_movl_seg_T0_vm(seg_reg);
2594	if (seg_reg == R_SS)
2595	s->is_jmp = 3;
2596	}
2597	}
2598
2599	static inline int svm_is_rep(int prefixes)
2600	{
2601	return ((prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) ? 8 : 0);
2602	}
2603
2604	static inline void
2605	gen_svm_check_intercept_param(DisasContext *s, target_ulong pc_start,
2606	uint32_t type, uint64_t param)
2607	{
2608	/* no SVM activated; fast case */
2609	if (likely(!(s->flags & HF_SVMI_MASK)))
2610	return;
2611	if (s->cc_op != CC_OP_DYNAMIC)
2612	gen_op_set_cc_op(s->cc_op);
2613	gen_jmp_im(pc_start - s->cs_base);
2614	gen_helper_svm_check_intercept_param(tcg_const_i32(type),
2615	tcg_const_i64(param));
2616	}
2617
2618	static inline void
2619	gen_svm_check_intercept(DisasContext *s, target_ulong pc_start, uint64_t type)
2620	{
2621	gen_svm_check_intercept_param(s, pc_start, type, 0);
2622	}
2623
2624	static inline void gen_stack_update(DisasContext *s, int addend)
2625	{
2626	#ifdef TARGET_X86_64
2627	if (CODE64(s)) {
2628	gen_op_add_reg_im(2, R_ESP, addend);
2629	} else
2630	#endif
2631	if (s->ss32) {
2632	gen_op_add_reg_im(1, R_ESP, addend);
2633	} else {
2634	gen_op_add_reg_im(0, R_ESP, addend);
2635	}
2636	}
2637
2638	/* generate a push. It depends on ss32, addseg and dflag */
2639	static void gen_push_T0(DisasContext *s)
2640	{
2641	#ifdef TARGET_X86_64
2642	if (CODE64(s)) {
2643	gen_op_movq_A0_reg(R_ESP);
2644	if (s->dflag) {
2645	gen_op_addq_A0_im(-8);
2646	gen_op_st_T0_A0(OT_QUAD + s->mem_index);
2647	} else {
2648	gen_op_addq_A0_im(-2);
2649	gen_op_st_T0_A0(OT_WORD + s->mem_index);
2650	}
2651	gen_op_mov_reg_A0(2, R_ESP);
2652	} else
2653	#endif
2654	{
2655	gen_op_movl_A0_reg(R_ESP);
2656	if (!s->dflag)
2657	gen_op_addl_A0_im(-2);
2658	else
2659	gen_op_addl_A0_im(-4);
2660	if (s->ss32) {
2661	if (s->addseg) {
2662	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2663	gen_op_addl_A0_seg(R_SS);
2664	}
2665	} else {
2666	gen_op_andl_A0_ffff();
2667	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2668	gen_op_addl_A0_seg(R_SS);
2669	}
2670	gen_op_st_T0_A0(s->dflag + 1 + s->mem_index);
2671	if (s->ss32 && !s->addseg)
2672	gen_op_mov_reg_A0(1, R_ESP);
2673	else
2674	gen_op_mov_reg_T1(s->ss32 + 1, R_ESP);
2675	}
2676	}
2677
2678	/* generate a push. It depends on ss32, addseg and dflag */
2679	/* slower version for T1, only used for call Ev */
2680	static void gen_push_T1(DisasContext *s)
2681	{
2682	#ifdef TARGET_X86_64
2683	if (CODE64(s)) {
2684	gen_op_movq_A0_reg(R_ESP);
2685	if (s->dflag) {
2686	gen_op_addq_A0_im(-8);
2687	gen_op_st_T1_A0(OT_QUAD + s->mem_index);
2688	} else {
2689	gen_op_addq_A0_im(-2);
2690	gen_op_st_T0_A0(OT_WORD + s->mem_index);
2691	}
2692	gen_op_mov_reg_A0(2, R_ESP);
2693	} else
2694	#endif
2695	{
2696	gen_op_movl_A0_reg(R_ESP);
2697	if (!s->dflag)
2698	gen_op_addl_A0_im(-2);
2699	else
2700	gen_op_addl_A0_im(-4);
2701	if (s->ss32) {
2702	if (s->addseg) {
2703	gen_op_addl_A0_seg(R_SS);
2704	}
2705	} else {
2706	gen_op_andl_A0_ffff();
2707	gen_op_addl_A0_seg(R_SS);
2708	}
2709	gen_op_st_T1_A0(s->dflag + 1 + s->mem_index);
2710
2711	if (s->ss32 && !s->addseg)
2712	gen_op_mov_reg_A0(1, R_ESP);
2713	else
2714	gen_stack_update(s, (-2) << s->dflag);
2715	}
2716	}
2717
2718	/* two step pop is necessary for precise exceptions */
2719	static void gen_pop_T0(DisasContext *s)
2720	{
2721	#ifdef TARGET_X86_64
2722	if (CODE64(s)) {
2723	gen_op_movq_A0_reg(R_ESP);
2724	gen_op_ld_T0_A0((s->dflag ? OT_QUAD : OT_WORD) + s->mem_index);
2725	} else
2726	#endif
2727	{
2728	gen_op_movl_A0_reg(R_ESP);
2729	if (s->ss32) {
2730	if (s->addseg)
2731	gen_op_addl_A0_seg(R_SS);
2732	} else {
2733	gen_op_andl_A0_ffff();
2734	gen_op_addl_A0_seg(R_SS);
2735	}
2736	gen_op_ld_T0_A0(s->dflag + 1 + s->mem_index);
2737	}
2738	}
2739
2740	static void gen_pop_update(DisasContext *s)
2741	{
2742	#ifdef TARGET_X86_64
2743	if (CODE64(s) && s->dflag) {
2744	gen_stack_update(s, 8);
2745	} else
2746	#endif
2747	{
2748	gen_stack_update(s, 2 << s->dflag);
2749	}
2750	}
2751
2752	static void gen_stack_A0(DisasContext *s)
2753	{
2754	gen_op_movl_A0_reg(R_ESP);
2755	if (!s->ss32)
2756	gen_op_andl_A0_ffff();
2757	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2758	if (s->addseg)
2759	gen_op_addl_A0_seg(R_SS);
2760	}
2761
2762	/* NOTE: wrap around in 16 bit not fully handled */
2763	static void gen_pusha(DisasContext *s)
2764	{
2765	int i;
2766	gen_op_movl_A0_reg(R_ESP);
2767	gen_op_addl_A0_im(-16 << s->dflag);
2768	if (!s->ss32)
2769	gen_op_andl_A0_ffff();
2770	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2771	if (s->addseg)
2772	gen_op_addl_A0_seg(R_SS);
2773	for(i = 0;i < 8; i++) {
2774	gen_op_mov_TN_reg(OT_LONG, 0, 7 - i);
2775	gen_op_st_T0_A0(OT_WORD + s->dflag + s->mem_index);
2776	gen_op_addl_A0_im(2 << s->dflag);
2777	}
2778	gen_op_mov_reg_T1(OT_WORD + s->ss32, R_ESP);
2779	}
2780
2781	/* NOTE: wrap around in 16 bit not fully handled */
2782	static void gen_popa(DisasContext *s)
2783	{
2784	int i;
2785	gen_op_movl_A0_reg(R_ESP);
2786	if (!s->ss32)
2787	gen_op_andl_A0_ffff();
2788	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2789	tcg_gen_addi_tl(cpu_T[1], cpu_T[1], 16 << s->dflag);
2790	if (s->addseg)
2791	gen_op_addl_A0_seg(R_SS);
2792	for(i = 0;i < 8; i++) {
2793	/* ESP is not reloaded */
2794	if (i != 3) {
2795	gen_op_ld_T0_A0(OT_WORD + s->dflag + s->mem_index);
2796	gen_op_mov_reg_T0(OT_WORD + s->dflag, 7 - i);
2797	}
2798	gen_op_addl_A0_im(2 << s->dflag);
2799	}
2800	gen_op_mov_reg_T1(OT_WORD + s->ss32, R_ESP);
2801	}
2802
2803	static void gen_enter(DisasContext *s, int esp_addend, int level)
2804	{
2805	int ot, opsize;
2806
2807	level &= 0x1f;
2808	#ifdef TARGET_X86_64
2809	if (CODE64(s)) {
2810	ot = s->dflag ? OT_QUAD : OT_WORD;
2811	opsize = 1 << ot;
2812
2813	gen_op_movl_A0_reg(R_ESP);
2814	gen_op_addq_A0_im(-opsize);
2815	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2816
2817	/* push bp */
2818	gen_op_mov_TN_reg(OT_LONG, 0, R_EBP);
2819	gen_op_st_T0_A0(ot + s->mem_index);
2820	if (level) {
2821	/* XXX: must save state */
2822	gen_helper_enter64_level(tcg_const_i32(level),
2823	tcg_const_i32((ot == OT_QUAD)),
2824	cpu_T[1]);
2825	}
2826	gen_op_mov_reg_T1(ot, R_EBP);
2827	tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
2828	gen_op_mov_reg_T1(OT_QUAD, R_ESP);
2829	} else
2830	#endif
2831	{
2832	ot = s->dflag + OT_WORD;
2833	opsize = 2 << s->dflag;
2834
2835	gen_op_movl_A0_reg(R_ESP);
2836	gen_op_addl_A0_im(-opsize);
2837	if (!s->ss32)
2838	gen_op_andl_A0_ffff();
2839	tcg_gen_mov_tl(cpu_T[1], cpu_A0);
2840	if (s->addseg)
2841	gen_op_addl_A0_seg(R_SS);
2842	/* push bp */
2843	gen_op_mov_TN_reg(OT_LONG, 0, R_EBP);
2844	gen_op_st_T0_A0(ot + s->mem_index);
2845	if (level) {
2846	/* XXX: must save state */
2847	gen_helper_enter_level(tcg_const_i32(level),
2848	tcg_const_i32(s->dflag),
2849	cpu_T[1]);
2850	}
2851	gen_op_mov_reg_T1(ot, R_EBP);
2852	tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
2853	gen_op_mov_reg_T1(OT_WORD + s->ss32, R_ESP);
2854	}
2855	}
2856
2857	static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
2858	{
2859	if (s->cc_op != CC_OP_DYNAMIC)
2860	gen_op_set_cc_op(s->cc_op);
2861	gen_jmp_im(cur_eip);
2862	gen_helper_raise_exception(tcg_const_i32(trapno));
2863	s->is_jmp = 3;
2864	}
2865
2866	/* an interrupt is different from an exception because of the
2867	privilege checks */
2868	static void gen_interrupt(DisasContext *s, int intno,
2869	target_ulong cur_eip, target_ulong next_eip)
2870	{
2871	if (s->cc_op != CC_OP_DYNAMIC)
2872	gen_op_set_cc_op(s->cc_op);
2873	gen_jmp_im(cur_eip);
2874	gen_helper_raise_interrupt(tcg_const_i32(intno),
2875	tcg_const_i32(next_eip - cur_eip));
2876	s->is_jmp = 3;
2877	}
2878
2879	static void gen_debug(DisasContext *s, target_ulong cur_eip)
2880	{
2881	if (s->cc_op != CC_OP_DYNAMIC)
2882	gen_op_set_cc_op(s->cc_op);
2883	gen_jmp_im(cur_eip);
2884	gen_helper_debug();
2885	s->is_jmp = 3;
2886	}
2887
2888	/* generate a generic end of block. Trace exception is also generated
2889	if needed */
2890	static void gen_eob(DisasContext *s)
2891	{
2892	if (s->cc_op != CC_OP_DYNAMIC)
2893	gen_op_set_cc_op(s->cc_op);
2894	if (s->tb->flags & HF_INHIBIT_IRQ_MASK) {
2895	gen_helper_reset_inhibit_irq();
2896	}
2897	if (s->tb->flags & HF_RF_MASK) {
2898	gen_helper_reset_rf();
2899	}
2900	if ( s->singlestep_enabled
2901	#ifdef VBOX
2902	&& ( !(cpu_single_env->state & CPU_EMULATE_SINGLE_STEP)
2903	\|\| !(s->prefix & (PREFIX_REPNZ \| PREFIX_REPZ) ))
2904	#endif
2905	) {
2906	gen_helper_debug();
2907	} else if (s->tf) {
2908	gen_helper_single_step();
2909	} else {
2910	tcg_gen_exit_tb(0);
2911	}
2912	s->is_jmp = 3;
2913	}
2914
2915	/* generate a jump to eip. No segment change must happen before as a
2916	direct call to the next block may occur */
2917	static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num)
2918	{
2919	if (s->jmp_opt) {
2920	if (s->cc_op != CC_OP_DYNAMIC) {
2921	gen_op_set_cc_op(s->cc_op);
2922	s->cc_op = CC_OP_DYNAMIC;
2923	}
2924	gen_goto_tb(s, tb_num, eip);
2925	s->is_jmp = 3;
2926	} else {
2927	gen_jmp_im(eip);
2928	gen_eob(s);
2929	}
2930	}
2931
2932	static void gen_jmp(DisasContext *s, target_ulong eip)
2933	{
2934	gen_jmp_tb(s, eip, 0);
2935	}
2936
2937	static inline void gen_ldq_env_A0(int idx, int offset)
2938	{
2939	int mem_index = (idx >> 2) - 1;
2940	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0, mem_index);
2941	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset);
2942	}
2943
2944	static inline void gen_stq_env_A0(int idx, int offset)
2945	{
2946	int mem_index = (idx >> 2) - 1;
2947	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset);
2948	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0, mem_index);
2949	}
2950
2951	static inline void gen_ldo_env_A0(int idx, int offset)
2952	{
2953	int mem_index = (idx >> 2) - 1;
2954	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0, mem_index);
2955	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
2956	tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
2957	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_tmp0, mem_index);
2958	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
2959	}
2960
2961	static inline void gen_sto_env_A0(int idx, int offset)
2962	{
2963	int mem_index = (idx >> 2) - 1;
2964	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
2965	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0, mem_index);
2966	tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
2967	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
2968	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_tmp0, mem_index);
2969	}
2970
2971	static inline void gen_op_movo(int d_offset, int s_offset)
2972	{
2973	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset);
2974	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset);
2975	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + 8);
2976	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + 8);
2977	}
2978
2979	static inline void gen_op_movq(int d_offset, int s_offset)
2980	{
2981	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset);
2982	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset);
2983	}
2984
2985	static inline void gen_op_movl(int d_offset, int s_offset)
2986	{
2987	tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env, s_offset);
2988	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, d_offset);
2989	}
2990
2991	static inline void gen_op_movq_env_0(int d_offset)
2992	{
2993	tcg_gen_movi_i64(cpu_tmp1_i64, 0);
2994	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset);
2995	}
2996
2997	#define SSE_SPECIAL ((void *)1)
2998	#define SSE_DUMMY ((void *)2)
2999
3000	#define MMX_OP2(x) { gen_helper_ ## x ## _mmx, gen_helper_ ## x ## _xmm }
3001	#define SSE_FOP(x) { gen_helper_ ## x ## ps, gen_helper_ ## x ## pd, \
3002	gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, }
3003
3004	static void *sse_op_table1[256][4] = {
3005	/* 3DNow! extensions */
3006	[0x0e] = { SSE_DUMMY }, /* femms */
3007	[0x0f] = { SSE_DUMMY }, /* pf... */
3008	/* pure SSE operations */
3009	[0x10] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movups, movupd, movss, movsd */
3010	[0x11] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movups, movupd, movss, movsd */
3011	[0x12] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movlps, movlpd, movsldup, movddup */
3012	[0x13] = { SSE_SPECIAL, SSE_SPECIAL }, /* movlps, movlpd */
3013	[0x14] = { gen_helper_punpckldq_xmm, gen_helper_punpcklqdq_xmm },
3014	[0x15] = { gen_helper_punpckhdq_xmm, gen_helper_punpckhqdq_xmm },
3015	[0x16] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movhps, movhpd, movshdup */
3016	[0x17] = { SSE_SPECIAL, SSE_SPECIAL }, /* movhps, movhpd */
3017
3018	[0x28] = { SSE_SPECIAL, SSE_SPECIAL }, /* movaps, movapd */
3019	[0x29] = { SSE_SPECIAL, SSE_SPECIAL }, /* movaps, movapd */
3020	[0x2a] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvtpi2ps, cvtpi2pd, cvtsi2ss, cvtsi2sd */
3021	[0x2b] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movntps, movntpd, movntss, movntsd */
3022	[0x2c] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvttps2pi, cvttpd2pi, cvttsd2si, cvttss2si */
3023	[0x2d] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvtps2pi, cvtpd2pi, cvtsd2si, cvtss2si */
3024	[0x2e] = { gen_helper_ucomiss, gen_helper_ucomisd },
3025	[0x2f] = { gen_helper_comiss, gen_helper_comisd },
3026	[0x50] = { SSE_SPECIAL, SSE_SPECIAL }, /* movmskps, movmskpd */
3027	[0x51] = SSE_FOP(sqrt),
3028	[0x52] = { gen_helper_rsqrtps, NULL, gen_helper_rsqrtss, NULL },
3029	[0x53] = { gen_helper_rcpps, NULL, gen_helper_rcpss, NULL },
3030	[0x54] = { gen_helper_pand_xmm, gen_helper_pand_xmm }, /* andps, andpd */
3031	[0x55] = { gen_helper_pandn_xmm, gen_helper_pandn_xmm }, /* andnps, andnpd */
3032	[0x56] = { gen_helper_por_xmm, gen_helper_por_xmm }, /* orps, orpd */
3033	[0x57] = { gen_helper_pxor_xmm, gen_helper_pxor_xmm }, /* xorps, xorpd */
3034	[0x58] = SSE_FOP(add),
3035	[0x59] = SSE_FOP(mul),
3036	[0x5a] = { gen_helper_cvtps2pd, gen_helper_cvtpd2ps,
3037	gen_helper_cvtss2sd, gen_helper_cvtsd2ss },
3038	[0x5b] = { gen_helper_cvtdq2ps, gen_helper_cvtps2dq, gen_helper_cvttps2dq },
3039	[0x5c] = SSE_FOP(sub),
3040	[0x5d] = SSE_FOP(min),
3041	[0x5e] = SSE_FOP(div),
3042	[0x5f] = SSE_FOP(max),
3043
3044	[0xc2] = SSE_FOP(cmpeq),
3045	[0xc6] = { gen_helper_shufps, gen_helper_shufpd },
3046
3047	[0x38] = { SSE_SPECIAL, SSE_SPECIAL, NULL, SSE_SPECIAL }, /* SSSE3/SSE4 */
3048	[0x3a] = { SSE_SPECIAL, SSE_SPECIAL }, /* SSSE3/SSE4 */
3049
3050	/* MMX ops and their SSE extensions */
3051	[0x60] = MMX_OP2(punpcklbw),
3052	[0x61] = MMX_OP2(punpcklwd),
3053	[0x62] = MMX_OP2(punpckldq),
3054	[0x63] = MMX_OP2(packsswb),
3055	[0x64] = MMX_OP2(pcmpgtb),
3056	[0x65] = MMX_OP2(pcmpgtw),
3057	[0x66] = MMX_OP2(pcmpgtl),
3058	[0x67] = MMX_OP2(packuswb),
3059	[0x68] = MMX_OP2(punpckhbw),
3060	[0x69] = MMX_OP2(punpckhwd),
3061	[0x6a] = MMX_OP2(punpckhdq),
3062	[0x6b] = MMX_OP2(packssdw),
3063	[0x6c] = { NULL, gen_helper_punpcklqdq_xmm },
3064	[0x6d] = { NULL, gen_helper_punpckhqdq_xmm },
3065	[0x6e] = { SSE_SPECIAL, SSE_SPECIAL }, /* movd mm, ea */
3066	[0x6f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, , movqdu */
3067	[0x70] = { gen_helper_pshufw_mmx,
3068	gen_helper_pshufd_xmm,
3069	gen_helper_pshufhw_xmm,
3070	gen_helper_pshuflw_xmm },
3071	[0x71] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftw */
3072	[0x72] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftd */
3073	[0x73] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftq */
3074	[0x74] = MMX_OP2(pcmpeqb),
3075	[0x75] = MMX_OP2(pcmpeqw),
3076	[0x76] = MMX_OP2(pcmpeql),
3077	[0x77] = { SSE_DUMMY }, /* emms */
3078	[0x78] = { NULL, SSE_SPECIAL, NULL, SSE_SPECIAL }, /* extrq_i, insertq_i */
3079	[0x79] = { NULL, gen_helper_extrq_r, NULL, gen_helper_insertq_r },
3080	[0x7c] = { NULL, gen_helper_haddpd, NULL, gen_helper_haddps },
3081	[0x7d] = { NULL, gen_helper_hsubpd, NULL, gen_helper_hsubps },
3082	[0x7e] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movd, movd, , movq */
3083	[0x7f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, movdqu */
3084	[0xc4] = { SSE_SPECIAL, SSE_SPECIAL }, /* pinsrw */
3085	[0xc5] = { SSE_SPECIAL, SSE_SPECIAL }, /* pextrw */
3086	[0xd0] = { NULL, gen_helper_addsubpd, NULL, gen_helper_addsubps },
3087	[0xd1] = MMX_OP2(psrlw),
3088	[0xd2] = MMX_OP2(psrld),
3089	[0xd3] = MMX_OP2(psrlq),
3090	[0xd4] = MMX_OP2(paddq),
3091	[0xd5] = MMX_OP2(pmullw),
3092	[0xd6] = { NULL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },
3093	[0xd7] = { SSE_SPECIAL, SSE_SPECIAL }, /* pmovmskb */
3094	[0xd8] = MMX_OP2(psubusb),
3095	[0xd9] = MMX_OP2(psubusw),
3096	[0xda] = MMX_OP2(pminub),
3097	[0xdb] = MMX_OP2(pand),
3098	[0xdc] = MMX_OP2(paddusb),
3099	[0xdd] = MMX_OP2(paddusw),
3100	[0xde] = MMX_OP2(pmaxub),
3101	[0xdf] = MMX_OP2(pandn),
3102	[0xe0] = MMX_OP2(pavgb),
3103	[0xe1] = MMX_OP2(psraw),
3104	[0xe2] = MMX_OP2(psrad),
3105	[0xe3] = MMX_OP2(pavgw),
3106	[0xe4] = MMX_OP2(pmulhuw),
3107	[0xe5] = MMX_OP2(pmulhw),
3108	[0xe6] = { NULL, gen_helper_cvttpd2dq, gen_helper_cvtdq2pd, gen_helper_cvtpd2dq },
3109	[0xe7] = { SSE_SPECIAL , SSE_SPECIAL }, /* movntq, movntq */
3110	[0xe8] = MMX_OP2(psubsb),
3111	[0xe9] = MMX_OP2(psubsw),
3112	[0xea] = MMX_OP2(pminsw),
3113	[0xeb] = MMX_OP2(por),
3114	[0xec] = MMX_OP2(paddsb),
3115	[0xed] = MMX_OP2(paddsw),
3116	[0xee] = MMX_OP2(pmaxsw),
3117	[0xef] = MMX_OP2(pxor),
3118	[0xf0] = { NULL, NULL, NULL, SSE_SPECIAL }, /* lddqu */
3119	[0xf1] = MMX_OP2(psllw),
3120	[0xf2] = MMX_OP2(pslld),
3121	[0xf3] = MMX_OP2(psllq),
3122	[0xf4] = MMX_OP2(pmuludq),
3123	[0xf5] = MMX_OP2(pmaddwd),
3124	[0xf6] = MMX_OP2(psadbw),
3125	[0xf7] = MMX_OP2(maskmov),
3126	[0xf8] = MMX_OP2(psubb),
3127	[0xf9] = MMX_OP2(psubw),
3128	[0xfa] = MMX_OP2(psubl),
3129	[0xfb] = MMX_OP2(psubq),
3130	[0xfc] = MMX_OP2(paddb),
3131	[0xfd] = MMX_OP2(paddw),
3132	[0xfe] = MMX_OP2(paddl),
3133	};
3134
3135	static void sse_op_table2[3 8][2] = {
3136	[0 + 2] = MMX_OP2(psrlw),
3137	[0 + 4] = MMX_OP2(psraw),
3138	[0 + 6] = MMX_OP2(psllw),
3139	[8 + 2] = MMX_OP2(psrld),
3140	[8 + 4] = MMX_OP2(psrad),
3141	[8 + 6] = MMX_OP2(pslld),
3142	[16 + 2] = MMX_OP2(psrlq),
3143	[16 + 3] = { NULL, gen_helper_psrldq_xmm },
3144	[16 + 6] = MMX_OP2(psllq),
3145	[16 + 7] = { NULL, gen_helper_pslldq_xmm },
3146	};
3147
3148	static void sse_op_table3[4 3] = {
3149	gen_helper_cvtsi2ss,
3150	gen_helper_cvtsi2sd,
3151	X86_64_ONLY(gen_helper_cvtsq2ss),
3152	X86_64_ONLY(gen_helper_cvtsq2sd),
3153
3154	gen_helper_cvttss2si,
3155	gen_helper_cvttsd2si,
3156	X86_64_ONLY(gen_helper_cvttss2sq),
3157	X86_64_ONLY(gen_helper_cvttsd2sq),
3158
3159	gen_helper_cvtss2si,
3160	gen_helper_cvtsd2si,
3161	X86_64_ONLY(gen_helper_cvtss2sq),
3162	X86_64_ONLY(gen_helper_cvtsd2sq),
3163	};
3164
3165	static void *sse_op_table4[8][4] = {
3166	SSE_FOP(cmpeq),
3167	SSE_FOP(cmplt),
3168	SSE_FOP(cmple),
3169	SSE_FOP(cmpunord),
3170	SSE_FOP(cmpneq),
3171	SSE_FOP(cmpnlt),
3172	SSE_FOP(cmpnle),
3173	SSE_FOP(cmpord),
3174	};
3175
3176	static void *sse_op_table5[256] = {
3177	[0x0c] = gen_helper_pi2fw,
3178	[0x0d] = gen_helper_pi2fd,
3179	[0x1c] = gen_helper_pf2iw,
3180	[0x1d] = gen_helper_pf2id,
3181	[0x8a] = gen_helper_pfnacc,
3182	[0x8e] = gen_helper_pfpnacc,
3183	[0x90] = gen_helper_pfcmpge,
3184	[0x94] = gen_helper_pfmin,
3185	[0x96] = gen_helper_pfrcp,
3186	[0x97] = gen_helper_pfrsqrt,
3187	[0x9a] = gen_helper_pfsub,
3188	[0x9e] = gen_helper_pfadd,
3189	[0xa0] = gen_helper_pfcmpgt,
3190	[0xa4] = gen_helper_pfmax,
3191	[0xa6] = gen_helper_movq, /* pfrcpit1; no need to actually increase precision */
3192	[0xa7] = gen_helper_movq, /* pfrsqit1 */
3193	[0xaa] = gen_helper_pfsubr,
3194	[0xae] = gen_helper_pfacc,
3195	[0xb0] = gen_helper_pfcmpeq,
3196	[0xb4] = gen_helper_pfmul,
3197	[0xb6] = gen_helper_movq, /* pfrcpit2 */
3198	[0xb7] = gen_helper_pmulhrw_mmx,
3199	[0xbb] = gen_helper_pswapd,
3200	[0xbf] = gen_helper_pavgb_mmx /* pavgusb */
3201	};
3202
3203	struct sse_op_helper_s {
3204	void *op[2]; uint32_t ext_mask;
3205	};
3206	#define SSSE3_OP(x) { MMX_OP2(x), CPUID_EXT_SSSE3 }
3207	#define SSE41_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE41 }
3208	#define SSE42_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE42 }
3209	#define SSE41_SPECIAL { { NULL, SSE_SPECIAL }, CPUID_EXT_SSE41 }
3210	static struct sse_op_helper_s sse_op_table6[256] = {
3211	[0x00] = SSSE3_OP(pshufb),
3212	[0x01] = SSSE3_OP(phaddw),
3213	[0x02] = SSSE3_OP(phaddd),
3214	[0x03] = SSSE3_OP(phaddsw),
3215	[0x04] = SSSE3_OP(pmaddubsw),
3216	[0x05] = SSSE3_OP(phsubw),
3217	[0x06] = SSSE3_OP(phsubd),
3218	[0x07] = SSSE3_OP(phsubsw),
3219	[0x08] = SSSE3_OP(psignb),
3220	[0x09] = SSSE3_OP(psignw),
3221	[0x0a] = SSSE3_OP(psignd),
3222	[0x0b] = SSSE3_OP(pmulhrsw),
3223	[0x10] = SSE41_OP(pblendvb),
3224	[0x14] = SSE41_OP(blendvps),
3225	[0x15] = SSE41_OP(blendvpd),
3226	[0x17] = SSE41_OP(ptest),
3227	[0x1c] = SSSE3_OP(pabsb),
3228	[0x1d] = SSSE3_OP(pabsw),
3229	[0x1e] = SSSE3_OP(pabsd),
3230	[0x20] = SSE41_OP(pmovsxbw),
3231	[0x21] = SSE41_OP(pmovsxbd),
3232	[0x22] = SSE41_OP(pmovsxbq),
3233	[0x23] = SSE41_OP(pmovsxwd),
3234	[0x24] = SSE41_OP(pmovsxwq),
3235	[0x25] = SSE41_OP(pmovsxdq),
3236	[0x28] = SSE41_OP(pmuldq),
3237	[0x29] = SSE41_OP(pcmpeqq),
3238	[0x2a] = SSE41_SPECIAL, /* movntqda */
3239	[0x2b] = SSE41_OP(packusdw),
3240	[0x30] = SSE41_OP(pmovzxbw),
3241	[0x31] = SSE41_OP(pmovzxbd),
3242	[0x32] = SSE41_OP(pmovzxbq),
3243	[0x33] = SSE41_OP(pmovzxwd),
3244	[0x34] = SSE41_OP(pmovzxwq),
3245	[0x35] = SSE41_OP(pmovzxdq),
3246	[0x37] = SSE42_OP(pcmpgtq),
3247	[0x38] = SSE41_OP(pminsb),
3248	[0x39] = SSE41_OP(pminsd),
3249	[0x3a] = SSE41_OP(pminuw),
3250	[0x3b] = SSE41_OP(pminud),
3251	[0x3c] = SSE41_OP(pmaxsb),
3252	[0x3d] = SSE41_OP(pmaxsd),
3253	[0x3e] = SSE41_OP(pmaxuw),
3254	[0x3f] = SSE41_OP(pmaxud),
3255	[0x40] = SSE41_OP(pmulld),
3256	[0x41] = SSE41_OP(phminposuw),
3257	};
3258
3259	static struct sse_op_helper_s sse_op_table7[256] = {
3260	[0x08] = SSE41_OP(roundps),
3261	[0x09] = SSE41_OP(roundpd),
3262	[0x0a] = SSE41_OP(roundss),
3263	[0x0b] = SSE41_OP(roundsd),
3264	[0x0c] = SSE41_OP(blendps),
3265	[0x0d] = SSE41_OP(blendpd),
3266	[0x0e] = SSE41_OP(pblendw),
3267	[0x0f] = SSSE3_OP(palignr),
3268	[0x14] = SSE41_SPECIAL, /* pextrb */
3269	[0x15] = SSE41_SPECIAL, /* pextrw */
3270	[0x16] = SSE41_SPECIAL, /* pextrd/pextrq */
3271	[0x17] = SSE41_SPECIAL, /* extractps */
3272	[0x20] = SSE41_SPECIAL, /* pinsrb */
3273	[0x21] = SSE41_SPECIAL, /* insertps */
3274	[0x22] = SSE41_SPECIAL, /* pinsrd/pinsrq */
3275	[0x40] = SSE41_OP(dpps),
3276	[0x41] = SSE41_OP(dppd),
3277	[0x42] = SSE41_OP(mpsadbw),
3278	[0x60] = SSE42_OP(pcmpestrm),
3279	[0x61] = SSE42_OP(pcmpestri),
3280	[0x62] = SSE42_OP(pcmpistrm),
3281	[0x63] = SSE42_OP(pcmpistri),
3282	};
3283
3284	static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
3285	{
3286	int b1, op1_offset, op2_offset, is_xmm, val, ot;
3287	int modrm, mod, rm, reg, reg_addr, offset_addr;
3288	void *sse_op2;
3289
3290	b &= 0xff;
3291	if (s->prefix & PREFIX_DATA)
3292	b1 = 1;
3293	else if (s->prefix & PREFIX_REPZ)
3294	b1 = 2;
3295	else if (s->prefix & PREFIX_REPNZ)
3296	b1 = 3;
3297	else
3298	b1 = 0;
3299	sse_op2 = sse_op_table1[b][b1];
3300	if (!sse_op2)
3301	goto illegal_op;
3302	if ((b <= 0x5f && b >= 0x10) \|\| b == 0xc6 \|\| b == 0xc2) {
3303	is_xmm = 1;
3304	} else {
3305	if (b1 == 0) {
3306	/* MMX case */
3307	is_xmm = 0;
3308	} else {
3309	is_xmm = 1;
3310	}
3311	}
3312	/* simple MMX/SSE operation */
3313	if (s->flags & HF_TS_MASK) {
3314	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
3315	return;
3316	}
3317	if (s->flags & HF_EM_MASK) {
3318	illegal_op:
3319	gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
3320	return;
3321	}
3322	if (is_xmm && !(s->flags & HF_OSFXSR_MASK))
3323	if ((b != 0x38 && b != 0x3a) \|\| (s->prefix & PREFIX_DATA))
3324	goto illegal_op;
3325	if (b == 0x0e) {
3326	if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
3327	goto illegal_op;
3328	/* femms */
3329	gen_helper_emms();
3330	return;
3331	}
3332	if (b == 0x77) {
3333	/* emms */
3334	gen_helper_emms();
3335	return;
3336	}
3337	/* prepare MMX state (XXX: optimize by storing fptt and fptags in
3338	the static cpu state) */
3339	if (!is_xmm) {
3340	gen_helper_enter_mmx();
3341	}
3342
3343	modrm = ldub_code(s->pc++);
3344	reg = ((modrm >> 3) & 7);
3345	if (is_xmm)
3346	reg \|= rex_r;
3347	mod = (modrm >> 6) & 3;
3348	if (sse_op2 == SSE_SPECIAL) {
3349	b \|= (b1 << 8);
3350	switch(b) {
3351	case 0x0e7: /* movntq */
3352	if (mod == 3)
3353	goto illegal_op;
3354	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3355	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
3356	break;
3357	case 0x1e7: /* movntdq */
3358	case 0x02b: /* movntps */
3359	case 0x12b: /* movntps */
3360	if (mod == 3)
3361	goto illegal_op;
3362	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3363	gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3364	break;
3365	case 0x3f0: /* lddqu */
3366	if (mod == 3)
3367	goto illegal_op;
3368	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3369	gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3370	break;
3371	case 0x22b: /* movntss */
3372	case 0x32b: /* movntsd */
3373	if (mod == 3)
3374	goto illegal_op;
3375	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3376	if (b1 & 1) {
3377	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,
3378	xmm_regs[reg]));
3379	} else {
3380	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
3381	xmm_regs[reg].XMM_L(0)));
3382	gen_op_st_T0_A0(OT_LONG + s->mem_index);
3383	}
3384	break;
3385	case 0x6e: /* movd mm, ea */
3386	#ifdef TARGET_X86_64
3387	if (s->dflag == 2) {
3388	gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
3389	tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
3390	} else
3391	#endif
3392	{
3393	gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
3394	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3395	offsetof(CPUX86State,fpregs[reg].mmx));
3396	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
3397	gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32);
3398	}
3399	break;
3400	case 0x16e: /* movd xmm, ea */
3401	#ifdef TARGET_X86_64
3402	if (s->dflag == 2) {
3403	gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
3404	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3405	offsetof(CPUX86State,xmm_regs[reg]));
3406	gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
3407	} else
3408	#endif
3409	{
3410	gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
3411	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3412	offsetof(CPUX86State,xmm_regs[reg]));
3413	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
3414	gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32);
3415	}
3416	break;
3417	case 0x6f: /* movq mm, ea */
3418	if (mod != 3) {
3419	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3420	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
3421	} else {
3422	rm = (modrm & 7);
3423	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env,
3424	offsetof(CPUX86State,fpregs[rm].mmx));
3425	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env,
3426	offsetof(CPUX86State,fpregs[reg].mmx));
3427	}
3428	break;
3429	case 0x010: /* movups */
3430	case 0x110: /* movupd */
3431	case 0x028: /* movaps */
3432	case 0x128: /* movapd */
3433	case 0x16f: /* movdqa xmm, ea */
3434	case 0x26f: /* movdqu xmm, ea */
3435	if (mod != 3) {
3436	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3437	gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3438	} else {
3439	rm = (modrm & 7) \| REX_B(s);
3440	gen_op_movo(offsetof(CPUX86State,xmm_regs[reg]),
3441	offsetof(CPUX86State,xmm_regs[rm]));
3442	}
3443	break;
3444	case 0x210: /* movss xmm, ea */
3445	if (mod != 3) {
3446	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3447	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
3448	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
3449	gen_op_movl_T0_0();
3450	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
3451	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
3452	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
3453	} else {
3454	rm = (modrm & 7) \| REX_B(s);
3455	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
3456	offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
3457	}
3458	break;
3459	case 0x310: /* movsd xmm, ea */
3460	if (mod != 3) {
3461	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3462	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3463	gen_op_movl_T0_0();
3464	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
3465	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
3466	} else {
3467	rm = (modrm & 7) \| REX_B(s);
3468	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
3469	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
3470	}
3471	break;
3472	case 0x012: /* movlps */
3473	case 0x112: /* movlpd */
3474	if (mod != 3) {
3475	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3476	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3477	} else {
3478	/* movhlps */
3479	rm = (modrm & 7) \| REX_B(s);
3480	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
3481	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
3482	}
3483	break;
3484	case 0x212: /* movsldup */
3485	if (mod != 3) {
3486	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3487	gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3488	} else {
3489	rm = (modrm & 7) \| REX_B(s);
3490	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
3491	offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
3492	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
3493	offsetof(CPUX86State,xmm_regs[rm].XMM_L(2)));
3494	}
3495	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
3496	offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
3497	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
3498	offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
3499	break;
3500	case 0x312: /* movddup */
3501	if (mod != 3) {
3502	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3503	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3504	} else {
3505	rm = (modrm & 7) \| REX_B(s);
3506	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
3507	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
3508	}
3509	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
3510	offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3511	break;
3512	case 0x016: /* movhps */
3513	case 0x116: /* movhpd */
3514	if (mod != 3) {
3515	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3516	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
3517	} else {
3518	/* movlhps */
3519	rm = (modrm & 7) \| REX_B(s);
3520	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
3521	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
3522	}
3523	break;
3524	case 0x216: /* movshdup */
3525	if (mod != 3) {
3526	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3527	gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3528	} else {
3529	rm = (modrm & 7) \| REX_B(s);
3530	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
3531	offsetof(CPUX86State,xmm_regs[rm].XMM_L(1)));
3532	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
3533	offsetof(CPUX86State,xmm_regs[rm].XMM_L(3)));
3534	}
3535	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
3536	offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
3537	gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
3538	offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
3539	break;
3540	case 0x178:
3541	case 0x378:
3542	{
3543	int bit_index, field_length;
3544
3545	if (b1 == 1 && reg != 0)
3546	goto illegal_op;
3547	field_length = ldub_code(s->pc++) & 0x3F;
3548	bit_index = ldub_code(s->pc++) & 0x3F;
3549	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3550	offsetof(CPUX86State,xmm_regs[reg]));
3551	if (b1 == 1)
3552	gen_helper_extrq_i(cpu_ptr0, tcg_const_i32(bit_index),
3553	tcg_const_i32(field_length));
3554	else
3555	gen_helper_insertq_i(cpu_ptr0, tcg_const_i32(bit_index),
3556	tcg_const_i32(field_length));
3557	}
3558	break;
3559	case 0x7e: /* movd ea, mm */
3560	#ifdef TARGET_X86_64
3561	if (s->dflag == 2) {
3562	tcg_gen_ld_i64(cpu_T[0], cpu_env,
3563	offsetof(CPUX86State,fpregs[reg].mmx));
3564	gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
3565	} else
3566	#endif
3567	{
3568	tcg_gen_ld32u_tl(cpu_T[0], cpu_env,
3569	offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
3570	gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
3571	}
3572	break;
3573	case 0x17e: /* movd ea, xmm */
3574	#ifdef TARGET_X86_64
3575	if (s->dflag == 2) {
3576	tcg_gen_ld_i64(cpu_T[0], cpu_env,
3577	offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3578	gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
3579	} else
3580	#endif
3581	{
3582	tcg_gen_ld32u_tl(cpu_T[0], cpu_env,
3583	offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
3584	gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
3585	}
3586	break;
3587	case 0x27e: /* movq xmm, ea */
3588	if (mod != 3) {
3589	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3590	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3591	} else {
3592	rm = (modrm & 7) \| REX_B(s);
3593	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
3594	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
3595	}
3596	gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
3597	break;
3598	case 0x7f: /* movq ea, mm */
3599	if (mod != 3) {
3600	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3601	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
3602	} else {
3603	rm = (modrm & 7);
3604	gen_op_movq(offsetof(CPUX86State,fpregs[rm].mmx),
3605	offsetof(CPUX86State,fpregs[reg].mmx));
3606	}
3607	break;
3608	case 0x011: /* movups */
3609	case 0x111: /* movupd */
3610	case 0x029: /* movaps */
3611	case 0x129: /* movapd */
3612	case 0x17f: /* movdqa ea, xmm */
3613	case 0x27f: /* movdqu ea, xmm */
3614	if (mod != 3) {
3615	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3616	gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
3617	} else {
3618	rm = (modrm & 7) \| REX_B(s);
3619	gen_op_movo(offsetof(CPUX86State,xmm_regs[rm]),
3620	offsetof(CPUX86State,xmm_regs[reg]));
3621	}
3622	break;
3623	case 0x211: /* movss ea, xmm */
3624	if (mod != 3) {
3625	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3626	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
3627	gen_op_st_T0_A0(OT_LONG + s->mem_index);
3628	} else {
3629	rm = (modrm & 7) \| REX_B(s);
3630	gen_op_movl(offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)),
3631	offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
3632	}
3633	break;
3634	case 0x311: /* movsd ea, xmm */
3635	if (mod != 3) {
3636	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3637	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3638	} else {
3639	rm = (modrm & 7) \| REX_B(s);
3640	gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
3641	offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3642	}
3643	break;
3644	case 0x013: /* movlps */
3645	case 0x113: /* movlpd */
3646	if (mod != 3) {
3647	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3648	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3649	} else {
3650	goto illegal_op;
3651	}
3652	break;
3653	case 0x017: /* movhps */
3654	case 0x117: /* movhpd */
3655	if (mod != 3) {
3656	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3657	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
3658	} else {
3659	goto illegal_op;
3660	}
3661	break;
3662	case 0x71: /* shift mm, im */
3663	case 0x72:
3664	case 0x73:
3665	case 0x171: /* shift xmm, im */
3666	case 0x172:
3667	case 0x173:
3668	val = ldub_code(s->pc++);
3669	if (is_xmm) {
3670	gen_op_movl_T0_im(val);
3671	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
3672	gen_op_movl_T0_0();
3673	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(1)));
3674	op1_offset = offsetof(CPUX86State,xmm_t0);
3675	} else {
3676	gen_op_movl_T0_im(val);
3677	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(0)));
3678	gen_op_movl_T0_0();
3679	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(1)));
3680	op1_offset = offsetof(CPUX86State,mmx_t0);
3681	}
3682	sse_op2 = sse_op_table2[((b - 1) & 3) * 8 + (((modrm >> 3)) & 7)][b1];
3683	if (!sse_op2)
3684	goto illegal_op;
3685	if (is_xmm) {
3686	rm = (modrm & 7) \| REX_B(s);
3687	op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
3688	} else {
3689	rm = (modrm & 7);
3690	op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
3691	}
3692	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op2_offset);
3693	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op1_offset);
3694	((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
3695	break;
3696	case 0x050: /* movmskps */
3697	rm = (modrm & 7) \| REX_B(s);
3698	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3699	offsetof(CPUX86State,xmm_regs[rm]));
3700	gen_helper_movmskps(cpu_tmp2_i32, cpu_ptr0);
3701	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
3702	gen_op_mov_reg_T0(OT_LONG, reg);
3703	break;
3704	case 0x150: /* movmskpd */
3705	rm = (modrm & 7) \| REX_B(s);
3706	tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
3707	offsetof(CPUX86State,xmm_regs[rm]));
3708	gen_helper_movmskpd(cpu_tmp2_i32, cpu_ptr0);
3709	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
3710	gen_op_mov_reg_T0(OT_LONG, reg);
3711	break;
3712	case 0x02a: /* cvtpi2ps */
3713	case 0x12a: /* cvtpi2pd */
3714	gen_helper_enter_mmx();
3715	if (mod != 3) {
3716	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3717	op2_offset = offsetof(CPUX86State,mmx_t0);
3718	gen_ldq_env_A0(s->mem_index, op2_offset);
3719	} else {
3720	rm = (modrm & 7);
3721	op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
3722	}
3723	op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
3724	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
3725	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
3726	switch(b >> 8) {
3727	case 0x0:
3728	gen_helper_cvtpi2ps(cpu_ptr0, cpu_ptr1);
3729	break;
3730	default:
3731	case 0x1:
3732	gen_helper_cvtpi2pd(cpu_ptr0, cpu_ptr1);
3733	break;
3734	}
3735	break;
3736	case 0x22a: /* cvtsi2ss */
3737	case 0x32a: /* cvtsi2sd */
3738	ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
3739	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
3740	op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
3741	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
3742	sse_op2 = sse_op_table3[(s->dflag == 2) * 2 + ((b >> 8) - 2)];
3743	if (ot == OT_LONG) {
3744	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
3745	((void (*)(TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_tmp2_i32);
3746	} else {
3747	((void (*)(TCGv_ptr, TCGv))sse_op2)(cpu_ptr0, cpu_T[0]);
3748	}
3749	break;
3750	case 0x02c: /* cvttps2pi */
3751	case 0x12c: /* cvttpd2pi */
3752	case 0x02d: /* cvtps2pi */
3753	case 0x12d: /* cvtpd2pi */
3754	gen_helper_enter_mmx();
3755	if (mod != 3) {
3756	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3757	op2_offset = offsetof(CPUX86State,xmm_t0);
3758	gen_ldo_env_A0(s->mem_index, op2_offset);
3759	} else {
3760	rm = (modrm & 7) \| REX_B(s);
3761	op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
3762	}
3763	op1_offset = offsetof(CPUX86State,fpregs[reg & 7].mmx);
3764	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
3765	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
3766	switch(b) {
3767	case 0x02c:
3768	gen_helper_cvttps2pi(cpu_ptr0, cpu_ptr1);
3769	break;
3770	case 0x12c:
3771	gen_helper_cvttpd2pi(cpu_ptr0, cpu_ptr1);
3772	break;
3773	case 0x02d:
3774	gen_helper_cvtps2pi(cpu_ptr0, cpu_ptr1);
3775	break;
3776	case 0x12d:
3777	gen_helper_cvtpd2pi(cpu_ptr0, cpu_ptr1);
3778	break;
3779	}
3780	break;
3781	case 0x22c: /* cvttss2si */
3782	case 0x32c: /* cvttsd2si */
3783	case 0x22d: /* cvtss2si */
3784	case 0x32d: /* cvtsd2si */
3785	ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
3786	if (mod != 3) {
3787	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3788	if ((b >> 8) & 1) {
3789	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_t0.XMM_Q(0)));
3790	} else {
3791	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
3792	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
3793	}
3794	op2_offset = offsetof(CPUX86State,xmm_t0);
3795	} else {
3796	rm = (modrm & 7) \| REX_B(s);
3797	op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
3798	}
3799	sse_op2 = sse_op_table3[(s->dflag == 2) * 2 + ((b >> 8) - 2) + 4 +
3800	(b & 1) * 4];
3801	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op2_offset);
3802	if (ot == OT_LONG) {
3803	((void (*)(TCGv_i32, TCGv_ptr))sse_op2)(cpu_tmp2_i32, cpu_ptr0);
3804	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
3805	} else {
3806	((void (*)(TCGv, TCGv_ptr))sse_op2)(cpu_T[0], cpu_ptr0);
3807	}
3808	gen_op_mov_reg_T0(ot, reg);
3809	break;
3810	case 0xc4: /* pinsrw */
3811	case 0x1c4:
3812	s->rip_offset = 1;
3813	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
3814	val = ldub_code(s->pc++);
3815	if (b1) {
3816	val &= 7;
3817	tcg_gen_st16_tl(cpu_T[0], cpu_env,
3818	offsetof(CPUX86State,xmm_regs[reg].XMM_W(val)));
3819	} else {
3820	val &= 3;
3821	tcg_gen_st16_tl(cpu_T[0], cpu_env,
3822	offsetof(CPUX86State,fpregs[reg].mmx.MMX_W(val)));
3823	}
3824	break;
3825	case 0xc5: /* pextrw */
3826	case 0x1c5:
3827	if (mod != 3)
3828	goto illegal_op;
3829	ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
3830	val = ldub_code(s->pc++);
3831	if (b1) {
3832	val &= 7;
3833	rm = (modrm & 7) \| REX_B(s);
3834	tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
3835	offsetof(CPUX86State,xmm_regs[rm].XMM_W(val)));
3836	} else {
3837	val &= 3;
3838	rm = (modrm & 7);
3839	tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
3840	offsetof(CPUX86State,fpregs[rm].mmx.MMX_W(val)));
3841	}
3842	reg = ((modrm >> 3) & 7) \| rex_r;
3843	gen_op_mov_reg_T0(ot, reg);
3844	break;
3845	case 0x1d6: /* movq ea, xmm */
3846	if (mod != 3) {
3847	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3848	gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3849	} else {
3850	rm = (modrm & 7) \| REX_B(s);
3851	gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
3852	offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
3853	gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
3854	}
3855	break;
3856	case 0x2d6: /* movq2dq */
3857	gen_helper_enter_mmx();
3858	rm = (modrm & 7);
3859	gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
3860	offsetof(CPUX86State,fpregs[rm].mmx));
3861	gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
3862	break;
3863	case 0x3d6: /* movdq2q */
3864	gen_helper_enter_mmx();
3865	rm = (modrm & 7) \| REX_B(s);
3866	gen_op_movq(offsetof(CPUX86State,fpregs[reg & 7].mmx),
3867	offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
3868	break;
3869	case 0xd7: /* pmovmskb */
3870	case 0x1d7:
3871	if (mod != 3)
3872	goto illegal_op;
3873	if (b1) {
3874	rm = (modrm & 7) \| REX_B(s);
3875	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, offsetof(CPUX86State,xmm_regs[rm]));
3876	gen_helper_pmovmskb_xmm(cpu_tmp2_i32, cpu_ptr0);
3877	} else {
3878	rm = (modrm & 7);
3879	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, offsetof(CPUX86State,fpregs[rm].mmx));
3880	gen_helper_pmovmskb_mmx(cpu_tmp2_i32, cpu_ptr0);
3881	}
3882	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
3883	reg = ((modrm >> 3) & 7) \| rex_r;
3884	gen_op_mov_reg_T0(OT_LONG, reg);
3885	break;
3886	case 0x138:
3887	if (s->prefix & PREFIX_REPNZ)
3888	goto crc32;
3889	case 0x038:
3890	b = modrm;
3891	modrm = ldub_code(s->pc++);
3892	rm = modrm & 7;
3893	reg = ((modrm >> 3) & 7) \| rex_r;
3894	mod = (modrm >> 6) & 3;
3895
3896	sse_op2 = sse_op_table6[b].op[b1];
3897	if (!sse_op2)
3898	goto illegal_op;
3899	if (!(s->cpuid_ext_features & sse_op_table6[b].ext_mask))
3900	goto illegal_op;
3901
3902	if (b1) {
3903	op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
3904	if (mod == 3) {
3905	op2_offset = offsetof(CPUX86State,xmm_regs[rm \| REX_B(s)]);
3906	} else {
3907	op2_offset = offsetof(CPUX86State,xmm_t0);
3908	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3909	switch (b) {
3910	case 0x20: case 0x30: /* pmovsxbw, pmovzxbw */
3911	case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
3912	case 0x25: case 0x35: /* pmovsxdq, pmovzxdq */
3913	gen_ldq_env_A0(s->mem_index, op2_offset +
3914	offsetof(XMMReg, XMM_Q(0)));
3915	break;
3916	case 0x21: case 0x31: /* pmovsxbd, pmovzxbd */
3917	case 0x24: case 0x34: /* pmovsxwq, pmovzxwq */
3918	tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
3919	(s->mem_index >> 2) - 1);
3920	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
3921	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, op2_offset +
3922	offsetof(XMMReg, XMM_L(0)));
3923	break;
3924	case 0x22: case 0x32: /* pmovsxbq, pmovzxbq */
3925	tcg_gen_qemu_ld16u(cpu_tmp0, cpu_A0,
3926	(s->mem_index >> 2) - 1);
3927	tcg_gen_st16_tl(cpu_tmp0, cpu_env, op2_offset +
3928	offsetof(XMMReg, XMM_W(0)));
3929	break;
3930	case 0x2a: /* movntqda */
3931	gen_ldo_env_A0(s->mem_index, op1_offset);
3932	return;
3933	default:
3934	gen_ldo_env_A0(s->mem_index, op2_offset);
3935	}
3936	}
3937	} else {
3938	op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
3939	if (mod == 3) {
3940	op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
3941	} else {
3942	op2_offset = offsetof(CPUX86State,mmx_t0);
3943	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
3944	gen_ldq_env_A0(s->mem_index, op2_offset);
3945	}
3946	}
3947	if (sse_op2 == SSE_SPECIAL)
3948	goto illegal_op;
3949
3950	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
3951	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
3952	((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
3953
3954	if (b == 0x17)
3955	s->cc_op = CC_OP_EFLAGS;
3956	break;
3957	case 0x338: /* crc32 */
3958	crc32:
3959	b = modrm;
3960	modrm = ldub_code(s->pc++);
3961	reg = ((modrm >> 3) & 7) \| rex_r;
3962
3963	if (b != 0xf0 && b != 0xf1)
3964	goto illegal_op;
3965	if (!(s->cpuid_ext_features & CPUID_EXT_SSE42))
3966	goto illegal_op;
3967
3968	if (b == 0xf0)
3969	ot = OT_BYTE;
3970	else if (b == 0xf1 && s->dflag != 2)
3971	if (s->prefix & PREFIX_DATA)
3972	ot = OT_WORD;
3973	else
3974	ot = OT_LONG;
3975	else
3976	ot = OT_QUAD;
3977
3978	gen_op_mov_TN_reg(OT_LONG, 0, reg);
3979	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
3980	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
3981	gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
3982	cpu_T[0], tcg_const_i32(8 << ot));
3983
3984	ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
3985	gen_op_mov_reg_T0(ot, reg);
3986	break;
3987	case 0x03a:
3988	case 0x13a:
3989	b = modrm;
3990	modrm = ldub_code(s->pc++);
3991	rm = modrm & 7;
3992	reg = ((modrm >> 3) & 7) \| rex_r;
3993	mod = (modrm >> 6) & 3;
3994
3995	sse_op2 = sse_op_table7[b].op[b1];
3996	if (!sse_op2)
3997	goto illegal_op;
3998	if (!(s->cpuid_ext_features & sse_op_table7[b].ext_mask))
3999	goto illegal_op;
4000
4001	if (sse_op2 == SSE_SPECIAL) {
4002	ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
4003	rm = (modrm & 7) \| REX_B(s);
4004	if (mod != 3)
4005	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4006	reg = ((modrm >> 3) & 7) \| rex_r;
4007	val = ldub_code(s->pc++);
4008	switch (b) {
4009	case 0x14: /* pextrb */
4010	tcg_gen_ld8u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
4011	xmm_regs[reg].XMM_B(val & 15)));
4012	if (mod == 3)
4013	gen_op_mov_reg_T0(ot, rm);
4014	else
4015	tcg_gen_qemu_st8(cpu_T[0], cpu_A0,
4016	(s->mem_index >> 2) - 1);
4017	break;
4018	case 0x15: /* pextrw */
4019	tcg_gen_ld16u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
4020	xmm_regs[reg].XMM_W(val & 7)));
4021	if (mod == 3)
4022	gen_op_mov_reg_T0(ot, rm);
4023	else
4024	tcg_gen_qemu_st16(cpu_T[0], cpu_A0,
4025	(s->mem_index >> 2) - 1);
4026	break;
4027	case 0x16:
4028	if (ot == OT_LONG) { /* pextrd */
4029	tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
4030	offsetof(CPUX86State,
4031	xmm_regs[reg].XMM_L(val & 3)));
4032	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
4033	if (mod == 3)
4034	gen_op_mov_reg_v(ot, rm, cpu_T[0]);
4035	else
4036	tcg_gen_qemu_st32(cpu_T[0], cpu_A0,
4037	(s->mem_index >> 2) - 1);
4038	} else { /* pextrq */
4039	#ifdef TARGET_X86_64
4040	tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env,
4041	offsetof(CPUX86State,
4042	xmm_regs[reg].XMM_Q(val & 1)));
4043	if (mod == 3)
4044	gen_op_mov_reg_v(ot, rm, cpu_tmp1_i64);
4045	else
4046	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
4047	(s->mem_index >> 2) - 1);
4048	#else
4049	goto illegal_op;
4050	#endif
4051	}
4052	break;
4053	case 0x17: /* extractps */
4054	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
4055	xmm_regs[reg].XMM_L(val & 3)));
4056	if (mod == 3)
4057	gen_op_mov_reg_T0(ot, rm);
4058	else
4059	tcg_gen_qemu_st32(cpu_T[0], cpu_A0,
4060	(s->mem_index >> 2) - 1);
4061	break;
4062	case 0x20: /* pinsrb */
4063	if (mod == 3)
4064	gen_op_mov_TN_reg(OT_LONG, 0, rm);
4065	else
4066	tcg_gen_qemu_ld8u(cpu_tmp0, cpu_A0,
4067	(s->mem_index >> 2) - 1);
4068	tcg_gen_st8_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State,
4069	xmm_regs[reg].XMM_B(val & 15)));
4070	break;
4071	case 0x21: /* insertps */
4072	if (mod == 3) {
4073	tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
4074	offsetof(CPUX86State,xmm_regs[rm]
4075	.XMM_L((val >> 6) & 3)));
4076	} else {
4077	tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
4078	(s->mem_index >> 2) - 1);
4079	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
4080	}
4081	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
4082	offsetof(CPUX86State,xmm_regs[reg]
4083	.XMM_L((val >> 4) & 3)));
4084	if ((val >> 0) & 1)
4085	tcg_gen_st_i32(tcg_const_i32(0 /float32_zero/),
4086	cpu_env, offsetof(CPUX86State,
4087	xmm_regs[reg].XMM_L(0)));
4088	if ((val >> 1) & 1)
4089	tcg_gen_st_i32(tcg_const_i32(0 /float32_zero/),
4090	cpu_env, offsetof(CPUX86State,
4091	xmm_regs[reg].XMM_L(1)));
4092	if ((val >> 2) & 1)
4093	tcg_gen_st_i32(tcg_const_i32(0 /float32_zero/),
4094	cpu_env, offsetof(CPUX86State,
4095	xmm_regs[reg].XMM_L(2)));
4096	if ((val >> 3) & 1)
4097	tcg_gen_st_i32(tcg_const_i32(0 /float32_zero/),
4098	cpu_env, offsetof(CPUX86State,
4099	xmm_regs[reg].XMM_L(3)));
4100	break;
4101	case 0x22:
4102	if (ot == OT_LONG) { /* pinsrd */
4103	if (mod == 3)
4104	gen_op_mov_v_reg(ot, cpu_tmp0, rm);
4105	else
4106	tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
4107	(s->mem_index >> 2) - 1);
4108	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
4109	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
4110	offsetof(CPUX86State,
4111	xmm_regs[reg].XMM_L(val & 3)));
4112	} else { /* pinsrq */
4113	#ifdef TARGET_X86_64
4114	if (mod == 3)
4115	gen_op_mov_v_reg(ot, cpu_tmp1_i64, rm);
4116	else
4117	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
4118	(s->mem_index >> 2) - 1);
4119	tcg_gen_st_i64(cpu_tmp1_i64, cpu_env,
4120	offsetof(CPUX86State,
4121	xmm_regs[reg].XMM_Q(val & 1)));
4122	#else
4123	goto illegal_op;
4124	#endif
4125	}
4126	break;
4127	}
4128	return;
4129	}
4130
4131	if (b1) {
4132	op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
4133	if (mod == 3) {
4134	op2_offset = offsetof(CPUX86State,xmm_regs[rm \| REX_B(s)]);
4135	} else {
4136	op2_offset = offsetof(CPUX86State,xmm_t0);
4137	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4138	gen_ldo_env_A0(s->mem_index, op2_offset);
4139	}
4140	} else {
4141	op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
4142	if (mod == 3) {
4143	op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
4144	} else {
4145	op2_offset = offsetof(CPUX86State,mmx_t0);
4146	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4147	gen_ldq_env_A0(s->mem_index, op2_offset);
4148	}
4149	}
4150	val = ldub_code(s->pc++);
4151
4152	if ((b & 0xfc) == 0x60) { /* pcmpXstrX */
4153	s->cc_op = CC_OP_EFLAGS;
4154
4155	if (s->dflag == 2)
4156	/* The helper must use entire 64-bit gp registers */
4157	val \|= 1 << 8;
4158	}
4159
4160	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4161	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4162	((void (*)(TCGv_ptr, TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
4163	break;
4164	default:
4165	goto illegal_op;
4166	}
4167	} else {
4168	/* generic MMX or SSE operation */
4169	switch(b) {
4170	case 0x70: /* pshufx insn */
4171	case 0xc6: /* pshufx insn */
4172	case 0xc2: /* compare insns */
4173	s->rip_offset = 1;
4174	break;
4175	default:
4176	break;
4177	}
4178	if (is_xmm) {
4179	op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
4180	if (mod != 3) {
4181	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4182	op2_offset = offsetof(CPUX86State,xmm_t0);
4183	if (b1 >= 2 && ((b >= 0x50 && b <= 0x5f && b != 0x5b) \|\|
4184	b == 0xc2)) {
4185	/* specific case for SSE single instructions */
4186	if (b1 == 2) {
4187	/* 32 bit access */
4188	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
4189	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
4190	} else {
4191	/* 64 bit access */
4192	gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_t0.XMM_D(0)));
4193	}
4194	} else {
4195	gen_ldo_env_A0(s->mem_index, op2_offset);
4196	}
4197	} else {
4198	rm = (modrm & 7) \| REX_B(s);
4199	op2_offset = offsetof(CPUX86State,xmm_regs[rm]);
4200	}
4201	} else {
4202	op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
4203	if (mod != 3) {
4204	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4205	op2_offset = offsetof(CPUX86State,mmx_t0);
4206	gen_ldq_env_A0(s->mem_index, op2_offset);
4207	} else {
4208	rm = (modrm & 7);
4209	op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
4210	}
4211	}
4212	switch(b) {
4213	case 0x0f: /* 3DNow! data insns */
4214	if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
4215	goto illegal_op;
4216	val = ldub_code(s->pc++);
4217	sse_op2 = sse_op_table5[val];
4218	if (!sse_op2)
4219	goto illegal_op;
4220	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4221	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4222	((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
4223	break;
4224	case 0x70: /* pshufx insn */
4225	case 0xc6: /* pshufx insn */
4226	val = ldub_code(s->pc++);
4227	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4228	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4229	((void (*)(TCGv_ptr, TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
4230	break;
4231	case 0xc2:
4232	/* compare insns */
4233	val = ldub_code(s->pc++);
4234	if (val >= 8)
4235	goto illegal_op;
4236	sse_op2 = sse_op_table4[val][b1];
4237	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4238	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4239	((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
4240	break;
4241	case 0xf7:
4242	/* maskmov : we must prepare A0 */
4243	if (mod != 3)
4244	goto illegal_op;
4245	#ifdef TARGET_X86_64
4246	if (s->aflag == 2) {
4247	gen_op_movq_A0_reg(R_EDI);
4248	} else
4249	#endif
4250	{
4251	gen_op_movl_A0_reg(R_EDI);
4252	if (s->aflag == 0)
4253	gen_op_andl_A0_ffff();
4254	}
4255	gen_add_A0_ds_seg(s);
4256
4257	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4258	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4259	((void (*)(TCGv_ptr, TCGv_ptr, TCGv))sse_op2)(cpu_ptr0, cpu_ptr1, cpu_A0);
4260	break;
4261	default:
4262	tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
4263	tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
4264	((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
4265	break;
4266	}
4267	if (b == 0x2e \|\| b == 0x2f) {
4268	s->cc_op = CC_OP_EFLAGS;
4269	}
4270	}
4271	}
4272
4273	#ifdef VBOX
4274	/* Checks if it's an invalid lock sequence. Only a few instructions
4275	can be used together with the lock prefix and of those only the
4276	form that write a memory operand. So, this is kind of annoying
4277	work to do...
4278	The AMD manual lists the following instructions.
4279	ADC
4280	ADD
4281	AND
4282	BTC
4283	BTR
4284	BTS
4285	CMPXCHG
4286	CMPXCHG8B
4287	CMPXCHG16B
4288	DEC
4289	INC
4290	NEG
4291	NOT
4292	OR
4293	SBB
4294	SUB
4295	XADD
4296	XCHG
4297	XOR */
4298	static bool is_invalid_lock_sequence(DisasContext *s, target_ulong pc_start, int b)
4299	{
4300	target_ulong pc = s->pc;
4301	int modrm, mod, op;
4302
4303	/* X={8,16,32,64} Y={16,32,64} */
4304	switch (b)
4305	{
4306	/* /2: ADC reg/memX, immX */
4307	/* /0: ADD reg/memX, immX */
4308	/* /4: AND reg/memX, immX */
4309	/* /1: OR reg/memX, immX */
4310	/* /3: SBB reg/memX, immX */
4311	/* /5: SUB reg/memX, immX */
4312	/* /6: XOR reg/memX, immX */
4313	case 0x80:
4314	case 0x81:
4315	case 0x83:
4316	modrm = ldub_code(pc++);
4317	op = (modrm >> 3) & 7;
4318	if (op == 7) /* /7: CMP */
4319	break;
4320	mod = (modrm >> 6) & 3;
4321	if (mod == 3) /* register destination */
4322	break;
4323	return false;
4324
4325	case 0x10: /* /r: ADC reg/mem8, reg8 */
4326	case 0x11: /* /r: ADC reg/memX, regY */
4327	case 0x00: /* /r: ADD reg/mem8, reg8 */
4328	case 0x01: /* /r: ADD reg/memX, regY */
4329	case 0x20: /* /r: AND reg/mem8, reg8 */
4330	case 0x21: /* /r: AND reg/memY, regY */
4331	case 0x08: /* /r: OR reg/mem8, reg8 */
4332	case 0x09: /* /r: OR reg/memY, regY */
4333	case 0x18: /* /r: SBB reg/mem8, reg8 */
4334	case 0x19: /* /r: SBB reg/memY, regY */
4335	case 0x28: /* /r: SUB reg/mem8, reg8 */
4336	case 0x29: /* /r: SUB reg/memY, regY */
4337	case 0x86: /* /r: XCHG reg/mem8, reg8 or XCHG reg8, reg/mem8 */
4338	case 0x87: /* /r: XCHG reg/memY, regY or XCHG regY, reg/memY */
4339	case 0x30: /* /r: XOR reg/mem8, reg8 */
4340	case 0x31: /* /r: XOR reg/memY, regY */
4341	modrm = ldub_code(pc++);
4342	mod = (modrm >> 6) & 3;
4343	if (mod == 3) /* register destination */
4344	break;
4345	return false;
4346
4347	/* /1: DEC reg/memX */
4348	/* /0: INC reg/memX */
4349	case 0xfe:
4350	case 0xff:
4351	modrm = ldub_code(pc++);
4352	mod = (modrm >> 6) & 3;
4353	if (mod == 3) /* register destination */
4354	break;
4355	return false;
4356
4357	/* /3: NEG reg/memX */
4358	/* /2: NOT reg/memX */
4359	case 0xf6:
4360	case 0xf7:
4361	modrm = ldub_code(pc++);
4362	mod = (modrm >> 6) & 3;
4363	if (mod == 3) /* register destination */
4364	break;
4365	return false;
4366
4367	case 0x0f:
4368	b = ldub_code(pc++);
4369	switch (b)
4370	{
4371	/* /7: BTC reg/memY, imm8 */
4372	/* /6: BTR reg/memY, imm8 */
4373	/* /5: BTS reg/memY, imm8 */
4374	case 0xba:
4375	modrm = ldub_code(pc++);
4376	op = (modrm >> 3) & 7;
4377	if (op < 5)
4378	break;
4379	mod = (modrm >> 6) & 3;
4380	if (mod == 3) /* register destination */
4381	break;
4382	return false;
4383
4384	case 0xbb: /* /r: BTC reg/memY, regY */
4385	case 0xb3: /* /r: BTR reg/memY, regY */
4386	case 0xab: /* /r: BTS reg/memY, regY */
4387	case 0xb0: /* /r: CMPXCHG reg/mem8, reg8 */
4388	case 0xb1: /* /r: CMPXCHG reg/memY, regY */
4389	case 0xc0: /* /r: XADD reg/mem8, reg8 */
4390	case 0xc1: /* /r: XADD reg/memY, regY */
4391	modrm = ldub_code(pc++);
4392	mod = (modrm >> 6) & 3;
4393	if (mod == 3) /* register destination */
4394	break;
4395	return false;
4396
4397	/* /1: CMPXCHG8B mem64 or CMPXCHG16B mem128 */
4398	case 0xc7:
4399	modrm = ldub_code(pc++);
4400	op = (modrm >> 3) & 7;
4401	if (op != 1)
4402	break;
4403	return false;
4404	}
4405	break;
4406	}
4407
4408	/* illegal sequence. The s->pc is past the lock prefix and that
4409	is sufficient for the TB, I think. */
4410	Log(("illegal lock sequence %RGv (b=%#x)\n", pc_start, b));
4411	return true;
4412	}
4413	#endif /* VBOX */
4414
4415	/* convert one instruction. s->is_jmp is set if the translation must
4416	be stopped. Return the next pc value */
4417	static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
4418	{
4419	int b, prefixes, aflag, dflag;
4420	int shift, ot;
4421	int modrm, reg, rm, mod, reg_addr, op, opreg, offset_addr, val;
4422	target_ulong next_eip, tval;
4423	int rex_w, rex_r;
4424
4425	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
4426	tcg_gen_debug_insn_start(pc_start);
4427	s->pc = pc_start;
4428	prefixes = 0;
4429	aflag = s->code32;
4430	dflag = s->code32;
4431	s->override = -1;
4432	rex_w = -1;
4433	rex_r = 0;
4434	#ifdef TARGET_X86_64
4435	s->rex_x = 0;
4436	s->rex_b = 0;
4437	x86_64_hregs = 0;
4438	#endif
4439	s->rip_offset = 0; /* for relative ip address */
4440	#ifdef VBOX
4441	/* nike: seems only slow down things */
4442	# if 0
4443	/* Always update EIP. Otherwise one must be very careful with generated code that can raise exceptions. */
4444
4445	gen_update_eip(pc_start - s->cs_base);
4446	# endif
4447	#endif /* VBOX */
4448
4449	next_byte:
4450	b = ldub_code(s->pc);
4451	s->pc++;
4452	/* check prefixes */
4453	#ifdef TARGET_X86_64
4454	if (CODE64(s)) {
4455	switch (b) {
4456	case 0xf3:
4457	prefixes \|= PREFIX_REPZ;
4458	goto next_byte;
4459	case 0xf2:
4460	prefixes \|= PREFIX_REPNZ;
4461	goto next_byte;
4462	case 0xf0:
4463	prefixes \|= PREFIX_LOCK;
4464	goto next_byte;
4465	case 0x2e:
4466	s->override = R_CS;
4467	goto next_byte;
4468	case 0x36:
4469	s->override = R_SS;
4470	goto next_byte;
4471	case 0x3e:
4472	s->override = R_DS;
4473	goto next_byte;
4474	case 0x26:
4475	s->override = R_ES;
4476	goto next_byte;
4477	case 0x64:
4478	s->override = R_FS;
4479	goto next_byte;
4480	case 0x65:
4481	s->override = R_GS;
4482	goto next_byte;
4483	case 0x66:
4484	prefixes \|= PREFIX_DATA;
4485	goto next_byte;
4486	case 0x67:
4487	prefixes \|= PREFIX_ADR;
4488	goto next_byte;
4489	case 0x40 ... 0x4f:
4490	/* REX prefix */
4491	rex_w = (b >> 3) & 1;
4492	rex_r = (b & 0x4) << 1;
4493	s->rex_x = (b & 0x2) << 2;
4494	REX_B(s) = (b & 0x1) << 3;
4495	x86_64_hregs = 1; /* select uniform byte register addressing */
4496	goto next_byte;
4497	}
4498	if (rex_w == 1) {
4499	/* 0x66 is ignored if rex.w is set */
4500	dflag = 2;
4501	} else {
4502	if (prefixes & PREFIX_DATA)
4503	dflag ^= 1;
4504	}
4505	if (!(prefixes & PREFIX_ADR))
4506	aflag = 2;
4507	} else
4508	#endif
4509	{
4510	switch (b) {
4511	case 0xf3:
4512	prefixes \|= PREFIX_REPZ;
4513	goto next_byte;
4514	case 0xf2:
4515	prefixes \|= PREFIX_REPNZ;
4516	goto next_byte;
4517	case 0xf0:
4518	prefixes \|= PREFIX_LOCK;
4519	goto next_byte;
4520	case 0x2e:
4521	s->override = R_CS;
4522	goto next_byte;
4523	case 0x36:
4524	s->override = R_SS;
4525	goto next_byte;
4526	case 0x3e:
4527	s->override = R_DS;
4528	goto next_byte;
4529	case 0x26:
4530	s->override = R_ES;
4531	goto next_byte;
4532	case 0x64:
4533	s->override = R_FS;
4534	goto next_byte;
4535	case 0x65:
4536	s->override = R_GS;
4537	goto next_byte;
4538	case 0x66:
4539	prefixes \|= PREFIX_DATA;
4540	goto next_byte;
4541	case 0x67:
4542	prefixes \|= PREFIX_ADR;
4543	goto next_byte;
4544	}
4545	if (prefixes & PREFIX_DATA)
4546	dflag ^= 1;
4547	if (prefixes & PREFIX_ADR)
4548	aflag ^= 1;
4549	}
4550
4551	s->prefix = prefixes;
4552	s->aflag = aflag;
4553	s->dflag = dflag;
4554
4555	/* lock generation */
4556	#ifndef VBOX
4557	if (prefixes & PREFIX_LOCK)
4558	gen_helper_lock();
4559	#else /* VBOX */
4560	if (prefixes & PREFIX_LOCK) {
4561	if (is_invalid_lock_sequence(s, pc_start, b)) {
4562	gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
4563	return s->pc;
4564	}
4565	gen_helper_lock();
4566	}
4567	#endif /* VBOX */
4568
4569	/* now check op code */
4570	reswitch:
4571	switch(b) {
4572	case 0x0f:
4573	/**************************/
4574	/* extended op code */
4575	b = ldub_code(s->pc++) \| 0x100;
4576	goto reswitch;
4577
4578	/**************************/
4579	/* arith & logic */
4580	case 0x00 ... 0x05:
4581	case 0x08 ... 0x0d:
4582	case 0x10 ... 0x15:
4583	case 0x18 ... 0x1d:
4584	case 0x20 ... 0x25:
4585	case 0x28 ... 0x2d:
4586	case 0x30 ... 0x35:
4587	case 0x38 ... 0x3d:
4588	{
4589	int op, f, val;
4590	op = (b >> 3) & 7;
4591	f = (b >> 1) & 3;
4592
4593	if ((b & 1) == 0)
4594	ot = OT_BYTE;
4595	else
4596	ot = dflag + OT_WORD;
4597
4598	switch(f) {
4599	case 0: /* OP Ev, Gv */
4600	modrm = ldub_code(s->pc++);
4601	reg = ((modrm >> 3) & 7) \| rex_r;
4602	mod = (modrm >> 6) & 3;
4603	rm = (modrm & 7) \| REX_B(s);
4604	if (mod != 3) {
4605	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4606	opreg = OR_TMP0;
4607	} else if (op == OP_XORL && rm == reg) {
4608	xor_zero:
4609	/* xor reg, reg optimisation */
4610	gen_op_movl_T0_0();
4611	s->cc_op = CC_OP_LOGICB + ot;
4612	gen_op_mov_reg_T0(ot, reg);
4613	gen_op_update1_cc();
4614	break;
4615	} else {
4616	opreg = rm;
4617	}
4618	gen_op_mov_TN_reg(ot, 1, reg);
4619	gen_op(s, op, ot, opreg);
4620	break;
4621	case 1: /* OP Gv, Ev */
4622	modrm = ldub_code(s->pc++);
4623	mod = (modrm >> 6) & 3;
4624	reg = ((modrm >> 3) & 7) \| rex_r;
4625	rm = (modrm & 7) \| REX_B(s);
4626	if (mod != 3) {
4627	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4628	gen_op_ld_T1_A0(ot + s->mem_index);
4629	} else if (op == OP_XORL && rm == reg) {
4630	goto xor_zero;
4631	} else {
4632	gen_op_mov_TN_reg(ot, 1, rm);
4633	}
4634	gen_op(s, op, ot, reg);
4635	break;
4636	case 2: /* OP A, Iv */
4637	val = insn_get(s, ot);
4638	gen_op_movl_T1_im(val);
4639	gen_op(s, op, ot, OR_EAX);
4640	break;
4641	}
4642	}
4643	break;
4644
4645	case 0x82:
4646	if (CODE64(s))
4647	goto illegal_op;
4648	case 0x80: /* GRP1 */
4649	case 0x81:
4650	case 0x83:
4651	{
4652	int val;
4653
4654	if ((b & 1) == 0)
4655	ot = OT_BYTE;
4656	else
4657	ot = dflag + OT_WORD;
4658
4659	modrm = ldub_code(s->pc++);
4660	mod = (modrm >> 6) & 3;
4661	rm = (modrm & 7) \| REX_B(s);
4662	op = (modrm >> 3) & 7;
4663
4664	if (mod != 3) {
4665	if (b == 0x83)
4666	s->rip_offset = 1;
4667	else
4668	s->rip_offset = insn_const_size(ot);
4669	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4670	opreg = OR_TMP0;
4671	} else {
4672	opreg = rm;
4673	}
4674
4675	switch(b) {
4676	default:
4677	case 0x80:
4678	case 0x81:
4679	case 0x82:
4680	val = insn_get(s, ot);
4681	break;
4682	case 0x83:
4683	val = (int8_t)insn_get(s, OT_BYTE);
4684	break;
4685	}
4686	gen_op_movl_T1_im(val);
4687	gen_op(s, op, ot, opreg);
4688	}
4689	break;
4690
4691	/**************************/
4692	/* inc, dec, and other misc arith */
4693	case 0x40 ... 0x47: /* inc Gv */
4694	ot = dflag ? OT_LONG : OT_WORD;
4695	gen_inc(s, ot, OR_EAX + (b & 7), 1);
4696	break;
4697	case 0x48 ... 0x4f: /* dec Gv */
4698	ot = dflag ? OT_LONG : OT_WORD;
4699	gen_inc(s, ot, OR_EAX + (b & 7), -1);
4700	break;
4701	case 0xf6: /* GRP3 */
4702	case 0xf7:
4703	if ((b & 1) == 0)
4704	ot = OT_BYTE;
4705	else
4706	ot = dflag + OT_WORD;
4707
4708	modrm = ldub_code(s->pc++);
4709	mod = (modrm >> 6) & 3;
4710	rm = (modrm & 7) \| REX_B(s);
4711	op = (modrm >> 3) & 7;
4712	if (mod != 3) {
4713	if (op == 0)
4714	s->rip_offset = insn_const_size(ot);
4715	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4716	gen_op_ld_T0_A0(ot + s->mem_index);
4717	} else {
4718	gen_op_mov_TN_reg(ot, 0, rm);
4719	}
4720
4721	switch(op) {
4722	case 0: /* test */
4723	val = insn_get(s, ot);
4724	gen_op_movl_T1_im(val);
4725	gen_op_testl_T0_T1_cc();
4726	s->cc_op = CC_OP_LOGICB + ot;
4727	break;
4728	case 2: /* not */
4729	tcg_gen_not_tl(cpu_T[0], cpu_T[0]);
4730	if (mod != 3) {
4731	gen_op_st_T0_A0(ot + s->mem_index);
4732	} else {
4733	gen_op_mov_reg_T0(ot, rm);
4734	}
4735	break;
4736	case 3: /* neg */
4737	tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
4738	if (mod != 3) {
4739	gen_op_st_T0_A0(ot + s->mem_index);
4740	} else {
4741	gen_op_mov_reg_T0(ot, rm);
4742	}
4743	gen_op_update_neg_cc();
4744	s->cc_op = CC_OP_SUBB + ot;
4745	break;
4746	case 4: /* mul */
4747	switch(ot) {
4748	case OT_BYTE:
4749	gen_op_mov_TN_reg(OT_BYTE, 1, R_EAX);
4750	tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
4751	tcg_gen_ext8u_tl(cpu_T[1], cpu_T[1]);
4752	/* XXX: use 32 bit mul which could be faster */
4753	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4754	gen_op_mov_reg_T0(OT_WORD, R_EAX);
4755	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4756	tcg_gen_andi_tl(cpu_cc_src, cpu_T[0], 0xff00);
4757	s->cc_op = CC_OP_MULB;
4758	break;
4759	case OT_WORD:
4760	gen_op_mov_TN_reg(OT_WORD, 1, R_EAX);
4761	tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
4762	tcg_gen_ext16u_tl(cpu_T[1], cpu_T[1]);
4763	/* XXX: use 32 bit mul which could be faster */
4764	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4765	gen_op_mov_reg_T0(OT_WORD, R_EAX);
4766	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4767	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
4768	gen_op_mov_reg_T0(OT_WORD, R_EDX);
4769	tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
4770	s->cc_op = CC_OP_MULW;
4771	break;
4772	default:
4773	case OT_LONG:
4774	#ifdef TARGET_X86_64
4775	gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
4776	tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
4777	tcg_gen_ext32u_tl(cpu_T[1], cpu_T[1]);
4778	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4779	gen_op_mov_reg_T0(OT_LONG, R_EAX);
4780	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4781	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 32);
4782	gen_op_mov_reg_T0(OT_LONG, R_EDX);
4783	tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
4784	#else
4785	{
4786	TCGv_i64 t0, t1;
4787	t0 = tcg_temp_new_i64();
4788	t1 = tcg_temp_new_i64();
4789	gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
4790	tcg_gen_extu_i32_i64(t0, cpu_T[0]);
4791	tcg_gen_extu_i32_i64(t1, cpu_T[1]);
4792	tcg_gen_mul_i64(t0, t0, t1);
4793	tcg_gen_trunc_i64_i32(cpu_T[0], t0);
4794	gen_op_mov_reg_T0(OT_LONG, R_EAX);
4795	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4796	tcg_gen_shri_i64(t0, t0, 32);
4797	tcg_gen_trunc_i64_i32(cpu_T[0], t0);
4798	gen_op_mov_reg_T0(OT_LONG, R_EDX);
4799	tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
4800	}
4801	#endif
4802	s->cc_op = CC_OP_MULL;
4803	break;
4804	#ifdef TARGET_X86_64
4805	case OT_QUAD:
4806	gen_helper_mulq_EAX_T0(cpu_T[0]);
4807	s->cc_op = CC_OP_MULQ;
4808	break;
4809	#endif
4810	}
4811	break;
4812	case 5: /* imul */
4813	switch(ot) {
4814	case OT_BYTE:
4815	gen_op_mov_TN_reg(OT_BYTE, 1, R_EAX);
4816	tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
4817	tcg_gen_ext8s_tl(cpu_T[1], cpu_T[1]);
4818	/* XXX: use 32 bit mul which could be faster */
4819	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4820	gen_op_mov_reg_T0(OT_WORD, R_EAX);
4821	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4822	tcg_gen_ext8s_tl(cpu_tmp0, cpu_T[0]);
4823	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
4824	s->cc_op = CC_OP_MULB;
4825	break;
4826	case OT_WORD:
4827	gen_op_mov_TN_reg(OT_WORD, 1, R_EAX);
4828	tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
4829	tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
4830	/* XXX: use 32 bit mul which could be faster */
4831	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4832	gen_op_mov_reg_T0(OT_WORD, R_EAX);
4833	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4834	tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
4835	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
4836	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
4837	gen_op_mov_reg_T0(OT_WORD, R_EDX);
4838	s->cc_op = CC_OP_MULW;
4839	break;
4840	default:
4841	case OT_LONG:
4842	#ifdef TARGET_X86_64
4843	gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
4844	tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
4845	tcg_gen_ext32s_tl(cpu_T[1], cpu_T[1]);
4846	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
4847	gen_op_mov_reg_T0(OT_LONG, R_EAX);
4848	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4849	tcg_gen_ext32s_tl(cpu_tmp0, cpu_T[0]);
4850	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
4851	tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 32);
4852	gen_op_mov_reg_T0(OT_LONG, R_EDX);
4853	#else
4854	{
4855	TCGv_i64 t0, t1;
4856	t0 = tcg_temp_new_i64();
4857	t1 = tcg_temp_new_i64();
4858	gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
4859	tcg_gen_ext_i32_i64(t0, cpu_T[0]);
4860	tcg_gen_ext_i32_i64(t1, cpu_T[1]);
4861	tcg_gen_mul_i64(t0, t0, t1);
4862	tcg_gen_trunc_i64_i32(cpu_T[0], t0);
4863	gen_op_mov_reg_T0(OT_LONG, R_EAX);
4864	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
4865	tcg_gen_sari_tl(cpu_tmp0, cpu_T[0], 31);
4866	tcg_gen_shri_i64(t0, t0, 32);
4867	tcg_gen_trunc_i64_i32(cpu_T[0], t0);
4868	gen_op_mov_reg_T0(OT_LONG, R_EDX);
4869	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
4870	}
4871	#endif
4872	s->cc_op = CC_OP_MULL;
4873	break;
4874	#ifdef TARGET_X86_64
4875	case OT_QUAD:
4876	gen_helper_imulq_EAX_T0(cpu_T[0]);
4877	s->cc_op = CC_OP_MULQ;
4878	break;
4879	#endif
4880	}
4881	break;
4882	case 6: /* div */
4883	switch(ot) {
4884	case OT_BYTE:
4885	gen_jmp_im(pc_start - s->cs_base);
4886	gen_helper_divb_AL(cpu_T[0]);
4887	break;
4888	case OT_WORD:
4889	gen_jmp_im(pc_start - s->cs_base);
4890	gen_helper_divw_AX(cpu_T[0]);
4891	break;
4892	default:
4893	case OT_LONG:
4894	gen_jmp_im(pc_start - s->cs_base);
4895	gen_helper_divl_EAX(cpu_T[0]);
4896	break;
4897	#ifdef TARGET_X86_64
4898	case OT_QUAD:
4899	gen_jmp_im(pc_start - s->cs_base);
4900	gen_helper_divq_EAX(cpu_T[0]);
4901	break;
4902	#endif
4903	}
4904	break;
4905	case 7: /* idiv */
4906	switch(ot) {
4907	case OT_BYTE:
4908	gen_jmp_im(pc_start - s->cs_base);
4909	gen_helper_idivb_AL(cpu_T[0]);
4910	break;
4911	case OT_WORD:
4912	gen_jmp_im(pc_start - s->cs_base);
4913	gen_helper_idivw_AX(cpu_T[0]);
4914	break;
4915	default:
4916	case OT_LONG:
4917	gen_jmp_im(pc_start - s->cs_base);
4918	gen_helper_idivl_EAX(cpu_T[0]);
4919	break;
4920	#ifdef TARGET_X86_64
4921	case OT_QUAD:
4922	gen_jmp_im(pc_start - s->cs_base);
4923	gen_helper_idivq_EAX(cpu_T[0]);
4924	break;
4925	#endif
4926	}
4927	break;
4928	default:
4929	goto illegal_op;
4930	}
4931	break;
4932
4933	case 0xfe: /* GRP4 */
4934	case 0xff: /* GRP5 */
4935	if ((b & 1) == 0)
4936	ot = OT_BYTE;
4937	else
4938	ot = dflag + OT_WORD;
4939
4940	modrm = ldub_code(s->pc++);
4941	mod = (modrm >> 6) & 3;
4942	rm = (modrm & 7) \| REX_B(s);
4943	op = (modrm >> 3) & 7;
4944	if (op >= 2 && b == 0xfe) {
4945	goto illegal_op;
4946	}
4947	if (CODE64(s)) {
4948	if (op == 2 \|\| op == 4) {
4949	/* operand size for jumps is 64 bit */
4950	ot = OT_QUAD;
4951	} else if (op == 3 \|\| op == 5) {
4952	ot = dflag ? OT_LONG + (rex_w == 1) : OT_WORD;
4953	} else if (op == 6) {
4954	/* default push size is 64 bit */
4955	ot = dflag ? OT_QUAD : OT_WORD;
4956	}
4957	}
4958	if (mod != 3) {
4959	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
4960	if (op >= 2 && op != 3 && op != 5)
4961	gen_op_ld_T0_A0(ot + s->mem_index);
4962	} else {
4963	gen_op_mov_TN_reg(ot, 0, rm);
4964	}
4965
4966	switch(op) {
4967	case 0: /* inc Ev */
4968	if (mod != 3)
4969	opreg = OR_TMP0;
4970	else
4971	opreg = rm;
4972	gen_inc(s, ot, opreg, 1);
4973	break;
4974	case 1: /* dec Ev */
4975	if (mod != 3)
4976	opreg = OR_TMP0;
4977	else
4978	opreg = rm;
4979	gen_inc(s, ot, opreg, -1);
4980	break;
4981	case 2: /* call Ev */
4982	/* XXX: optimize if memory (no 'and' is necessary) */
4983	#ifdef VBOX_WITH_CALL_RECORD
4984	if (s->record_call)
4985	gen_op_record_call();
4986	#endif
4987	if (s->dflag == 0)
4988	gen_op_andl_T0_ffff();
4989	next_eip = s->pc - s->cs_base;
4990	gen_movtl_T1_im(next_eip);
4991	gen_push_T1(s);
4992	gen_op_jmp_T0();
4993	gen_eob(s);
4994	break;
4995	case 3: /* lcall Ev */
4996	gen_op_ld_T1_A0(ot + s->mem_index);
4997	gen_add_A0_im(s, 1 << (ot - OT_WORD + 1));
4998	gen_op_ldu_T0_A0(OT_WORD + s->mem_index);
4999	do_lcall:
5000	if (s->pe && !s->vm86) {
5001	if (s->cc_op != CC_OP_DYNAMIC)
5002	gen_op_set_cc_op(s->cc_op);
5003	gen_jmp_im(pc_start - s->cs_base);
5004	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5005	gen_helper_lcall_protected(cpu_tmp2_i32, cpu_T[1],
5006	tcg_const_i32(dflag),
5007	tcg_const_i32(s->pc - pc_start));
5008	} else {
5009	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5010	gen_helper_lcall_real(cpu_tmp2_i32, cpu_T[1],
5011	tcg_const_i32(dflag),
5012	tcg_const_i32(s->pc - s->cs_base));
5013	}
5014	gen_eob(s);
5015	break;
5016	case 4: /* jmp Ev */
5017	if (s->dflag == 0)
5018	gen_op_andl_T0_ffff();
5019	gen_op_jmp_T0();
5020	gen_eob(s);
5021	break;
5022	case 5: /* ljmp Ev */
5023	gen_op_ld_T1_A0(ot + s->mem_index);
5024	gen_add_A0_im(s, 1 << (ot - OT_WORD + 1));
5025	gen_op_ldu_T0_A0(OT_WORD + s->mem_index);
5026	do_ljmp:
5027	if (s->pe && !s->vm86) {
5028	if (s->cc_op != CC_OP_DYNAMIC)
5029	gen_op_set_cc_op(s->cc_op);
5030	gen_jmp_im(pc_start - s->cs_base);
5031	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5032	gen_helper_ljmp_protected(cpu_tmp2_i32, cpu_T[1],
5033	tcg_const_i32(s->pc - pc_start));
5034	} else {
5035	gen_op_movl_seg_T0_vm(R_CS);
5036	gen_op_movl_T0_T1();
5037	gen_op_jmp_T0();
5038	}
5039	gen_eob(s);
5040	break;
5041	case 6: /* push Ev */
5042	gen_push_T0(s);
5043	break;
5044	default:
5045	goto illegal_op;
5046	}
5047	break;
5048
5049	case 0x84: /* test Ev, Gv */
5050	case 0x85:
5051	if ((b & 1) == 0)
5052	ot = OT_BYTE;
5053	else
5054	ot = dflag + OT_WORD;
5055
5056	modrm = ldub_code(s->pc++);
5057	mod = (modrm >> 6) & 3;
5058	rm = (modrm & 7) \| REX_B(s);
5059	reg = ((modrm >> 3) & 7) \| rex_r;
5060
5061	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
5062	gen_op_mov_TN_reg(ot, 1, reg);
5063	gen_op_testl_T0_T1_cc();
5064	s->cc_op = CC_OP_LOGICB + ot;
5065	break;
5066
5067	case 0xa8: /* test eAX, Iv */
5068	case 0xa9:
5069	if ((b & 1) == 0)
5070	ot = OT_BYTE;
5071	else
5072	ot = dflag + OT_WORD;
5073	val = insn_get(s, ot);
5074
5075	gen_op_mov_TN_reg(ot, 0, OR_EAX);
5076	gen_op_movl_T1_im(val);
5077	gen_op_testl_T0_T1_cc();
5078	s->cc_op = CC_OP_LOGICB + ot;
5079	break;
5080
5081	case 0x98: /* CWDE/CBW */
5082	#ifdef TARGET_X86_64
5083	if (dflag == 2) {
5084	gen_op_mov_TN_reg(OT_LONG, 0, R_EAX);
5085	tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
5086	gen_op_mov_reg_T0(OT_QUAD, R_EAX);
5087	} else
5088	#endif
5089	if (dflag == 1) {
5090	gen_op_mov_TN_reg(OT_WORD, 0, R_EAX);
5091	tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
5092	gen_op_mov_reg_T0(OT_LONG, R_EAX);
5093	} else {
5094	gen_op_mov_TN_reg(OT_BYTE, 0, R_EAX);
5095	tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
5096	gen_op_mov_reg_T0(OT_WORD, R_EAX);
5097	}
5098	break;
5099	case 0x99: /* CDQ/CWD */
5100	#ifdef TARGET_X86_64
5101	if (dflag == 2) {
5102	gen_op_mov_TN_reg(OT_QUAD, 0, R_EAX);
5103	tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 63);
5104	gen_op_mov_reg_T0(OT_QUAD, R_EDX);
5105	} else
5106	#endif
5107	if (dflag == 1) {
5108	gen_op_mov_TN_reg(OT_LONG, 0, R_EAX);
5109	tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
5110	tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 31);
5111	gen_op_mov_reg_T0(OT_LONG, R_EDX);
5112	} else {
5113	gen_op_mov_TN_reg(OT_WORD, 0, R_EAX);
5114	tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
5115	tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 15);
5116	gen_op_mov_reg_T0(OT_WORD, R_EDX);
5117	}
5118	break;
5119	case 0x1af: /* imul Gv, Ev */
5120	case 0x69: /* imul Gv, Ev, I */
5121	case 0x6b:
5122	ot = dflag + OT_WORD;
5123	modrm = ldub_code(s->pc++);
5124	reg = ((modrm >> 3) & 7) \| rex_r;
5125	if (b == 0x69)
5126	s->rip_offset = insn_const_size(ot);
5127	else if (b == 0x6b)
5128	s->rip_offset = 1;
5129	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
5130	if (b == 0x69) {
5131	val = insn_get(s, ot);
5132	gen_op_movl_T1_im(val);
5133	} else if (b == 0x6b) {
5134	val = (int8_t)insn_get(s, OT_BYTE);
5135	gen_op_movl_T1_im(val);
5136	} else {
5137	gen_op_mov_TN_reg(ot, 1, reg);
5138	}
5139
5140	#ifdef TARGET_X86_64
5141	if (ot == OT_QUAD) {
5142	gen_helper_imulq_T0_T1(cpu_T[0], cpu_T[0], cpu_T[1]);
5143	} else
5144	#endif
5145	if (ot == OT_LONG) {
5146	#ifdef TARGET_X86_64
5147	tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
5148	tcg_gen_ext32s_tl(cpu_T[1], cpu_T[1]);
5149	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
5150	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
5151	tcg_gen_ext32s_tl(cpu_tmp0, cpu_T[0]);
5152	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
5153	#else
5154	{
5155	TCGv_i64 t0, t1;
5156	t0 = tcg_temp_new_i64();
5157	t1 = tcg_temp_new_i64();
5158	tcg_gen_ext_i32_i64(t0, cpu_T[0]);
5159	tcg_gen_ext_i32_i64(t1, cpu_T[1]);
5160	tcg_gen_mul_i64(t0, t0, t1);
5161	tcg_gen_trunc_i64_i32(cpu_T[0], t0);
5162	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
5163	tcg_gen_sari_tl(cpu_tmp0, cpu_T[0], 31);
5164	tcg_gen_shri_i64(t0, t0, 32);
5165	tcg_gen_trunc_i64_i32(cpu_T[1], t0);
5166	tcg_gen_sub_tl(cpu_cc_src, cpu_T[1], cpu_tmp0);
5167	}
5168	#endif
5169	} else {
5170	tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
5171	tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
5172	/* XXX: use 32 bit mul which could be faster */
5173	tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
5174	tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
5175	tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
5176	tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
5177	}
5178	gen_op_mov_reg_T0(ot, reg);
5179	s->cc_op = CC_OP_MULB + ot;
5180	break;
5181	case 0x1c0:
5182	case 0x1c1: /* xadd Ev, Gv */
5183	if ((b & 1) == 0)
5184	ot = OT_BYTE;
5185	else
5186	ot = dflag + OT_WORD;
5187	modrm = ldub_code(s->pc++);
5188	reg = ((modrm >> 3) & 7) \| rex_r;
5189	mod = (modrm >> 6) & 3;
5190	if (mod == 3) {
5191	rm = (modrm & 7) \| REX_B(s);
5192	gen_op_mov_TN_reg(ot, 0, reg);
5193	gen_op_mov_TN_reg(ot, 1, rm);
5194	gen_op_addl_T0_T1();
5195	gen_op_mov_reg_T1(ot, reg);
5196	gen_op_mov_reg_T0(ot, rm);
5197	} else {
5198	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5199	gen_op_mov_TN_reg(ot, 0, reg);
5200	gen_op_ld_T1_A0(ot + s->mem_index);
5201	gen_op_addl_T0_T1();
5202	gen_op_st_T0_A0(ot + s->mem_index);
5203	gen_op_mov_reg_T1(ot, reg);
5204	}
5205	gen_op_update2_cc();
5206	s->cc_op = CC_OP_ADDB + ot;
5207	break;
5208	case 0x1b0:
5209	case 0x1b1: /* cmpxchg Ev, Gv */
5210	{
5211	int label1, label2;
5212	TCGv t0, t1, t2, a0;
5213
5214	if ((b & 1) == 0)
5215	ot = OT_BYTE;
5216	else
5217	ot = dflag + OT_WORD;
5218	modrm = ldub_code(s->pc++);
5219	reg = ((modrm >> 3) & 7) \| rex_r;
5220	mod = (modrm >> 6) & 3;
5221	t0 = tcg_temp_local_new();
5222	t1 = tcg_temp_local_new();
5223	t2 = tcg_temp_local_new();
5224	a0 = tcg_temp_local_new();
5225	gen_op_mov_v_reg(ot, t1, reg);
5226	if (mod == 3) {
5227	rm = (modrm & 7) \| REX_B(s);
5228	gen_op_mov_v_reg(ot, t0, rm);
5229	} else {
5230	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5231	tcg_gen_mov_tl(a0, cpu_A0);
5232	gen_op_ld_v(ot + s->mem_index, t0, a0);
5233	rm = 0; /* avoid warning */
5234	}
5235	label1 = gen_new_label();
5236	tcg_gen_sub_tl(t2, cpu_regs[R_EAX], t0);
5237	gen_extu(ot, t2);
5238	tcg_gen_brcondi_tl(TCG_COND_EQ, t2, 0, label1);
5239	if (mod == 3) {
5240	label2 = gen_new_label();
5241	gen_op_mov_reg_v(ot, R_EAX, t0);
5242	tcg_gen_br(label2);
5243	gen_set_label(label1);
5244	gen_op_mov_reg_v(ot, rm, t1);
5245	gen_set_label(label2);
5246	} else {
5247	tcg_gen_mov_tl(t1, t0);
5248	gen_op_mov_reg_v(ot, R_EAX, t0);
5249	gen_set_label(label1);
5250	/* always store */
5251	gen_op_st_v(ot + s->mem_index, t1, a0);
5252	}
5253	tcg_gen_mov_tl(cpu_cc_src, t0);
5254	tcg_gen_mov_tl(cpu_cc_dst, t2);
5255	s->cc_op = CC_OP_SUBB + ot;
5256	tcg_temp_free(t0);
5257	tcg_temp_free(t1);
5258	tcg_temp_free(t2);
5259	tcg_temp_free(a0);
5260	}
5261	break;
5262	case 0x1c7: /* cmpxchg8b */
5263	modrm = ldub_code(s->pc++);
5264	mod = (modrm >> 6) & 3;
5265	if ((mod == 3) \|\| ((modrm & 0x38) != 0x8))
5266	goto illegal_op;
5267	#ifdef TARGET_X86_64
5268	if (dflag == 2) {
5269	if (!(s->cpuid_ext_features & CPUID_EXT_CX16))
5270	goto illegal_op;
5271	gen_jmp_im(pc_start - s->cs_base);
5272	if (s->cc_op != CC_OP_DYNAMIC)
5273	gen_op_set_cc_op(s->cc_op);
5274	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5275	gen_helper_cmpxchg16b(cpu_A0);
5276	} else
5277	#endif
5278	{
5279	if (!(s->cpuid_features & CPUID_CX8))
5280	goto illegal_op;
5281	gen_jmp_im(pc_start - s->cs_base);
5282	if (s->cc_op != CC_OP_DYNAMIC)
5283	gen_op_set_cc_op(s->cc_op);
5284	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5285	gen_helper_cmpxchg8b(cpu_A0);
5286	}
5287	s->cc_op = CC_OP_EFLAGS;
5288	break;
5289
5290	/**************************/
5291	/* push/pop */
5292	case 0x50 ... 0x57: /* push */
5293	gen_op_mov_TN_reg(OT_LONG, 0, (b & 7) \| REX_B(s));
5294	gen_push_T0(s);
5295	break;
5296	case 0x58 ... 0x5f: /* pop */
5297	if (CODE64(s)) {
5298	ot = dflag ? OT_QUAD : OT_WORD;
5299	} else {
5300	ot = dflag + OT_WORD;
5301	}
5302	gen_pop_T0(s);
5303	/* NOTE: order is important for pop %sp */
5304	gen_pop_update(s);
5305	gen_op_mov_reg_T0(ot, (b & 7) \| REX_B(s));
5306	break;
5307	case 0x60: /* pusha */
5308	if (CODE64(s))
5309	goto illegal_op;
5310	gen_pusha(s);
5311	break;
5312	case 0x61: /* popa */
5313	if (CODE64(s))
5314	goto illegal_op;
5315	gen_popa(s);
5316	break;
5317	case 0x68: /* push Iv */
5318	case 0x6a:
5319	if (CODE64(s)) {
5320	ot = dflag ? OT_QUAD : OT_WORD;
5321	} else {
5322	ot = dflag + OT_WORD;
5323	}
5324	if (b == 0x68)
5325	val = insn_get(s, ot);
5326	else
5327	val = (int8_t)insn_get(s, OT_BYTE);
5328	gen_op_movl_T0_im(val);
5329	gen_push_T0(s);
5330	break;
5331	case 0x8f: /* pop Ev */
5332	if (CODE64(s)) {
5333	ot = dflag ? OT_QUAD : OT_WORD;
5334	} else {
5335	ot = dflag + OT_WORD;
5336	}
5337	modrm = ldub_code(s->pc++);
5338	mod = (modrm >> 6) & 3;
5339	gen_pop_T0(s);
5340	if (mod == 3) {
5341	/* NOTE: order is important for pop %sp */
5342	gen_pop_update(s);
5343	rm = (modrm & 7) \| REX_B(s);
5344	gen_op_mov_reg_T0(ot, rm);
5345	} else {
5346	/* NOTE: order is important too for MMU exceptions */
5347	s->popl_esp_hack = 1 << ot;
5348	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
5349	s->popl_esp_hack = 0;
5350	gen_pop_update(s);
5351	}
5352	break;
5353	case 0xc8: /* enter */
5354	{
5355	int level;
5356	val = lduw_code(s->pc);
5357	s->pc += 2;
5358	level = ldub_code(s->pc++);
5359	gen_enter(s, val, level);
5360	}
5361	break;
5362	case 0xc9: /* leave */
5363	/* XXX: exception not precise (ESP is updated before potential exception) */
5364	if (CODE64(s)) {
5365	gen_op_mov_TN_reg(OT_QUAD, 0, R_EBP);
5366	gen_op_mov_reg_T0(OT_QUAD, R_ESP);
5367	} else if (s->ss32) {
5368	gen_op_mov_TN_reg(OT_LONG, 0, R_EBP);
5369	gen_op_mov_reg_T0(OT_LONG, R_ESP);
5370	} else {
5371	gen_op_mov_TN_reg(OT_WORD, 0, R_EBP);
5372	gen_op_mov_reg_T0(OT_WORD, R_ESP);
5373	}
5374	gen_pop_T0(s);
5375	if (CODE64(s)) {
5376	ot = dflag ? OT_QUAD : OT_WORD;
5377	} else {
5378	ot = dflag + OT_WORD;
5379	}
5380	gen_op_mov_reg_T0(ot, R_EBP);
5381	gen_pop_update(s);
5382	break;
5383	case 0x06: /* push es */
5384	case 0x0e: /* push cs */
5385	case 0x16: /* push ss */
5386	case 0x1e: /* push ds */
5387	if (CODE64(s))
5388	goto illegal_op;
5389	gen_op_movl_T0_seg(b >> 3);
5390	gen_push_T0(s);
5391	break;
5392	case 0x1a0: /* push fs */
5393	case 0x1a8: /* push gs */
5394	gen_op_movl_T0_seg((b >> 3) & 7);
5395	gen_push_T0(s);
5396	break;
5397	case 0x07: /* pop es */
5398	case 0x17: /* pop ss */
5399	case 0x1f: /* pop ds */
5400	if (CODE64(s))
5401	goto illegal_op;
5402	reg = b >> 3;
5403	gen_pop_T0(s);
5404	gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
5405	gen_pop_update(s);
5406	if (reg == R_SS) {
5407	/* if reg == SS, inhibit interrupts/trace. */
5408	/* If several instructions disable interrupts, only the
5409	_first_ does it */
5410	if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
5411	gen_helper_set_inhibit_irq();
5412	s->tf = 0;
5413	}
5414	if (s->is_jmp) {
5415	gen_jmp_im(s->pc - s->cs_base);
5416	gen_eob(s);
5417	}
5418	break;
5419	case 0x1a1: /* pop fs */
5420	case 0x1a9: /* pop gs */
5421	gen_pop_T0(s);
5422	gen_movl_seg_T0(s, (b >> 3) & 7, pc_start - s->cs_base);
5423	gen_pop_update(s);
5424	if (s->is_jmp) {
5425	gen_jmp_im(s->pc - s->cs_base);
5426	gen_eob(s);
5427	}
5428	break;
5429
5430	/**************************/
5431	/* mov */
5432	case 0x88:
5433	case 0x89: /* mov Gv, Ev */
5434	if ((b & 1) == 0)
5435	ot = OT_BYTE;
5436	else
5437	ot = dflag + OT_WORD;
5438	modrm = ldub_code(s->pc++);
5439	reg = ((modrm >> 3) & 7) \| rex_r;
5440
5441	/* generate a generic store */
5442	gen_ldst_modrm(s, modrm, ot, reg, 1);
5443	break;
5444	case 0xc6:
5445	case 0xc7: /* mov Ev, Iv */
5446	if ((b & 1) == 0)
5447	ot = OT_BYTE;
5448	else
5449	ot = dflag + OT_WORD;
5450	modrm = ldub_code(s->pc++);
5451	mod = (modrm >> 6) & 3;
5452	if (mod != 3) {
5453	s->rip_offset = insn_const_size(ot);
5454	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5455	}
5456	val = insn_get(s, ot);
5457	gen_op_movl_T0_im(val);
5458	if (mod != 3)
5459	gen_op_st_T0_A0(ot + s->mem_index);
5460	else
5461	gen_op_mov_reg_T0(ot, (modrm & 7) \| REX_B(s));
5462	break;
5463	case 0x8a:
5464	case 0x8b: /* mov Ev, Gv */
5465	#ifdef VBOX /* dtrace hot fix */
5466	if (prefixes & PREFIX_LOCK)
5467	goto illegal_op;
5468	#endif
5469	if ((b & 1) == 0)
5470	ot = OT_BYTE;
5471	else
5472	ot = OT_WORD + dflag;
5473	modrm = ldub_code(s->pc++);
5474	reg = ((modrm >> 3) & 7) \| rex_r;
5475
5476	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
5477	gen_op_mov_reg_T0(ot, reg);
5478	break;
5479	case 0x8e: /* mov seg, Gv */
5480	modrm = ldub_code(s->pc++);
5481	reg = (modrm >> 3) & 7;
5482	if (reg >= 6 \|\| reg == R_CS)
5483	goto illegal_op;
5484	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
5485	gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
5486	if (reg == R_SS) {
5487	/* if reg == SS, inhibit interrupts/trace */
5488	/* If several instructions disable interrupts, only the
5489	_first_ does it */
5490	if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
5491	gen_helper_set_inhibit_irq();
5492	s->tf = 0;
5493	}
5494	if (s->is_jmp) {
5495	gen_jmp_im(s->pc - s->cs_base);
5496	gen_eob(s);
5497	}
5498	break;
5499	case 0x8c: /* mov Gv, seg */
5500	modrm = ldub_code(s->pc++);
5501	reg = (modrm >> 3) & 7;
5502	mod = (modrm >> 6) & 3;
5503	if (reg >= 6)
5504	goto illegal_op;
5505	gen_op_movl_T0_seg(reg);
5506	if (mod == 3)
5507	ot = OT_WORD + dflag;
5508	else
5509	ot = OT_WORD;
5510	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
5511	break;
5512
5513	case 0x1b6: /* movzbS Gv, Eb */
5514	case 0x1b7: /* movzwS Gv, Eb */
5515	case 0x1be: /* movsbS Gv, Eb */
5516	case 0x1bf: /* movswS Gv, Eb */
5517	{
5518	int d_ot;
5519	/* d_ot is the size of destination */
5520	d_ot = dflag + OT_WORD;
5521	/* ot is the size of source */
5522	ot = (b & 1) + OT_BYTE;
5523	modrm = ldub_code(s->pc++);
5524	reg = ((modrm >> 3) & 7) \| rex_r;
5525	mod = (modrm >> 6) & 3;
5526	rm = (modrm & 7) \| REX_B(s);
5527
5528	if (mod == 3) {
5529	gen_op_mov_TN_reg(ot, 0, rm);
5530	switch(ot \| (b & 8)) {
5531	case OT_BYTE:
5532	tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
5533	break;
5534	case OT_BYTE \| 8:
5535	tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
5536	break;
5537	case OT_WORD:
5538	tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
5539	break;
5540	default:
5541	case OT_WORD \| 8:
5542	tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
5543	break;
5544	}
5545	gen_op_mov_reg_T0(d_ot, reg);
5546	} else {
5547	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5548	if (b & 8) {
5549	gen_op_lds_T0_A0(ot + s->mem_index);
5550	} else {
5551	gen_op_ldu_T0_A0(ot + s->mem_index);
5552	}
5553	gen_op_mov_reg_T0(d_ot, reg);
5554	}
5555	}
5556	break;
5557
5558	case 0x8d: /* lea */
5559	ot = dflag + OT_WORD;
5560	modrm = ldub_code(s->pc++);
5561	mod = (modrm >> 6) & 3;
5562	if (mod == 3)
5563	goto illegal_op;
5564	reg = ((modrm >> 3) & 7) \| rex_r;
5565	/* we must ensure that no segment is added */
5566	s->override = -1;
5567	val = s->addseg;
5568	s->addseg = 0;
5569	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5570	s->addseg = val;
5571	gen_op_mov_reg_A0(ot - OT_WORD, reg);
5572	break;
5573
5574	case 0xa0: /* mov EAX, Ov */
5575	case 0xa1:
5576	case 0xa2: /* mov Ov, EAX */
5577	case 0xa3:
5578	{
5579	target_ulong offset_addr;
5580
5581	if ((b & 1) == 0)
5582	ot = OT_BYTE;
5583	else
5584	ot = dflag + OT_WORD;
5585	#ifdef TARGET_X86_64
5586	if (s->aflag == 2) {
5587	offset_addr = ldq_code(s->pc);
5588	s->pc += 8;
5589	gen_op_movq_A0_im(offset_addr);
5590	} else
5591	#endif
5592	{
5593	if (s->aflag) {
5594	offset_addr = insn_get(s, OT_LONG);
5595	} else {
5596	offset_addr = insn_get(s, OT_WORD);
5597	}
5598	gen_op_movl_A0_im(offset_addr);
5599	}
5600	gen_add_A0_ds_seg(s);
5601	if ((b & 2) == 0) {
5602	gen_op_ld_T0_A0(ot + s->mem_index);
5603	gen_op_mov_reg_T0(ot, R_EAX);
5604	} else {
5605	gen_op_mov_TN_reg(ot, 0, R_EAX);
5606	gen_op_st_T0_A0(ot + s->mem_index);
5607	}
5608	}
5609	break;
5610	case 0xd7: /* xlat */
5611	#ifdef TARGET_X86_64
5612	if (s->aflag == 2) {
5613	gen_op_movq_A0_reg(R_EBX);
5614	gen_op_mov_TN_reg(OT_QUAD, 0, R_EAX);
5615	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xff);
5616	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T[0]);
5617	} else
5618	#endif
5619	{
5620	gen_op_movl_A0_reg(R_EBX);
5621	gen_op_mov_TN_reg(OT_LONG, 0, R_EAX);
5622	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xff);
5623	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T[0]);
5624	if (s->aflag == 0)
5625	gen_op_andl_A0_ffff();
5626	else
5627	tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
5628	}
5629	gen_add_A0_ds_seg(s);
5630	gen_op_ldu_T0_A0(OT_BYTE + s->mem_index);
5631	gen_op_mov_reg_T0(OT_BYTE, R_EAX);
5632	break;
5633	case 0xb0 ... 0xb7: /* mov R, Ib */
5634	val = insn_get(s, OT_BYTE);
5635	gen_op_movl_T0_im(val);
5636	gen_op_mov_reg_T0(OT_BYTE, (b & 7) \| REX_B(s));
5637	break;
5638	case 0xb8 ... 0xbf: /* mov R, Iv */
5639	#ifdef TARGET_X86_64
5640	if (dflag == 2) {
5641	uint64_t tmp;
5642	/* 64 bit case */
5643	tmp = ldq_code(s->pc);
5644	s->pc += 8;
5645	reg = (b & 7) \| REX_B(s);
5646	gen_movtl_T0_im(tmp);
5647	gen_op_mov_reg_T0(OT_QUAD, reg);
5648	} else
5649	#endif
5650	{
5651	ot = dflag ? OT_LONG : OT_WORD;
5652	val = insn_get(s, ot);
5653	reg = (b & 7) \| REX_B(s);
5654	gen_op_movl_T0_im(val);
5655	gen_op_mov_reg_T0(ot, reg);
5656	}
5657	break;
5658
5659	case 0x91 ... 0x97: /* xchg R, EAX */
5660	do_xchg_reg_eax:
5661	ot = dflag + OT_WORD;
5662	reg = (b & 7) \| REX_B(s);
5663	rm = R_EAX;
5664	goto do_xchg_reg;
5665	case 0x86:
5666	case 0x87: /* xchg Ev, Gv */
5667	if ((b & 1) == 0)
5668	ot = OT_BYTE;
5669	else
5670	ot = dflag + OT_WORD;
5671	modrm = ldub_code(s->pc++);
5672	reg = ((modrm >> 3) & 7) \| rex_r;
5673	mod = (modrm >> 6) & 3;
5674	if (mod == 3) {
5675	rm = (modrm & 7) \| REX_B(s);
5676	do_xchg_reg:
5677	gen_op_mov_TN_reg(ot, 0, reg);
5678	gen_op_mov_TN_reg(ot, 1, rm);
5679	gen_op_mov_reg_T0(ot, rm);
5680	gen_op_mov_reg_T1(ot, reg);
5681	} else {
5682	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5683	gen_op_mov_TN_reg(ot, 0, reg);
5684	/* for xchg, lock is implicit */
5685	if (!(prefixes & PREFIX_LOCK))
5686	gen_helper_lock();
5687	gen_op_ld_T1_A0(ot + s->mem_index);
5688	gen_op_st_T0_A0(ot + s->mem_index);
5689	if (!(prefixes & PREFIX_LOCK))
5690	gen_helper_unlock();
5691	gen_op_mov_reg_T1(ot, reg);
5692	}
5693	break;
5694	case 0xc4: /* les Gv */
5695	if (CODE64(s))
5696	goto illegal_op;
5697	op = R_ES;
5698	goto do_lxx;
5699	case 0xc5: /* lds Gv */
5700	if (CODE64(s))
5701	goto illegal_op;
5702	op = R_DS;
5703	goto do_lxx;
5704	case 0x1b2: /* lss Gv */
5705	op = R_SS;
5706	goto do_lxx;
5707	case 0x1b4: /* lfs Gv */
5708	op = R_FS;
5709	goto do_lxx;
5710	case 0x1b5: /* lgs Gv */
5711	op = R_GS;
5712	do_lxx:
5713	ot = dflag ? OT_LONG : OT_WORD;
5714	modrm = ldub_code(s->pc++);
5715	reg = ((modrm >> 3) & 7) \| rex_r;
5716	mod = (modrm >> 6) & 3;
5717	if (mod == 3)
5718	goto illegal_op;
5719	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5720	gen_op_ld_T1_A0(ot + s->mem_index);
5721	gen_add_A0_im(s, 1 << (ot - OT_WORD + 1));
5722	/* load the segment first to handle exceptions properly */
5723	gen_op_ldu_T0_A0(OT_WORD + s->mem_index);
5724	gen_movl_seg_T0(s, op, pc_start - s->cs_base);
5725	/* then put the data */
5726	gen_op_mov_reg_T1(ot, reg);
5727	if (s->is_jmp) {
5728	gen_jmp_im(s->pc - s->cs_base);
5729	gen_eob(s);
5730	}
5731	break;
5732
5733	/************************/
5734	/* shifts */
5735	case 0xc0:
5736	case 0xc1:
5737	/* shift Ev,Ib */
5738	shift = 2;
5739	grp2:
5740	{
5741	if ((b & 1) == 0)
5742	ot = OT_BYTE;
5743	else
5744	ot = dflag + OT_WORD;
5745
5746	modrm = ldub_code(s->pc++);
5747	mod = (modrm >> 6) & 3;
5748	op = (modrm >> 3) & 7;
5749
5750	if (mod != 3) {
5751	if (shift == 2) {
5752	s->rip_offset = 1;
5753	}
5754	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5755	opreg = OR_TMP0;
5756	} else {
5757	opreg = (modrm & 7) \| REX_B(s);
5758	}
5759
5760	/* simpler op */
5761	if (shift == 0) {
5762	gen_shift(s, op, ot, opreg, OR_ECX);
5763	} else {
5764	if (shift == 2) {
5765	shift = ldub_code(s->pc++);
5766	}
5767	gen_shifti(s, op, ot, opreg, shift);
5768	}
5769	}
5770	break;
5771	case 0xd0:
5772	case 0xd1:
5773	/* shift Ev,1 */
5774	shift = 1;
5775	goto grp2;
5776	case 0xd2:
5777	case 0xd3:
5778	/* shift Ev,cl */
5779	shift = 0;
5780	goto grp2;
5781
5782	case 0x1a4: /* shld imm */
5783	op = 0;
5784	shift = 1;
5785	goto do_shiftd;
5786	case 0x1a5: /* shld cl */
5787	op = 0;
5788	shift = 0;
5789	goto do_shiftd;
5790	case 0x1ac: /* shrd imm */
5791	op = 1;
5792	shift = 1;
5793	goto do_shiftd;
5794	case 0x1ad: /* shrd cl */
5795	op = 1;
5796	shift = 0;
5797	do_shiftd:
5798	ot = dflag + OT_WORD;
5799	modrm = ldub_code(s->pc++);
5800	mod = (modrm >> 6) & 3;
5801	rm = (modrm & 7) \| REX_B(s);
5802	reg = ((modrm >> 3) & 7) \| rex_r;
5803	if (mod != 3) {
5804	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5805	opreg = OR_TMP0;
5806	} else {
5807	opreg = rm;
5808	}
5809	gen_op_mov_TN_reg(ot, 1, reg);
5810
5811	if (shift) {
5812	val = ldub_code(s->pc++);
5813	tcg_gen_movi_tl(cpu_T3, val);
5814	} else {
5815	tcg_gen_mov_tl(cpu_T3, cpu_regs[R_ECX]);
5816	}
5817	gen_shiftd_rm_T1_T3(s, ot, opreg, op);
5818	break;
5819
5820	/************************/
5821	/* floats */
5822	case 0xd8 ... 0xdf:
5823	if (s->flags & (HF_EM_MASK \| HF_TS_MASK)) {
5824	/* if CR0.EM or CR0.TS are set, generate an FPU exception */
5825	/* XXX: what to do if illegal op ? */
5826	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
5827	break;
5828	}
5829	modrm = ldub_code(s->pc++);
5830	mod = (modrm >> 6) & 3;
5831	rm = modrm & 7;
5832	op = ((b & 7) << 3) \| ((modrm >> 3) & 7);
5833	if (mod != 3) {
5834	/* memory op */
5835	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
5836	switch(op) {
5837	case 0x00 ... 0x07: /* fxxxs */
5838	case 0x10 ... 0x17: /* fixxxl */
5839	case 0x20 ... 0x27: /* fxxxl */
5840	case 0x30 ... 0x37: /* fixxx */
5841	{
5842	int op1;
5843	op1 = op & 7;
5844
5845	switch(op >> 4) {
5846	case 0:
5847	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
5848	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5849	gen_helper_flds_FT0(cpu_tmp2_i32);
5850	break;
5851	case 1:
5852	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
5853	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5854	gen_helper_fildl_FT0(cpu_tmp2_i32);
5855	break;
5856	case 2:
5857	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
5858	(s->mem_index >> 2) - 1);
5859	gen_helper_fldl_FT0(cpu_tmp1_i64);
5860	break;
5861	case 3:
5862	default:
5863	gen_op_lds_T0_A0(OT_WORD + s->mem_index);
5864	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5865	gen_helper_fildl_FT0(cpu_tmp2_i32);
5866	break;
5867	}
5868
5869	gen_helper_fp_arith_ST0_FT0(op1);
5870	if (op1 == 3) {
5871	/* fcomp needs pop */
5872	gen_helper_fpop();
5873	}
5874	}
5875	break;
5876	case 0x08: /* flds */
5877	case 0x0a: /* fsts */
5878	case 0x0b: /* fstps */
5879	case 0x18 ... 0x1b: /* fildl, fisttpl, fistl, fistpl */
5880	case 0x28 ... 0x2b: /* fldl, fisttpll, fstl, fstpl */
5881	case 0x38 ... 0x3b: /* filds, fisttps, fists, fistps */
5882	switch(op & 7) {
5883	case 0:
5884	switch(op >> 4) {
5885	case 0:
5886	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
5887	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5888	gen_helper_flds_ST0(cpu_tmp2_i32);
5889	break;
5890	case 1:
5891	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
5892	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5893	gen_helper_fildl_ST0(cpu_tmp2_i32);
5894	break;
5895	case 2:
5896	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
5897	(s->mem_index >> 2) - 1);
5898	gen_helper_fldl_ST0(cpu_tmp1_i64);
5899	break;
5900	case 3:
5901	default:
5902	gen_op_lds_T0_A0(OT_WORD + s->mem_index);
5903	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5904	gen_helper_fildl_ST0(cpu_tmp2_i32);
5905	break;
5906	}
5907	break;
5908	case 1:
5909	/* XXX: the corresponding CPUID bit must be tested ! */
5910	switch(op >> 4) {
5911	case 1:
5912	gen_helper_fisttl_ST0(cpu_tmp2_i32);
5913	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5914	gen_op_st_T0_A0(OT_LONG + s->mem_index);
5915	break;
5916	case 2:
5917	gen_helper_fisttll_ST0(cpu_tmp1_i64);
5918	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
5919	(s->mem_index >> 2) - 1);
5920	break;
5921	case 3:
5922	default:
5923	gen_helper_fistt_ST0(cpu_tmp2_i32);
5924	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5925	gen_op_st_T0_A0(OT_WORD + s->mem_index);
5926	break;
5927	}
5928	gen_helper_fpop();
5929	break;
5930	default:
5931	switch(op >> 4) {
5932	case 0:
5933	gen_helper_fsts_ST0(cpu_tmp2_i32);
5934	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5935	gen_op_st_T0_A0(OT_LONG + s->mem_index);
5936	break;
5937	case 1:
5938	gen_helper_fistl_ST0(cpu_tmp2_i32);
5939	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5940	gen_op_st_T0_A0(OT_LONG + s->mem_index);
5941	break;
5942	case 2:
5943	gen_helper_fstl_ST0(cpu_tmp1_i64);
5944	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
5945	(s->mem_index >> 2) - 1);
5946	break;
5947	case 3:
5948	default:
5949	gen_helper_fist_ST0(cpu_tmp2_i32);
5950	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5951	gen_op_st_T0_A0(OT_WORD + s->mem_index);
5952	break;
5953	}
5954	if ((op & 7) == 3)
5955	gen_helper_fpop();
5956	break;
5957	}
5958	break;
5959	case 0x0c: /* fldenv mem */
5960	if (s->cc_op != CC_OP_DYNAMIC)
5961	gen_op_set_cc_op(s->cc_op);
5962	gen_jmp_im(pc_start - s->cs_base);
5963	gen_helper_fldenv(
5964	cpu_A0, tcg_const_i32(s->dflag));
5965	break;
5966	case 0x0d: /* fldcw mem */
5967	gen_op_ld_T0_A0(OT_WORD + s->mem_index);
5968	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
5969	gen_helper_fldcw(cpu_tmp2_i32);
5970	break;
5971	case 0x0e: /* fnstenv mem */
5972	if (s->cc_op != CC_OP_DYNAMIC)
5973	gen_op_set_cc_op(s->cc_op);
5974	gen_jmp_im(pc_start - s->cs_base);
5975	gen_helper_fstenv(cpu_A0, tcg_const_i32(s->dflag));
5976	break;
5977	case 0x0f: /* fnstcw mem */
5978	gen_helper_fnstcw(cpu_tmp2_i32);
5979	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
5980	gen_op_st_T0_A0(OT_WORD + s->mem_index);
5981	break;
5982	case 0x1d: /* fldt mem */
5983	if (s->cc_op != CC_OP_DYNAMIC)
5984	gen_op_set_cc_op(s->cc_op);
5985	gen_jmp_im(pc_start - s->cs_base);
5986	gen_helper_fldt_ST0(cpu_A0);
5987	break;
5988	case 0x1f: /* fstpt mem */
5989	if (s->cc_op != CC_OP_DYNAMIC)
5990	gen_op_set_cc_op(s->cc_op);
5991	gen_jmp_im(pc_start - s->cs_base);
5992	gen_helper_fstt_ST0(cpu_A0);
5993	gen_helper_fpop();
5994	break;
5995	case 0x2c: /* frstor mem */
5996	if (s->cc_op != CC_OP_DYNAMIC)
5997	gen_op_set_cc_op(s->cc_op);
5998	gen_jmp_im(pc_start - s->cs_base);
5999	gen_helper_frstor(cpu_A0, tcg_const_i32(s->dflag));
6000	break;
6001	case 0x2e: /* fnsave mem */
6002	if (s->cc_op != CC_OP_DYNAMIC)
6003	gen_op_set_cc_op(s->cc_op);
6004	gen_jmp_im(pc_start - s->cs_base);
6005	gen_helper_fsave(cpu_A0, tcg_const_i32(s->dflag));
6006	break;
6007	case 0x2f: /* fnstsw mem */
6008	gen_helper_fnstsw(cpu_tmp2_i32);
6009	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
6010	gen_op_st_T0_A0(OT_WORD + s->mem_index);
6011	break;
6012	case 0x3c: /* fbld */
6013	if (s->cc_op != CC_OP_DYNAMIC)
6014	gen_op_set_cc_op(s->cc_op);
6015	gen_jmp_im(pc_start - s->cs_base);
6016	gen_helper_fbld_ST0(cpu_A0);
6017	break;
6018	case 0x3e: /* fbstp */
6019	if (s->cc_op != CC_OP_DYNAMIC)
6020	gen_op_set_cc_op(s->cc_op);
6021	gen_jmp_im(pc_start - s->cs_base);
6022	gen_helper_fbst_ST0(cpu_A0);
6023	gen_helper_fpop();
6024	break;
6025	case 0x3d: /* fildll */
6026	tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
6027	(s->mem_index >> 2) - 1);
6028	gen_helper_fildll_ST0(cpu_tmp1_i64);
6029	break;
6030	case 0x3f: /* fistpll */
6031	gen_helper_fistll_ST0(cpu_tmp1_i64);
6032	tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
6033	(s->mem_index >> 2) - 1);
6034	gen_helper_fpop();
6035	break;
6036	default:
6037	goto illegal_op;
6038	}
6039	} else {
6040	/* register float ops */
6041	opreg = rm;
6042
6043	switch(op) {
6044	case 0x08: /* fld sti */
6045	gen_helper_fpush();
6046	gen_helper_fmov_ST0_STN(tcg_const_i32((opreg + 1) & 7));
6047	break;
6048	case 0x09: /* fxchg sti */
6049	case 0x29: /* fxchg4 sti, undocumented op */
6050	case 0x39: /* fxchg7 sti, undocumented op */
6051	gen_helper_fxchg_ST0_STN(tcg_const_i32(opreg));
6052	break;
6053	case 0x0a: /* grp d9/2 */
6054	switch(rm) {
6055	case 0: /* fnop */
6056	/* check exceptions (FreeBSD FPU probe) */
6057	if (s->cc_op != CC_OP_DYNAMIC)
6058	gen_op_set_cc_op(s->cc_op);
6059	gen_jmp_im(pc_start - s->cs_base);
6060	gen_helper_fwait();
6061	break;
6062	default:
6063	goto illegal_op;
6064	}
6065	break;
6066	case 0x0c: /* grp d9/4 */
6067	switch(rm) {
6068	case 0: /* fchs */
6069	gen_helper_fchs_ST0();
6070	break;
6071	case 1: /* fabs */
6072	gen_helper_fabs_ST0();
6073	break;
6074	case 4: /* ftst */
6075	gen_helper_fldz_FT0();
6076	gen_helper_fcom_ST0_FT0();
6077	break;
6078	case 5: /* fxam */
6079	gen_helper_fxam_ST0();
6080	break;
6081	default:
6082	goto illegal_op;
6083	}
6084	break;
6085	case 0x0d: /* grp d9/5 */
6086	{
6087	switch(rm) {
6088	case 0:
6089	gen_helper_fpush();
6090	gen_helper_fld1_ST0();
6091	break;
6092	case 1:
6093	gen_helper_fpush();
6094	gen_helper_fldl2t_ST0();
6095	break;
6096	case 2:
6097	gen_helper_fpush();
6098	gen_helper_fldl2e_ST0();
6099	break;
6100	case 3:
6101	gen_helper_fpush();
6102	gen_helper_fldpi_ST0();
6103	break;
6104	case 4:
6105	gen_helper_fpush();
6106	gen_helper_fldlg2_ST0();
6107	break;
6108	case 5:
6109	gen_helper_fpush();
6110	gen_helper_fldln2_ST0();
6111	break;
6112	case 6:
6113	gen_helper_fpush();
6114	gen_helper_fldz_ST0();
6115	break;
6116	default:
6117	goto illegal_op;
6118	}
6119	}
6120	break;
6121	case 0x0e: /* grp d9/6 */
6122	switch(rm) {
6123	case 0: /* f2xm1 */
6124	gen_helper_f2xm1();
6125	break;
6126	case 1: /* fyl2x */
6127	gen_helper_fyl2x();
6128	break;
6129	case 2: /* fptan */
6130	gen_helper_fptan();
6131	break;
6132	case 3: /* fpatan */
6133	gen_helper_fpatan();
6134	break;
6135	case 4: /* fxtract */
6136	gen_helper_fxtract();
6137	break;
6138	case 5: /* fprem1 */
6139	gen_helper_fprem1();
6140	break;
6141	case 6: /* fdecstp */
6142	gen_helper_fdecstp();
6143	break;
6144	default:
6145	case 7: /* fincstp */
6146	gen_helper_fincstp();
6147	break;
6148	}
6149	break;
6150	case 0x0f: /* grp d9/7 */
6151	switch(rm) {
6152	case 0: /* fprem */
6153	gen_helper_fprem();
6154	break;
6155	case 1: /* fyl2xp1 */
6156	gen_helper_fyl2xp1();
6157	break;
6158	case 2: /* fsqrt */
6159	gen_helper_fsqrt();
6160	break;
6161	case 3: /* fsincos */
6162	gen_helper_fsincos();
6163	break;
6164	case 5: /* fscale */
6165	gen_helper_fscale();
6166	break;
6167	case 4: /* frndint */
6168	gen_helper_frndint();
6169	break;
6170	case 6: /* fsin */
6171	gen_helper_fsin();
6172	break;
6173	default:
6174	case 7: /* fcos */
6175	gen_helper_fcos();
6176	break;
6177	}
6178	break;
6179	case 0x00: case 0x01: case 0x04 ... 0x07: /* fxxx st, sti */
6180	case 0x20: case 0x21: case 0x24 ... 0x27: /* fxxx sti, st */
6181	case 0x30: case 0x31: case 0x34 ... 0x37: /* fxxxp sti, st */
6182	{
6183	int op1;
6184
6185	op1 = op & 7;
6186	if (op >= 0x20) {
6187	gen_helper_fp_arith_STN_ST0(op1, opreg);
6188	if (op >= 0x30)
6189	gen_helper_fpop();
6190	} else {
6191	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6192	gen_helper_fp_arith_ST0_FT0(op1);
6193	}
6194	}
6195	break;
6196	case 0x02: /* fcom */
6197	case 0x22: /* fcom2, undocumented op */
6198	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6199	gen_helper_fcom_ST0_FT0();
6200	break;
6201	case 0x03: /* fcomp */
6202	case 0x23: /* fcomp3, undocumented op */
6203	case 0x32: /* fcomp5, undocumented op */
6204	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6205	gen_helper_fcom_ST0_FT0();
6206	gen_helper_fpop();
6207	break;
6208	case 0x15: /* da/5 */
6209	switch(rm) {
6210	case 1: /* fucompp */
6211	gen_helper_fmov_FT0_STN(tcg_const_i32(1));
6212	gen_helper_fucom_ST0_FT0();
6213	gen_helper_fpop();
6214	gen_helper_fpop();
6215	break;
6216	default:
6217	goto illegal_op;
6218	}
6219	break;
6220	case 0x1c:
6221	switch(rm) {
6222	case 0: /* feni (287 only, just do nop here) */
6223	break;
6224	case 1: /* fdisi (287 only, just do nop here) */
6225	break;
6226	case 2: /* fclex */
6227	gen_helper_fclex();
6228	break;
6229	case 3: /* fninit */
6230	gen_helper_fninit();
6231	break;
6232	case 4: /* fsetpm (287 only, just do nop here) */
6233	break;
6234	default:
6235	goto illegal_op;
6236	}
6237	break;
6238	case 0x1d: /* fucomi */
6239	if (s->cc_op != CC_OP_DYNAMIC)
6240	gen_op_set_cc_op(s->cc_op);
6241	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6242	gen_helper_fucomi_ST0_FT0();
6243	s->cc_op = CC_OP_EFLAGS;
6244	break;
6245	case 0x1e: /* fcomi */
6246	if (s->cc_op != CC_OP_DYNAMIC)
6247	gen_op_set_cc_op(s->cc_op);
6248	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6249	gen_helper_fcomi_ST0_FT0();
6250	s->cc_op = CC_OP_EFLAGS;
6251	break;
6252	case 0x28: /* ffree sti */
6253	gen_helper_ffree_STN(tcg_const_i32(opreg));
6254	break;
6255	case 0x2a: /* fst sti */
6256	gen_helper_fmov_STN_ST0(tcg_const_i32(opreg));
6257	break;
6258	case 0x2b: /* fstp sti */
6259	case 0x0b: /* fstp1 sti, undocumented op */
6260	case 0x3a: /* fstp8 sti, undocumented op */
6261	case 0x3b: /* fstp9 sti, undocumented op */
6262	gen_helper_fmov_STN_ST0(tcg_const_i32(opreg));
6263	gen_helper_fpop();
6264	break;
6265	case 0x2c: /* fucom st(i) */
6266	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6267	gen_helper_fucom_ST0_FT0();
6268	break;
6269	case 0x2d: /* fucomp st(i) */
6270	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6271	gen_helper_fucom_ST0_FT0();
6272	gen_helper_fpop();
6273	break;
6274	case 0x33: /* de/3 */
6275	switch(rm) {
6276	case 1: /* fcompp */
6277	gen_helper_fmov_FT0_STN(tcg_const_i32(1));
6278	gen_helper_fcom_ST0_FT0();
6279	gen_helper_fpop();
6280	gen_helper_fpop();
6281	break;
6282	default:
6283	goto illegal_op;
6284	}
6285	break;
6286	case 0x38: /* ffreep sti, undocumented op */
6287	gen_helper_ffree_STN(tcg_const_i32(opreg));
6288	gen_helper_fpop();
6289	break;
6290	case 0x3c: /* df/4 */
6291	switch(rm) {
6292	case 0:
6293	gen_helper_fnstsw(cpu_tmp2_i32);
6294	tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
6295	gen_op_mov_reg_T0(OT_WORD, R_EAX);
6296	break;
6297	default:
6298	goto illegal_op;
6299	}
6300	break;
6301	case 0x3d: /* fucomip */
6302	if (s->cc_op != CC_OP_DYNAMIC)
6303	gen_op_set_cc_op(s->cc_op);
6304	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6305	gen_helper_fucomi_ST0_FT0();
6306	gen_helper_fpop();
6307	s->cc_op = CC_OP_EFLAGS;
6308	break;
6309	case 0x3e: /* fcomip */
6310	if (s->cc_op != CC_OP_DYNAMIC)
6311	gen_op_set_cc_op(s->cc_op);
6312	gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
6313	gen_helper_fcomi_ST0_FT0();
6314	gen_helper_fpop();
6315	s->cc_op = CC_OP_EFLAGS;
6316	break;
6317	case 0x10 ... 0x13: /* fcmovxx */
6318	case 0x18 ... 0x1b:
6319	{
6320	int op1, l1;
6321	static const uint8_t fcmov_cc[8] = {
6322	(JCC_B << 1),
6323	(JCC_Z << 1),
6324	(JCC_BE << 1),
6325	(JCC_P << 1),
6326	};
6327	op1 = fcmov_cc[op & 3] \| (((op >> 3) & 1) ^ 1);
6328	l1 = gen_new_label();
6329	gen_jcc1(s, s->cc_op, op1, l1);
6330	gen_helper_fmov_ST0_STN(tcg_const_i32(opreg));
6331	gen_set_label(l1);
6332	}
6333	break;
6334	default:
6335	goto illegal_op;
6336	}
6337	}
6338	break;
6339	/************************/
6340	/* string ops */
6341
6342	case 0xa4: /* movsS */
6343	case 0xa5:
6344	if ((b & 1) == 0)
6345	ot = OT_BYTE;
6346	else
6347	ot = dflag + OT_WORD;
6348
6349	if (prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) {
6350	gen_repz_movs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
6351	} else {
6352	gen_movs(s, ot);
6353	}
6354	break;
6355
6356	case 0xaa: /* stosS */
6357	case 0xab:
6358	if ((b & 1) == 0)
6359	ot = OT_BYTE;
6360	else
6361	ot = dflag + OT_WORD;
6362
6363	if (prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) {
6364	gen_repz_stos(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
6365	} else {
6366	gen_stos(s, ot);
6367	}
6368	break;
6369	case 0xac: /* lodsS */
6370	case 0xad:
6371	if ((b & 1) == 0)
6372	ot = OT_BYTE;
6373	else
6374	ot = dflag + OT_WORD;
6375	if (prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) {
6376	gen_repz_lods(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
6377	} else {
6378	gen_lods(s, ot);
6379	}
6380	break;
6381	case 0xae: /* scasS */
6382	case 0xaf:
6383	if ((b & 1) == 0)
6384	ot = OT_BYTE;
6385	else
6386	ot = dflag + OT_WORD;
6387	if (prefixes & PREFIX_REPNZ) {
6388	gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
6389	} else if (prefixes & PREFIX_REPZ) {
6390	gen_repz_scas(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0);
6391	} else {
6392	gen_scas(s, ot);
6393	s->cc_op = CC_OP_SUBB + ot;
6394	}
6395	break;
6396
6397	case 0xa6: /* cmpsS */
6398	case 0xa7:
6399	if ((b & 1) == 0)
6400	ot = OT_BYTE;
6401	else
6402	ot = dflag + OT_WORD;
6403	if (prefixes & PREFIX_REPNZ) {
6404	gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 1);
6405	} else if (prefixes & PREFIX_REPZ) {
6406	gen_repz_cmps(s, ot, pc_start - s->cs_base, s->pc - s->cs_base, 0);
6407	} else {
6408	gen_cmps(s, ot);
6409	s->cc_op = CC_OP_SUBB + ot;
6410	}
6411	break;
6412	case 0x6c: /* insS */
6413	case 0x6d:
6414	if ((b & 1) == 0)
6415	ot = OT_BYTE;
6416	else
6417	ot = dflag ? OT_LONG : OT_WORD;
6418	gen_op_mov_TN_reg(OT_WORD, 0, R_EDX);
6419	gen_op_andl_T0_ffff();
6420	gen_check_io(s, ot, pc_start - s->cs_base,
6421	SVM_IOIO_TYPE_MASK \| svm_is_rep(prefixes) \| 4);
6422	if (prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) {
6423	gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
6424	} else {
6425	gen_ins(s, ot);
6426	if (use_icount) {
6427	gen_jmp(s, s->pc - s->cs_base);
6428	}
6429	}
6430	break;
6431	case 0x6e: /* outsS */
6432	case 0x6f:
6433	if ((b & 1) == 0)
6434	ot = OT_BYTE;
6435	else
6436	ot = dflag ? OT_LONG : OT_WORD;
6437	gen_op_mov_TN_reg(OT_WORD, 0, R_EDX);
6438	gen_op_andl_T0_ffff();
6439	gen_check_io(s, ot, pc_start - s->cs_base,
6440	svm_is_rep(prefixes) \| 4);
6441	if (prefixes & (PREFIX_REPZ \| PREFIX_REPNZ)) {
6442	gen_repz_outs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
6443	} else {
6444	gen_outs(s, ot);
6445	if (use_icount) {
6446	gen_jmp(s, s->pc - s->cs_base);
6447	}
6448	}
6449	break;
6450
6451	/************************/
6452	/* port I/O */
6453
6454	case 0xe4:
6455	case 0xe5:
6456	if ((b & 1) == 0)
6457	ot = OT_BYTE;
6458	else
6459	ot = dflag ? OT_LONG : OT_WORD;
6460	val = ldub_code(s->pc++);
6461	gen_op_movl_T0_im(val);
6462	gen_check_io(s, ot, pc_start - s->cs_base,
6463	SVM_IOIO_TYPE_MASK \| svm_is_rep(prefixes));
6464	if (use_icount)
6465	gen_io_start();
6466	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
6467	gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
6468	gen_op_mov_reg_T1(ot, R_EAX);
6469	if (use_icount) {
6470	gen_io_end();
6471	gen_jmp(s, s->pc - s->cs_base);
6472	}
6473	break;
6474	case 0xe6:
6475	case 0xe7:
6476	if ((b & 1) == 0)
6477	ot = OT_BYTE;
6478	else
6479	ot = dflag ? OT_LONG : OT_WORD;
6480	val = ldub_code(s->pc++);
6481	gen_op_movl_T0_im(val);
6482	gen_check_io(s, ot, pc_start - s->cs_base,
6483	svm_is_rep(prefixes));
6484	gen_op_mov_TN_reg(ot, 1, R_EAX);
6485
6486	if (use_icount)
6487	gen_io_start();
6488	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
6489	tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
6490	tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
6491	gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
6492	if (use_icount) {
6493	gen_io_end();
6494	gen_jmp(s, s->pc - s->cs_base);
6495	}
6496	break;
6497	case 0xec:
6498	case 0xed:
6499	if ((b & 1) == 0)
6500	ot = OT_BYTE;
6501	else
6502	ot = dflag ? OT_LONG : OT_WORD;
6503	gen_op_mov_TN_reg(OT_WORD, 0, R_EDX);
6504	gen_op_andl_T0_ffff();
6505	gen_check_io(s, ot, pc_start - s->cs_base,
6506	SVM_IOIO_TYPE_MASK \| svm_is_rep(prefixes));
6507	if (use_icount)
6508	gen_io_start();
6509	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
6510	gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
6511	gen_op_mov_reg_T1(ot, R_EAX);
6512	if (use_icount) {
6513	gen_io_end();
6514	gen_jmp(s, s->pc - s->cs_base);
6515	}
6516	break;
6517	case 0xee:
6518	case 0xef:
6519	if ((b & 1) == 0)
6520	ot = OT_BYTE;
6521	else
6522	ot = dflag ? OT_LONG : OT_WORD;
6523	gen_op_mov_TN_reg(OT_WORD, 0, R_EDX);
6524	gen_op_andl_T0_ffff();
6525	gen_check_io(s, ot, pc_start - s->cs_base,
6526	svm_is_rep(prefixes));
6527	gen_op_mov_TN_reg(ot, 1, R_EAX);
6528
6529	if (use_icount)
6530	gen_io_start();
6531	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
6532	tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
6533	tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
6534	gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
6535	if (use_icount) {
6536	gen_io_end();
6537	gen_jmp(s, s->pc - s->cs_base);
6538	}
6539	break;
6540
6541	/************************/
6542	/* control */
6543	case 0xc2: /* ret im */
6544	val = ldsw_code(s->pc);
6545	s->pc += 2;
6546	gen_pop_T0(s);
6547	if (CODE64(s) && s->dflag)
6548	s->dflag = 2;
6549	gen_stack_update(s, val + (2 << s->dflag));
6550	if (s->dflag == 0)
6551	gen_op_andl_T0_ffff();
6552	gen_op_jmp_T0();
6553	gen_eob(s);
6554	break;
6555	case 0xc3: /* ret */
6556	gen_pop_T0(s);
6557	gen_pop_update(s);
6558	if (s->dflag == 0)
6559	gen_op_andl_T0_ffff();
6560	gen_op_jmp_T0();
6561	gen_eob(s);
6562	break;
6563	case 0xca: /* lret im */
6564	val = ldsw_code(s->pc);
6565	s->pc += 2;
6566	do_lret:
6567	if (s->pe && !s->vm86) {
6568	if (s->cc_op != CC_OP_DYNAMIC)
6569	gen_op_set_cc_op(s->cc_op);
6570	gen_jmp_im(pc_start - s->cs_base);
6571	gen_helper_lret_protected(tcg_const_i32(s->dflag),
6572	tcg_const_i32(val));
6573	} else {
6574	gen_stack_A0(s);
6575	/* pop offset */
6576	gen_op_ld_T0_A0(1 + s->dflag + s->mem_index);
6577	if (s->dflag == 0)
6578	gen_op_andl_T0_ffff();
6579	/* NOTE: keeping EIP updated is not a problem in case of
6580	exception */
6581	gen_op_jmp_T0();
6582	/* pop selector */
6583	gen_op_addl_A0_im(2 << s->dflag);
6584	gen_op_ld_T0_A0(1 + s->dflag + s->mem_index);
6585	gen_op_movl_seg_T0_vm(R_CS);
6586	/* add stack offset */
6587	gen_stack_update(s, val + (4 << s->dflag));
6588	}
6589	gen_eob(s);
6590	break;
6591	case 0xcb: /* lret */
6592	val = 0;
6593	goto do_lret;
6594	case 0xcf: /* iret */
6595	gen_svm_check_intercept(s, pc_start, SVM_EXIT_IRET);
6596	if (!s->pe) {
6597	/* real mode */
6598	gen_helper_iret_real(tcg_const_i32(s->dflag));
6599	s->cc_op = CC_OP_EFLAGS;
6600	} else if (s->vm86) {
6601	#ifdef VBOX
6602	if (s->iopl != 3 && (!s->vme \|\| s->dflag)) {
6603	#else
6604	if (s->iopl != 3) {
6605	#endif
6606	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
6607	} else {
6608	gen_helper_iret_real(tcg_const_i32(s->dflag));
6609	s->cc_op = CC_OP_EFLAGS;
6610	}
6611	} else {
6612	if (s->cc_op != CC_OP_DYNAMIC)
6613	gen_op_set_cc_op(s->cc_op);
6614	gen_jmp_im(pc_start - s->cs_base);
6615	gen_helper_iret_protected(tcg_const_i32(s->dflag),
6616	tcg_const_i32(s->pc - s->cs_base));
6617	s->cc_op = CC_OP_EFLAGS;
6618	}
6619	gen_eob(s);
6620	break;
6621	case 0xe8: /* call im */
6622	{
6623	if (dflag)
6624	tval = (int32_t)insn_get(s, OT_LONG);
6625	else
6626	tval = (int16_t)insn_get(s, OT_WORD);
6627	next_eip = s->pc - s->cs_base;
6628	tval += next_eip;
6629	if (s->dflag == 0)
6630	tval &= 0xffff;
6631	else if(!CODE64(s))
6632	tval &= 0xffffffff;
6633	gen_movtl_T0_im(next_eip);
6634	gen_push_T0(s);
6635	gen_jmp(s, tval);
6636	}
6637	break;
6638	case 0x9a: /* lcall im */
6639	{
6640	unsigned int selector, offset;
6641
6642	if (CODE64(s))
6643	goto illegal_op;
6644	ot = dflag ? OT_LONG : OT_WORD;
6645	offset = insn_get(s, ot);
6646	selector = insn_get(s, OT_WORD);
6647
6648	gen_op_movl_T0_im(selector);
6649	gen_op_movl_T1_imu(offset);
6650	}
6651	goto do_lcall;
6652	case 0xe9: /* jmp im */
6653	if (dflag)
6654	tval = (int32_t)insn_get(s, OT_LONG);
6655	else
6656	tval = (int16_t)insn_get(s, OT_WORD);
6657	tval += s->pc - s->cs_base;
6658	if (s->dflag == 0)
6659	tval &= 0xffff;
6660	else if(!CODE64(s))
6661	tval &= 0xffffffff;
6662	gen_jmp(s, tval);
6663	break;
6664	case 0xea: /* ljmp im */
6665	{
6666	unsigned int selector, offset;
6667
6668	if (CODE64(s))
6669	goto illegal_op;
6670	ot = dflag ? OT_LONG : OT_WORD;
6671	offset = insn_get(s, ot);
6672	selector = insn_get(s, OT_WORD);
6673
6674	gen_op_movl_T0_im(selector);
6675	gen_op_movl_T1_imu(offset);
6676	}
6677	goto do_ljmp;
6678	case 0xeb: /* jmp Jb */
6679	tval = (int8_t)insn_get(s, OT_BYTE);
6680	tval += s->pc - s->cs_base;
6681	if (s->dflag == 0)
6682	tval &= 0xffff;
6683	gen_jmp(s, tval);
6684	break;
6685	case 0x70 ... 0x7f: /* jcc Jb */
6686	tval = (int8_t)insn_get(s, OT_BYTE);
6687	goto do_jcc;
6688	case 0x180 ... 0x18f: /* jcc Jv */
6689	if (dflag) {
6690	tval = (int32_t)insn_get(s, OT_LONG);
6691	} else {
6692	tval = (int16_t)insn_get(s, OT_WORD);
6693	}
6694	do_jcc:
6695	next_eip = s->pc - s->cs_base;
6696	tval += next_eip;
6697	if (s->dflag == 0)
6698	tval &= 0xffff;
6699	gen_jcc(s, b, tval, next_eip);
6700	break;
6701
6702	case 0x190 ... 0x19f: /* setcc Gv */
6703	modrm = ldub_code(s->pc++);
6704	gen_setcc(s, b);
6705	gen_ldst_modrm(s, modrm, OT_BYTE, OR_TMP0, 1);
6706	break;
6707	case 0x140 ... 0x14f: /* cmov Gv, Ev */
6708	{
6709	int l1;
6710	TCGv t0;
6711
6712	ot = dflag + OT_WORD;
6713	modrm = ldub_code(s->pc++);
6714	reg = ((modrm >> 3) & 7) \| rex_r;
6715	mod = (modrm >> 6) & 3;
6716	t0 = tcg_temp_local_new();
6717	if (mod != 3) {
6718	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
6719	gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
6720	} else {
6721	rm = (modrm & 7) \| REX_B(s);
6722	gen_op_mov_v_reg(ot, t0, rm);
6723	}
6724	#ifdef TARGET_X86_64
6725	if (ot == OT_LONG) {
6726	/* XXX: specific Intel behaviour ? */
6727	l1 = gen_new_label();
6728	gen_jcc1(s, s->cc_op, b ^ 1, l1);
6729	tcg_gen_mov_tl(cpu_regs[reg], t0);
6730	gen_set_label(l1);
6731	tcg_gen_ext32u_tl(cpu_regs[reg], cpu_regs[reg]);
6732	} else
6733	#endif
6734	{
6735	l1 = gen_new_label();
6736	gen_jcc1(s, s->cc_op, b ^ 1, l1);
6737	gen_op_mov_reg_v(ot, reg, t0);
6738	gen_set_label(l1);
6739	}
6740	tcg_temp_free(t0);
6741	}
6742	break;
6743
6744	/************************/
6745	/* flags */
6746	case 0x9c: /* pushf */
6747	gen_svm_check_intercept(s, pc_start, SVM_EXIT_PUSHF);
6748	#ifdef VBOX
6749	if (s->vm86 && s->iopl != 3 && (!s->vme \|\| s->dflag)) {
6750	#else
6751	if (s->vm86 && s->iopl != 3) {
6752	#endif
6753	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
6754	} else {
6755	if (s->cc_op != CC_OP_DYNAMIC)
6756	gen_op_set_cc_op(s->cc_op);
6757	#ifdef VBOX
6758	if (s->vm86 && s->vme && s->iopl != 3)
6759	gen_helper_read_eflags_vme(cpu_T[0]);
6760	else
6761	#endif
6762	gen_helper_read_eflags(cpu_T[0]);
6763	gen_push_T0(s);
6764	}
6765	break;
6766	case 0x9d: /* popf */
6767	gen_svm_check_intercept(s, pc_start, SVM_EXIT_POPF);
6768	#ifdef VBOX
6769	if (s->vm86 && s->iopl != 3 && (!s->vme \|\| s->dflag)) {
6770	#else
6771	if (s->vm86 && s->iopl != 3) {
6772	#endif
6773	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
6774	} else {
6775	gen_pop_T0(s);
6776	if (s->cpl == 0) {
6777	if (s->dflag) {
6778	gen_helper_write_eflags(cpu_T[0],
6779	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK \| IF_MASK \| IOPL_MASK)));
6780	} else {
6781	gen_helper_write_eflags(cpu_T[0],
6782	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK \| IF_MASK \| IOPL_MASK) & 0xffff));
6783	}
6784	} else {
6785	if (s->cpl <= s->iopl) {
6786	if (s->dflag) {
6787	gen_helper_write_eflags(cpu_T[0],
6788	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK \| IF_MASK)));
6789	} else {
6790	gen_helper_write_eflags(cpu_T[0],
6791	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK \| IF_MASK) & 0xffff));
6792	}
6793	} else {
6794	if (s->dflag) {
6795	gen_helper_write_eflags(cpu_T[0],
6796	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK)));
6797	} else {
6798	#ifdef VBOX
6799	if (s->vm86 && s->vme)
6800	gen_helper_write_eflags_vme(cpu_T[0]);
6801	else
6802	#endif
6803	gen_helper_write_eflags(cpu_T[0],
6804	tcg_const_i32((TF_MASK \| AC_MASK \| ID_MASK \| NT_MASK) & 0xffff));
6805	}
6806	}
6807	}
6808	gen_pop_update(s);
6809	s->cc_op = CC_OP_EFLAGS;
6810	/* abort translation because TF flag may change */
6811	gen_jmp_im(s->pc - s->cs_base);
6812	gen_eob(s);
6813	}
6814	break;
6815	case 0x9e: /* sahf */
6816	if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
6817	goto illegal_op;
6818	gen_op_mov_TN_reg(OT_BYTE, 0, R_AH);
6819	if (s->cc_op != CC_OP_DYNAMIC)
6820	gen_op_set_cc_op(s->cc_op);
6821	gen_compute_eflags(cpu_cc_src);
6822	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O);
6823	tcg_gen_andi_tl(cpu_T[0], cpu_T[0], CC_S \| CC_Z \| CC_A \| CC_P \| CC_C);
6824	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_T[0]);
6825	s->cc_op = CC_OP_EFLAGS;
6826	break;
6827	case 0x9f: /* lahf */
6828	if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
6829	goto illegal_op;
6830	if (s->cc_op != CC_OP_DYNAMIC)
6831	gen_op_set_cc_op(s->cc_op);
6832	gen_compute_eflags(cpu_T[0]);
6833	/* Note: gen_compute_eflags() only gives the condition codes */
6834	tcg_gen_ori_tl(cpu_T[0], cpu_T[0], 0x02);
6835	gen_op_mov_reg_T0(OT_BYTE, R_AH);
6836	break;
6837	case 0xf5: /* cmc */
6838	if (s->cc_op != CC_OP_DYNAMIC)
6839	gen_op_set_cc_op(s->cc_op);
6840	gen_compute_eflags(cpu_cc_src);
6841	tcg_gen_xori_tl(cpu_cc_src, cpu_cc_src, CC_C);
6842	s->cc_op = CC_OP_EFLAGS;
6843	break;
6844	case 0xf8: /* clc */
6845	if (s->cc_op != CC_OP_DYNAMIC)
6846	gen_op_set_cc_op(s->cc_op);
6847	gen_compute_eflags(cpu_cc_src);
6848	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_C);
6849	s->cc_op = CC_OP_EFLAGS;
6850	break;
6851	case 0xf9: /* stc */
6852	if (s->cc_op != CC_OP_DYNAMIC)
6853	gen_op_set_cc_op(s->cc_op);
6854	gen_compute_eflags(cpu_cc_src);
6855	tcg_gen_ori_tl(cpu_cc_src, cpu_cc_src, CC_C);
6856	s->cc_op = CC_OP_EFLAGS;
6857	break;
6858	case 0xfc: /* cld */
6859	tcg_gen_movi_i32(cpu_tmp2_i32, 1);
6860	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, offsetof(CPUState, df));
6861	break;
6862	case 0xfd: /* std */
6863	tcg_gen_movi_i32(cpu_tmp2_i32, -1);
6864	tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, offsetof(CPUState, df));
6865	break;
6866
6867	/************************/
6868	/* bit operations */
6869	case 0x1ba: /* bt/bts/btr/btc Gv, im */
6870	ot = dflag + OT_WORD;
6871	modrm = ldub_code(s->pc++);
6872	op = (modrm >> 3) & 7;
6873	mod = (modrm >> 6) & 3;
6874	rm = (modrm & 7) \| REX_B(s);
6875	if (mod != 3) {
6876	s->rip_offset = 1;
6877	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
6878	gen_op_ld_T0_A0(ot + s->mem_index);
6879	} else {
6880	gen_op_mov_TN_reg(ot, 0, rm);
6881	}
6882	/* load shift */
6883	val = ldub_code(s->pc++);
6884	gen_op_movl_T1_im(val);
6885	if (op < 4)
6886	goto illegal_op;
6887	op -= 4;
6888	goto bt_op;
6889	case 0x1a3: /* bt Gv, Ev */
6890	op = 0;
6891	goto do_btx;
6892	case 0x1ab: /* bts */
6893	op = 1;
6894	goto do_btx;
6895	case 0x1b3: /* btr */
6896	op = 2;
6897	goto do_btx;
6898	case 0x1bb: /* btc */
6899	op = 3;
6900	do_btx:
6901	ot = dflag + OT_WORD;
6902	modrm = ldub_code(s->pc++);
6903	reg = ((modrm >> 3) & 7) \| rex_r;
6904	mod = (modrm >> 6) & 3;
6905	rm = (modrm & 7) \| REX_B(s);
6906	gen_op_mov_TN_reg(OT_LONG, 1, reg);
6907	if (mod != 3) {
6908	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
6909	/* specific case: we need to add a displacement */
6910	gen_exts(ot, cpu_T[1]);
6911	tcg_gen_sari_tl(cpu_tmp0, cpu_T[1], 3 + ot);
6912	tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, ot);
6913	tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
6914	gen_op_ld_T0_A0(ot + s->mem_index);
6915	} else {
6916	gen_op_mov_TN_reg(ot, 0, rm);
6917	}
6918	bt_op:
6919	tcg_gen_andi_tl(cpu_T[1], cpu_T[1], (1 << (3 + ot)) - 1);
6920	switch(op) {
6921	case 0:
6922	tcg_gen_shr_tl(cpu_cc_src, cpu_T[0], cpu_T[1]);
6923	tcg_gen_movi_tl(cpu_cc_dst, 0);
6924	break;
6925	case 1:
6926	tcg_gen_shr_tl(cpu_tmp4, cpu_T[0], cpu_T[1]);
6927	tcg_gen_movi_tl(cpu_tmp0, 1);
6928	tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
6929	tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
6930	break;
6931	case 2:
6932	tcg_gen_shr_tl(cpu_tmp4, cpu_T[0], cpu_T[1]);
6933	tcg_gen_movi_tl(cpu_tmp0, 1);
6934	tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
6935	tcg_gen_not_tl(cpu_tmp0, cpu_tmp0);
6936	tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
6937	break;
6938	default:
6939	case 3:
6940	tcg_gen_shr_tl(cpu_tmp4, cpu_T[0], cpu_T[1]);
6941	tcg_gen_movi_tl(cpu_tmp0, 1);
6942	tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
6943	tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
6944	break;
6945	}
6946	s->cc_op = CC_OP_SARB + ot;
6947	if (op != 0) {
6948	if (mod != 3)
6949	gen_op_st_T0_A0(ot + s->mem_index);
6950	else
6951	gen_op_mov_reg_T0(ot, rm);
6952	tcg_gen_mov_tl(cpu_cc_src, cpu_tmp4);
6953	tcg_gen_movi_tl(cpu_cc_dst, 0);
6954	}
6955	break;
6956	case 0x1bc: /* bsf */
6957	case 0x1bd: /* bsr */
6958	{
6959	int label1;
6960	TCGv t0;
6961
6962	ot = dflag + OT_WORD;
6963	modrm = ldub_code(s->pc++);
6964	reg = ((modrm >> 3) & 7) \| rex_r;
6965	gen_ldst_modrm(s,modrm, ot, OR_TMP0, 0);
6966	gen_extu(ot, cpu_T[0]);
6967	t0 = tcg_temp_local_new();
6968	tcg_gen_mov_tl(t0, cpu_T[0]);
6969	if ((b & 1) && (prefixes & PREFIX_REPZ) &&
6970	(s->cpuid_ext3_features & CPUID_EXT3_ABM)) {
6971	switch(ot) {
6972	case OT_WORD: gen_helper_lzcnt(cpu_T[0], t0,
6973	tcg_const_i32(16)); break;
6974	case OT_LONG: gen_helper_lzcnt(cpu_T[0], t0,
6975	tcg_const_i32(32)); break;
6976	case OT_QUAD: gen_helper_lzcnt(cpu_T[0], t0,
6977	tcg_const_i32(64)); break;
6978	}
6979	gen_op_mov_reg_T0(ot, reg);
6980	} else {
6981	label1 = gen_new_label();
6982	tcg_gen_movi_tl(cpu_cc_dst, 0);
6983	tcg_gen_brcondi_tl(TCG_COND_EQ, t0, 0, label1);
6984	if (b & 1) {
6985	gen_helper_bsr(cpu_T[0], t0);
6986	} else {
6987	gen_helper_bsf(cpu_T[0], t0);
6988	}
6989	gen_op_mov_reg_T0(ot, reg);
6990	tcg_gen_movi_tl(cpu_cc_dst, 1);
6991	gen_set_label(label1);
6992	tcg_gen_discard_tl(cpu_cc_src);
6993	s->cc_op = CC_OP_LOGICB + ot;
6994	}
6995	tcg_temp_free(t0);
6996	}
6997	break;
6998	/************************/
6999	/* bcd */
7000	case 0x27: /* daa */
7001	if (CODE64(s))
7002	goto illegal_op;
7003	if (s->cc_op != CC_OP_DYNAMIC)
7004	gen_op_set_cc_op(s->cc_op);
7005	gen_helper_daa();
7006	s->cc_op = CC_OP_EFLAGS;
7007	break;
7008	case 0x2f: /* das */
7009	if (CODE64(s))
7010	goto illegal_op;
7011	if (s->cc_op != CC_OP_DYNAMIC)
7012	gen_op_set_cc_op(s->cc_op);
7013	gen_helper_das();
7014	s->cc_op = CC_OP_EFLAGS;
7015	break;
7016	case 0x37: /* aaa */
7017	if (CODE64(s))
7018	goto illegal_op;
7019	if (s->cc_op != CC_OP_DYNAMIC)
7020	gen_op_set_cc_op(s->cc_op);
7021	gen_helper_aaa();
7022	s->cc_op = CC_OP_EFLAGS;
7023	break;
7024	case 0x3f: /* aas */
7025	if (CODE64(s))
7026	goto illegal_op;
7027	if (s->cc_op != CC_OP_DYNAMIC)
7028	gen_op_set_cc_op(s->cc_op);
7029	gen_helper_aas();
7030	s->cc_op = CC_OP_EFLAGS;
7031	break;
7032	case 0xd4: /* aam */
7033	if (CODE64(s))
7034	goto illegal_op;
7035	val = ldub_code(s->pc++);
7036	if (val == 0) {
7037	gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base);
7038	} else {
7039	gen_helper_aam(tcg_const_i32(val));
7040	s->cc_op = CC_OP_LOGICB;
7041	}
7042	break;
7043	case 0xd5: /* aad */
7044	if (CODE64(s))
7045	goto illegal_op;
7046	val = ldub_code(s->pc++);
7047	gen_helper_aad(tcg_const_i32(val));
7048	s->cc_op = CC_OP_LOGICB;
7049	break;
7050	/************************/
7051	/* misc */
7052	case 0x90: /* nop */
7053	/* XXX: correct lock test for all insn */
7054	if (prefixes & PREFIX_LOCK) {
7055	goto illegal_op;
7056	}
7057	/* If REX_B is set, then this is xchg eax, r8d, not a nop. */
7058	if (REX_B(s)) {
7059	goto do_xchg_reg_eax;
7060	}
7061	if (prefixes & PREFIX_REPZ) {
7062	gen_svm_check_intercept(s, pc_start, SVM_EXIT_PAUSE);
7063	}
7064	break;
7065	case 0x9b: /* fwait */
7066	if ((s->flags & (HF_MP_MASK \| HF_TS_MASK)) ==
7067	(HF_MP_MASK \| HF_TS_MASK)) {
7068	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
7069	} else {
7070	if (s->cc_op != CC_OP_DYNAMIC)
7071	gen_op_set_cc_op(s->cc_op);
7072	gen_jmp_im(pc_start - s->cs_base);
7073	gen_helper_fwait();
7074	}
7075	break;
7076	case 0xcc: /* int3 */
7077	#ifdef VBOX
7078	if (s->vm86 && s->iopl != 3 && !s->vme) {
7079	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7080	} else
7081	#endif
7082	gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base);
7083	break;
7084	case 0xcd: /* int N */
7085	val = ldub_code(s->pc++);
7086	#ifdef VBOX
7087	if (s->vm86 && s->iopl != 3 && !s->vme) {
7088	#else
7089	if (s->vm86 && s->iopl != 3) {
7090	#endif
7091	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7092	} else {
7093	gen_interrupt(s, val, pc_start - s->cs_base, s->pc - s->cs_base);
7094	}
7095	break;
7096	case 0xce: /* into */
7097	if (CODE64(s))
7098	goto illegal_op;
7099	if (s->cc_op != CC_OP_DYNAMIC)
7100	gen_op_set_cc_op(s->cc_op);
7101	gen_jmp_im(pc_start - s->cs_base);
7102	gen_helper_into(tcg_const_i32(s->pc - pc_start));
7103	break;
7104	#ifdef WANT_ICEBP
7105	case 0xf1: /* icebp (undocumented, exits to external debugger) */
7106	gen_svm_check_intercept(s, pc_start, SVM_EXIT_ICEBP);
7107	#if 1
7108	gen_debug(s, pc_start - s->cs_base);
7109	#else
7110	/* start debug */
7111	tb_flush(cpu_single_env);
7112	cpu_set_log(CPU_LOG_INT \| CPU_LOG_TB_IN_ASM);
7113	#endif
7114	break;
7115	#endif
7116	case 0xfa: /* cli */
7117	if (!s->vm86) {
7118	if (s->cpl <= s->iopl) {
7119	gen_helper_cli();
7120	} else {
7121	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7122	}
7123	} else {
7124	if (s->iopl == 3) {
7125	gen_helper_cli();
7126	#ifdef VBOX
7127	} else if (s->iopl != 3 && s->vme) {
7128	gen_helper_cli_vme();
7129	#endif
7130	} else {
7131	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7132	}
7133	}
7134	break;
7135	case 0xfb: /* sti */
7136	if (!s->vm86) {
7137	if (s->cpl <= s->iopl) {
7138	gen_sti:
7139	gen_helper_sti();
7140	/* interruptions are enabled only the first insn after sti */
7141	/* If several instructions disable interrupts, only the
7142	_first_ does it */
7143	if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
7144	gen_helper_set_inhibit_irq();
7145	/* give a chance to handle pending irqs */
7146	gen_jmp_im(s->pc - s->cs_base);
7147	gen_eob(s);
7148	} else {
7149	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7150	}
7151	} else {
7152	if (s->iopl == 3) {
7153	goto gen_sti;
7154	#ifdef VBOX
7155	} else if (s->iopl != 3 && s->vme) {
7156	gen_helper_sti_vme();
7157	/* give a chance to handle pending irqs */
7158	gen_jmp_im(s->pc - s->cs_base);
7159	gen_eob(s);
7160	#endif
7161	} else {
7162	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7163	}
7164	}
7165	break;
7166	case 0x62: /* bound */
7167	if (CODE64(s))
7168	goto illegal_op;
7169	ot = dflag ? OT_LONG : OT_WORD;
7170	modrm = ldub_code(s->pc++);
7171	reg = (modrm >> 3) & 7;
7172	mod = (modrm >> 6) & 3;
7173	if (mod == 3)
7174	goto illegal_op;
7175	gen_op_mov_TN_reg(ot, 0, reg);
7176	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7177	gen_jmp_im(pc_start - s->cs_base);
7178	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
7179	if (ot == OT_WORD)
7180	gen_helper_boundw(cpu_A0, cpu_tmp2_i32);
7181	else
7182	gen_helper_boundl(cpu_A0, cpu_tmp2_i32);
7183	break;
7184	case 0x1c8 ... 0x1cf: /* bswap reg */
7185	reg = (b & 7) \| REX_B(s);
7186	#ifdef TARGET_X86_64
7187	if (dflag == 2) {
7188	gen_op_mov_TN_reg(OT_QUAD, 0, reg);
7189	tcg_gen_bswap64_i64(cpu_T[0], cpu_T[0]);
7190	gen_op_mov_reg_T0(OT_QUAD, reg);
7191	} else
7192	#endif
7193	{
7194	gen_op_mov_TN_reg(OT_LONG, 0, reg);
7195	tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
7196	tcg_gen_bswap32_tl(cpu_T[0], cpu_T[0]);
7197	gen_op_mov_reg_T0(OT_LONG, reg);
7198	}
7199	break;
7200	case 0xd6: /* salc */
7201	if (CODE64(s))
7202	goto illegal_op;
7203	if (s->cc_op != CC_OP_DYNAMIC)
7204	gen_op_set_cc_op(s->cc_op);
7205	gen_compute_eflags_c(cpu_T[0]);
7206	tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
7207	gen_op_mov_reg_T0(OT_BYTE, R_EAX);
7208	break;
7209	case 0xe0: /* loopnz */
7210	case 0xe1: /* loopz */
7211	case 0xe2: /* loop */
7212	case 0xe3: /* jecxz */
7213	{
7214	int l1, l2, l3;
7215
7216	tval = (int8_t)insn_get(s, OT_BYTE);
7217	next_eip = s->pc - s->cs_base;
7218	tval += next_eip;
7219	if (s->dflag == 0)
7220	tval &= 0xffff;
7221
7222	l1 = gen_new_label();
7223	l2 = gen_new_label();
7224	l3 = gen_new_label();
7225	b &= 3;
7226	switch(b) {
7227	case 0: /* loopnz */
7228	case 1: /* loopz */
7229	if (s->cc_op != CC_OP_DYNAMIC)
7230	gen_op_set_cc_op(s->cc_op);
7231	gen_op_add_reg_im(s->aflag, R_ECX, -1);
7232	gen_op_jz_ecx(s->aflag, l3);
7233	gen_compute_eflags(cpu_tmp0);
7234	tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, CC_Z);
7235	if (b == 0) {
7236	tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_tmp0, 0, l1);
7237	} else {
7238	tcg_gen_brcondi_tl(TCG_COND_NE, cpu_tmp0, 0, l1);
7239	}
7240	break;
7241	case 2: /* loop */
7242	gen_op_add_reg_im(s->aflag, R_ECX, -1);
7243	gen_op_jnz_ecx(s->aflag, l1);
7244	break;
7245	default:
7246	case 3: /* jcxz */
7247	gen_op_jz_ecx(s->aflag, l1);
7248	break;
7249	}
7250
7251	gen_set_label(l3);
7252	gen_jmp_im(next_eip);
7253	tcg_gen_br(l2);
7254
7255	gen_set_label(l1);
7256	gen_jmp_im(tval);
7257	gen_set_label(l2);
7258	gen_eob(s);
7259	}
7260	break;
7261	case 0x130: /* wrmsr */
7262	case 0x132: /* rdmsr */
7263	if (s->cpl != 0) {
7264	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7265	} else {
7266	if (s->cc_op != CC_OP_DYNAMIC)
7267	gen_op_set_cc_op(s->cc_op);
7268	gen_jmp_im(pc_start - s->cs_base);
7269	if (b & 2) {
7270	gen_helper_rdmsr();
7271	} else {
7272	gen_helper_wrmsr();
7273	}
7274	}
7275	break;
7276	case 0x131: /* rdtsc */
7277	if (s->cc_op != CC_OP_DYNAMIC)
7278	gen_op_set_cc_op(s->cc_op);
7279	gen_jmp_im(pc_start - s->cs_base);
7280	if (use_icount)
7281	gen_io_start();
7282	gen_helper_rdtsc();
7283	if (use_icount) {
7284	gen_io_end();
7285	gen_jmp(s, s->pc - s->cs_base);
7286	}
7287	break;
7288	case 0x133: /* rdpmc */
7289	if (s->cc_op != CC_OP_DYNAMIC)
7290	gen_op_set_cc_op(s->cc_op);
7291	gen_jmp_im(pc_start - s->cs_base);
7292	gen_helper_rdpmc();
7293	break;
7294	case 0x134: /* sysenter */
7295	#ifndef VBOX
7296	/* For Intel SYSENTER is valid on 64-bit */
7297	if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
7298	#else
7299	/** @todo: make things right */
7300	if (CODE64(s))
7301	#endif
7302	goto illegal_op;
7303	if (!s->pe) {
7304	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7305	} else {
7306	if (s->cc_op != CC_OP_DYNAMIC) {
7307	gen_op_set_cc_op(s->cc_op);
7308	s->cc_op = CC_OP_DYNAMIC;
7309	}
7310	gen_jmp_im(pc_start - s->cs_base);
7311	gen_helper_sysenter();
7312	gen_eob(s);
7313	}
7314	break;
7315	case 0x135: /* sysexit */
7316	#ifndef VBOX
7317	/* For Intel SYSEXIT is valid on 64-bit */
7318	if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
7319	#else
7320	/** @todo: make things right */
7321	if (CODE64(s))
7322	#endif
7323	goto illegal_op;
7324	if (!s->pe) {
7325	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7326	} else {
7327	if (s->cc_op != CC_OP_DYNAMIC) {
7328	gen_op_set_cc_op(s->cc_op);
7329	s->cc_op = CC_OP_DYNAMIC;
7330	}
7331	gen_jmp_im(pc_start - s->cs_base);
7332	gen_helper_sysexit(tcg_const_i32(dflag));
7333	gen_eob(s);
7334	}
7335	break;
7336	#ifdef TARGET_X86_64
7337	case 0x105: /* syscall */
7338	/* XXX: is it usable in real mode ? */
7339	if (s->cc_op != CC_OP_DYNAMIC) {
7340	gen_op_set_cc_op(s->cc_op);
7341	s->cc_op = CC_OP_DYNAMIC;
7342	}
7343	gen_jmp_im(pc_start - s->cs_base);
7344	gen_helper_syscall(tcg_const_i32(s->pc - pc_start));
7345	gen_eob(s);
7346	break;
7347	case 0x107: /* sysret */
7348	if (!s->pe) {
7349	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7350	} else {
7351	if (s->cc_op != CC_OP_DYNAMIC) {
7352	gen_op_set_cc_op(s->cc_op);
7353	s->cc_op = CC_OP_DYNAMIC;
7354	}
7355	gen_jmp_im(pc_start - s->cs_base);
7356	gen_helper_sysret(tcg_const_i32(s->dflag));
7357	/* condition codes are modified only in long mode */
7358	if (s->lma)
7359	s->cc_op = CC_OP_EFLAGS;
7360	gen_eob(s);
7361	}
7362	break;
7363	#endif
7364	case 0x1a2: /* cpuid */
7365	if (s->cc_op != CC_OP_DYNAMIC)
7366	gen_op_set_cc_op(s->cc_op);
7367	gen_jmp_im(pc_start - s->cs_base);
7368	gen_helper_cpuid();
7369	break;
7370	case 0xf4: /* hlt */
7371	if (s->cpl != 0) {
7372	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7373	} else {
7374	if (s->cc_op != CC_OP_DYNAMIC)
7375	gen_op_set_cc_op(s->cc_op);
7376	gen_jmp_im(pc_start - s->cs_base);
7377	gen_helper_hlt(tcg_const_i32(s->pc - pc_start));
7378	s->is_jmp = 3;
7379	}
7380	break;
7381	case 0x100:
7382	modrm = ldub_code(s->pc++);
7383	mod = (modrm >> 6) & 3;
7384	op = (modrm >> 3) & 7;
7385	switch(op) {
7386	case 0: /* sldt */
7387	if (!s->pe \|\| s->vm86)
7388	goto illegal_op;
7389	gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ);
7390	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,ldt.selector));
7391	ot = OT_WORD;
7392	if (mod == 3)
7393	ot += s->dflag;
7394	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
7395	break;
7396	case 2: /* lldt */
7397	if (!s->pe \|\| s->vm86)
7398	goto illegal_op;
7399	if (s->cpl != 0) {
7400	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7401	} else {
7402	gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
7403	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
7404	gen_jmp_im(pc_start - s->cs_base);
7405	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
7406	gen_helper_lldt(cpu_tmp2_i32);
7407	}
7408	break;
7409	case 1: /* str */
7410	if (!s->pe \|\| s->vm86)
7411	goto illegal_op;
7412	gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ);
7413	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,tr.selector));
7414	ot = OT_WORD;
7415	if (mod == 3)
7416	ot += s->dflag;
7417	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
7418	break;
7419	case 3: /* ltr */
7420	if (!s->pe \|\| s->vm86)
7421	goto illegal_op;
7422	if (s->cpl != 0) {
7423	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7424	} else {
7425	gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
7426	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
7427	gen_jmp_im(pc_start - s->cs_base);
7428	tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
7429	gen_helper_ltr(cpu_tmp2_i32);
7430	}
7431	break;
7432	case 4: /* verr */
7433	case 5: /* verw */
7434	if (!s->pe \|\| s->vm86)
7435	goto illegal_op;
7436	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
7437	if (s->cc_op != CC_OP_DYNAMIC)
7438	gen_op_set_cc_op(s->cc_op);
7439	if (op == 4)
7440	gen_helper_verr(cpu_T[0]);
7441	else
7442	gen_helper_verw(cpu_T[0]);
7443	s->cc_op = CC_OP_EFLAGS;
7444	break;
7445	default:
7446	goto illegal_op;
7447	}
7448	break;
7449	case 0x101:
7450	modrm = ldub_code(s->pc++);
7451	mod = (modrm >> 6) & 3;
7452	op = (modrm >> 3) & 7;
7453	rm = modrm & 7;
7454	#ifdef VBOX
7455	/* 0f 01 f9 */
7456	if (modrm == 0xf9)
7457	{
7458	if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP))
7459	goto illegal_op;
7460	gen_jmp_im(pc_start - s->cs_base);
7461	gen_helper_rdtscp();
7462	break;
7463	}
7464	#endif /* VBOX */
7465	switch(op) {
7466	case 0: /* sgdt */
7467	if (mod == 3)
7468	goto illegal_op;
7469	gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
7470	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7471	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
7472	gen_op_st_T0_A0(OT_WORD + s->mem_index);
7473	gen_add_A0_im(s, 2);
7474	tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.base));
7475	if (!s->dflag)
7476	gen_op_andl_T0_im(0xffffff);
7477	gen_op_st_T0_A0(CODE64(s) + OT_LONG + s->mem_index);
7478	break;
7479	case 1:
7480	if (mod == 3) {
7481	switch (rm) {
7482	case 0: /* monitor */
7483	if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) \|\|
7484	s->cpl != 0)
7485	goto illegal_op;
7486	if (s->cc_op != CC_OP_DYNAMIC)
7487	gen_op_set_cc_op(s->cc_op);
7488	gen_jmp_im(pc_start - s->cs_base);
7489	#ifdef TARGET_X86_64
7490	if (s->aflag == 2) {
7491	gen_op_movq_A0_reg(R_EAX);
7492	} else
7493	#endif
7494	{
7495	gen_op_movl_A0_reg(R_EAX);
7496	if (s->aflag == 0)
7497	gen_op_andl_A0_ffff();
7498	}
7499	gen_add_A0_ds_seg(s);
7500	gen_helper_monitor(cpu_A0);
7501	break;
7502	case 1: /* mwait */
7503	if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) \|\|
7504	s->cpl != 0)
7505	goto illegal_op;
7506	if (s->cc_op != CC_OP_DYNAMIC) {
7507	gen_op_set_cc_op(s->cc_op);
7508	s->cc_op = CC_OP_DYNAMIC;
7509	}
7510	gen_jmp_im(pc_start - s->cs_base);
7511	gen_helper_mwait(tcg_const_i32(s->pc - pc_start));
7512	gen_eob(s);
7513	break;
7514	default:
7515	goto illegal_op;
7516	}
7517	} else { /* sidt */
7518	gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
7519	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7520	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
7521	gen_op_st_T0_A0(OT_WORD + s->mem_index);
7522	gen_add_A0_im(s, 2);
7523	tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.base));
7524	if (!s->dflag)
7525	gen_op_andl_T0_im(0xffffff);
7526	gen_op_st_T0_A0(CODE64(s) + OT_LONG + s->mem_index);
7527	}
7528	break;
7529	case 2: /* lgdt */
7530	case 3: /* lidt */
7531	if (mod == 3) {
7532	if (s->cc_op != CC_OP_DYNAMIC)
7533	gen_op_set_cc_op(s->cc_op);
7534	gen_jmp_im(pc_start - s->cs_base);
7535	switch(rm) {
7536	case 0: /* VMRUN */
7537	if (!(s->flags & HF_SVME_MASK) \|\| !s->pe)
7538	goto illegal_op;
7539	if (s->cpl != 0) {
7540	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7541	break;
7542	} else {
7543	gen_helper_vmrun(tcg_const_i32(s->aflag),
7544	tcg_const_i32(s->pc - pc_start));
7545	tcg_gen_exit_tb(0);
7546	s->is_jmp = 3;
7547	}
7548	break;
7549	case 1: /* VMMCALL */
7550	if (!(s->flags & HF_SVME_MASK))
7551	goto illegal_op;
7552	gen_helper_vmmcall();
7553	break;
7554	case 2: /* VMLOAD */
7555	if (!(s->flags & HF_SVME_MASK) \|\| !s->pe)
7556	goto illegal_op;
7557	if (s->cpl != 0) {
7558	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7559	break;
7560	} else {
7561	gen_helper_vmload(tcg_const_i32(s->aflag));
7562	}
7563	break;
7564	case 3: /* VMSAVE */
7565	if (!(s->flags & HF_SVME_MASK) \|\| !s->pe)
7566	goto illegal_op;
7567	if (s->cpl != 0) {
7568	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7569	break;
7570	} else {
7571	gen_helper_vmsave(tcg_const_i32(s->aflag));
7572	}
7573	break;
7574	case 4: /* STGI */
7575	if ((!(s->flags & HF_SVME_MASK) &&
7576	!(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) \|\|
7577	!s->pe)
7578	goto illegal_op;
7579	if (s->cpl != 0) {
7580	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7581	break;
7582	} else {
7583	gen_helper_stgi();
7584	}
7585	break;
7586	case 5: /* CLGI */
7587	if (!(s->flags & HF_SVME_MASK) \|\| !s->pe)
7588	goto illegal_op;
7589	if (s->cpl != 0) {
7590	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7591	break;
7592	} else {
7593	gen_helper_clgi();
7594	}
7595	break;
7596	case 6: /* SKINIT */
7597	if ((!(s->flags & HF_SVME_MASK) &&
7598	!(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) \|\|
7599	!s->pe)
7600	goto illegal_op;
7601	gen_helper_skinit();
7602	break;
7603	case 7: /* INVLPGA */
7604	if (!(s->flags & HF_SVME_MASK) \|\| !s->pe)
7605	goto illegal_op;
7606	if (s->cpl != 0) {
7607	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7608	break;
7609	} else {
7610	gen_helper_invlpga(tcg_const_i32(s->aflag));
7611	}
7612	break;
7613	default:
7614	goto illegal_op;
7615	}
7616	} else if (s->cpl != 0) {
7617	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7618	} else {
7619	gen_svm_check_intercept(s, pc_start,
7620	op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
7621	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7622	gen_op_ld_T1_A0(OT_WORD + s->mem_index);
7623	gen_add_A0_im(s, 2);
7624	gen_op_ld_T0_A0(CODE64(s) + OT_LONG + s->mem_index);
7625	if (!s->dflag)
7626	gen_op_andl_T0_im(0xffffff);
7627	if (op == 2) {
7628	tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,gdt.base));
7629	tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,gdt.limit));
7630	} else {
7631	tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,idt.base));
7632	tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,idt.limit));
7633	}
7634	}
7635	break;
7636	case 4: /* smsw */
7637	gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
7638	#if defined TARGET_X86_64 && defined HOST_WORDS_BIGENDIAN
7639	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]) + 4);
7640	#else
7641	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
7642	#endif
7643	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 1);
7644	break;
7645	case 6: /* lmsw */
7646	if (s->cpl != 0) {
7647	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7648	} else {
7649	gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
7650	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
7651	gen_helper_lmsw(cpu_T[0]);
7652	gen_jmp_im(s->pc - s->cs_base);
7653	gen_eob(s);
7654	}
7655	break;
7656	case 7:
7657	if (mod != 3) { /* invlpg */
7658	if (s->cpl != 0) {
7659	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7660	} else {
7661	if (s->cc_op != CC_OP_DYNAMIC)
7662	gen_op_set_cc_op(s->cc_op);
7663	gen_jmp_im(pc_start - s->cs_base);
7664	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7665	gen_helper_invlpg(cpu_A0);
7666	gen_jmp_im(s->pc - s->cs_base);
7667	gen_eob(s);
7668	}
7669	} else {
7670	switch (rm) {
7671	case 0: /* swapgs */
7672	#ifdef TARGET_X86_64
7673	if (CODE64(s)) {
7674	if (s->cpl != 0) {
7675	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7676	} else {
7677	tcg_gen_ld_tl(cpu_T[0], cpu_env,
7678	offsetof(CPUX86State,segs[R_GS].base));
7679	tcg_gen_ld_tl(cpu_T[1], cpu_env,
7680	offsetof(CPUX86State,kernelgsbase));
7681	tcg_gen_st_tl(cpu_T[1], cpu_env,
7682	offsetof(CPUX86State,segs[R_GS].base));
7683	tcg_gen_st_tl(cpu_T[0], cpu_env,
7684	offsetof(CPUX86State,kernelgsbase));
7685	}
7686	} else
7687	#endif
7688	{
7689	goto illegal_op;
7690	}
7691	break;
7692	case 1: /* rdtscp */
7693	if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP))
7694	goto illegal_op;
7695	if (s->cc_op != CC_OP_DYNAMIC)
7696	gen_op_set_cc_op(s->cc_op);
7697	gen_jmp_im(pc_start - s->cs_base);
7698	if (use_icount)
7699	gen_io_start();
7700	gen_helper_rdtscp();
7701	if (use_icount) {
7702	gen_io_end();
7703	gen_jmp(s, s->pc - s->cs_base);
7704	}
7705	break;
7706	default:
7707	goto illegal_op;
7708	}
7709	}
7710	break;
7711	default:
7712	goto illegal_op;
7713	}
7714	break;
7715	case 0x108: /* invd */
7716	case 0x109: /* wbinvd */
7717	if (s->cpl != 0) {
7718	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7719	} else {
7720	gen_svm_check_intercept(s, pc_start, (b & 2) ? SVM_EXIT_INVD : SVM_EXIT_WBINVD);
7721	/* nothing to do */
7722	}
7723	break;
7724	case 0x63: /* arpl or movslS (x86_64) */
7725	#ifdef TARGET_X86_64
7726	if (CODE64(s)) {
7727	int d_ot;
7728	/* d_ot is the size of destination */
7729	d_ot = dflag + OT_WORD;
7730
7731	modrm = ldub_code(s->pc++);
7732	reg = ((modrm >> 3) & 7) \| rex_r;
7733	mod = (modrm >> 6) & 3;
7734	rm = (modrm & 7) \| REX_B(s);
7735
7736	if (mod == 3) {
7737	gen_op_mov_TN_reg(OT_LONG, 0, rm);
7738	/* sign extend */
7739	if (d_ot == OT_QUAD)
7740	tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
7741	gen_op_mov_reg_T0(d_ot, reg);
7742	} else {
7743	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7744	if (d_ot == OT_QUAD) {
7745	gen_op_lds_T0_A0(OT_LONG + s->mem_index);
7746	} else {
7747	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
7748	}
7749	gen_op_mov_reg_T0(d_ot, reg);
7750	}
7751	} else
7752	#endif
7753	{
7754	int label1;
7755	TCGv t0, t1, t2, a0;
7756
7757	if (!s->pe \|\| s->vm86)
7758	goto illegal_op;
7759	t0 = tcg_temp_local_new();
7760	t1 = tcg_temp_local_new();
7761	t2 = tcg_temp_local_new();
7762	ot = OT_WORD;
7763	modrm = ldub_code(s->pc++);
7764	reg = (modrm >> 3) & 7;
7765	mod = (modrm >> 6) & 3;
7766	rm = modrm & 7;
7767	if (mod != 3) {
7768	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7769	gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
7770	a0 = tcg_temp_local_new();
7771	tcg_gen_mov_tl(a0, cpu_A0);
7772	} else {
7773	gen_op_mov_v_reg(ot, t0, rm);
7774	TCGV_UNUSED(a0);
7775	}
7776	gen_op_mov_v_reg(ot, t1, reg);
7777	tcg_gen_andi_tl(cpu_tmp0, t0, 3);
7778	tcg_gen_andi_tl(t1, t1, 3);
7779	tcg_gen_movi_tl(t2, 0);
7780	label1 = gen_new_label();
7781	tcg_gen_brcond_tl(TCG_COND_GE, cpu_tmp0, t1, label1);
7782	tcg_gen_andi_tl(t0, t0, ~3);
7783	tcg_gen_or_tl(t0, t0, t1);
7784	tcg_gen_movi_tl(t2, CC_Z);
7785	gen_set_label(label1);
7786	if (mod != 3) {
7787	gen_op_st_v(ot + s->mem_index, t0, a0);
7788	tcg_temp_free(a0);
7789	} else {
7790	gen_op_mov_reg_v(ot, rm, t0);
7791	}
7792	if (s->cc_op != CC_OP_DYNAMIC)
7793	gen_op_set_cc_op(s->cc_op);
7794	gen_compute_eflags(cpu_cc_src);
7795	tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, ~CC_Z);
7796	tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, t2);
7797	s->cc_op = CC_OP_EFLAGS;
7798	tcg_temp_free(t0);
7799	tcg_temp_free(t1);
7800	tcg_temp_free(t2);
7801	}
7802	break;
7803	case 0x102: /* lar */
7804	case 0x103: /* lsl */
7805	{
7806	int label1;
7807	TCGv t0;
7808	if (!s->pe \|\| s->vm86)
7809	goto illegal_op;
7810	ot = dflag ? OT_LONG : OT_WORD;
7811	modrm = ldub_code(s->pc++);
7812	reg = ((modrm >> 3) & 7) \| rex_r;
7813	gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
7814	t0 = tcg_temp_local_new();
7815	if (s->cc_op != CC_OP_DYNAMIC)
7816	gen_op_set_cc_op(s->cc_op);
7817	if (b == 0x102)
7818	gen_helper_lar(t0, cpu_T[0]);
7819	else
7820	gen_helper_lsl(t0, cpu_T[0]);
7821	tcg_gen_andi_tl(cpu_tmp0, cpu_cc_src, CC_Z);
7822	label1 = gen_new_label();
7823	tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_tmp0, 0, label1);
7824	gen_op_mov_reg_v(ot, reg, t0);
7825	gen_set_label(label1);
7826	s->cc_op = CC_OP_EFLAGS;
7827	tcg_temp_free(t0);
7828	}
7829	break;
7830	case 0x118:
7831	modrm = ldub_code(s->pc++);
7832	mod = (modrm >> 6) & 3;
7833	op = (modrm >> 3) & 7;
7834	switch(op) {
7835	case 0: /* prefetchnta */
7836	case 1: /* prefetchnt0 */
7837	case 2: /* prefetchnt0 */
7838	case 3: /* prefetchnt0 */
7839	if (mod == 3)
7840	goto illegal_op;
7841	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7842	/* nothing more to do */
7843	break;
7844	default: /* nop (multi byte) */
7845	gen_nop_modrm(s, modrm);
7846	break;
7847	}
7848	break;
7849	case 0x119 ... 0x11f: /* nop (multi byte) */
7850	modrm = ldub_code(s->pc++);
7851	gen_nop_modrm(s, modrm);
7852	break;
7853	case 0x120: /* mov reg, crN */
7854	case 0x122: /* mov crN, reg */
7855	if (s->cpl != 0) {
7856	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7857	} else {
7858	modrm = ldub_code(s->pc++);
7859	#ifndef VBOX /* mod bits are always understood to be 11 (0xc0) regardless of actual content; see AMD manuals */
7860	if ((modrm & 0xc0) != 0xc0)
7861	goto illegal_op;
7862	#endif
7863	rm = (modrm & 7) \| REX_B(s);
7864	reg = ((modrm >> 3) & 7) \| rex_r;
7865	if (CODE64(s))
7866	ot = OT_QUAD;
7867	else
7868	ot = OT_LONG;
7869	if ((prefixes & PREFIX_LOCK) && (reg == 0) &&
7870	(s->cpuid_ext3_features & CPUID_EXT3_CR8LEG)) {
7871	reg = 8;
7872	}
7873	switch(reg) {
7874	case 0:
7875	case 2:
7876	case 3:
7877	case 4:
7878	case 8:
7879	if (s->cc_op != CC_OP_DYNAMIC)
7880	gen_op_set_cc_op(s->cc_op);
7881	gen_jmp_im(pc_start - s->cs_base);
7882	if (b & 2) {
7883	gen_op_mov_TN_reg(ot, 0, rm);
7884	gen_helper_write_crN(tcg_const_i32(reg), cpu_T[0]);
7885	gen_jmp_im(s->pc - s->cs_base);
7886	gen_eob(s);
7887	} else {
7888	gen_helper_read_crN(cpu_T[0], tcg_const_i32(reg));
7889	gen_op_mov_reg_T0(ot, rm);
7890	}
7891	break;
7892	default:
7893	goto illegal_op;
7894	}
7895	}
7896	break;
7897	case 0x121: /* mov reg, drN */
7898	case 0x123: /* mov drN, reg */
7899	if (s->cpl != 0) {
7900	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7901	} else {
7902	modrm = ldub_code(s->pc++);
7903	#ifndef VBOX /* mod bits are always understood to be 11 (0xc0) regardless of actual content; see AMD manuals */
7904	if ((modrm & 0xc0) != 0xc0)
7905	goto illegal_op;
7906	#endif
7907	rm = (modrm & 7) \| REX_B(s);
7908	reg = ((modrm >> 3) & 7) \| rex_r;
7909	if (CODE64(s))
7910	ot = OT_QUAD;
7911	else
7912	ot = OT_LONG;
7913	/* XXX: do it dynamically with CR4.DE bit */
7914	if (reg == 4 \|\| reg == 5 \|\| reg >= 8)
7915	goto illegal_op;
7916	if (b & 2) {
7917	gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
7918	gen_op_mov_TN_reg(ot, 0, rm);
7919	gen_helper_movl_drN_T0(tcg_const_i32(reg), cpu_T[0]);
7920	gen_jmp_im(s->pc - s->cs_base);
7921	gen_eob(s);
7922	} else {
7923	gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_DR0 + reg);
7924	tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,dr[reg]));
7925	gen_op_mov_reg_T0(ot, rm);
7926	}
7927	}
7928	break;
7929	case 0x106: /* clts */
7930	if (s->cpl != 0) {
7931	gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
7932	} else {
7933	gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
7934	gen_helper_clts();
7935	/* abort block because static cpu state changed */
7936	gen_jmp_im(s->pc - s->cs_base);
7937	gen_eob(s);
7938	}
7939	break;
7940	/* MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4 support */
7941	case 0x1c3: /* MOVNTI reg, mem */
7942	if (!(s->cpuid_features & CPUID_SSE2))
7943	goto illegal_op;
7944	ot = s->dflag == 2 ? OT_QUAD : OT_LONG;
7945	modrm = ldub_code(s->pc++);
7946	mod = (modrm >> 6) & 3;
7947	if (mod == 3)
7948	goto illegal_op;
7949	reg = ((modrm >> 3) & 7) \| rex_r;
7950	/* generate a generic store */
7951	gen_ldst_modrm(s, modrm, ot, reg, 1);
7952	break;
7953	case 0x1ae:
7954	modrm = ldub_code(s->pc++);
7955	mod = (modrm >> 6) & 3;
7956	op = (modrm >> 3) & 7;
7957	switch(op) {
7958	case 0: /* fxsave */
7959	if (mod == 3 \|\| !(s->cpuid_features & CPUID_FXSR) \|\|
7960	(s->prefix & PREFIX_LOCK))
7961	goto illegal_op;
7962	if ((s->flags & HF_EM_MASK) \|\| (s->flags & HF_TS_MASK)) {
7963	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
7964	break;
7965	}
7966	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7967	if (s->cc_op != CC_OP_DYNAMIC)
7968	gen_op_set_cc_op(s->cc_op);
7969	gen_jmp_im(pc_start - s->cs_base);
7970	gen_helper_fxsave(cpu_A0, tcg_const_i32((s->dflag == 2)));
7971	break;
7972	case 1: /* fxrstor */
7973	if (mod == 3 \|\| !(s->cpuid_features & CPUID_FXSR) \|\|
7974	(s->prefix & PREFIX_LOCK))
7975	goto illegal_op;
7976	if ((s->flags & HF_EM_MASK) \|\| (s->flags & HF_TS_MASK)) {
7977	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
7978	break;
7979	}
7980	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7981	if (s->cc_op != CC_OP_DYNAMIC)
7982	gen_op_set_cc_op(s->cc_op);
7983	gen_jmp_im(pc_start - s->cs_base);
7984	gen_helper_fxrstor(cpu_A0, tcg_const_i32((s->dflag == 2)));
7985	break;
7986	case 2: /* ldmxcsr */
7987	case 3: /* stmxcsr */
7988	if (s->flags & HF_TS_MASK) {
7989	gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
7990	break;
7991	}
7992	if ((s->flags & HF_EM_MASK) \|\| !(s->flags & HF_OSFXSR_MASK) \|\|
7993	mod == 3)
7994	goto illegal_op;
7995	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
7996	if (op == 2) {
7997	gen_op_ld_T0_A0(OT_LONG + s->mem_index);
7998	tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, mxcsr));
7999	} else {
8000	tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, mxcsr));
8001	gen_op_st_T0_A0(OT_LONG + s->mem_index);
8002	}
8003	break;
8004	case 5: /* lfence */
8005	case 6: /* mfence */
8006	if ((modrm & 0xc7) != 0xc0 \|\| !(s->cpuid_features & CPUID_SSE))
8007	goto illegal_op;
8008	break;
8009	case 7: /* sfence / clflush */
8010	if ((modrm & 0xc7) == 0xc0) {
8011	/* sfence */
8012	/* XXX: also check for cpuid_ext2_features & CPUID_EXT2_EMMX */
8013	if (!(s->cpuid_features & CPUID_SSE))
8014	goto illegal_op;
8015	} else {
8016	/* clflush */
8017	if (!(s->cpuid_features & CPUID_CLFLUSH))
8018	goto illegal_op;
8019	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
8020	}
8021	break;
8022	default:
8023	goto illegal_op;
8024	}
8025	break;
8026	case 0x10d: /* 3DNow! prefetch(w) */
8027	modrm = ldub_code(s->pc++);
8028	mod = (modrm >> 6) & 3;
8029	if (mod == 3)
8030	goto illegal_op;
8031	gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
8032	/* ignore for now */
8033	break;
8034	case 0x1aa: /* rsm */
8035	gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM);
8036	if (!(s->flags & HF_SMM_MASK))
8037	goto illegal_op;
8038	if (s->cc_op != CC_OP_DYNAMIC) {
8039	gen_op_set_cc_op(s->cc_op);
8040	s->cc_op = CC_OP_DYNAMIC;
8041	}
8042	gen_jmp_im(s->pc - s->cs_base);
8043	gen_helper_rsm();
8044	gen_eob(s);
8045	break;
8046	case 0x1b8: /* SSE4.2 popcnt */
8047	if ((prefixes & (PREFIX_REPZ \| PREFIX_LOCK \| PREFIX_REPNZ)) !=
8048	PREFIX_REPZ)
8049	goto illegal_op;
8050	if (!(s->cpuid_ext_features & CPUID_EXT_POPCNT))
8051	goto illegal_op;
8052
8053	modrm = ldub_code(s->pc++);
8054	reg = ((modrm >> 3) & 7);
8055
8056	if (s->prefix & PREFIX_DATA)
8057	ot = OT_WORD;
8058	else if (s->dflag != 2)
8059	ot = OT_LONG;
8060	else
8061	ot = OT_QUAD;
8062
8063	gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
8064	gen_helper_popcnt(cpu_T[0], cpu_T[0], tcg_const_i32(ot));
8065	gen_op_mov_reg_T0(ot, reg);
8066
8067	s->cc_op = CC_OP_EFLAGS;
8068	break;
8069	case 0x10e ... 0x10f:
8070	/* 3DNow! instructions, ignore prefixes */
8071	s->prefix &= ~(PREFIX_REPZ \| PREFIX_REPNZ \| PREFIX_DATA);
8072	case 0x110 ... 0x117:
8073	case 0x128 ... 0x12f:
8074	case 0x138 ... 0x13a:
8075	case 0x150 ... 0x179:
8076	case 0x17c ... 0x17f:
8077	case 0x1c2:
8078	case 0x1c4 ... 0x1c6:
8079	case 0x1d0 ... 0x1fe:
8080	gen_sse(s, b, pc_start, rex_r);
8081	break;
8082	default:
8083	goto illegal_op;
8084	}
8085	/* lock generation */
8086	if (s->prefix & PREFIX_LOCK)
8087	gen_helper_unlock();
8088	return s->pc;
8089	illegal_op:
8090	if (s->prefix & PREFIX_LOCK)
8091	gen_helper_unlock();
8092	/* XXX: ensure that no lock was generated */
8093	gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
8094	return s->pc;
8095	}
8096
8097	void optimize_flags_init(void)
8098	{
8099	#if TCG_TARGET_REG_BITS == 32
8100	assert(sizeof(CCTable) == (1 << 3));
8101	#else
8102	assert(sizeof(CCTable) == (1 << 4));
8103	#endif
8104	cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
8105	cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0,
8106	offsetof(CPUState, cc_op), "cc_op");
8107	cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_src),
8108	"cc_src");
8109	cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_dst),
8110	"cc_dst");
8111	cpu_cc_tmp = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_tmp),
8112	"cc_tmp");
8113
8114	#ifdef TARGET_X86_64
8115	cpu_regs[R_EAX] = tcg_global_mem_new_i64(TCG_AREG0,
8116	offsetof(CPUState, regs[R_EAX]), "rax");
8117	cpu_regs[R_ECX] = tcg_global_mem_new_i64(TCG_AREG0,
8118	offsetof(CPUState, regs[R_ECX]), "rcx");
8119	cpu_regs[R_EDX] = tcg_global_mem_new_i64(TCG_AREG0,
8120	offsetof(CPUState, regs[R_EDX]), "rdx");
8121	cpu_regs[R_EBX] = tcg_global_mem_new_i64(TCG_AREG0,
8122	offsetof(CPUState, regs[R_EBX]), "rbx");
8123	cpu_regs[R_ESP] = tcg_global_mem_new_i64(TCG_AREG0,
8124	offsetof(CPUState, regs[R_ESP]), "rsp");
8125	cpu_regs[R_EBP] = tcg_global_mem_new_i64(TCG_AREG0,
8126	offsetof(CPUState, regs[R_EBP]), "rbp");
8127	cpu_regs[R_ESI] = tcg_global_mem_new_i64(TCG_AREG0,
8128	offsetof(CPUState, regs[R_ESI]), "rsi");
8129	cpu_regs[R_EDI] = tcg_global_mem_new_i64(TCG_AREG0,
8130	offsetof(CPUState, regs[R_EDI]), "rdi");
8131	cpu_regs[8] = tcg_global_mem_new_i64(TCG_AREG0,
8132	offsetof(CPUState, regs[8]), "r8");
8133	cpu_regs[9] = tcg_global_mem_new_i64(TCG_AREG0,
8134	offsetof(CPUState, regs[9]), "r9");
8135	cpu_regs[10] = tcg_global_mem_new_i64(TCG_AREG0,
8136	offsetof(CPUState, regs[10]), "r10");
8137	cpu_regs[11] = tcg_global_mem_new_i64(TCG_AREG0,
8138	offsetof(CPUState, regs[11]), "r11");
8139	cpu_regs[12] = tcg_global_mem_new_i64(TCG_AREG0,
8140	offsetof(CPUState, regs[12]), "r12");
8141	cpu_regs[13] = tcg_global_mem_new_i64(TCG_AREG0,
8142	offsetof(CPUState, regs[13]), "r13");
8143	cpu_regs[14] = tcg_global_mem_new_i64(TCG_AREG0,
8144	offsetof(CPUState, regs[14]), "r14");
8145	cpu_regs[15] = tcg_global_mem_new_i64(TCG_AREG0,
8146	offsetof(CPUState, regs[15]), "r15");
8147	#else
8148	cpu_regs[R_EAX] = tcg_global_mem_new_i32(TCG_AREG0,
8149	offsetof(CPUState, regs[R_EAX]), "eax");
8150	cpu_regs[R_ECX] = tcg_global_mem_new_i32(TCG_AREG0,
8151	offsetof(CPUState, regs[R_ECX]), "ecx");
8152	cpu_regs[R_EDX] = tcg_global_mem_new_i32(TCG_AREG0,
8153	offsetof(CPUState, regs[R_EDX]), "edx");
8154	cpu_regs[R_EBX] = tcg_global_mem_new_i32(TCG_AREG0,
8155	offsetof(CPUState, regs[R_EBX]), "ebx");
8156	cpu_regs[R_ESP] = tcg_global_mem_new_i32(TCG_AREG0,
8157	offsetof(CPUState, regs[R_ESP]), "esp");
8158	cpu_regs[R_EBP] = tcg_global_mem_new_i32(TCG_AREG0,
8159	offsetof(CPUState, regs[R_EBP]), "ebp");
8160	cpu_regs[R_ESI] = tcg_global_mem_new_i32(TCG_AREG0,
8161	offsetof(CPUState, regs[R_ESI]), "esi");
8162	cpu_regs[R_EDI] = tcg_global_mem_new_i32(TCG_AREG0,
8163	offsetof(CPUState, regs[R_EDI]), "edi");
8164	#endif
8165
8166	/* register helpers */
8167	#define GEN_HELPER 2
8168	#include "helper.h"
8169	}
8170
8171	/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
8172	basic block 'tb'. If search_pc is TRUE, also generate PC
8173	information for each intermediate instruction. */
8174	static inline void gen_intermediate_code_internal(CPUState *env,
8175	TranslationBlock *tb,
8176	int search_pc)
8177	{
8178	DisasContext dc1, *dc = &dc1;
8179	target_ulong pc_ptr;
8180	uint16_t *gen_opc_end;
8181	CPUBreakpoint *bp;
8182	int j, lj, cflags;
8183	uint64_t flags;
8184	target_ulong pc_start;
8185	target_ulong cs_base;
8186	int num_insns;
8187	int max_insns;
8188	#ifdef VBOX
8189	int const singlestep = env->state & CPU_EMULATE_SINGLE_STEP;
8190	#endif
8191
8192	/* generate intermediate code */
8193	pc_start = tb->pc;
8194	cs_base = tb->cs_base;
8195	flags = tb->flags;
8196	cflags = tb->cflags;
8197
8198	dc->pe = (flags >> HF_PE_SHIFT) & 1;
8199	dc->code32 = (flags >> HF_CS32_SHIFT) & 1;
8200	dc->ss32 = (flags >> HF_SS32_SHIFT) & 1;
8201	dc->addseg = (flags >> HF_ADDSEG_SHIFT) & 1;
8202	dc->f_st = 0;
8203	dc->vm86 = (flags >> VM_SHIFT) & 1;
8204	#ifdef VBOX
8205	dc->vme = !!(env->cr[4] & CR4_VME_MASK);
8206	dc->pvi = !!(env->cr[4] & CR4_PVI_MASK);
8207	# ifdef VBOX_WITH_CALL_RECORD
8208	if ( !(env->state & CPU_RAW_RING0)
8209	&& (env->cr[0] & CR0_PG_MASK)
8210	&& !(env->eflags & X86_EFL_IF)
8211	&& dc->code32)
8212	dc->record_call = 1;
8213	else
8214	dc->record_call = 0;
8215	# endif
8216	#endif /* VBOX */
8217	dc->cpl = (flags >> HF_CPL_SHIFT) & 3;
8218	dc->iopl = (flags >> IOPL_SHIFT) & 3;
8219	dc->tf = (flags >> TF_SHIFT) & 1;
8220	dc->singlestep_enabled = env->singlestep_enabled;
8221	dc->cc_op = CC_OP_DYNAMIC;
8222	dc->cs_base = cs_base;
8223	dc->tb = tb;
8224	dc->popl_esp_hack = 0;
8225	/* select memory access functions */
8226	dc->mem_index = 0;
8227	if (flags & HF_SOFTMMU_MASK) {
8228	if (dc->cpl == 3)
8229	dc->mem_index = 2 * 4;
8230	else
8231	dc->mem_index = 1 * 4;
8232	}
8233	dc->cpuid_features = env->cpuid_features;
8234	dc->cpuid_ext_features = env->cpuid_ext_features;
8235	dc->cpuid_ext2_features = env->cpuid_ext2_features;
8236	dc->cpuid_ext3_features = env->cpuid_ext3_features;
8237	#ifdef TARGET_X86_64
8238	dc->lma = (flags >> HF_LMA_SHIFT) & 1;
8239	dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
8240	#endif
8241	dc->flags = flags;
8242	dc->jmp_opt = !(dc->tf \|\| env->singlestep_enabled \|\|
8243	(flags & HF_INHIBIT_IRQ_MASK)
8244	#ifndef CONFIG_SOFTMMU
8245	\|\| (flags & HF_SOFTMMU_MASK)
8246	#endif
8247	);
8248	#if 0
8249	/* check addseg logic */
8250	if (!dc->addseg && (dc->vm86 \|\| !dc->pe \|\| !dc->code32))
8251	printf("ERROR addseg\n");
8252	#endif
8253
8254	cpu_T[0] = tcg_temp_new();
8255	cpu_T[1] = tcg_temp_new();
8256	cpu_A0 = tcg_temp_new();
8257	cpu_T3 = tcg_temp_new();
8258
8259	cpu_tmp0 = tcg_temp_new();
8260	cpu_tmp1_i64 = tcg_temp_new_i64();
8261	cpu_tmp2_i32 = tcg_temp_new_i32();
8262	cpu_tmp3_i32 = tcg_temp_new_i32();
8263	cpu_tmp4 = tcg_temp_new();
8264	cpu_tmp5 = tcg_temp_new();
8265	cpu_ptr0 = tcg_temp_new_ptr();
8266	cpu_ptr1 = tcg_temp_new_ptr();
8267
8268	gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
8269
8270	dc->is_jmp = DISAS_NEXT;
8271	pc_ptr = pc_start;
8272	lj = -1;
8273	num_insns = 0;
8274	max_insns = tb->cflags & CF_COUNT_MASK;
8275	if (max_insns == 0)
8276	max_insns = CF_COUNT_MASK;
8277
8278	gen_icount_start();
8279	for(;;) {
8280	if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
8281	QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
8282	if (bp->pc == pc_ptr &&
8283	!((bp->flags & BP_CPU) && (tb->flags & HF_RF_MASK))) {
8284	gen_debug(dc, pc_ptr - dc->cs_base);
8285	break;
8286	}
8287	}
8288	}
8289	if (search_pc) {
8290	j = gen_opc_ptr - gen_opc_buf;
8291	if (lj < j) {
8292	lj++;
8293	while (lj < j)
8294	gen_opc_instr_start[lj++] = 0;
8295	}
8296	gen_opc_pc[lj] = pc_ptr;
8297	gen_opc_cc_op[lj] = dc->cc_op;
8298	gen_opc_instr_start[lj] = 1;
8299	gen_opc_icount[lj] = num_insns;
8300	}
8301	if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
8302	gen_io_start();
8303
8304	pc_ptr = disas_insn(dc, pc_ptr);
8305	num_insns++;
8306	/* stop translation if indicated */
8307	if (dc->is_jmp)
8308	break;
8309	#ifdef VBOX
8310	# ifdef DEBUG
8311	/*
8312	if(cpu_check_code_raw(env, pc_ptr, env->hflags \| (env->eflags & (IOPL_MASK \| TF_MASK \| VM_MASK))) == ERROR_SUCCESS)
8313	{
8314	//should never happen as the jump to the patch code terminates the translation block
8315	dprintf(("QEmu is about to execute instructions in our patch block at %08X!!\n", pc_ptr));
8316	}
8317	*/
8318	# endif /* DEBUG */
8319	if (env->state & CPU_EMULATE_SINGLE_INSTR)
8320	{
8321	env->state &= ~CPU_EMULATE_SINGLE_INSTR;
8322	gen_jmp_im(pc_ptr - dc->cs_base);
8323	gen_eob(dc);
8324	break;
8325	}
8326	#endif /* VBOX */
8327
8328	/* if single step mode, we generate only one instruction and
8329	generate an exception */
8330	/* if irq were inhibited with HF_INHIBIT_IRQ_MASK, we clear
8331	the flag and abort the translation to give the irqs a
8332	change to be happen */
8333	if (dc->tf \|\| dc->singlestep_enabled \|\|
8334	(flags & HF_INHIBIT_IRQ_MASK)) {
8335	gen_jmp_im(pc_ptr - dc->cs_base);
8336	gen_eob(dc);
8337	break;
8338	}
8339	/* if too long translation, stop generation too */
8340	if (gen_opc_ptr >= gen_opc_end \|\|
8341	(pc_ptr - pc_start) >= (TARGET_PAGE_SIZE - 32) \|\|
8342	num_insns >= max_insns) {
8343	gen_jmp_im(pc_ptr - dc->cs_base);
8344	gen_eob(dc);
8345	break;
8346	}
8347	if (singlestep) {
8348	gen_jmp_im(pc_ptr - dc->cs_base);
8349	gen_eob(dc);
8350	break;
8351	}
8352	}
8353	if (tb->cflags & CF_LAST_IO)
8354	gen_io_end();
8355	gen_icount_end(tb, num_insns);
8356	*gen_opc_ptr = INDEX_op_end;
8357	/* we don't forget to fill the last values */
8358	if (search_pc) {
8359	j = gen_opc_ptr - gen_opc_buf;
8360	lj++;
8361	while (lj <= j)
8362	gen_opc_instr_start[lj++] = 0;
8363	}
8364
8365	#ifdef DEBUG_DISAS
8366	log_cpu_state_mask(CPU_LOG_TB_CPU, env, X86_DUMP_CCOP);
8367	if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
8368	int disas_flags;
8369	qemu_log("----------------\n");
8370	qemu_log("IN: %s\n", lookup_symbol(pc_start));
8371	#ifdef TARGET_X86_64
8372	if (dc->code64)
8373	disas_flags = 2;
8374	else
8375	#endif
8376	disas_flags = !dc->code32;
8377	log_target_disas(pc_start, pc_ptr - pc_start, disas_flags);
8378	qemu_log("\n");
8379	}
8380	#endif
8381
8382	if (!search_pc) {
8383	tb->size = pc_ptr - pc_start;
8384	tb->icount = num_insns;
8385	}
8386	}
8387
8388	void gen_intermediate_code(CPUState env, TranslationBlock tb)
8389	{
8390	gen_intermediate_code_internal(env, tb, 0);
8391	}
8392
8393	void gen_intermediate_code_pc(CPUState env, TranslationBlock tb)
8394	{
8395	gen_intermediate_code_internal(env, tb, 1);
8396	}
8397
8398	void gen_pc_load(CPUState env, TranslationBlock tb,
8399	unsigned long searched_pc, int pc_pos, void *puc)
8400	{
8401	int cc_op;
8402	#ifdef DEBUG_DISAS
8403	if (qemu_loglevel_mask(CPU_LOG_TB_OP)) {
8404	int i;
8405	qemu_log("RESTORE:\n");
8406	for(i = 0;i <= pc_pos; i++) {
8407	if (gen_opc_instr_start[i]) {
8408	qemu_log("0x%04x: " TARGET_FMT_lx "\n", i, gen_opc_pc[i]);
8409	}
8410	}
8411	qemu_log("spc=0x%08lx pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
8412	searched_pc, pc_pos, gen_opc_pc[pc_pos] - tb->cs_base,
8413	(uint32_t)tb->cs_base);
8414	}
8415	#endif
8416	env->eip = gen_opc_pc[pc_pos] - tb->cs_base;
8417	cc_op = gen_opc_cc_op[pc_pos];
8418	if (cc_op != CC_OP_DYNAMIC)
8419	env->cc_op = cc_op;
8420	}

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/recompiler/target-i386/translate.c@ 37675

以其他格式下載: