@@ -232,9 +232,7 @@ static inline void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx,
#define TCG_TARGET_HAS_MEMORY_BSWAP have_movbe
-#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
-#endif
#define TCG_TARGET_NEED_POOL_LABELS
#endif
@@ -22,6 +22,7 @@
* THE SOFTWARE.
*/
+#include "../tcg-ldst.c.inc"
#include "../tcg-pool.c.inc"
#ifdef CONFIG_DEBUG_TCG
@@ -421,8 +422,9 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
#define OPC_VZEROUPPER (0x77 | P_EXT)
#define OPC_XCHG_ax_r32 (0x90)
-#define OPC_GRP3_Ev (0xf7)
-#define OPC_GRP5 (0xff)
+#define OPC_GRP3_Eb (0xf6)
+#define OPC_GRP3_Ev (0xf7)
+#define OPC_GRP5 (0xff)
#define OPC_GRP14 (0x73 | P_EXT | P_DATA16)
/* Group 1 opcode extensions for 0x80-0x83.
@@ -444,6 +446,7 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct)
#define SHIFT_SAR 7
/* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */
+#define EXT3_TESTi 0
#define EXT3_NOT 2
#define EXT3_NEG 3
#define EXT3_MUL 4
@@ -1606,8 +1609,6 @@ static void tcg_out_nopn(TCGContext *s, int n)
}
#if defined(CONFIG_SOFTMMU)
-#include "../tcg-ldst.c.inc"
-
/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
* int mmu_idx, uintptr_t ra)
*/
@@ -1916,7 +1917,84 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
return true;
}
-#elif TCG_TARGET_REG_BITS == 32
+#else
+
+static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addrlo,
+ TCGReg addrhi, unsigned a_bits)
+{
+ unsigned a_mask = (1 << a_bits) - 1;
+ TCGLabelQemuLdst *label;
+
+ /*
+ * We are expecting a_bits to max out at 7, so we can usually use testb.
+ * For i686, we have to use testl for %esi/%edi.
+ */
+ if (a_mask <= 0xff && (TCG_TARGET_REG_BITS == 64 || addrlo < 4)) {
+ tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, addrlo);
+ tcg_out8(s, a_mask);
+ } else {
+ tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, addrlo);
+ tcg_out32(s, a_mask);
+ }
+
+ /* jne slow_path */
+ tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
+
+ label = new_ldst_label(s);
+ label->is_ld = is_ld;
+ label->addrlo_reg = addrlo;
+ label->addrhi_reg = addrhi;
+ label->raddr = tcg_splitwx_to_rx(s->code_ptr + 4);
+ label->label_ptr[0] = s->code_ptr;
+
+ s->code_ptr += 4;
+}
+
+static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
+{
+ /* resolve label address */
+ tcg_patch32(l->label_ptr[0], s->code_ptr - l->label_ptr[0] - 4);
+
+ if (TCG_TARGET_REG_BITS == 32) {
+ int ofs = 0;
+
+ tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
+ ofs += 4;
+
+ tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
+ ofs += 4;
+ if (TARGET_LONG_BITS == 64) {
+ tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
+ ofs += 4;
+ }
+
+ tcg_out_pushi(s, (uintptr_t)l->raddr);
+ } else {
+ tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
+ l->addrlo_reg);
+ tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+
+ tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, (uintptr_t)l->raddr);
+ tcg_out_push(s, TCG_REG_RAX);
+ }
+
+ /* "Tail call" to the helper, with the return address back inline. */
+ tcg_out_jmp(s, (const void *)(l->is_ld ? helper_unaligned_ld
+ : helper_unaligned_st));
+ return true;
+}
+
+static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+ return tcg_out_fail_alignment(s, l);
+}
+
+static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+ return tcg_out_fail_alignment(s, l);
+}
+
+#if TCG_TARGET_REG_BITS == 32
# define x86_guest_base_seg 0
# define x86_guest_base_index -1
# define x86_guest_base_offset guest_base
@@ -1950,6 +2028,7 @@ static inline int setup_guest_base_seg(void)
return 0;
}
# endif
+#endif
#endif /* SOFTMMU */
static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
@@ -2059,6 +2138,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
#if defined(CONFIG_SOFTMMU)
int mem_index;
tcg_insn_unit *label_ptr[2];
+#else
+ unsigned a_bits;
#endif
datalo = *args++;
@@ -2081,6 +2162,11 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
add_qemu_ldst_label(s, true, is64, oi, datalo, datahi, addrlo, addrhi,
s->code_ptr, label_ptr);
#else
+ a_bits = get_alignment_bits(opc);
+ if (a_bits) {
+ tcg_out_test_alignment(s, true, addrlo, addrhi, a_bits);
+ }
+
tcg_out_qemu_ld_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
x86_guest_base_offset, x86_guest_base_seg,
is64, opc);
@@ -2148,6 +2234,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
#if defined(CONFIG_SOFTMMU)
int mem_index;
tcg_insn_unit *label_ptr[2];
+#else
+ unsigned a_bits;
#endif
datalo = *args++;
@@ -2170,6 +2258,11 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
add_qemu_ldst_label(s, false, is64, oi, datalo, datahi, addrlo, addrhi,
s->code_ptr, label_ptr);
#else
+ a_bits = get_alignment_bits(opc);
+ if (a_bits) {
+ tcg_out_test_alignment(s, false, addrlo, addrhi, a_bits);
+ }
+
tcg_out_qemu_st_direct(s, datalo, datahi, addrlo, x86_guest_base_index,
x86_guest_base_offset, x86_guest_base_seg, opc);
#endif