@@ -66,10 +66,6 @@ DEF_HELPER_1(rsm, void, env)
#endif /* !CONFIG_USER_ONLY */
DEF_HELPER_2(into, void, env, int)
-#ifdef TARGET_X86_64
-DEF_HELPER_2(cmpxchg16b_unlocked, void, env, tl)
-DEF_HELPER_2(cmpxchg16b, void, env, tl)
-#endif
DEF_HELPER_FLAGS_1(single_step, TCG_CALL_NO_WG, noreturn, env)
DEF_HELPER_1(rechecking_single_step, void, env)
DEF_HELPER_1(cpuid, void, env)
@@ -27,75 +27,6 @@
#include "tcg/tcg.h"
#include "helper-tcg.h"
-#ifdef TARGET_X86_64
-void helper_cmpxchg16b_unlocked(CPUX86State *env, target_ulong a0)
-{
- uintptr_t ra = GETPC();
- Int128 oldv, cmpv, newv;
- uint64_t o0, o1;
- int eflags;
- bool success;
-
- if ((a0 & 0xf) != 0) {
- raise_exception_ra(env, EXCP0D_GPF, GETPC());
- }
- eflags = cpu_cc_compute_all(env, CC_OP);
-
- cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]);
- newv = int128_make128(env->regs[R_EBX], env->regs[R_ECX]);
-
- o0 = cpu_ldq_data_ra(env, a0 + 0, ra);
- o1 = cpu_ldq_data_ra(env, a0 + 8, ra);
-
- oldv = int128_make128(o0, o1);
- success = int128_eq(oldv, cmpv);
- if (!success) {
- newv = oldv;
- }
-
- cpu_stq_data_ra(env, a0 + 0, int128_getlo(newv), ra);
- cpu_stq_data_ra(env, a0 + 8, int128_gethi(newv), ra);
-
- if (success) {
- eflags |= CC_Z;
- } else {
- env->regs[R_EAX] = int128_getlo(oldv);
- env->regs[R_EDX] = int128_gethi(oldv);
- eflags &= ~CC_Z;
- }
- CC_SRC = eflags;
-}
-
-void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
-{
- uintptr_t ra = GETPC();
-
- if ((a0 & 0xf) != 0) {
- raise_exception_ra(env, EXCP0D_GPF, ra);
- } else if (HAVE_CMPXCHG128) {
- int eflags = cpu_cc_compute_all(env, CC_OP);
-
- Int128 cmpv = int128_make128(env->regs[R_EAX], env->regs[R_EDX]);
- Int128 newv = int128_make128(env->regs[R_EBX], env->regs[R_ECX]);
-
- int mem_idx = cpu_mmu_index(env, false);
- MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
- Int128 oldv = cpu_atomic_cmpxchgo_le_mmu(env, a0, cmpv, newv, oi, ra);
-
- if (int128_eq(oldv, cmpv)) {
- eflags |= CC_Z;
- } else {
- env->regs[R_EAX] = int128_getlo(oldv);
- env->regs[R_EDX] = int128_gethi(oldv);
- eflags &= ~CC_Z;
- }
- CC_SRC = eflags;
- } else {
- cpu_loop_exit_atomic(env_cpu(env), ra);
- }
-}
-#endif
-
void helper_boundw(CPUX86State *env, target_ulong a0, int v)
{
int low, high;
@@ -3053,15 +3053,49 @@ static void gen_cmpxchg8b(DisasContext *s, CPUX86State *env, int modrm)
#ifdef TARGET_X86_64
static void gen_cmpxchg16b(DisasContext *s, CPUX86State *env, int modrm)
{
+ MemOp mop = MO_TE | MO_128 | MO_ALIGN;
+ TCGv_i64 t0, t1;
+ TCGv_i128 cmp, val;
+
gen_lea_modrm(env, s, modrm);
- if ((s->prefix & PREFIX_LOCK) &&
- (tb_cflags(s->base.tb) & CF_PARALLEL)) {
- gen_helper_cmpxchg16b(cpu_env, s->A0);
+ cmp = tcg_temp_new_i128();
+ val = tcg_temp_new_i128();
+ tcg_gen_concat_i64_i128(cmp, cpu_regs[R_EAX], cpu_regs[R_EDX]);
+ tcg_gen_concat_i64_i128(val, cpu_regs[R_EBX], cpu_regs[R_ECX]);
+
+ /* Only require atomic with LOCK; non-parallel handled in generator. */
+ if (s->prefix & PREFIX_LOCK) {
+ tcg_gen_atomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop);
} else {
- gen_helper_cmpxchg16b_unlocked(cpu_env, s->A0);
+ tcg_gen_nonatomic_cmpxchg_i128(val, s->A0, cmp, val, s->mem_index, mop);
}
- set_cc_op(s, CC_OP_EFLAGS);
+
+ tcg_gen_extr_i128_i64(s->T0, s->T1, val);
+ tcg_temp_free_i128(cmp);
+ tcg_temp_free_i128(val);
+
+ /* Determine success after the fact. */
+ t0 = tcg_temp_new_i64();
+ t1 = tcg_temp_new_i64();
+ tcg_gen_xor_i64(t0, s->T0, cpu_regs[R_EAX]);
+ tcg_gen_xor_i64(t1, s->T1, cpu_regs[R_EDX]);
+ tcg_gen_or_i64(t0, t0, t1);
+ tcg_temp_free_i64(t1);
+
+ /* Update Z. */
+ gen_compute_eflags(s);
+ tcg_gen_setcondi_i64(TCG_COND_EQ, t0, t0, 0);
+ tcg_gen_deposit_tl(cpu_cc_src, cpu_cc_src, t0, ctz32(CC_Z), 1);
+ tcg_temp_free_i64(t0);
+
+ /*
+ * Extract the result values for the register pair. We may do this
+ * unconditionally, because on success (Z=1), the old value matches
+ * the previous value in RDX:RAX.
+ */
+ tcg_gen_mov_i64(cpu_regs[R_EAX], s->T0);
+ tcg_gen_mov_i64(cpu_regs[R_EDX], s->T1);
}
#endif
Use tcg_gen_atomic_cmpxchg_i128 for the atomic case, and tcg_gen_qemu_ld/st_i128 otherwise. Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Eduardo Habkost <eduardo@habkost.net> --- target/i386/helper.h | 4 --- target/i386/tcg/mem_helper.c | 69 ------------------------------------ target/i386/tcg/translate.c | 44 ++++++++++++++++++++--- 3 files changed, 39 insertions(+), 78 deletions(-)