diff mbox series

[v3,23/66] tcg: Expand MO_SIZE to 3 bits

Message ID 20210818191920.390759-24-richard.henderson@linaro.org
State Superseded
Headers show
Series Unaligned access for user-only | expand

Commit Message

Richard Henderson Aug. 18, 2021, 7:18 p.m. UTC 
We have lacked expressive support for memory sizes larger
than 64-bits for a while.  Fixing that requires adjustment
to several points where we used this for array indexing,
and two places that develop -Wswitch warnings after the change.

Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>

---
 include/exec/memop.h                | 14 +++++++++-----
 target/arm/translate-a64.c          |  2 +-
 tcg/tcg-op.c                        | 13 ++++++++-----
 target/s390x/tcg/translate_vx.c.inc |  2 +-
 tcg/aarch64/tcg-target.c.inc        |  4 ++--
 tcg/arm/tcg-target.c.inc            |  4 ++--
 tcg/i386/tcg-target.c.inc           |  4 ++--
 tcg/mips/tcg-target.c.inc           |  4 ++--
 tcg/ppc/tcg-target.c.inc            |  8 ++++----
 tcg/riscv/tcg-target.c.inc          |  4 ++--
 tcg/s390/tcg-target.c.inc           |  4 ++--
 tcg/sparc/tcg-target.c.inc          | 16 ++++++++--------
 12 files changed, 43 insertions(+), 36 deletions(-)

-- 
2.25.1

Comments

Alistair Francis Aug. 19, 2021, 6:17 a.m. UTC | #1 
On Thu, Aug 19, 2021 at 5:41 AM Richard Henderson
<richard.henderson@linaro.org> wrote:
>

> We have lacked expressive support for memory sizes larger

> than 64-bits for a while.  Fixing that requires adjustment

> to several points where we used this for array indexing,

> and two places that develop -Wswitch warnings after the change.

>

> Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>

> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>


Reviewed-by: Alistair Francis <alistair.francis@wdc.com>


Alistair

> ---

>  include/exec/memop.h                | 14 +++++++++-----

>  target/arm/translate-a64.c          |  2 +-

>  tcg/tcg-op.c                        | 13 ++++++++-----

>  target/s390x/tcg/translate_vx.c.inc |  2 +-

>  tcg/aarch64/tcg-target.c.inc        |  4 ++--

>  tcg/arm/tcg-target.c.inc            |  4 ++--

>  tcg/i386/tcg-target.c.inc           |  4 ++--

>  tcg/mips/tcg-target.c.inc           |  4 ++--

>  tcg/ppc/tcg-target.c.inc            |  8 ++++----

>  tcg/riscv/tcg-target.c.inc          |  4 ++--

>  tcg/s390/tcg-target.c.inc           |  4 ++--

>  tcg/sparc/tcg-target.c.inc          | 16 ++++++++--------

>  12 files changed, 43 insertions(+), 36 deletions(-)

>

> diff --git a/include/exec/memop.h b/include/exec/memop.h

> index 529d07b02d..04264ffd6b 100644

> --- a/include/exec/memop.h

> +++ b/include/exec/memop.h

> @@ -19,11 +19,15 @@ typedef enum MemOp {

>      MO_16    = 1,

>      MO_32    = 2,

>      MO_64    = 3,

> -    MO_SIZE  = 3,   /* Mask for the above.  */

> +    MO_128   = 4,

> +    MO_256   = 5,

> +    MO_512   = 6,

> +    MO_1024  = 7,

> +    MO_SIZE  = 0x07,   /* Mask for the above.  */

>

> -    MO_SIGN  = 4,   /* Sign-extended, otherwise zero-extended.  */

> +    MO_SIGN  = 0x08,   /* Sign-extended, otherwise zero-extended.  */

>

> -    MO_BSWAP = 8,   /* Host reverse endian.  */

> +    MO_BSWAP = 0x10,   /* Host reverse endian.  */

>  #ifdef HOST_WORDS_BIGENDIAN

>      MO_LE    = MO_BSWAP,

>      MO_BE    = 0,

> @@ -59,8 +63,8 @@ typedef enum MemOp {

>       * - an alignment to a specified size, which may be more or less than

>       *   the access size (MO_ALIGN_x where 'x' is a size in bytes);

>       */

> -    MO_ASHIFT = 4,

> -    MO_AMASK = 7 << MO_ASHIFT,

> +    MO_ASHIFT = 5,

> +    MO_AMASK = 0x7 << MO_ASHIFT,

>  #ifdef NEED_CPU_H

>  #ifdef TARGET_ALIGNED_ONLY

>      MO_ALIGN = 0,

> diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c

> index 422e2ac0c9..247c9672be 100644

> --- a/target/arm/translate-a64.c

> +++ b/target/arm/translate-a64.c

> @@ -1045,7 +1045,7 @@ static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,

>                               int element, MemOp memop)

>  {

>      int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);

> -    switch (memop) {

> +    switch ((unsigned)memop) {

>      case MO_8:

>          tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off);

>          break;

> diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c

> index c754396575..e01f68f44d 100644

> --- a/tcg/tcg-op.c

> +++ b/tcg/tcg-op.c

> @@ -2780,10 +2780,13 @@ static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)

>          }

>          break;

>      case MO_64:

> -        if (!is64) {

> -            tcg_abort();

> +        if (is64) {

> +            op &= ~MO_SIGN;

> +            break;

>          }

> -        break;

> +        /* fall through */

> +    default:

> +        g_assert_not_reached();

>      }

>      if (st) {

>          op &= ~MO_SIGN;

> @@ -3095,7 +3098,7 @@ typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,

>  # define WITH_ATOMIC64(X)

>  #endif

>

> -static void * const table_cmpxchg[16] = {

> +static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_8] = gen_helper_atomic_cmpxchgb,

>      [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,

>      [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,

> @@ -3297,7 +3300,7 @@ static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,

>  }

>

>  #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \

> -static void * const table_##NAME[16] = {                                \

> +static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \

>      [MO_8] = gen_helper_atomic_##NAME##b,                               \

>      [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \

>      [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \

> diff --git a/target/s390x/tcg/translate_vx.c.inc b/target/s390x/tcg/translate_vx.c.inc

> index 0afa46e463..28bf5a23b6 100644

> --- a/target/s390x/tcg/translate_vx.c.inc

> +++ b/target/s390x/tcg/translate_vx.c.inc

> @@ -67,7 +67,7 @@ static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,

>  {

>      const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);

>

> -    switch (memop) {

> +    switch ((unsigned)memop) {

>      case ES_8:

>          tcg_gen_ld8u_i64(dst, cpu_env, offs);

>          break;

> diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc

> index 5924977b42..6f43c048a5 100644

> --- a/tcg/aarch64/tcg-target.c.inc

> +++ b/tcg/aarch64/tcg-target.c.inc

> @@ -1547,7 +1547,7 @@ static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d,

>  /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,

>   *                                     TCGMemOpIdx oi, uintptr_t ra)

>   */

> -static void * const qemu_ld_helpers[4] = {

> +static void * const qemu_ld_helpers[MO_SIZE + 1] = {

>      [MO_8]  = helper_ret_ldub_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

>      [MO_16] = helper_be_lduw_mmu,

> @@ -1564,7 +1564,7 @@ static void * const qemu_ld_helpers[4] = {

>   *                                     uintxx_t val, TCGMemOpIdx oi,

>   *                                     uintptr_t ra)

>   */

> -static void * const qemu_st_helpers[4] = {

> +static void * const qemu_st_helpers[MO_SIZE + 1] = {

>      [MO_8]  = helper_ret_stb_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

>      [MO_16] = helper_be_stw_mmu,

> diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc

> index 007ceee68e..8939b2c2da 100644

> --- a/tcg/arm/tcg-target.c.inc

> +++ b/tcg/arm/tcg-target.c.inc

> @@ -1393,7 +1393,7 @@ static void tcg_out_vldst(TCGContext *s, ARMInsn insn,

>  /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,

>   *                                     int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_ld_helpers[8] = {

> +static void * const qemu_ld_helpers[MO_SSIZE + 1] = {

>      [MO_UB]   = helper_ret_ldub_mmu,

>      [MO_SB]   = helper_ret_ldsb_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

> @@ -1414,7 +1414,7 @@ static void * const qemu_ld_helpers[8] = {

>  /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,

>   *                                     uintxx_t val, int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_st_helpers[4] = {

> +static void * const qemu_st_helpers[MO_SIZE + 1] = {

>      [MO_8]   = helper_ret_stb_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

>      [MO_16] = helper_be_stw_mmu,

> diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc

> index 98d924b91a..5fd4e4392f 100644

> --- a/tcg/i386/tcg-target.c.inc

> +++ b/tcg/i386/tcg-target.c.inc

> @@ -1610,7 +1610,7 @@ static void tcg_out_nopn(TCGContext *s, int n)

>  /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,

>   *                                     int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_ld_helpers[16] = {

> +static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_ldub_mmu,

>      [MO_LEUW] = helper_le_lduw_mmu,

>      [MO_LEUL] = helper_le_ldul_mmu,

> @@ -1623,7 +1623,7 @@ static void * const qemu_ld_helpers[16] = {

>  /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,

>   *                                     uintxx_t val, int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_st_helpers[16] = {

> +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_stb_mmu,

>      [MO_LEUW] = helper_le_stw_mmu,

>      [MO_LEUL] = helper_le_stl_mmu,

> diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc

> index bf0eb84e2d..cc279205d6 100644

> --- a/tcg/mips/tcg-target.c.inc

> +++ b/tcg/mips/tcg-target.c.inc

> @@ -1037,7 +1037,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)

>  #if defined(CONFIG_SOFTMMU)

>  #include "../tcg-ldst.c.inc"

>

> -static void * const qemu_ld_helpers[16] = {

> +static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_ldub_mmu,

>      [MO_SB]   = helper_ret_ldsb_mmu,

>      [MO_LEUW] = helper_le_lduw_mmu,

> @@ -1054,7 +1054,7 @@ static void * const qemu_ld_helpers[16] = {

>  #endif

>  };

>

> -static void * const qemu_st_helpers[16] = {

> +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_stb_mmu,

>      [MO_LEUW] = helper_le_stw_mmu,

>      [MO_LEUL] = helper_le_stl_mmu,

> diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc

> index e0f4665213..3fef2aa6b2 100644

> --- a/tcg/ppc/tcg-target.c.inc

> +++ b/tcg/ppc/tcg-target.c.inc

> @@ -1916,7 +1916,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)

>  #endif

>  }

>

> -static const uint32_t qemu_ldx_opc[16] = {

> +static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = {

>      [MO_UB] = LBZX,

>      [MO_UW] = LHZX,

>      [MO_UL] = LWZX,

> @@ -1929,7 +1929,7 @@ static const uint32_t qemu_ldx_opc[16] = {

>      [MO_BSWAP | MO_Q]  = LDBRX,

>  };

>

> -static const uint32_t qemu_stx_opc[16] = {

> +static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {

>      [MO_UB] = STBX,

>      [MO_UW] = STHX,

>      [MO_UL] = STWX,

> @@ -1950,7 +1950,7 @@ static const uint32_t qemu_exts_opc[4] = {

>  /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,

>   *                                 int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_ld_helpers[16] = {

> +static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_ldub_mmu,

>      [MO_LEUW] = helper_le_lduw_mmu,

>      [MO_LEUL] = helper_le_ldul_mmu,

> @@ -1963,7 +1963,7 @@ static void * const qemu_ld_helpers[16] = {

>  /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,

>   *                                 uintxx_t val, int mmu_idx, uintptr_t ra)

>   */

> -static void * const qemu_st_helpers[16] = {

> +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_stb_mmu,

>      [MO_LEUW] = helper_le_stw_mmu,

>      [MO_LEUL] = helper_le_stl_mmu,

> diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc

> index c16f96b401..6264e58b3a 100644

> --- a/tcg/riscv/tcg-target.c.inc

> +++ b/tcg/riscv/tcg-target.c.inc

> @@ -852,7 +852,7 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)

>  /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,

>   *                                     TCGMemOpIdx oi, uintptr_t ra)

>   */

> -static void * const qemu_ld_helpers[8] = {

> +static void * const qemu_ld_helpers[MO_SSIZE + 1] = {

>      [MO_UB] = helper_ret_ldub_mmu,

>      [MO_SB] = helper_ret_ldsb_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

> @@ -878,7 +878,7 @@ static void * const qemu_ld_helpers[8] = {

>   *                                     uintxx_t val, TCGMemOpIdx oi,

>   *                                     uintptr_t ra)

>   */

> -static void * const qemu_st_helpers[4] = {

> +static void * const qemu_st_helpers[MO_SIZE + 1] = {

>      [MO_8]   = helper_ret_stb_mmu,

>  #ifdef HOST_WORDS_BIGENDIAN

>      [MO_16] = helper_be_stw_mmu,

> diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc

> index b82cf19f09..67a2ba5ff3 100644

> --- a/tcg/s390/tcg-target.c.inc

> +++ b/tcg/s390/tcg-target.c.inc

> @@ -350,7 +350,7 @@ static const uint8_t tcg_cond_to_ltr_cond[] = {

>  };

>

>  #ifdef CONFIG_SOFTMMU

> -static void * const qemu_ld_helpers[16] = {

> +static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_ldub_mmu,

>      [MO_SB]   = helper_ret_ldsb_mmu,

>      [MO_LEUW] = helper_le_lduw_mmu,

> @@ -365,7 +365,7 @@ static void * const qemu_ld_helpers[16] = {

>      [MO_BEQ]  = helper_be_ldq_mmu,

>  };

>

> -static void * const qemu_st_helpers[16] = {

> +static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = helper_ret_stb_mmu,

>      [MO_LEUW] = helper_le_stw_mmu,

>      [MO_LEUL] = helper_le_stl_mmu,

> diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc

> index 688827968b..b9bce29282 100644

> --- a/tcg/sparc/tcg-target.c.inc

> +++ b/tcg/sparc/tcg-target.c.inc

> @@ -847,8 +847,8 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)

>  }

>

>  #ifdef CONFIG_SOFTMMU

> -static const tcg_insn_unit *qemu_ld_trampoline[16];

> -static const tcg_insn_unit *qemu_st_trampoline[16];

> +static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1];

> +static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1];

>

>  static void emit_extend(TCGContext *s, TCGReg r, int op)

>  {

> @@ -875,7 +875,7 @@ static void emit_extend(TCGContext *s, TCGReg r, int op)

>

>  static void build_trampolines(TCGContext *s)

>  {

> -    static void * const qemu_ld_helpers[16] = {

> +    static void * const qemu_ld_helpers[] = {

>          [MO_UB]   = helper_ret_ldub_mmu,

>          [MO_SB]   = helper_ret_ldsb_mmu,

>          [MO_LEUW] = helper_le_lduw_mmu,

> @@ -887,7 +887,7 @@ static void build_trampolines(TCGContext *s)

>          [MO_BEUL] = helper_be_ldul_mmu,

>          [MO_BEQ]  = helper_be_ldq_mmu,

>      };

> -    static void * const qemu_st_helpers[16] = {

> +    static void * const qemu_st_helpers[] = {

>          [MO_UB]   = helper_ret_stb_mmu,

>          [MO_LEUW] = helper_le_stw_mmu,

>          [MO_LEUL] = helper_le_stl_mmu,

> @@ -900,7 +900,7 @@ static void build_trampolines(TCGContext *s)

>      int i;

>      TCGReg ra;

>

> -    for (i = 0; i < 16; ++i) {

> +    for (i = 0; i < ARRAY_SIZE(qemu_ld_helpers); ++i) {

>          if (qemu_ld_helpers[i] == NULL) {

>              continue;

>          }

> @@ -928,7 +928,7 @@ static void build_trampolines(TCGContext *s)

>          tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);

>      }

>

> -    for (i = 0; i < 16; ++i) {

> +    for (i = 0; i < ARRAY_SIZE(qemu_st_helpers); ++i) {

>          if (qemu_st_helpers[i] == NULL) {

>              continue;

>          }

> @@ -1110,7 +1110,7 @@ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,

>  }

>  #endif /* CONFIG_SOFTMMU */

>

> -static const int qemu_ld_opc[16] = {

> +static const int qemu_ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = LDUB,

>      [MO_SB]   = LDSB,

>

> @@ -1127,7 +1127,7 @@ static const int qemu_ld_opc[16] = {

>      [MO_LEQ]  = LDX_LE,

>  };

>

> -static const int qemu_st_opc[16] = {

> +static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = {

>      [MO_UB]   = STB,

>

>      [MO_BEUW] = STH,

> --

> 2.25.1

>

>
diff mbox series

Patch

diff --git a/include/exec/memop.h b/include/exec/memop.h
index 529d07b02d..04264ffd6b 100644
--- a/include/exec/memop.h
+++ b/include/exec/memop.h
@@ -19,11 +19,15 @@  typedef enum MemOp {
     MO_16    = 1,
     MO_32    = 2,
     MO_64    = 3,
-    MO_SIZE  = 3,   /* Mask for the above.  */
+    MO_128   = 4,
+    MO_256   = 5,
+    MO_512   = 6,
+    MO_1024  = 7,
+    MO_SIZE  = 0x07,   /* Mask for the above.  */
 
-    MO_SIGN  = 4,   /* Sign-extended, otherwise zero-extended.  */
+    MO_SIGN  = 0x08,   /* Sign-extended, otherwise zero-extended.  */
 
-    MO_BSWAP = 8,   /* Host reverse endian.  */
+    MO_BSWAP = 0x10,   /* Host reverse endian.  */
 #ifdef HOST_WORDS_BIGENDIAN
     MO_LE    = MO_BSWAP,
     MO_BE    = 0,
@@ -59,8 +63,8 @@  typedef enum MemOp {
      * - an alignment to a specified size, which may be more or less than
      *   the access size (MO_ALIGN_x where 'x' is a size in bytes);
      */
-    MO_ASHIFT = 4,
-    MO_AMASK = 7 << MO_ASHIFT,
+    MO_ASHIFT = 5,
+    MO_AMASK = 0x7 << MO_ASHIFT,
 #ifdef NEED_CPU_H
 #ifdef TARGET_ALIGNED_ONLY
     MO_ALIGN = 0,
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 422e2ac0c9..247c9672be 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -1045,7 +1045,7 @@  static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
                              int element, MemOp memop)
 {
     int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
-    switch (memop) {
+    switch ((unsigned)memop) {
     case MO_8:
         tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off);
         break;
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index c754396575..e01f68f44d 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -2780,10 +2780,13 @@  static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
         }
         break;
     case MO_64:
-        if (!is64) {
-            tcg_abort();
+        if (is64) {
+            op &= ~MO_SIGN;
+            break;
         }
-        break;
+        /* fall through */
+    default:
+        g_assert_not_reached();
     }
     if (st) {
         op &= ~MO_SIGN;
@@ -3095,7 +3098,7 @@  typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
 # define WITH_ATOMIC64(X)
 #endif
 
-static void * const table_cmpxchg[16] = {
+static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_8] = gen_helper_atomic_cmpxchgb,
     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
@@ -3297,7 +3300,7 @@  static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
 }
 
 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
-static void * const table_##NAME[16] = {                                \
+static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = {          \
     [MO_8] = gen_helper_atomic_##NAME##b,                               \
     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
diff --git a/target/s390x/tcg/translate_vx.c.inc b/target/s390x/tcg/translate_vx.c.inc
index 0afa46e463..28bf5a23b6 100644
--- a/target/s390x/tcg/translate_vx.c.inc
+++ b/target/s390x/tcg/translate_vx.c.inc
@@ -67,7 +67,7 @@  static void read_vec_element_i64(TCGv_i64 dst, uint8_t reg, uint8_t enr,
 {
     const int offs = vec_reg_offset(reg, enr, memop & MO_SIZE);
 
-    switch (memop) {
+    switch ((unsigned)memop) {
     case ES_8:
         tcg_gen_ld8u_i64(dst, cpu_env, offs);
         break;
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 5924977b42..6f43c048a5 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -1547,7 +1547,7 @@  static void tcg_out_cltz(TCGContext *s, TCGType ext, TCGReg d,
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     TCGMemOpIdx oi, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[4] = {
+static void * const qemu_ld_helpers[MO_SIZE + 1] = {
     [MO_8]  = helper_ret_ldub_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
     [MO_16] = helper_be_lduw_mmu,
@@ -1564,7 +1564,7 @@  static void * const qemu_ld_helpers[4] = {
  *                                     uintxx_t val, TCGMemOpIdx oi,
  *                                     uintptr_t ra)
  */
-static void * const qemu_st_helpers[4] = {
+static void * const qemu_st_helpers[MO_SIZE + 1] = {
     [MO_8]  = helper_ret_stb_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
     [MO_16] = helper_be_stw_mmu,
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 007ceee68e..8939b2c2da 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -1393,7 +1393,7 @@  static void tcg_out_vldst(TCGContext *s, ARMInsn insn,
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[8] = {
+static void * const qemu_ld_helpers[MO_SSIZE + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
@@ -1414,7 +1414,7 @@  static void * const qemu_ld_helpers[8] = {
 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
  *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[4] = {
+static void * const qemu_st_helpers[MO_SIZE + 1] = {
     [MO_8]   = helper_ret_stb_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
     [MO_16] = helper_be_stw_mmu,
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index 98d924b91a..5fd4e4392f 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -1610,7 +1610,7 @@  static void tcg_out_nopn(TCGContext *s, int n)
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
     [MO_LEUL] = helper_le_ldul_mmu,
@@ -1623,7 +1623,7 @@  static void * const qemu_ld_helpers[16] = {
 /* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
  *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index bf0eb84e2d..cc279205d6 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -1037,7 +1037,7 @@  static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
 #if defined(CONFIG_SOFTMMU)
 #include "../tcg-ldst.c.inc"
 
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -1054,7 +1054,7 @@  static void * const qemu_ld_helpers[16] = {
 #endif
 };
 
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index e0f4665213..3fef2aa6b2 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -1916,7 +1916,7 @@  static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
 #endif
 }
 
-static const uint32_t qemu_ldx_opc[16] = {
+static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = {
     [MO_UB] = LBZX,
     [MO_UW] = LHZX,
     [MO_UL] = LWZX,
@@ -1929,7 +1929,7 @@  static const uint32_t qemu_ldx_opc[16] = {
     [MO_BSWAP | MO_Q]  = LDBRX,
 };
 
-static const uint32_t qemu_stx_opc[16] = {
+static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
     [MO_UB] = STBX,
     [MO_UW] = STHX,
     [MO_UL] = STWX,
@@ -1950,7 +1950,7 @@  static const uint32_t qemu_exts_opc[4] = {
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
  *                                 int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
     [MO_LEUL] = helper_le_ldul_mmu,
@@ -1963,7 +1963,7 @@  static void * const qemu_ld_helpers[16] = {
 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
  *                                 uintxx_t val, int mmu_idx, uintptr_t ra)
  */
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index c16f96b401..6264e58b3a 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -852,7 +852,7 @@  static void tcg_out_mb(TCGContext *s, TCGArg a0)
 /* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
  *                                     TCGMemOpIdx oi, uintptr_t ra)
  */
-static void * const qemu_ld_helpers[8] = {
+static void * const qemu_ld_helpers[MO_SSIZE + 1] = {
     [MO_UB] = helper_ret_ldub_mmu,
     [MO_SB] = helper_ret_ldsb_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
@@ -878,7 +878,7 @@  static void * const qemu_ld_helpers[8] = {
  *                                     uintxx_t val, TCGMemOpIdx oi,
  *                                     uintptr_t ra)
  */
-static void * const qemu_st_helpers[4] = {
+static void * const qemu_st_helpers[MO_SIZE + 1] = {
     [MO_8]   = helper_ret_stb_mmu,
 #ifdef HOST_WORDS_BIGENDIAN
     [MO_16] = helper_be_stw_mmu,
diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc
index b82cf19f09..67a2ba5ff3 100644
--- a/tcg/s390/tcg-target.c.inc
+++ b/tcg/s390/tcg-target.c.inc
@@ -350,7 +350,7 @@  static const uint8_t tcg_cond_to_ltr_cond[] = {
 };
 
 #ifdef CONFIG_SOFTMMU
-static void * const qemu_ld_helpers[16] = {
+static void * const qemu_ld_helpers[(MO_SSIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_ldub_mmu,
     [MO_SB]   = helper_ret_ldsb_mmu,
     [MO_LEUW] = helper_le_lduw_mmu,
@@ -365,7 +365,7 @@  static void * const qemu_ld_helpers[16] = {
     [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
-static void * const qemu_st_helpers[16] = {
+static void * const qemu_st_helpers[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = helper_ret_stb_mmu,
     [MO_LEUW] = helper_le_stw_mmu,
     [MO_LEUL] = helper_le_stl_mmu,
diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc
index 688827968b..b9bce29282 100644
--- a/tcg/sparc/tcg-target.c.inc
+++ b/tcg/sparc/tcg-target.c.inc
@@ -847,8 +847,8 @@  static void tcg_out_mb(TCGContext *s, TCGArg a0)
 }
 
 #ifdef CONFIG_SOFTMMU
-static const tcg_insn_unit *qemu_ld_trampoline[16];
-static const tcg_insn_unit *qemu_st_trampoline[16];
+static const tcg_insn_unit *qemu_ld_trampoline[(MO_SSIZE | MO_BSWAP) + 1];
+static const tcg_insn_unit *qemu_st_trampoline[(MO_SIZE | MO_BSWAP) + 1];
 
 static void emit_extend(TCGContext *s, TCGReg r, int op)
 {
@@ -875,7 +875,7 @@  static void emit_extend(TCGContext *s, TCGReg r, int op)
 
 static void build_trampolines(TCGContext *s)
 {
-    static void * const qemu_ld_helpers[16] = {
+    static void * const qemu_ld_helpers[] = {
         [MO_UB]   = helper_ret_ldub_mmu,
         [MO_SB]   = helper_ret_ldsb_mmu,
         [MO_LEUW] = helper_le_lduw_mmu,
@@ -887,7 +887,7 @@  static void build_trampolines(TCGContext *s)
         [MO_BEUL] = helper_be_ldul_mmu,
         [MO_BEQ]  = helper_be_ldq_mmu,
     };
-    static void * const qemu_st_helpers[16] = {
+    static void * const qemu_st_helpers[] = {
         [MO_UB]   = helper_ret_stb_mmu,
         [MO_LEUW] = helper_le_stw_mmu,
         [MO_LEUL] = helper_le_stl_mmu,
@@ -900,7 +900,7 @@  static void build_trampolines(TCGContext *s)
     int i;
     TCGReg ra;
 
-    for (i = 0; i < 16; ++i) {
+    for (i = 0; i < ARRAY_SIZE(qemu_ld_helpers); ++i) {
         if (qemu_ld_helpers[i] == NULL) {
             continue;
         }
@@ -928,7 +928,7 @@  static void build_trampolines(TCGContext *s)
         tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);
     }
 
-    for (i = 0; i < 16; ++i) {
+    for (i = 0; i < ARRAY_SIZE(qemu_st_helpers); ++i) {
         if (qemu_st_helpers[i] == NULL) {
             continue;
         }
@@ -1110,7 +1110,7 @@  static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,
 }
 #endif /* CONFIG_SOFTMMU */
 
-static const int qemu_ld_opc[16] = {
+static const int qemu_ld_opc[(MO_SSIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = LDUB,
     [MO_SB]   = LDSB,
 
@@ -1127,7 +1127,7 @@  static const int qemu_ld_opc[16] = {
     [MO_LEQ]  = LDX_LE,
 };
 
-static const int qemu_st_opc[16] = {
+static const int qemu_st_opc[(MO_SIZE | MO_BSWAP) + 1] = {
     [MO_UB]   = STB,
 
     [MO_BEUW] = STH,