@@ -104,8 +104,7 @@ DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)
-DEF_HELPER_FLAGS_3(mte_check1, TCG_CALL_NO_WG, i64, env, i32, i64)
-DEF_HELPER_FLAGS_3(mte_checkN, TCG_CALL_NO_WG, i64, env, i32, i64)
+DEF_HELPER_FLAGS_3(mte_check, TCG_CALL_NO_WG, i64, env, i32, i64)
DEF_HELPER_FLAGS_3(mte_check_zva, TCG_CALL_NO_WG, i64, env, i32, i64)
DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
@@ -1137,19 +1137,16 @@ FIELD(PREDDESC, DATA, 8, 24)
*/
#define SVE_MTEDESC_SHIFT 5
-/* Bits within a descriptor passed to the helper_mte_check* functions. */
+/* Bits within a descriptor passed to the helper_mte_check function. */
FIELD(MTEDESC, MIDX, 0, 4)
FIELD(MTEDESC, TBI, 4, 2)
FIELD(MTEDESC, TCMA, 6, 2)
FIELD(MTEDESC, WRITE, 8, 1)
-FIELD(MTEDESC, ESIZE, 9, 5)
-FIELD(MTEDESC, TSIZE, 14, 10) /* mte_checkN only */
+FIELD(MTEDESC, SIZEM1, 12, 10) /* size - 1 */
-bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr);
-uint64_t mte_check1(CPUARMState *env, uint32_t desc,
- uint64_t ptr, uintptr_t ra);
-uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
- uint64_t ptr, uintptr_t ra);
+bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr);
+uint64_t mte_check(CPUARMState *env, uint32_t desc,
+ uint64_t ptr, uintptr_t ra);
static inline int allocation_tag_from_addr(uint64_t ptr)
{
@@ -44,7 +44,7 @@ TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr);
TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
bool tag_checked, int log2_size);
TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
- bool tag_checked, int count, int log2_esize);
+ bool tag_checked, int total_size);
/* We should have at some point before trying to access an FP register
* done the necessary access check, so assert that
@@ -121,7 +121,7 @@ static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
* exception for inaccessible pages, and resolves the virtual address
* into the softmmu tlb.
*
- * When RA == 0, this is for mte_probe1. The page is expected to be
+ * When RA == 0, this is for mte_probe. The page is expected to be
* valid. Indicate to probe_access_flags no-fault, then assert that
* we received a valid page.
*/
@@ -617,80 +617,6 @@ static void mte_check_fail(CPUARMState *env, uint32_t desc,
}
}
-/*
- * Perform an MTE checked access for a single logical or atomic access.
- */
-static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
- uintptr_t ra, int bit55)
-{
- int mem_tag, mmu_idx, ptr_tag, size;
- MMUAccessType type;
- uint8_t *mem;
-
- ptr_tag = allocation_tag_from_addr(ptr);
-
- if (tcma_check(desc, bit55, ptr_tag)) {
- return true;
- }
-
- mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
- type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
- size = FIELD_EX32(desc, MTEDESC, ESIZE);
-
- mem = allocation_tag_mem(env, mmu_idx, ptr, type, size,
- MMU_DATA_LOAD, 1, ra);
- if (!mem) {
- return true;
- }
-
- mem_tag = load_tag1(ptr, mem);
- return ptr_tag == mem_tag;
-}
-
-/*
- * No-fault version of mte_check1, to be used by SVE for MemSingleNF.
- * Returns false if the access is Checked and the check failed. This
- * is only intended to probe the tag -- the validity of the page must
- * be checked beforehand.
- */
-bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr)
-{
- int bit55 = extract64(ptr, 55, 1);
-
- /* If TBI is disabled, the access is unchecked. */
- if (unlikely(!tbi_check(desc, bit55))) {
- return true;
- }
-
- return mte_probe1_int(env, desc, ptr, 0, bit55);
-}
-
-uint64_t mte_check1(CPUARMState *env, uint32_t desc,
- uint64_t ptr, uintptr_t ra)
-{
- int bit55 = extract64(ptr, 55, 1);
-
- /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
- if (unlikely(!tbi_check(desc, bit55))) {
- return ptr;
- }
-
- if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) {
- mte_check_fail(env, desc, ptr, ra);
- }
-
- return useronly_clean_ptr(ptr);
-}
-
-uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
-{
- return mte_check1(env, desc, ptr, GETPC());
-}
-
-/*
- * Perform an MTE checked access for multiple logical accesses.
- */
-
/**
* checkN:
* @tag: tag memory to test
@@ -753,38 +679,49 @@ static int checkN(uint8_t *mem, int odd, int cmp, int count)
return n;
}
-uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
- uint64_t ptr, uintptr_t ra)
+/*
+ * mte_check_int:
+ * @env: CPU environment
+ * @desc: MTEDESC descriptor
+ * @ptr: virtual address of the base of the access
+ * @fault: return virtual address of the first check failure
+ *
+ * Internal routine for both mte_probe and mte_check.
+ * Return zero on failure, filling in *fault.
+ * Return negative on trivial success for tbi disabled.
+ * Return positive on success with tbi enabled.
+ */
+static int mte_check_int(CPUARMState *env, uint32_t desc,
+ uint64_t ptr, uintptr_t ra, uint64_t *fault)
{
int mmu_idx, ptr_tag, bit55;
- uint64_t ptr_last, ptr_end, prev_page, next_page;
+ uint64_t ptr_last, prev_page, next_page;
uint64_t tag_first, tag_end;
uint64_t tag_byte_first, tag_byte_end;
- uint32_t esize, total, tag_count, tag_size, n, c;
+ uint32_t sizem1, tag_count, tag_size, n, c;
uint8_t *mem1, *mem2;
MMUAccessType type;
bit55 = extract64(ptr, 55, 1);
+ *fault = ptr;
/* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
if (unlikely(!tbi_check(desc, bit55))) {
- return ptr;
+ return -1;
}
ptr_tag = allocation_tag_from_addr(ptr);
if (tcma_check(desc, bit55, ptr_tag)) {
- goto done;
+ return 1;
}
mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
- esize = FIELD_EX32(desc, MTEDESC, ESIZE);
- total = FIELD_EX32(desc, MTEDESC, TSIZE);
+ sizem1 = FIELD_EX32(desc, MTEDESC, SIZEM1);
- /* Find the addr of the end of the access, and of the last element. */
- ptr_end = ptr + total;
- ptr_last = ptr_end - esize;
+ /* Find the addr of the last byte of the access. */
+ ptr_last = ptr + sizem1;
/* Round the bounds to the tag granule, and compute the number of tags. */
tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
@@ -802,12 +739,19 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) {
/* Memory access stays on one page. */
tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE);
- mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total,
+ mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, sizem1 + 1,
MMU_DATA_LOAD, tag_size, ra);
if (!mem1) {
- goto done;
+ return 1;
+ }
+
+ /*
+ * Perform all of the comparisons, recognizing that most are
+ * aligned operations that do not cross granule boundaries.
+ */
+ if (likely(tag_count == 1)) {
+ return ptr_tag == load_tag1(ptr, mem1);
}
- /* Perform all of the comparisons. */
n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
} else {
/* Memory access crosses to next page. */
@@ -817,7 +761,7 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE);
mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
- ptr_end - next_page,
+ ptr_last - next_page + 1,
MMU_DATA_LOAD, tag_size, ra);
/*
@@ -831,31 +775,54 @@ uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
}
if (n == c) {
if (!mem2) {
- goto done;
+ return 1;
}
n += checkN(mem2, 0, ptr_tag, tag_count - c);
}
}
/*
- * If we failed, we know which granule. Compute the element that
- * is first in that granule, and signal failure on that element.
+ * If we failed, we know which granule -- signal failure at that address.
*/
if (unlikely(n < tag_count)) {
- uint64_t fail_ofs;
-
- fail_ofs = tag_first + n * TAG_GRANULE - ptr;
- fail_ofs = ROUND_UP(fail_ofs, esize);
- mte_check_fail(env, desc, ptr + fail_ofs, ra);
+ if (n > 0) {
+ *fault = tag_first + n * TAG_GRANULE;
+ }
+ return 0;
}
- done:
+ return 1;
+}
+
+/*
+ * No-fault probe, to be used by SVE for MemSingleNF.
+ * Returns false if the access is Checked and the check failed. This
+ * is only intended to probe the tag -- the validity of the page must
+ * be checked beforehand.
+ */
+bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+ uint64_t discard;
+ return mte_check_int(env, desc, ptr, 0, &discard) != 0;
+}
+
+uint64_t mte_check(CPUARMState *env, uint32_t desc,
+ uint64_t ptr, uintptr_t ra)
+{
+ uint64_t fault;
+ int ret = mte_check_int(env, desc, ptr, ra, &fault);
+
+ if (unlikely(ret == 0)) {
+ mte_check_fail(env, desc, fault, ra);
+ } else if (ret < 0) {
+ return ptr;
+ }
return useronly_clean_ptr(ptr);
}
-uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+uint64_t HELPER(mte_check)(CPUARMState *env, uint32_t desc, uint64_t ptr)
{
- return mte_checkN(env, desc, ptr, GETPC());
+ return mte_check(env, desc, ptr, GETPC());
}
/*
@@ -4382,13 +4382,9 @@ static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env,
#endif
}
-typedef uint64_t mte_check_fn(CPUARMState *, uint32_t, uint64_t, uintptr_t);
-
-static inline QEMU_ALWAYS_INLINE
-void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env,
- uint64_t *vg, target_ulong addr, int esize,
- int msize, uint32_t mtedesc, uintptr_t ra,
- mte_check_fn *check)
+static void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env,
+ uint64_t *vg, target_ulong addr, int esize,
+ int msize, uint32_t mtedesc, uintptr_t ra)
{
intptr_t mem_off, reg_off, reg_last;
@@ -4405,7 +4401,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env,
uint64_t pg = vg[reg_off >> 6];
do {
if ((pg >> (reg_off & 63)) & 1) {
- check(env, mtedesc, addr, ra);
+ mte_check(env, mtedesc, addr, ra);
}
reg_off += esize;
mem_off += msize;
@@ -4422,7 +4418,7 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env,
uint64_t pg = vg[reg_off >> 6];
do {
if ((pg >> (reg_off & 63)) & 1) {
- check(env, mtedesc, addr, ra);
+ mte_check(env, mtedesc, addr, ra);
}
reg_off += esize;
mem_off += msize;
@@ -4431,30 +4427,6 @@ void sve_cont_ldst_mte_check_int(SVEContLdSt *info, CPUARMState *env,
}
}
-typedef void sve_cont_ldst_mte_check_fn(SVEContLdSt *info, CPUARMState *env,
- uint64_t *vg, target_ulong addr,
- int esize, int msize, uint32_t mtedesc,
- uintptr_t ra);
-
-static void sve_cont_ldst_mte_check1(SVEContLdSt *info, CPUARMState *env,
- uint64_t *vg, target_ulong addr,
- int esize, int msize, uint32_t mtedesc,
- uintptr_t ra)
-{
- sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize,
- mtedesc, ra, mte_check1);
-}
-
-static void sve_cont_ldst_mte_checkN(SVEContLdSt *info, CPUARMState *env,
- uint64_t *vg, target_ulong addr,
- int esize, int msize, uint32_t mtedesc,
- uintptr_t ra)
-{
- sve_cont_ldst_mte_check_int(info, env, vg, addr, esize, msize,
- mtedesc, ra, mte_checkN);
-}
-
-
/*
* Common helper for all contiguous 1,2,3,4-register predicated stores.
*/
@@ -4463,8 +4435,7 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr,
uint32_t desc, const uintptr_t retaddr,
const int esz, const int msz, const int N, uint32_t mtedesc,
sve_ldst1_host_fn *host_fn,
- sve_ldst1_tlb_fn *tlb_fn,
- sve_cont_ldst_mte_check_fn *mte_check_fn)
+ sve_ldst1_tlb_fn *tlb_fn)
{
const unsigned rd = simd_data(desc);
const intptr_t reg_max = simd_oprsz(desc);
@@ -4493,9 +4464,9 @@ void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr,
* Handle mte checks for all active elements.
* Since TBI must be set for MTE, !mtedesc => !mte_active.
*/
- if (mte_check_fn && mtedesc) {
- mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz,
- mtedesc, retaddr);
+ if (mtedesc) {
+ sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz,
+ mtedesc, retaddr);
}
flags = info.page[0].flags | info.page[1].flags;
@@ -4621,8 +4592,7 @@ void sve_ldN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr,
mtedesc = 0;
}
- sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn,
- N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN);
+ sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn);
}
#define DO_LD1_1(NAME, ESZ) \
@@ -4630,7 +4600,7 @@ void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, 1, 0, \
- sve_##NAME##_host, sve_##NAME##_tlb, NULL); \
+ sve_##NAME##_host, sve_##NAME##_tlb); \
} \
void HELPER(sve_##NAME##_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -4644,13 +4614,13 @@ void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \
- sve_##NAME##_le_host, sve_##NAME##_le_tlb, NULL); \
+ sve_##NAME##_le_host, sve_##NAME##_le_tlb); \
} \
void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \
- sve_##NAME##_be_host, sve_##NAME##_be_tlb, NULL); \
+ sve_##NAME##_be_host, sve_##NAME##_be_tlb); \
} \
void HELPER(sve_##NAME##_le_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -4693,7 +4663,7 @@ void HELPER(sve_ld##N##bb_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), MO_8, MO_8, N, 0, \
- sve_ld1bb_host, sve_ld1bb_tlb, NULL); \
+ sve_ld1bb_host, sve_ld1bb_tlb); \
} \
void HELPER(sve_ld##N##bb_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -4707,13 +4677,13 @@ void HELPER(sve_ld##N##SUFF##_le_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \
- sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb, NULL); \
+ sve_ld1##SUFF##_le_host, sve_ld1##SUFF##_le_tlb); \
} \
void HELPER(sve_ld##N##SUFF##_be_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \
- sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb, NULL); \
+ sve_ld1##SUFF##_be_host, sve_ld1##SUFF##_be_tlb); \
} \
void HELPER(sve_ld##N##SUFF##_le_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -4826,7 +4796,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr,
if (fault == FAULT_FIRST) {
/* Trapping mte check for the first-fault element. */
if (mtedesc) {
- mte_check1(env, mtedesc, addr + mem_off, retaddr);
+ mte_check(env, mtedesc, addr + mem_off, retaddr);
}
/*
@@ -4869,7 +4839,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr,
/* Watchpoint hit, see below. */
goto do_fault;
}
- if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) {
+ if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) {
goto do_fault;
}
/*
@@ -4919,7 +4889,7 @@ void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr,
& BP_MEM_READ)) {
goto do_fault;
}
- if (mtedesc && !mte_probe1(env, mtedesc, addr + mem_off)) {
+ if (mtedesc && !mte_probe(env, mtedesc, addr + mem_off)) {
goto do_fault;
}
host_fn(vd, reg_off, host + mem_off);
@@ -5090,8 +5060,7 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr,
uint32_t desc, const uintptr_t retaddr,
const int esz, const int msz, const int N, uint32_t mtedesc,
sve_ldst1_host_fn *host_fn,
- sve_ldst1_tlb_fn *tlb_fn,
- sve_cont_ldst_mte_check_fn *mte_check_fn)
+ sve_ldst1_tlb_fn *tlb_fn)
{
const unsigned rd = simd_data(desc);
const intptr_t reg_max = simd_oprsz(desc);
@@ -5117,9 +5086,9 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr,
* Handle mte checks for all active elements.
* Since TBI must be set for MTE, !mtedesc => !mte_active.
*/
- if (mte_check_fn && mtedesc) {
- mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz,
- mtedesc, retaddr);
+ if (mtedesc) {
+ sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz,
+ mtedesc, retaddr);
}
flags = info.page[0].flags | info.page[1].flags;
@@ -5233,8 +5202,7 @@ void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr,
mtedesc = 0;
}
- sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn,
- N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN);
+ sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn);
}
#define DO_STN_1(N, NAME, ESZ) \
@@ -5242,7 +5210,7 @@ void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0, \
- sve_st1##NAME##_host, sve_st1##NAME##_tlb, NULL); \
+ sve_st1##NAME##_host, sve_st1##NAME##_tlb); \
} \
void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -5256,13 +5224,13 @@ void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \
- sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb, NULL); \
+ sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb); \
} \
void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
{ \
sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \
- sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb, NULL); \
+ sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb); \
} \
void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg, \
target_ulong addr, uint32_t desc) \
@@ -5373,7 +5341,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
info.attrs, BP_MEM_READ, retaddr);
}
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
- mte_check1(env, mtedesc, addr, retaddr);
+ mte_check(env, mtedesc, addr, retaddr);
}
host_fn(&scratch, reg_off, info.host);
} else {
@@ -5386,7 +5354,7 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
BP_MEM_READ, retaddr);
}
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
- mte_check1(env, mtedesc, addr, retaddr);
+ mte_check(env, mtedesc, addr, retaddr);
}
tlb_fn(env, &scratch, reg_off, addr, retaddr);
}
@@ -5552,7 +5520,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
*/
addr = base + (off_fn(vm, reg_off) << scale);
if (mtedesc) {
- mte_check1(env, mtedesc, addr, retaddr);
+ mte_check(env, mtedesc, addr, retaddr);
}
tlb_fn(env, vd, reg_off, addr, retaddr);
@@ -5588,7 +5556,7 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
}
if (mtedesc &&
arm_tlb_mte_tagged(&info.attrs) &&
- !mte_probe1(env, mtedesc, addr)) {
+ !mte_probe(env, mtedesc, addr)) {
goto fault;
}
@@ -5773,7 +5741,7 @@ void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
}
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
- mte_check1(env, mtedesc, addr, retaddr);
+ mte_check(env, mtedesc, addr, retaddr);
}
}
i += 1;
@@ -253,10 +253,7 @@ static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
}
/*
- * For MTE, check a single logical or atomic access. This probes a single
- * address, the exact one specified. The size and alignment of the access
- * is not relevant to MTE, per se, but watchpoints do require the size,
- * and we want to recognize those before making any other changes to state.
+ * For MTE, check a single logical or atomic access.
*/
static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
bool is_write, bool tag_checked,
@@ -272,11 +269,11 @@ static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
- desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_size);
+ desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << log2_size) - 1);
tcg_desc = tcg_const_i32(desc);
ret = new_tmp_a64(s);
- gen_helper_mte_check1(ret, cpu_env, tcg_desc, addr);
+ gen_helper_mte_check(ret, cpu_env, tcg_desc, addr);
tcg_temp_free_i32(tcg_desc);
return ret;
@@ -295,9 +292,9 @@ TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
* For MTE, check multiple logical sequential accesses.
*/
TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
- bool tag_checked, int log2_esize, int total_size)
+ bool tag_checked, int total_size)
{
- if (tag_checked && s->mte_active[0] && total_size != (1 << log2_esize)) {
+ if (tag_checked && s->mte_active[0]) {
TCGv_i32 tcg_desc;
TCGv_i64 ret;
int desc = 0;
@@ -306,17 +303,16 @@ TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
- desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << log2_esize);
- desc = FIELD_DP32(desc, MTEDESC, TSIZE, total_size);
+ desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
tcg_desc = tcg_const_i32(desc);
ret = new_tmp_a64(s);
- gen_helper_mte_checkN(ret, cpu_env, tcg_desc, addr);
+ gen_helper_mte_check(ret, cpu_env, tcg_desc, addr);
tcg_temp_free_i32(tcg_desc);
return ret;
}
- return gen_mte_check1(s, addr, is_write, tag_checked, log2_esize);
+ return clean_data_tbi(s, addr);
}
typedef struct DisasCompare64 {
@@ -2966,8 +2962,7 @@ static void disas_ldst_pair(DisasContext *s, uint32_t insn)
}
clean_addr = gen_mte_checkN(s, dirty_addr, !is_load,
- (wback || rn != 31) && !set_tag,
- size, 2 << size);
+ (wback || rn != 31) && !set_tag, 2 << size);
if (is_vector) {
if (is_load) {
@@ -3713,8 +3708,8 @@ static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn)
* Issue the MTE check vs the logical repeat count, before we
* promote consecutive little-endian elements below.
*/
- clean_addr = gen_mte_checkN(s, tcg_rn, is_store, is_postidx || rn != 31,
- size, total);
+ clean_addr = gen_mte_checkN(s, tcg_rn, is_store,
+ is_postidx || rn != 31, total);
/*
* Consecutive little-endian elements from a single register
@@ -3866,8 +3861,8 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
total = selem << scale;
tcg_rn = cpu_reg_sp(s, rn);
- clean_addr = gen_mte_checkN(s, tcg_rn, !is_load, is_postidx || rn != 31,
- scale, total);
+ clean_addr = gen_mte_checkN(s, tcg_rn, !is_load,
+ is_postidx || rn != 31, total);
tcg_ebytes = tcg_const_i64(1 << scale);
for (xs = 0; xs < selem; xs++) {
@@ -4264,7 +4264,7 @@ static void do_ldr(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
dirty_addr = tcg_temp_new_i64();
tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
- clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
+ clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
tcg_temp_free_i64(dirty_addr);
/*
@@ -4352,7 +4352,7 @@ static void do_str(DisasContext *s, uint32_t vofs, int len, int rn, int imm)
dirty_addr = tcg_temp_new_i64();
tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
- clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
+ clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len);
tcg_temp_free_i64(dirty_addr);
/* Note that unpredicated load/store of vector/predicate registers
@@ -4509,8 +4509,7 @@ static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
- desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz);
- desc = FIELD_DP32(desc, MTEDESC, TSIZE, mte_n << msz);
+ desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1);
desc <<= SVE_MTEDESC_SHIFT;
} else {
addr = clean_data_tbi(s, addr);
@@ -5189,7 +5188,7 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
- desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz);
+ desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1);
desc <<= SVE_MTEDESC_SHIFT;
}
desc = simd_desc(vsz, vsz, desc | scale);
new file mode 100644
@@ -0,0 +1,44 @@
+/*
+ * Memory tagging, faulting unaligned access.
+ *
+ * Copyright (c) 2021 Linaro Ltd
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "mte.h"
+
+void pass(int sig, siginfo_t *info, void *uc)
+{
+ assert(info->si_code == SEGV_MTESERR);
+ exit(0);
+}
+
+int main(int ac, char **av)
+{
+ struct sigaction sa;
+ void *p0, *p1, *p2;
+ long excl = 1;
+
+ enable_mte(PR_MTE_TCF_SYNC);
+ p0 = alloc_mte_mem(sizeof(*p0));
+
+ /* Create two differently tagged pointers. */
+ asm("irg %0,%1,%2" : "=r"(p1) : "r"(p0), "r"(excl));
+ asm("gmi %0,%1,%0" : "+r"(excl) : "r" (p1));
+ assert(excl != 1);
+ asm("irg %0,%1,%2" : "=r"(p2) : "r"(p0), "r"(excl));
+ assert(p1 != p2);
+
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_sigaction = pass;
+ sa.sa_flags = SA_SIGINFO;
+ sigaction(SIGSEGV, &sa, NULL);
+
+ /* Store store two different tags in sequential granules. */
+ asm("stg %0, [%0]" : : "r"(p1));
+ asm("stg %0, [%0]" : : "r"(p2 + 16));
+
+ /* Perform an unaligned load crossing the granules. */
+ asm volatile("ldr %0, [%1]" : "=r"(p0) : "r"(p1 + 12));
+ abort();
+}
@@ -37,7 +37,7 @@ AARCH64_TESTS += bti-2
# MTE Tests
ifneq ($(DOCKER_IMAGE)$(CROSS_CC_HAS_ARMV8_MTE),)
-AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4
+AARCH64_TESTS += mte-1 mte-2 mte-3 mte-4 mte-5
mte-%: CFLAGS += -march=armv8.5-a+memtag
endif
We were incorrectly assuming that only the first byte of an MTE access is checked against the tags. But per the ARM, unaligned accesses are pre-decomposed into single-byte accesses. So by the time we reach the actual MTE check in the ARM pseudocode, all accesses are aligned. Therefore, drop mte_check1, since we cannot know a priori that an access is aligned. Rename mte_checkN to mte_check, which now handles all accesses. Rename mte_probe1 to mte_probe, and use a common helper. Drop the computation of the faulting nth element, since all accesses can be considered to devolve to bytes, and simply compute the faulting address. Buglink: https://bugs.launchpad.net/bugs/1921948 Signed-off-by: Richard Henderson <richard.henderson@linaro.org> --- target/arm/helper-a64.h | 3 +- target/arm/internals.h | 13 +-- target/arm/translate-a64.h | 2 +- target/arm/mte_helper.c | 169 ++++++++++++------------------ target/arm/sve_helper.c | 96 ++++++----------- target/arm/translate-a64.c | 31 +++--- target/arm/translate-sve.c | 9 +- tests/tcg/aarch64/mte-5.c | 44 ++++++++ tests/tcg/aarch64/Makefile.target | 2 +- 9 files changed, 169 insertions(+), 200 deletions(-) create mode 100644 tests/tcg/aarch64/mte-5.c -- 2.25.1