new file mode 100644
@@ -0,0 +1,53 @@
+/*
+ * ARM v8.5-MemTag Operations
+ *
+ * Copyright (c) 2020 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MTE_HELPER_H
+#define MTE_HELPER_H
+
+/**
+ * allocation_tag_mem:
+ * @env: the cpu environment
+ * @ptr_mmu_idx: the addressing regime to use for the virtual address
+ * @ptr: the virtual address for which to look up tag memory
+ * @ptr_access: the access to use for the virtual address
+ * @ptr_size: the number of bytes in the normal memory access
+ * @tag_access: the access to use for the tag memory
+ * @tag_size: the number of bytes in the tag memory access
+ * @ra: the return address for exception handling
+ *
+ * Our tag memory is formatted as a sequence of little-endian nibbles.
+ * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
+ * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
+ * for the higher addr.
+ *
+ * Here, resolve the physical address from the virtual address, and return
+ * a pointer to the corresponding tag byte. Exit with exception if the
+ * virtual address is not accessible for @ptr_access.
+ *
+ * The @ptr_size and @tag_size values may not have an obvious relation
+ * due to the alignment of @ptr, and the number of tag checks required.
+ *
+ * If there is no tag storage corresponding to @ptr, return NULL.
+ */
+uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
+ uint64_t ptr, MMUAccessType ptr_access,
+ int ptr_size, MMUAccessType tag_access,
+ int tag_size, uintptr_t ra);
+
+#endif /* MTE_HELPER_H */
@@ -26,7 +26,7 @@
#include "exec/helper-proto.h"
#include "qapi/error.h"
#include "qemu/guest-random.h"
-
+#include "tcg/mte_helper.h"
static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
{
@@ -47,195 +47,6 @@ static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
return tag;
}
-/**
- * allocation_tag_mem:
- * @env: the cpu environment
- * @ptr_mmu_idx: the addressing regime to use for the virtual address
- * @ptr: the virtual address for which to look up tag memory
- * @ptr_access: the access to use for the virtual address
- * @ptr_size: the number of bytes in the normal memory access
- * @tag_access: the access to use for the tag memory
- * @tag_size: the number of bytes in the tag memory access
- * @ra: the return address for exception handling
- *
- * Our tag memory is formatted as a sequence of little-endian nibbles.
- * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
- * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
- * for the higher addr.
- *
- * Here, resolve the physical address from the virtual address, and return
- * a pointer to the corresponding tag byte. Exit with exception if the
- * virtual address is not accessible for @ptr_access.
- *
- * The @ptr_size and @tag_size values may not have an obvious relation
- * due to the alignment of @ptr, and the number of tag checks required.
- *
- * If there is no tag storage corresponding to @ptr, return NULL.
- */
-static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
- uint64_t ptr, MMUAccessType ptr_access,
- int ptr_size, MMUAccessType tag_access,
- int tag_size, uintptr_t ra)
-{
-#ifdef CONFIG_USER_ONLY
- uint64_t clean_ptr = useronly_clean_ptr(ptr);
- int flags = page_get_flags(clean_ptr);
- uint8_t *tags;
- uintptr_t index;
-
- if (!(flags & (ptr_access == MMU_DATA_STORE ? PAGE_WRITE_ORG : PAGE_READ))) {
- /* SIGSEGV */
- arm_cpu_tlb_fill(env_cpu(env), ptr, ptr_size, ptr_access,
- ptr_mmu_idx, false, ra);
- g_assert_not_reached();
- }
-
- /* Require both MAP_ANON and PROT_MTE for the page. */
- if (!(flags & PAGE_ANON) || !(flags & PAGE_MTE)) {
- return NULL;
- }
-
- tags = page_get_target_data(clean_ptr);
- if (tags == NULL) {
- size_t alloc_size = TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1);
- tags = page_alloc_target_data(clean_ptr, alloc_size);
- assert(tags != NULL);
- }
-
- index = extract32(ptr, LOG2_TAG_GRANULE + 1,
- TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1);
- return tags + index;
-#else
- uintptr_t index;
- CPUIOTLBEntry *iotlbentry;
- int in_page, flags;
- ram_addr_t ptr_ra;
- hwaddr ptr_paddr, tag_paddr, xlat;
- MemoryRegion *mr;
- ARMASIdx tag_asi;
- AddressSpace *tag_as;
- void *host;
-
- /*
- * Probe the first byte of the virtual address. This raises an
- * exception for inaccessible pages, and resolves the virtual address
- * into the softmmu tlb.
- *
- * When RA == 0, this is for mte_probe. The page is expected to be
- * valid. Indicate to probe_access_flags no-fault, then assert that
- * we received a valid page.
- */
- flags = probe_access_flags(env, ptr, ptr_access, ptr_mmu_idx,
- ra == 0, &host, ra);
- assert(!(flags & TLB_INVALID_MASK));
-
- /*
- * Find the iotlbentry for ptr. This *must* be present in the TLB
- * because we just found the mapping.
- * TODO: Perhaps there should be a cputlb helper that returns a
- * matching tlb entry + iotlb entry.
- */
- index = tlb_index(env, ptr_mmu_idx, ptr);
-# ifdef CONFIG_DEBUG_TCG
- {
- CPUTLBEntry *entry = tlb_entry(env, ptr_mmu_idx, ptr);
- target_ulong comparator = (ptr_access == MMU_DATA_LOAD
- ? entry->addr_read
- : tlb_addr_write(entry));
- g_assert(tlb_hit(comparator, ptr));
- }
-# endif
- iotlbentry = &env_tlb(env)->d[ptr_mmu_idx].iotlb[index];
-
- /* If the virtual page MemAttr != Tagged, access unchecked. */
- if (!arm_tlb_mte_tagged(&iotlbentry->attrs)) {
- return NULL;
- }
-
- /*
- * If not backed by host ram, there is no tag storage: access unchecked.
- * This is probably a guest os bug though, so log it.
- */
- if (unlikely(flags & TLB_MMIO)) {
- qemu_log_mask(LOG_GUEST_ERROR,
- "Page @ 0x%" PRIx64 " indicates Tagged Normal memory "
- "but is not backed by host ram\n", ptr);
- return NULL;
- }
-
- /*
- * The Normal memory access can extend to the next page. E.g. a single
- * 8-byte access to the last byte of a page will check only the last
- * tag on the first page.
- * Any page access exception has priority over tag check exception.
- */
- in_page = -(ptr | TARGET_PAGE_MASK);
- if (unlikely(ptr_size > in_page)) {
- void *ignore;
- flags |= probe_access_flags(env, ptr + in_page, ptr_access,
- ptr_mmu_idx, ra == 0, &ignore, ra);
- assert(!(flags & TLB_INVALID_MASK));
- }
-
- /* Any debug exception has priority over a tag check exception. */
- if (unlikely(flags & TLB_WATCHPOINT)) {
- int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE;
- assert(ra != 0);
- cpu_check_watchpoint(env_cpu(env), ptr, ptr_size,
- iotlbentry->attrs, wp, ra);
- }
-
- /*
- * Find the physical address within the normal mem space.
- * The memory region lookup must succeed because TLB_MMIO was
- * not set in the cputlb lookup above.
- */
- mr = memory_region_from_host(host, &ptr_ra);
- tcg_debug_assert(mr != NULL);
- tcg_debug_assert(memory_region_is_ram(mr));
- ptr_paddr = ptr_ra;
- do {
- ptr_paddr += mr->addr;
- mr = mr->container;
- } while (mr);
-
- /* Convert to the physical address in tag space. */
- tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1);
-
- /* Look up the address in tag space. */
- tag_asi = iotlbentry->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS;
- tag_as = cpu_get_address_space(env_cpu(env), tag_asi);
- mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL,
- tag_access == MMU_DATA_STORE,
- iotlbentry->attrs);
-
- /*
- * Note that @mr will never be NULL. If there is nothing in the address
- * space at @tag_paddr, the translation will return the unallocated memory
- * region. For our purposes, the result must be ram.
- */
- if (unlikely(!memory_region_is_ram(mr))) {
- /* ??? Failure is a board configuration error. */
- qemu_log_mask(LOG_UNIMP,
- "Tag Memory @ 0x%" HWADDR_PRIx " not found for "
- "Normal Memory @ 0x%" HWADDR_PRIx "\n",
- tag_paddr, ptr_paddr);
- return NULL;
- }
-
- /*
- * Ensure the tag memory is dirty on write, for migration.
- * Tag memory can never contain code or display memory (vga).
- */
- if (tag_access == MMU_DATA_STORE) {
- ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat;
- cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION);
- }
-
- return memory_region_get_ram_ptr(mr) + xlat;
-#endif
-}
-
uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
{
uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
new file mode 100644
@@ -0,0 +1,159 @@
+/*
+ * ARM v8.5-MemTag Operations - System Emulation
+ *
+ * Copyright (c) 2020 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "exec/exec-all.h"
+#include "exec/ram_addr.h"
+#include "tcg/mte_helper.h"
+
+uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
+ uint64_t ptr, MMUAccessType ptr_access,
+ int ptr_size, MMUAccessType tag_access,
+ int tag_size, uintptr_t ra)
+{
+ uintptr_t index;
+ CPUIOTLBEntry *iotlbentry;
+ int in_page, flags;
+ ram_addr_t ptr_ra;
+ hwaddr ptr_paddr, tag_paddr, xlat;
+ MemoryRegion *mr;
+ ARMASIdx tag_asi;
+ AddressSpace *tag_as;
+ void *host;
+
+ /*
+ * Probe the first byte of the virtual address. This raises an
+ * exception for inaccessible pages, and resolves the virtual address
+ * into the softmmu tlb.
+ *
+ * When RA == 0, this is for mte_probe. The page is expected to be
+ * valid. Indicate to probe_access_flags no-fault, then assert that
+ * we received a valid page.
+ */
+ flags = probe_access_flags(env, ptr, ptr_access, ptr_mmu_idx,
+ ra == 0, &host, ra);
+ assert(!(flags & TLB_INVALID_MASK));
+
+ /*
+ * Find the iotlbentry for ptr. This *must* be present in the TLB
+ * because we just found the mapping.
+ * TODO: Perhaps there should be a cputlb helper that returns a
+ * matching tlb entry + iotlb entry.
+ */
+ index = tlb_index(env, ptr_mmu_idx, ptr);
+# ifdef CONFIG_DEBUG_TCG
+ {
+ CPUTLBEntry *entry = tlb_entry(env, ptr_mmu_idx, ptr);
+ target_ulong comparator = (ptr_access == MMU_DATA_LOAD
+ ? entry->addr_read
+ : tlb_addr_write(entry));
+ g_assert(tlb_hit(comparator, ptr));
+ }
+# endif
+ iotlbentry = &env_tlb(env)->d[ptr_mmu_idx].iotlb[index];
+
+ /* If the virtual page MemAttr != Tagged, access unchecked. */
+ if (!arm_tlb_mte_tagged(&iotlbentry->attrs)) {
+ return NULL;
+ }
+
+ /*
+ * If not backed by host ram, there is no tag storage: access unchecked.
+ * This is probably a guest os bug though, so log it.
+ */
+ if (unlikely(flags & TLB_MMIO)) {
+ qemu_log_mask(LOG_GUEST_ERROR,
+ "Page @ 0x%" PRIx64 " indicates Tagged Normal memory "
+ "but is not backed by host ram\n", ptr);
+ return NULL;
+ }
+
+ /*
+ * The Normal memory access can extend to the next page. E.g. a single
+ * 8-byte access to the last byte of a page will check only the last
+ * tag on the first page.
+ * Any page access exception has priority over tag check exception.
+ */
+ in_page = -(ptr | TARGET_PAGE_MASK);
+ if (unlikely(ptr_size > in_page)) {
+ void *ignore;
+ flags |= probe_access_flags(env, ptr + in_page, ptr_access,
+ ptr_mmu_idx, ra == 0, &ignore, ra);
+ assert(!(flags & TLB_INVALID_MASK));
+ }
+
+ /* Any debug exception has priority over a tag check exception. */
+ if (unlikely(flags & TLB_WATCHPOINT)) {
+ int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE;
+ assert(ra != 0);
+ cpu_check_watchpoint(env_cpu(env), ptr, ptr_size,
+ iotlbentry->attrs, wp, ra);
+ }
+
+ /*
+ * Find the physical address within the normal mem space.
+ * The memory region lookup must succeed because TLB_MMIO was
+ * not set in the cputlb lookup above.
+ */
+ mr = memory_region_from_host(host, &ptr_ra);
+ tcg_debug_assert(mr != NULL);
+ tcg_debug_assert(memory_region_is_ram(mr));
+ ptr_paddr = ptr_ra;
+ do {
+ ptr_paddr += mr->addr;
+ mr = mr->container;
+ } while (mr);
+
+ /* Convert to the physical address in tag space. */
+ tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1);
+
+ /* Look up the address in tag space. */
+ tag_asi = iotlbentry->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS;
+ tag_as = cpu_get_address_space(env_cpu(env), tag_asi);
+ mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL,
+ tag_access == MMU_DATA_STORE,
+ iotlbentry->attrs);
+
+ /*
+ * Note that @mr will never be NULL. If there is nothing in the address
+ * space at @tag_paddr, the translation will return the unallocated memory
+ * region. For our purposes, the result must be ram.
+ */
+ if (unlikely(!memory_region_is_ram(mr))) {
+ /* ??? Failure is a board configuration error. */
+ qemu_log_mask(LOG_UNIMP,
+ "Tag Memory @ 0x%" HWADDR_PRIx " not found for "
+ "Normal Memory @ 0x%" HWADDR_PRIx "\n",
+ tag_paddr, ptr_paddr);
+ return NULL;
+ }
+
+ /*
+ * Ensure the tag memory is dirty on write, for migration.
+ * Tag memory can never contain code or display memory (vga).
+ */
+ if (tag_access == MMU_DATA_STORE) {
+ ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat;
+ cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION);
+ }
+
+ return memory_region_get_ram_ptr(mr) + xlat;
+}
new file mode 100644
@@ -0,0 +1,57 @@
+/*
+ * ARM v8.5-MemTag Operations - User-mode
+ *
+ * Copyright (c) 2020 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "tcg/mte_helper.h"
+
+uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
+ uint64_t ptr, MMUAccessType ptr_access,
+ int ptr_size, MMUAccessType tag_access,
+ int tag_size, uintptr_t ra)
+{
+ uint64_t clean_ptr = useronly_clean_ptr(ptr);
+ int flags = page_get_flags(clean_ptr);
+ uint8_t *tags;
+ uintptr_t index;
+
+ if (!(flags & (ptr_access == MMU_DATA_STORE ? PAGE_WRITE_ORG : PAGE_READ))) {
+ /* SIGSEGV */
+ arm_cpu_tlb_fill(env_cpu(env), ptr, ptr_size, ptr_access,
+ ptr_mmu_idx, false, ra);
+ g_assert_not_reached();
+ }
+
+ /* Require both MAP_ANON and PROT_MTE for the page. */
+ if (!(flags & PAGE_ANON) || !(flags & PAGE_MTE)) {
+ return NULL;
+ }
+
+ tags = page_get_target_data(clean_ptr);
+ if (tags == NULL) {
+ size_t alloc_size = TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1);
+ tags = page_alloc_target_data(clean_ptr, alloc_size);
+ assert(tags != NULL);
+ }
+
+ index = extract32(ptr, LOG2_TAG_GRANULE + 1,
+ TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1);
+ return tags + index;
+}
@@ -1,2 +1,3 @@
arm_softmmu_ss.add(when: 'CONFIG_TCG', if_true: files(
+ 'mte_helper.c',
))
@@ -1,2 +1,3 @@
arm_user_ss.add(when: 'CONFIG_TCG', if_true: files(
+ 'mte_helper.c',
))