[v5,5/7] arm64: head: Clean the ID map and the HYP text to the PoC if needed

If we enter with the MMU and caches enabled, the bootloader may not have
performed any cache maintenance to the PoC. So clean the ID mapped page
to the PoC, to ensure that instruction and data accesses with the MMU
off see the correct data. For similar reasons, clean all the HYP text to
the PoC as well when entering at EL2 with the MMU and caches enabled.

Note that this means primary_entry() itself needs to be moved into the
ID map as well, as we will return from init_kernel_el() with the MMU and
caches off.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
 arch/arm64/kernel/head.S  | 31 +++++++++++++++++---
 arch/arm64/kernel/sleep.S |  1 +
 2 files changed, 28 insertions(+), 4 deletions(-)

On Tue, 8 Nov 2022 at 19:22, Ard Biesheuvel <ardb@kernel.org> wrote:
>
> If we enter with the MMU and caches enabled, the bootloader may not have
> performed any cache maintenance to the PoC. So clean the ID mapped page
> to the PoC, to ensure that instruction and data accesses with the MMU
> off see the correct data. For similar reasons, clean all the HYP text to
> the PoC as well when entering at EL2 with the MMU and caches enabled.
>
> Note that this means primary_entry() itself needs to be moved into the
> ID map as well, as we will return from init_kernel_el() with the MMU and
> caches off.
>
> Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
> ---
>  arch/arm64/kernel/head.S  | 31 +++++++++++++++++---
>  arch/arm64/kernel/sleep.S |  1 +
>  2 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index a7c84cde67c5c652..825f1d0549661030 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -70,7 +70,7 @@
>
>         __EFI_PE_HEADER
>
> -       __INIT
> +       .section ".idmap.text","awx"
>
>         /*
>          * The following callee saved general purpose registers are used on the
> @@ -90,6 +90,17 @@ SYM_CODE_START(primary_entry)
>         bl      record_mmu_state
>         bl      preserve_boot_args
>         bl      create_idmap
> +
> +       /*
> +        * If we entered with the MMU and caches on, clean the ID mapped part
> +        * of the primary boot code to the PoC so we can safely execute it with
> +        * the MMU off.
> +        */
> +       cbz     x19, 0f
> +       adrp    x0, __idmap_text_start
> +       adr_l   x1, __idmap_text_end
> +       bl      dcache_clean_poc
> +0:     mov     x19, x0

This is wrong, it should be

mov x0, x19


>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mov     x20, x0
>
> @@ -111,6 +122,7 @@ SYM_CODE_START(primary_entry)
>         b       __primary_switch
>  SYM_CODE_END(primary_entry)
>
> +       __INIT
>  SYM_CODE_START_LOCAL(record_mmu_state)
>         mrs     x19, CurrentEL
>         cmp     x19, #CurrentEL_EL2
> @@ -505,10 +517,12 @@ SYM_FUNC_END(__primary_switched)
>   * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x0 if
>   * booted in EL1 or EL2 respectively, with the top 32 bits containing
>   * potential context flags. These flags are *not* stored in __boot_cpu_mode.
> + *
> + * x0: whether we are being called from the primary boot path with the MMU on
>   */
>  SYM_FUNC_START(init_kernel_el)
> -       mrs     x0, CurrentEL
> -       cmp     x0, #CurrentEL_EL2
> +       mrs     x1, CurrentEL
> +       cmp     x1, #CurrentEL_EL2
>         b.eq    init_el2
>
>  SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
> @@ -523,6 +537,14 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
>         eret
>
>  SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
> +       msr     elr_el2, lr
> +
> +       // clean all HYP code to the PoC if we booted at EL2 with the MMU on
> +       cbz     x0, 0f
> +       adrp    x0, __hyp_idmap_text_start
> +       adr_l   x1, __hyp_text_end
> +       bl      dcache_clean_poc
> +0:
>         mov_q   x0, HCR_HOST_NVHE_FLAGS
>         msr     hcr_el2, x0
>         isb
> @@ -556,7 +578,6 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
>         msr     sctlr_el1, x1
>         mov     x2, xzr
>  2:
> -       msr     elr_el2, lr
>         mov     w0, #BOOT_CPU_MODE_EL2
>         orr     x0, x0, x2
>         eret
> @@ -567,6 +588,7 @@ SYM_FUNC_END(init_kernel_el)
>          * cores are held until we're ready for them to initialise.
>          */
>  SYM_FUNC_START(secondary_holding_pen)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         mrs     x2, mpidr_el1
>         mov_q   x1, MPIDR_HWID_BITMASK
> @@ -584,6 +606,7 @@ SYM_FUNC_END(secondary_holding_pen)
>          * be used where CPUs are brought online dynamically by the kernel.
>          */
>  SYM_FUNC_START(secondary_entry)
> +       mov     x0, xzr
>         bl      init_kernel_el                  // w0=cpu_boot_mode
>         b       secondary_startup
>  SYM_FUNC_END(secondary_entry)
> diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
> index 7b7c56e048346e97..2ae7cff1953aaf87 100644
> --- a/arch/arm64/kernel/sleep.S
> +++ b/arch/arm64/kernel/sleep.S
> @@ -99,6 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter)
>
>         .pushsection ".idmap.text", "awx"
>  SYM_CODE_START(cpu_resume)
> +       mov     x0, xzr
>         bl      init_kernel_el
>         mov     x19, x0                 // preserve boot mode
>  #if VA_BITS > 48
> --
> 2.35.1
>