* [PATCH 1/4] arm64/mm: migrate swapper_pg_dir
2018-05-30 9:12 [PATCH 0/4] arm64/mm: migrate swapper_pg_dir YaoJun
@ 2018-05-30 9:12 ` YaoJun
2018-05-30 9:46 ` Greg KH
2018-05-30 9:12 ` [PATCH 2/4] " YaoJun
` (2 subsequent siblings)
3 siblings, 1 reply; 9+ messages in thread
From: YaoJun @ 2018-05-30 9:12 UTC (permalink / raw)
To: kernel-hardening
Cc: catalin.marinas, will.deacon, linux-arm-kernel, linux-kernel,
mark.rutland
Introduce __pa_swapper_pg_dir to save physical address
of swapper_pg_dir. And pass it as an argument to
__enable_mmu().
Signed-off-by: YaoJun <yaojun8558363@gmail.com>
---
arch/arm64/include/asm/mmu_context.h | 4 +---
arch/arm64/include/asm/pgtable.h | 1 +
arch/arm64/kernel/cpufeature.c | 2 +-
arch/arm64/kernel/head.S | 10 ++++++----
arch/arm64/kernel/hibernate.c | 2 +-
arch/arm64/kernel/sleep.S | 2 ++
arch/arm64/mm/kasan_init.c | 4 ++--
arch/arm64/mm/mmu.c | 8 ++++++--
8 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 39ec0b8a689e..3eddb871f251 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -141,14 +141,12 @@ static inline void cpu_install_idmap(void)
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
* avoiding the possibility of conflicting TLB entries being allocated.
*/
-static inline void cpu_replace_ttbr1(pgd_t *pgdp)
+static inline void cpu_replace_ttbr1(phys_addr_t pgd_phys)
{
typedef void (ttbr_replace_func)(phys_addr_t);
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
ttbr_replace_func *replace_phys;
- phys_addr_t pgd_phys = virt_to_phys(pgdp);
-
replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
cpu_install_idmap();
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 7c4c8f318ba9..14ba344b1af7 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -722,6 +722,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
extern pgd_t swapper_pg_end[];
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
+extern phys_addr_t __pa_swapper_pg_dir;
/*
* Encode and decode a swap entry:
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 9d1b06d67c53..5b9448688d80 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -917,7 +917,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
cpu_install_idmap();
- remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
+ remap_fn(cpu, num_online_cpus(), __pa_swapper_pg_dir);
cpu_uninstall_idmap();
if (!cpu)
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index b0853069702f..e3bb44b4b6c6 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -706,6 +706,8 @@ secondary_startup:
* Common entry point for secondary CPUs.
*/
bl __cpu_setup // initialise processor
+ adrp x25, idmap_pg_dir
+ ldr_l x26, __pa_swapper_pg_dir
bl __enable_mmu
ldr x8, =__secondary_switched
br x8
@@ -761,10 +763,8 @@ ENTRY(__enable_mmu)
cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
b.ne __no_granule_support
update_early_cpu_boot_status 0, x1, x2
- adrp x1, idmap_pg_dir
- adrp x2, swapper_pg_dir
- phys_to_ttbr x3, x1
- phys_to_ttbr x4, x2
+ phys_to_ttbr x3, x25
+ phys_to_ttbr x4, x26
msr ttbr0_el1, x3 // load TTBR0
msr ttbr1_el1, x4 // load TTBR1
isb
@@ -823,6 +823,8 @@ __primary_switch:
mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value
#endif
+ adrp x25, idmap_pg_dir
+ adrp x26, swapper_pg_dir
bl __enable_mmu
#ifdef CONFIG_RELOCATABLE
bl __relocate_kernel
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 1ec5f28c39fc..12948949202c 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -125,7 +125,7 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size)
return -EOVERFLOW;
arch_hdr_invariants(&hdr->invariants);
- hdr->ttbr1_el1 = __pa_symbol(swapper_pg_dir);
+ hdr->ttbr1_el1 = __pa_swapper_pg_dir;
hdr->reenter_kernel = _cpu_resume;
/* We can't use __hyp_get_vectors() because kvm may still be loaded */
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index bebec8ef9372..860d46395be1 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -101,6 +101,8 @@ ENTRY(cpu_resume)
bl el2_setup // if in EL2 drop to EL1 cleanly
bl __cpu_setup
/* enable the MMU early - so we can access sleep_save_stash by va */
+ adrp x25, idmap_pg_dir
+ ldr_l x26, __pa_swapper_pg_dir
bl __enable_mmu
ldr x8, =_cpu_resume
br x8
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index 12145874c02b..dd4f28c19165 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -199,7 +199,7 @@ void __init kasan_init(void)
*/
memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
dsb(ishst);
- cpu_replace_ttbr1(lm_alias(tmp_pg_dir));
+ cpu_replace_ttbr1(__pa_symbol(tmp_pg_dir));
clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
@@ -236,7 +236,7 @@ void __init kasan_init(void)
pfn_pte(sym_to_pfn(kasan_zero_page), PAGE_KERNEL_RO));
memset(kasan_zero_page, 0, PAGE_SIZE);
- cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
+ cpu_replace_ttbr1(__pa_swapper_pg_dir);
/* At this point kasan is fully initialized. Enable error messages */
init_task.kasan_depth = 0;
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 2dbb2c9f1ec1..41eee333f91a 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -55,6 +55,8 @@ u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
u64 kimage_voffset __ro_after_init;
EXPORT_SYMBOL(kimage_voffset);
+phys_addr_t __pa_swapper_pg_dir;
+
/*
* Empty_zero_page is a special page that is used for zero-initialized data
* and COW.
@@ -631,6 +633,8 @@ void __init paging_init(void)
phys_addr_t pgd_phys = early_pgtable_alloc();
pgd_t *pgdp = pgd_set_fixmap(pgd_phys);
+ __pa_swapper_pg_dir = __pa_symbol(swapper_pg_dir);
+
map_kernel(pgdp);
map_mem(pgdp);
@@ -642,9 +646,9 @@ void __init paging_init(void)
*
* To do this we need to go via a temporary pgd.
*/
- cpu_replace_ttbr1(__va(pgd_phys));
+ cpu_replace_ttbr1(pgd_phys);
memcpy(swapper_pg_dir, pgdp, PGD_SIZE);
- cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
+ cpu_replace_ttbr1(__pa_swapper_pg_dir);
pgd_clear_fixmap();
memblock_free(pgd_phys, PAGE_SIZE);
--
2.17.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/4] arm64/mm: migrate swapper_pg_dir
2018-05-30 9:12 ` [PATCH 1/4] " YaoJun
@ 2018-05-30 9:46 ` Greg KH
0 siblings, 0 replies; 9+ messages in thread
From: Greg KH @ 2018-05-30 9:46 UTC (permalink / raw)
To: YaoJun
Cc: kernel-hardening, catalin.marinas, will.deacon, linux-arm-kernel,
linux-kernel, mark.rutland
On Wed, May 30, 2018 at 05:12:56PM +0800, YaoJun wrote:
> Introduce __pa_swapper_pg_dir to save physical address
> of swapper_pg_dir. And pass it as an argument to
> __enable_mmu().
>
> Signed-off-by: YaoJun <yaojun8558363@gmail.com>
This is better, but your subject line is still identical for all 4
patches (which doesn't make sense as they do different things), and I
think you need to put a space in your name somewhere, right?
thanks,
greg k-h
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH 2/4] arm64/mm: migrate swapper_pg_dir
2018-05-30 9:12 [PATCH 0/4] arm64/mm: migrate swapper_pg_dir YaoJun
2018-05-30 9:12 ` [PATCH 1/4] " YaoJun
@ 2018-05-30 9:12 ` YaoJun
2018-05-30 9:12 ` [PATCH 3/4] " YaoJun
2018-05-30 9:12 ` [PATCH 4/4] " YaoJun
3 siblings, 0 replies; 9+ messages in thread
From: YaoJun @ 2018-05-30 9:12 UTC (permalink / raw)
To: kernel-hardening
Cc: catalin.marinas, will.deacon, linux-arm-kernel, linux-kernel,
mark.rutland
Introduce new_swapper_pg_dir to save virtual address of
new swapper_pg_dir.
Signed-off-by: YaoJun <yaojun8558363@gmail.com>
---
arch/arm64/include/asm/mmu_context.h | 2 +-
arch/arm64/include/asm/pgtable.h | 1 +
arch/arm64/mm/kasan_init.c | 2 +-
arch/arm64/mm/mmu.c | 1 +
4 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 3eddb871f251..481c2d16adeb 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -57,7 +57,7 @@ static inline void cpu_set_reserved_ttbr0(void)
static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm)
{
- BUG_ON(pgd == swapper_pg_dir);
+ BUG_ON(pgd == new_swapper_pg_dir);
cpu_set_reserved_ttbr0();
cpu_do_switch_mm(virt_to_phys(pgd),mm);
}
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 14ba344b1af7..7abec25cedd2 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -723,6 +723,7 @@ extern pgd_t swapper_pg_end[];
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
extern phys_addr_t __pa_swapper_pg_dir;
+extern pgd_t *new_swapper_pg_dir;
/*
* Encode and decode a swap entry:
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index dd4f28c19165..08bcaae4725e 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -197,7 +197,7 @@ void __init kasan_init(void)
* tmp_pg_dir used to keep early shadow mapped until full shadow
* setup will be finished.
*/
- memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
+ memcpy(tmp_pg_dir, new_swapper_pg_dir, sizeof(tmp_pg_dir));
dsb(ishst);
cpu_replace_ttbr1(__pa_symbol(tmp_pg_dir));
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 41eee333f91a..26ba3e70a91c 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -56,6 +56,7 @@ u64 kimage_voffset __ro_after_init;
EXPORT_SYMBOL(kimage_voffset);
phys_addr_t __pa_swapper_pg_dir;
+pgd_t *new_swapper_pg_dir = swapper_pg_dir;
/*
* Empty_zero_page is a special page that is used for zero-initialized data
--
2.17.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH 3/4] arm64/mm: migrate swapper_pg_dir
2018-05-30 9:12 [PATCH 0/4] arm64/mm: migrate swapper_pg_dir YaoJun
2018-05-30 9:12 ` [PATCH 1/4] " YaoJun
2018-05-30 9:12 ` [PATCH 2/4] " YaoJun
@ 2018-05-30 9:12 ` YaoJun
2018-05-30 9:12 ` [PATCH 4/4] " YaoJun
3 siblings, 0 replies; 9+ messages in thread
From: YaoJun @ 2018-05-30 9:12 UTC (permalink / raw)
To: kernel-hardening
Cc: catalin.marinas, will.deacon, linux-arm-kernel, linux-kernel,
mark.rutland
Make tramp_pg_dir and swapper_pg_dir adjacent. So we can migrate
them together.
Signed-off-by: YaoJun <yaojun8558363@gmail.com>
---
arch/arm64/kernel/entry.S | 4 ++--
arch/arm64/kernel/vmlinux.lds.S | 10 +++++-----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index ec2ee720e33e..b35425feaf56 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -1004,7 +1004,7 @@ __ni_sys_trace:
.macro tramp_map_kernel, tmp
mrs \tmp, ttbr1_el1
- add \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
+ add \tmp, \tmp, #(PAGE_SIZE)
bic \tmp, \tmp, #USER_ASID_FLAG
msr ttbr1_el1, \tmp
#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
@@ -1023,7 +1023,7 @@ alternative_else_nop_endif
.macro tramp_unmap_kernel, tmp
mrs \tmp, ttbr1_el1
- sub \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
+ sub \tmp, \tmp, #(PAGE_SIZE)
orr \tmp, \tmp, #USER_ASID_FLAG
msr ttbr1_el1, \tmp
/*
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 0221aca6493d..a094156e05a4 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -219,15 +219,15 @@ SECTIONS
idmap_pg_dir = .;
. += IDMAP_DIR_SIZE;
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
- tramp_pg_dir = .;
- . += PAGE_SIZE;
-#endif
-
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
reserved_ttbr0 = .;
. += RESERVED_TTBR0_SIZE;
#endif
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ tramp_pg_dir = .;
+ . += PAGE_SIZE;
+#endif
swapper_pg_dir = .;
. += SWAPPER_DIR_SIZE;
swapper_pg_end = .;
--
2.17.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* [PATCH 4/4] arm64/mm: migrate swapper_pg_dir
2018-05-30 9:12 [PATCH 0/4] arm64/mm: migrate swapper_pg_dir YaoJun
` (2 preceding siblings ...)
2018-05-30 9:12 ` [PATCH 3/4] " YaoJun
@ 2018-05-30 9:12 ` YaoJun
3 siblings, 0 replies; 9+ messages in thread
From: YaoJun @ 2018-05-30 9:12 UTC (permalink / raw)
To: kernel-hardening
Cc: catalin.marinas, will.deacon, linux-arm-kernel, linux-kernel,
mark.rutland
Migrate swapper_pg_dir and tramp_pg_dir. And its placement in
the virtual address space does not correlate with the placement
of the kernel.
Signed-off-by: YaoJun <yaojun8558363@gmail.com>
---
arch/arm64/mm/mmu.c | 67 +++++++++++++++++++++++++++++++--------------
1 file changed, 46 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 26ba3e70a91c..b508de2cc6c4 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -57,6 +57,9 @@ EXPORT_SYMBOL(kimage_voffset);
phys_addr_t __pa_swapper_pg_dir;
pgd_t *new_swapper_pg_dir = swapper_pg_dir;
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+pgd_t *new_tramp_pg_dir;
+#endif
/*
* Empty_zero_page is a special page that is used for zero-initialized data
@@ -105,6 +108,25 @@ static phys_addr_t __init early_pgtable_alloc(void)
return phys;
}
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static phys_addr_t __init early_pgtables_alloc(int num)
+{
+ int i;
+ phys_addr_t phys;
+ void *ptr;
+
+ phys = memblock_alloc(PAGE_SIZE * num, PAGE_SIZE);
+
+ for (i = 0; i < num; i++) {
+ ptr = pte_set_fixmap(phys + i * PAGE_SIZE);
+ memset(ptr, 0, PAGE_SIZE);
+ pte_clear_fixmap();
+ }
+
+ return phys;
+}
+#endif
+
static bool pgattr_change_is_safe(u64 old, u64 new)
{
/*
@@ -554,6 +576,10 @@ static int __init map_entry_trampoline(void)
__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
prot, pgd_pgtable_alloc, 0);
+ memcpy(new_tramp_pg_dir, tramp_pg_dir, PGD_SIZE);
+ memblock_free(__pa_symbol(tramp_pg_dir),
+ __pa_symbol(swapper_pg_dir) - __pa_symbol(tramp_pg_dir));
+
/* Map both the text and data into the kernel page table */
__set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
@@ -631,36 +657,35 @@ static void __init map_kernel(pgd_t *pgdp)
*/
void __init paging_init(void)
{
- phys_addr_t pgd_phys = early_pgtable_alloc();
- pgd_t *pgdp = pgd_set_fixmap(pgd_phys);
+ phys_addr_t pgd_phys;
+ pgd_t *pgdp;
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ int pages;
+
+ pages = (__pa_symbol(swapper_pg_dir) - __pa_symbol(tramp_pg_dir) +
+ PAGE_SIZE) >> PAGE_SHIFT;
+ pgd_phys = early_pgtables_alloc(pages);
+ new_tramp_pg_dir = __va(pgd_phys);
+ __pa_swapper_pg_dir = pgd_phys + PAGE_SIZE;
+#else
+ pgd_phys = early_pgtable_alloc();
+ __pa_swapper_pg_dir = pgd_phys;
+#endif
+ new_swapper_pg_dir = __va(__pa_swapper_pg_dir);
- __pa_swapper_pg_dir = __pa_symbol(swapper_pg_dir);
+ pgdp = pgd_set_fixmap(__pa_swapper_pg_dir);
map_kernel(pgdp);
map_mem(pgdp);
- /*
- * We want to reuse the original swapper_pg_dir so we don't have to
- * communicate the new address to non-coherent secondaries in
- * secondary_entry, and so cpu_switch_mm can generate the address with
- * adrp+add rather than a load from some global variable.
- *
- * To do this we need to go via a temporary pgd.
- */
- cpu_replace_ttbr1(pgd_phys);
- memcpy(swapper_pg_dir, pgdp, PGD_SIZE);
cpu_replace_ttbr1(__pa_swapper_pg_dir);
+ init_mm.pgd = new_swapper_pg_dir;
pgd_clear_fixmap();
- memblock_free(pgd_phys, PAGE_SIZE);
- /*
- * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd
- * allocated with it.
- */
- memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
- __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir)
- - PAGE_SIZE);
+ memblock_free(__pa_symbol(swapper_pg_dir),
+ __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir));
}
/*
--
2.17.0
^ permalink raw reply [flat|nested] 9+ messages in thread