ARC: Proper kernel mapping

Use proper page flags for different memory sections,
for example X is used only for text section.

Previously we mapped kernel as an one big chunk.

Now kernel mapping is page-based, so it requires to have
memblock initialized to be able to allocate page entries
dynamically.

Author: xxkent

arch/arc/include/asm/pgtable-bits-arcv3.h

@@ -73,7 +73,8 @@
 /* TBD: kernel is RWX by default, split it to code/data */
 #define _PAGE_KERNEL	(_PAGE_TABLE        |			\
 			 /* writable */				\
-			 _PAGE_NOTEXEC_U    | 	/* exec k */	\
+			 _PAGE_NOTEXEC_U    | 		\
+			 _PAGE_NOTEXEC_K 	|  		\
 			 /* AP kernel only  |      global */	\
 			 _PAGE_ACCESSED     |			\
 			 _PAGE_SHARED_INNER |			\
@@ -82,7 +83,12 @@
 #define PAGE_NONE	__pgprot(_PAGE_BASE)	/* TBD */
 #define PAGE_TABLE	__pgprot(_PAGE_TABLE)
 #define PAGE_KERNEL	__pgprot(_PAGE_KERNEL)
-#define PAGE_KERNEL_BLK	__pgprot(_PAGE_KERNEL & ~_PAGE_LINK)
+#define PAGE_KERNEL_RW	__pgprot(_PAGE_KERNEL)
+#define PAGE_KERNEL_RWX	__pgprot(_PAGE_KERNEL & ~_PAGE_NOTEXEC_K)
+
+#define PAGE_KERNEL_BLK		__pgprot(_PAGE_KERNEL & ~_PAGE_LINK)
+#define PAGE_KERNEL_BLK_RW	__pgprot(_PAGE_KERNEL & ~_PAGE_LINK)
+#define PAGE_KERNEL_BLK_RWX	__pgprot(_PAGE_KERNEL & ~_PAGE_NOTEXEC_K & ~_PAGE_LINK)
 
 #define PAGE_R		__pgprot(_PAGE_BASE)
 #define PAGE_RW		__pgprot(_PAGE_RW)

arch/arc/kernel/head.S

@@ -91,14 +91,12 @@
 ;       All registers are clobbered.
 .macro BUILD_PAGE_TABLE, link_addr, phy_addr, link_end, pgd, pud, pmd, cur, tmp, val, foo
 
-; FIXME: we need asserts about kernel size <= allocated size
-; FIXME: we need a better way to generalize this
 #if defined(CONFIG_ARC_MMU_V6_48) && defined(CONFIG_ARC_PAGE_SIZE_4K)
 
 ; Three-level page table, PGD + PUD + PMD
-#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pud, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL
-#define BUILD_PUD_ENTRY BUILD_PAGE_TABLE_ENTRY \pud, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PUD, PUD_SHIFT, PAGE_KERNEL
-#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK
+#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pud, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL_RWX
+#define BUILD_PUD_ENTRY BUILD_PAGE_TABLE_ENTRY \pud, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PUD, PUD_SHIFT, PAGE_KERNEL_RWX
+#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK_RWX
 #define ENTRY_SIZE	PMD_SIZE
 #define ENTRY_LABEL	.Lbuild_pmd_entry\@
 #define NEXT_SIZE	PUD_SIZE
@@ -107,9 +105,9 @@
 #elif defined(CONFIG_ARC_MMU_V6_48) && defined(CONFIG_ARC_PAGE_SIZE_16K)
 
 ; Three-level page table, PGD + PUD + PMD
-#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pud, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL
-#define BUILD_PUD_ENTRY BUILD_PAGE_TABLE_ENTRY \pud, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PUD, PUD_SHIFT, PAGE_KERNEL
-#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK
+#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pud, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL_RWX
+#define BUILD_PUD_ENTRY BUILD_PAGE_TABLE_ENTRY \pud, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PUD, PUD_SHIFT, PAGE_KERNEL_RWX
+#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK_RWX
 #define ENTRY_SIZE	PMD_SIZE
 #define ENTRY_LABEL	.Lbuild_pmd_entry\@
 #define NEXT_SIZE	PUD_SIZE
@@ -118,17 +116,17 @@
 #elif defined(CONFIG_ARC_MMU_V6_48) && defined(CONFIG_ARC_PAGE_SIZE_64K) || defined(CONFIG_ARC_MMU_V6_52) || defined(CONFIG_ARC_MMU_V6_32)
 
 ; Two-level page table, PGD + PMD
-#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL
+#define BUILD_PGD_ENTRY BUILD_PAGE_TABLE_ENTRY \pgd, \link_addr, \pmd, \cur, \tmp, PTRS_PER_PGD, PGDIR_SHIFT, PAGE_KERNEL_RWX
 #define BUILD_PUD_ENTRY
-#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK
+#define BUILD_PMD_ENTRY BUILD_PAGE_TABLE_ENTRY \pmd, \link_addr, \phy_addr, \cur, \tmp, PTRS_PER_PMD, PMD_SHIFT, PAGE_KERNEL_BLK_RWX
 #define ENTRY_SIZE	PMD_SIZE
 #define ENTRY_LABEL	.Lbuild_pmd_entry\@
 #define NEXT_SIZE	PGDIR_SIZE
 #define ENTRY_LABEL_NEXT	.Lbuild_pgd_entry\@
 
 #endif
 
-	ADDR	\val, \link_addr, NEXT_SIZE
+	ADDR	\val, \link_addr, (NEXT_SIZE-1)
 	MOVR	\foo, \pmd
 
 .Lbuild_pgd_entry\@:
@@ -148,13 +146,13 @@
 	CMPR	\link_addr, \link_end
 	bge		.Lbuild_end\@
 	CMPR	\link_addr, \val
-	bge.d	.Linc\@
+	bgt.d	.Linc\@
 	ADDR	\foo, \foo, 0x8
 	b	ENTRY_LABEL
 
 .Linc\@:
 	MOVR	\pmd, \foo
-	ADDR	\val, \foo, NEXT_SIZE
+	ADDR	\val, \link_addr, (NEXT_SIZE-1)
 	b 		ENTRY_LABEL_NEXT
 
 .Lbuild_end\@:

arch/arc/kernel/setup.c

@@ -499,8 +499,8 @@ void setup_processor(void)
 	arc_chk_core_config(&info);
 
 	arc_init_IRQ();
-	/* No ned to setup MMU for secondary CPU for ARCv3. */
-	if (!IS_ENABLED(CONFIG_ISA_ARCV3) || c == 0)
+	/* ARCv3 MMU must be initialized after setup_arch_memory. */
+	if (!IS_ENABLED(CONFIG_ISA_ARCV3))
 		arc_mmu_init();
 	arc_cache_init();
 }
@@ -603,6 +603,8 @@ void __init setup_arch(char **cmdline_p)
 
 	setup_processor();
 	setup_arch_memory();
+	if (IS_ENABLED(CONFIG_ISA_ARCV3))
+		arc_mmu_init();
 
 	/* copy flat DT out of .init and then unflatten it */
 	unflatten_and_copy_device_tree();

arch/arc/kernel/vmlinux.lds.S

@@ -56,15 +56,12 @@ SECTIONS
 	 *	decompress_inflate.c:gunzip( ) -> zlib_inflate_workspace( )
 	 */
 
+	. = ALIGN(PAGE_SIZE);
 	__init_begin = .;
+	__init_data_begin = .;
 
 	.init.ramfs : { INIT_RAM_FS }
 
-	. = ALIGN(PAGE_SIZE);
-
-	HEAD_TEXT_SECTION
-	INIT_TEXT_SECTION(L1_CACHE_BYTES)
-
 	/* INIT_DATA_SECTION open-coded: special INIT_RAM_FS handling */
 	.init.data : {
 		INIT_DATA
@@ -82,6 +79,14 @@ SECTIONS
 	PERCPU_SECTION(L1_CACHE_BYTES)
 
 	. = ALIGN(PAGE_SIZE);
+	__init_data_end = .;
+	__init_text_begin = .;
+
+	HEAD_TEXT_SECTION
+	INIT_TEXT_SECTION(L1_CACHE_BYTES)
+
+	. = ALIGN(PAGE_SIZE);
+	__init_text_end = .;
 	__init_end = .;
 
 	.text : {
@@ -98,6 +103,7 @@ SECTIONS
 		*(.gnu.warning)
 	}
 	EXCEPTION_TABLE(L1_CACHE_BYTES)
+	. = ALIGN(PAGE_SIZE);
 	_etext = .;
 
 	_sdata = .;

arch/arc/mm/tlb-arcv3.c

@@ -2,12 +2,14 @@
 
 #include <linux/mm.h>
 #include <linux/types.h>
+#include <linux/memblock.h>
 
 #include <asm/arcregs.h>
 #include <asm/mmu_context.h>
 #include <asm/mmu.h>
 #include <asm/setup.h>
 #include <asm/fixmap.h>
+#include <asm/pgalloc.h>
 
 /* A copy of the ASID from the PID reg is kept in asid_cache */
 DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE;
@@ -230,89 +232,147 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot)
 	set_pte(pte, pfn_pte(PFN_DOWN(phys), prot));
 }
 
-/*
- * Map the kernel code/data into page tables for a given @mm
- *
- * Assumes
- *  - pgd, pud and pmd are already allocated
- *  - pud is wired up to pgd and pmd to pud
- *
- * TODO: assumes 4 levels, implement properly using p*d_addr_end loops
- */
-int arc_map_kernel_in_mm(struct mm_struct *mm)
+static phys_addr_t __init arc_map_early_alloc_page(void)
 {
-	unsigned long addr = PAGE_OFFSET;
-	unsigned long end = PAGE_OFFSET + CONFIG_LINUX_MAP_SIZE;
+	phys_addr_t phys;
+
+	/* At early stage we have mapped PAGE_OFFSET + CONFIG_LINUX_MAP_SIZE */
+	phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0,
+					 __pa(PAGE_OFFSET + CONFIG_LINUX_MAP_SIZE));
+
+	memset(__va(phys), 0, PAGE_SIZE);
+
+	return phys;
+}
+
+static int __init arc_map_segment_in_mm(struct mm_struct *mm,
+					unsigned long start,
+					unsigned long end,
+					pgprot_t prot)
+{
+	unsigned long addr;
+	phys_addr_t phys;
 	pgd_t *pgd;
 	p4d_t *p4d;
 	pud_t *pud;
 	pmd_t *pmd;
+	pte_t *pte;
 
-	do {
-		pgprot_t prot = PAGE_KERNEL_BLK;
+	BUG_ON(start & (PAGE_SIZE-1));
+	BUG_ON(end & (PAGE_SIZE-1));
 
+	/* TODO: Use bigger blocks if possible */
+	addr = start;
+	do {
 		pgd = pgd_offset(mm, addr);
-		if (pgd_none(*pgd) || !pgd_present(*pgd))
+		if (pgd_none(*pgd)){
 			return 1;
+		}
 
 		p4d = p4d_offset(pgd, addr);
-		if (p4d_none(*p4d) || !p4d_present(*p4d))
-			return 1;
+		if (p4d_none(*p4d)) {
+			phys = arc_map_early_alloc_page();
+			p4d_populate(mm, p4d, __va(phys));
+		}
 
 		pud = pud_offset(p4d, addr);
-		if (pud_none(*pud) || !pud_present(*pud))
-			return 1;
+		if (pud_none(*pud)) {
+			phys = arc_map_early_alloc_page();
+			pud_populate(mm, pud, __va(phys));
+		}
 
 		pmd = pmd_offset(pud, addr);
-		if (!pmd_none(*pmd) || pmd_present(*pmd))
-			return 1;
+		if (pmd_none(*pmd)) {
+			phys = arc_map_early_alloc_page();
+			pmd_populate_kernel(mm, pmd, __va(phys));
+		}
 
-		set_pmd(pmd, pfn_pmd(virt_to_pfn(addr), prot));
-		addr = pmd_addr_end(addr, end);
-	}
-	while (addr != end);
+		pte = pte_offset_kernel(pmd, addr);
+
+		set_pte(pte, pfn_pte(virt_to_pfn(addr), prot));
+
+		addr += PAGE_SIZE;
+	} while (addr < end);
 
 	return 0;
 }
 
-void arc_paging_init(void)
+/*
+ * Map the kernel code/data into page tables for a given @mm
+ */
+int __init arc_map_kernel_in_mm(struct mm_struct *mm)
 {
-#if CONFIG_PGTABLE_LEVELS == 4
-	unsigned int idx;
+	extern char __init_text_begin[];
+	extern char __init_data_begin[];
+	extern char __init_text_end[];
+	extern char __init_data_end[];
+
+	arc_map_segment_in_mm(mm,
+			      PAGE_OFFSET,
+			      (unsigned long) __init_data_begin,
+			      PAGE_KERNEL_RWX);
+	arc_map_segment_in_mm(mm,
+			      (unsigned long) __init_data_begin,
+			      (unsigned long) __init_data_end,
+			      PAGE_KERNEL_RW);
+	arc_map_segment_in_mm(mm,
+			      (unsigned long) __init_text_begin,
+			      (unsigned long) __init_text_end,
+			      PAGE_KERNEL_RWX);
+	arc_map_segment_in_mm(mm,
+			      (unsigned long) _stext,
+			      (unsigned long) _etext,
+			      PAGE_KERNEL_RWX);
+	arc_map_segment_in_mm(mm,
+			      (unsigned long) _sdata,
+			      (unsigned long) _end,
+			      PAGE_KERNEL_RW);
 
-	idx = pgd_index(PAGE_OFFSET);
-	swapper_pg_dir[idx] = pfn_pgd(virt_to_pfn(swapper_pud), PAGE_TABLE);
-	ptw_flush(&swapper_pg_dir[idx]);
+	return 0;
+}
 
-	idx = pud_index(PAGE_OFFSET);
-	swapper_pud[idx] = pfn_pud(virt_to_pfn(swapper_pmd), PAGE_TABLE);
-	ptw_flush(&swapper_pud[idx]);
+int __init arc_map_memory_in_mm(struct mm_struct *mm)
+{
+	u64 i;
+	phys_addr_t start, end;
 
-	idx = pud_index(PAGE_OFFSET + 0x40000000);
-	swapper_pud[idx] = pfn_pud(virt_to_pfn(&swapper_pmd[PTRS_PER_PMD]), PAGE_TABLE);
-	ptw_flush(&swapper_pud[idx]);
+	/*
+	 * Kernel (__pa(PAGE_OFFSET) to __pa(_end) is already mapped by
+	 * arc_map_kernel_in_mm(), so map only >= __pa(_end).
+	 *
+	 * We expect that kernel is mapped to the start of physical memory,
+	 * so start >= __pa(PAGE_OFFSET).
+	 */
+	for_each_mem_range(i, &start, &end) {
+		if (start >= end)
+			break;
 
-#elif CONFIG_PGTABLE_LEVELS == 3
-	unsigned int idx;
+		if (end <= __pa(_end))
+			continue;
 
-	idx = pgd_index(PAGE_OFFSET);
-	swapper_pg_dir[idx] = pfn_pgd(virt_to_pfn(swapper_pmd), PAGE_TABLE);
-	ptw_flush(&swapper_pg_dir[idx]);
+		if (start < __pa(_end))
+			start = __pa(_end);
 
-	idx = pgd_index(PAGE_OFFSET + 0x40000000);
-	swapper_pg_dir[idx] = pfn_pgd(virt_to_pfn(&swapper_pmd[PTRS_PER_PMD]), PAGE_TABLE);
-	ptw_flush(&swapper_pg_dir[idx]);
+		arc_map_segment_in_mm(mm,
+				      (unsigned long)__va(start),
+				      (unsigned long)__va(end),
+				      PAGE_KERNEL_RW);
+	}
 
-#endif
+	return 0;
+}
 
+void __init arc_paging_init(void)
+{
 	arc_map_kernel_in_mm(&init_mm);
+	arc_map_memory_in_mm(&init_mm);
 
 	arc_mmu_rtp_set(0, 0, 0);
 	arc_mmu_rtp_set(1, __pa(swapper_pg_dir), 0);
 	local_flush_tlb_all();
 }
 
-void arc_mmu_init(void)
+void __init arc_mmu_init(void)
 {
 	u64 memattr;
 
@@ -329,8 +389,6 @@ void arc_mmu_init(void)
 	if ((unsigned long)_end - PAGE_OFFSET > PUD_SIZE)
 		panic("kernel doesn't fit in PUD (%lu Mb)\n", TO_MB(PUD_SIZE));
 
-	write_aux_reg(ARC_REG_MMU_TTBC, MMU_TTBC);
-
 	memattr = MEMATTR_NORMAL << (MEMATTR_IDX_NORMAL * 8);
 	memattr |= MEMATTR_UNCACHED << (MEMATTR_IDX_UNCACHED * 8);
 	memattr |= MEMATTR_VOLATILE << (MEMATTR_IDX_VOLATILE * 8);