From: Christophe Leroy <christophe.leroy@c-s.fr>
To: Benjamin Herrenschmidt <benh@kernel.crashing.org>,
Paul Mackerras <paulus@samba.org>,
Michael Ellerman <mpe@ellerman.id.au>,
Scott Wood <oss@buserror.net>
Cc: linux-kernel@vger.kernel.org, linuxppc-dev@lists.ozlabs.org
Subject: [PATCH v5 08/23] powerpc/8xx: Map IMMR area with 512k page at a fixed address
Date: Wed, 3 Feb 2016 23:54:06 +0100 (CET) [thread overview]
Message-ID: <d8b91efcec9b54593dec6ddd492f1f86db4ce95f.1454538976.git.christophe.leroy@c-s.fr> (raw)
In-Reply-To: <cover.1454538974.git.christophe.leroy@c-s.fr>
Once the linear memory space has been mapped with 8Mb pages, as
seen in the related commit, we get 11 millions DTLB missed during
the reference 600s period. 77% of the misses are on user addresses
and 23% are on kernel addresses (1 fourth for linear address space
and 3 fourth for virtual address space)
Traditionaly, each driver manages one computer board which has its
own components with its own memory maps.
But on embedded chips like the MPC8xx, the SOC has all registers
located in the same IO area.
When looking at ioremaps done during startup, we see that
many drivers are re-mapping small parts of the IMMR for their own use
and all those small pieces gets their own 4k page, amplifying the
number of TLB misses: in our system we get 0xff000000 mapped 31 times
and 0xff003000 mapped 9 times.
Even if each part of IMMR was mapped only once with 4k pages, it would
still be several small mappings towards linear area.
With the patch, on the same principle as what was done for the RAM,
the IMMR gets mapped by a 512k page.
In 4k pages mode, we reserve a 4Mb area for mapping IMMR. The TLB
miss handler checks that we are within the first 512k and bail out
with page not marked valid if we are outside
In 16k pages mode, it is not realistic to reserve a 64Mb area, so
we do a standard mapping of the 512k area using 32 pages of 16k.
The CPM will be mapped via the first two pages, and the SEC engine
will be mapped via the 16th and 17th pages. As the pages are marked
guarded, there will be no speculative accesses.
With this patch applied, the number of DTLB misses during the 10 min
period is reduced to 11.8 millions for a duration of 5.8s, which
represents 2% of the non-idle time hence yet another 10% reduction.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
v2:
- using bt instead of blt/bgt
- reorganised in order to have only one taken branch for both 512k
and 8M instead of a first branch for both 8M and 512k then a second
branch for 512k
v3:
- using fixmap
- using the new x_block_mapped() functions
v4: no change
v5: no change
arch/powerpc/include/asm/fixmap.h | 9 ++++++-
arch/powerpc/kernel/head_8xx.S | 36 +++++++++++++++++++++++++-
arch/powerpc/mm/8xx_mmu.c | 53 +++++++++++++++++++++++++++++++++++++++
arch/powerpc/mm/mmu_decl.h | 3 ++-
4 files changed, 98 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h
index d7dd8fb..b954dc3 100644
--- a/arch/powerpc/include/asm/fixmap.h
+++ b/arch/powerpc/include/asm/fixmap.h
@@ -52,12 +52,19 @@ enum fixed_addresses {
FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
#endif
#ifdef CONFIG_PPC_8xx
- /* For IMMR we need an aligned 512K area */
FIX_IMMR_START,
+#ifdef CONFIG_PPC_4K_PAGES
+ /* For IMMR we need an aligned 4M area (full PGD entry) */
+ FIX_IMMR_TOP = (FIX_IMMR_START - 1 + ((4 * 1024 * 1024) / PAGE_SIZE)) &
+ ~(((4 * 1024 * 1024) / PAGE_SIZE) - 1),
+ FIX_IMMR_BASE = FIX_IMMR_TOP - 1 + ((4 * 1024 * 1024) / PAGE_SIZE),
+#else
+ /* For IMMR we need an aligned 512K area */
FIX_IMMR_TOP = (FIX_IMMR_START - 1 + ((512 * 1024) / PAGE_SIZE)) &
~(((512 * 1024) / PAGE_SIZE) - 1),
FIX_IMMR_BASE = FIX_IMMR_TOP - 1 + ((512 * 1024) / PAGE_SIZE),
#endif
+#endif
/* FIX_PCIE_MCFG, */
__end_of_fixed_addresses
};
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 09173ae..ae721a1 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -254,6 +254,37 @@ DataAccess:
. = 0x400
InstructionAccess:
+/*
+ * Bottom part of DTLBMiss handler for 512k pages
+ * not enough space in the primary location
+ */
+#ifdef CONFIG_PPC_4K_PAGES
+/*
+ * 512k pages are only used for mapping IMMR area in 4K pages mode.
+ * Only map the first 512k page of the 4M area covered by the PGD entry.
+ * This should not happen, but if we are called for another page of that
+ * area, don't mark it valid
+ *
+ * In 16k pages mode, IMMR is directly mapped with 16k pages
+ */
+DTLBMiss512k:
+ rlwinm. r10, r10, 0, 0x00380000
+ bne- 1f
+ ori r11, r11, MD_SVALID
+1: mtcr r3
+ MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
+ rlwinm r10, r11, 0, 0xffc00000
+ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \
+ _PAGE_PRESENT | _PAGE_NO_CACHE
+ MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */
+
+ li r11, RPN_PATTERN
+ mfspr r3, SPRN_SPRG_SCRATCH2
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+ EXCEPTION_EPILOG_0
+ rfi
+#endif
+
/* External interrupt */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
@@ -405,6 +436,9 @@ DataStoreTLBMiss:
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
mtcr r11
bt- 28,DTLBMiss8M /* bit 28 = Large page (8M) */
+#ifdef CONFIG_PPC_4K_PAGES
+ bt- 29,DTLBMiss512k /* bit 29 = Large page (8M or 512K) */
+#endif
mtcr r3
/* We have a pte table, so load fetch the pte from the table.
@@ -559,7 +593,7 @@ FixupDAR:/* Entry point for dcbx workaround. */
3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
mtcr r11
- bt 28,200f /* bit 28 = Large page (8M) */
+ bt 29,200f /* bit 29 = Large page (8M or 512K) */
rlwinm r11, r11,0,0,19 /* Extract page descriptor page address */
/* Insert level 2 index */
rlwimi r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index a84f5eb..4de93b2 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -13,10 +13,61 @@
*/
#include <linux/memblock.h>
+#include <asm/fixmap.h>
#include "mmu_decl.h"
+#define IMMR_SIZE (__fix_to_virt(FIX_IMMR_TOP) + PAGE_SIZE - VIRT_IMMR_BASE)
+
extern int __map_without_ltlbs;
+
+/*
+ * Return PA for this VA if it is in IMMR area, or 0
+ */
+phys_addr_t v_block_mapped(unsigned long va)
+{
+ unsigned long p = PHYS_IMMR_BASE;
+
+ if (__map_without_ltlbs)
+ return 0;
+ if (va >= VIRT_IMMR_BASE && va < VIRT_IMMR_BASE + IMMR_SIZE)
+ return p + va - VIRT_IMMR_BASE;
+ return 0;
+}
+
+/*
+ * Return VA for a given PA or 0 if not mapped
+ */
+unsigned long p_block_mapped(phys_addr_t pa)
+{
+ unsigned long p = PHYS_IMMR_BASE;
+
+ if (__map_without_ltlbs)
+ return 0;
+ if (pa >= p && pa < p + IMMR_SIZE)
+ return VIRT_IMMR_BASE + pa - p;
+ return 0;
+}
+
+static void mmu_mapin_immr(void)
+{
+ unsigned long p = PHYS_IMMR_BASE;
+ unsigned long v = VIRT_IMMR_BASE;
+#ifdef CONFIG_PPC_4K_PAGES
+ pmd_t *pmdp;
+ unsigned long val = p | MD_PS512K | MD_GUARDED;
+
+ pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);
+ pmd_val(*pmdp) = val;
+#else /* CONFIG_PPC_16K_PAGES */
+ unsigned long f = pgprot_val(PAGE_KERNEL_NCG);
+ int offset;
+
+ for (offset = 0; offset < IMMR_SIZE; offset += PAGE_SIZE)
+ map_page(v + offset, p + offset, f);
+#endif
+}
+
/*
* MMU_init_hw does the chip-specific initialization of the MMU hardware.
*/
@@ -79,6 +130,8 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
*/
memblock_set_current_limit(mapped);
+ mmu_mapin_immr();
+
return mapped;
}
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index e7228b7..3872332 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -166,9 +166,10 @@ struct tlbcam {
};
#endif
-#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE)
+#if defined(CONFIG_6xx) || defined(CONFIG_FSL_BOOKE) || defined(CONFIG_PPC_8xx)
/* 6xx have BATS */
/* FSL_BOOKE have TLBCAM */
+/* 8xx have LTLB */
phys_addr_t v_block_mapped(unsigned long va);
unsigned long p_block_mapped(phys_addr_t pa);
#else
--
2.1.0
next prev parent reply other threads:[~2016-02-03 22:54 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-02-03 22:53 [PATCH v5 00/23] powerpc/8xx: Use large pages for RAM and IMMR and other improvments Christophe Leroy
2016-02-03 22:53 ` [PATCH v5 01/23] powerpc/8xx: Save r3 all the time in DTLB miss handler Christophe Leroy
2016-02-03 22:53 ` [PATCH v5 02/23] powerpc/8xx: Map linear kernel RAM with 8M pages Christophe Leroy
2016-02-03 22:53 ` [PATCH v5 03/23] powerpc: Update documentation for noltlbs kernel parameter Christophe Leroy
2016-02-03 22:53 ` [PATCH v5 04/23] powerpc/8xx: move setup_initial_memory_limit() into 8xx_mmu.c Christophe Leroy
2016-02-03 22:53 ` [PATCH v5 05/23] powerpc32: Fix pte_offset_kernel() to return NULL for bad pages Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 06/23] powerpc32: refactor x_mapped_by_bats() and x_mapped_by_tlbcam() together Christophe Leroy
2016-02-07 9:42 ` kbuild test robot
2016-02-03 22:54 ` [PATCH v5 07/23] powerpc/8xx: Fix vaddr for IMMR early remap Christophe Leroy
2016-02-03 22:54 ` Christophe Leroy [this message]
2016-02-04 9:58 ` [PATCH v5 08/23] powerpc/8xx: Map IMMR area with 512k page at a fixed address kbuild test robot
2016-02-03 22:54 ` [PATCH v5 09/23] powerpc/8xx: CONFIG_PIN_TLB unneeded for CONFIG_PPC_EARLY_DEBUG_CPM Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 10/23] powerpc/8xx: map more RAM at startup when needed Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 11/23] powerpc32: Remove useless/wrong MMU:setio progress message Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 12/23] powerpc32: remove ioremap_base Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 13/23] powerpc/8xx: Add missing SPRN defines into reg_8xx.h Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 14/23] powerpc/8xx: Handle CPU6 ERRATA directly in mtspr() macro Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 15/23] powerpc/8xx: remove special handling of CPU6 errata in set_dec() Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 16/23] powerpc/8xx: rewrite set_context() in C Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 17/23] powerpc/8xx: rewrite flush_instruction_cache() " Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 18/23] powerpc: add inline functions for cache related instructions Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 19/23] powerpc32: Remove clear_pages() and define clear_page() inline Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 20/23] powerpc32: move xxxxx_dcache_range() functions inline Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 21/23] powerpc: Simplify test in __dma_sync() Christophe Leroy
2016-02-04 11:37 ` Denis Kirjanov
2016-02-04 13:42 ` Christophe Leroy
2016-02-05 7:52 ` Denis Kirjanov
2016-02-05 7:56 ` Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 22/23] powerpc32: small optimisation in flush_icache_range() Christophe Leroy
2016-02-03 22:54 ` [PATCH v5 23/23] powerpc32: Remove one insn in mulhdu Christophe Leroy
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=d8b91efcec9b54593dec6ddd492f1f86db4ce95f.1454538976.git.christophe.leroy@c-s.fr \
--to=christophe.leroy@c-s.fr \
--cc=benh@kernel.crashing.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=mpe@ellerman.id.au \
--cc=oss@buserror.net \
--cc=paulus@samba.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).