From: guoren@kernel.org To: guoren@kernel.org, anup.patel@wdc.com, palmerdabbelt@google.com, arnd@arndb.de, wens@csie.org, maxime@cerno.tech, drew@beagleboard.org, liush@allwinnertech.com, lazyparser@gmail.com, wefu@redhat.com Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, linux-sunxi@lists.linux.dev, Guo Ren <guoren@linux.alibaba.com>, Christoph Hellwig <hch@lst.de> Subject: [RFC PATCH v2 02/11] riscv: asid: Add ASID-based tlbflushing methods Date: Sun, 6 Jun 2021 09:03:59 +0000 [thread overview] Message-ID: <1622970249-50770-5-git-send-email-guoren@kernel.org> (raw) In-Reply-To: <1622970249-50770-1-git-send-email-guoren@kernel.org> From: Guo Ren <guoren@linux.alibaba.com> Implement optimized version of the tlb flushing routines for systems using ASIDs. These are behind the use_asid_allocator static branch to not affect existing systems not using ASIDs. Signed-off-by: Guo Ren <guoren@linux.alibaba.com> Signed-off-by: Liu Shaohua <liush@allwinnertech.com> Reviewed-by: Anup Patel <anup.patel@wdc.com> Cc: Christoph Hellwig <hch@lst.de> Cc: Chen-Yu Tsai <wens@csie.org> Cc: Drew Fustini <drew@beagleboard.org> Cc: Maxime Ripard <maxime@cerno.tech> Cc: Palmer Dabbelt <palmerdabbelt@google.com> Cc: Wei Fu <wefu@redhat.com> Cc: Wei Wu <lazyparser@gmail.com> --- arch/riscv/include/asm/mmu_context.h | 2 ++ arch/riscv/include/asm/tlbflush.h | 22 +++++++++++++++++ arch/riscv/mm/context.c | 2 +- arch/riscv/mm/tlbflush.c | 46 +++++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index b065941..7030837 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk, return 0; } +DECLARE_STATIC_KEY_FALSE(use_asid_allocator); + #include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index c84218a..894cf75 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -22,9 +22,31 @@ static inline void local_flush_tlb_page(unsigned long addr) { ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory")); } + +static inline void local_flush_tlb_all_asid(unsigned long asid) +{ + __asm__ __volatile__ ("sfence.vma x0, %0" + : + : "r" (asid) + : "memory"); +} + +static inline void local_flush_tlb_range_asid(unsigned long start, + unsigned long size, unsigned long asid) +{ + unsigned long tmp, end = ALIGN(start + size, PAGE_SIZE); + + for (tmp = start & PAGE_MASK; tmp < end; tmp += PAGE_SIZE) { + __asm__ __volatile__ ("sfence.vma %0, %1" + : + : "r" (tmp), "r" (asid) + : "memory"); + } +} #else /* CONFIG_MMU */ #define local_flush_tlb_all() do { } while (0) #define local_flush_tlb_page(addr) do { } while (0) +#define local_flush_tlb_range_asid(addr) do { } while (0) #endif /* CONFIG_MMU */ #if defined(CONFIG_SMP) && defined(CONFIG_MMU) diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 68aa312..45c1b04 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -18,7 +18,7 @@ #ifdef CONFIG_MMU -static DEFINE_STATIC_KEY_FALSE(use_asid_allocator); +DEFINE_STATIC_KEY_FALSE(use_asid_allocator); static unsigned long asid_bits; static unsigned long num_asids; diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 720b443..87b4e52 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -4,6 +4,7 @@ #include <linux/smp.h> #include <linux/sched.h> #include <asm/sbi.h> +#include <asm/mmu_context.h> void flush_tlb_all(void) { @@ -39,18 +40,57 @@ static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start, put_cpu(); } +static void __sbi_tlb_flush_range_asid(struct cpumask *cmask, + unsigned long start, + unsigned long size, + unsigned long asid) +{ + struct cpumask hmask; + unsigned int cpuid; + + if (cpumask_empty(cmask)) + return; + + cpuid = get_cpu(); + + if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) { + if (size == -1) + local_flush_tlb_all_asid(asid); + else + local_flush_tlb_range_asid(start, size, asid); + } else { + riscv_cpuid_to_hartid_mask(cmask, &hmask); + sbi_remote_sfence_vma_asid(cpumask_bits(&hmask), + start, size, asid); + } + + put_cpu(); +} + void flush_tlb_mm(struct mm_struct *mm) { - __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(mm), 0, -1, + atomic_long_read(&mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE, + atomic_long_read(&vma->vm_mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), start, end - start, + atomic_long_read(&vma->vm_mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start); } -- 2.7.4
WARNING: multiple messages have this Message-ID (diff)
From: guoren@kernel.org To: guoren@kernel.org, anup.patel@wdc.com, palmerdabbelt@google.com, arnd@arndb.de, wens@csie.org, maxime@cerno.tech, drew@beagleboard.org, liush@allwinnertech.com, lazyparser@gmail.com, wefu@redhat.com Cc: linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, linux-sunxi@lists.linux.dev, Guo Ren <guoren@linux.alibaba.com>, Christoph Hellwig <hch@lst.de> Subject: [RFC PATCH v2 02/11] riscv: asid: Add ASID-based tlbflushing methods Date: Sun, 6 Jun 2021 09:03:59 +0000 [thread overview] Message-ID: <1622970249-50770-5-git-send-email-guoren@kernel.org> (raw) In-Reply-To: <1622970249-50770-1-git-send-email-guoren@kernel.org> From: Guo Ren <guoren@linux.alibaba.com> Implement optimized version of the tlb flushing routines for systems using ASIDs. These are behind the use_asid_allocator static branch to not affect existing systems not using ASIDs. Signed-off-by: Guo Ren <guoren@linux.alibaba.com> Signed-off-by: Liu Shaohua <liush@allwinnertech.com> Reviewed-by: Anup Patel <anup.patel@wdc.com> Cc: Christoph Hellwig <hch@lst.de> Cc: Chen-Yu Tsai <wens@csie.org> Cc: Drew Fustini <drew@beagleboard.org> Cc: Maxime Ripard <maxime@cerno.tech> Cc: Palmer Dabbelt <palmerdabbelt@google.com> Cc: Wei Fu <wefu@redhat.com> Cc: Wei Wu <lazyparser@gmail.com> --- arch/riscv/include/asm/mmu_context.h | 2 ++ arch/riscv/include/asm/tlbflush.h | 22 +++++++++++++++++ arch/riscv/mm/context.c | 2 +- arch/riscv/mm/tlbflush.c | 46 +++++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index b065941..7030837 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk, return 0; } +DECLARE_STATIC_KEY_FALSE(use_asid_allocator); + #include <asm-generic/mmu_context.h> #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index c84218a..894cf75 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -22,9 +22,31 @@ static inline void local_flush_tlb_page(unsigned long addr) { ALT_FLUSH_TLB_PAGE(__asm__ __volatile__ ("sfence.vma %0" : : "r" (addr) : "memory")); } + +static inline void local_flush_tlb_all_asid(unsigned long asid) +{ + __asm__ __volatile__ ("sfence.vma x0, %0" + : + : "r" (asid) + : "memory"); +} + +static inline void local_flush_tlb_range_asid(unsigned long start, + unsigned long size, unsigned long asid) +{ + unsigned long tmp, end = ALIGN(start + size, PAGE_SIZE); + + for (tmp = start & PAGE_MASK; tmp < end; tmp += PAGE_SIZE) { + __asm__ __volatile__ ("sfence.vma %0, %1" + : + : "r" (tmp), "r" (asid) + : "memory"); + } +} #else /* CONFIG_MMU */ #define local_flush_tlb_all() do { } while (0) #define local_flush_tlb_page(addr) do { } while (0) +#define local_flush_tlb_range_asid(addr) do { } while (0) #endif /* CONFIG_MMU */ #if defined(CONFIG_SMP) && defined(CONFIG_MMU) diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 68aa312..45c1b04 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -18,7 +18,7 @@ #ifdef CONFIG_MMU -static DEFINE_STATIC_KEY_FALSE(use_asid_allocator); +DEFINE_STATIC_KEY_FALSE(use_asid_allocator); static unsigned long asid_bits; static unsigned long num_asids; diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 720b443..87b4e52 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -4,6 +4,7 @@ #include <linux/smp.h> #include <linux/sched.h> #include <asm/sbi.h> +#include <asm/mmu_context.h> void flush_tlb_all(void) { @@ -39,18 +40,57 @@ static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start, put_cpu(); } +static void __sbi_tlb_flush_range_asid(struct cpumask *cmask, + unsigned long start, + unsigned long size, + unsigned long asid) +{ + struct cpumask hmask; + unsigned int cpuid; + + if (cpumask_empty(cmask)) + return; + + cpuid = get_cpu(); + + if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) { + if (size == -1) + local_flush_tlb_all_asid(asid); + else + local_flush_tlb_range_asid(start, size, asid); + } else { + riscv_cpuid_to_hartid_mask(cmask, &hmask); + sbi_remote_sfence_vma_asid(cpumask_bits(&hmask), + start, size, asid); + } + + put_cpu(); +} + void flush_tlb_mm(struct mm_struct *mm) { - __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(mm), 0, -1, + atomic_long_read(&mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE, + atomic_long_read(&vma->vm_mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE); } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start); + if (static_branch_unlikely(&use_asid_allocator)) + __sbi_tlb_flush_range_asid(mm_cpumask(vma->vm_mm), start, end - start, + atomic_long_read(&vma->vm_mm->context.id)); + else + __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start); } -- 2.7.4 _______________________________________________ linux-riscv mailing list linux-riscv@lists.infradead.org http://lists.infradead.org/mailman/listinfo/linux-riscv
next prev parent reply other threads:[~2021-06-06 9:05 UTC|newest] Thread overview: 104+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-06-06 9:03 [RFC PATCH v2 00/11] riscv: Add DMA_COHERENT support for Allwinner D1 guoren 2021-06-06 9:03 ` guoren 2021-06-06 9:03 ` [RFC PATCH v2 01/11] riscv: asid: Use global mappings for kernel pages guoren 2021-06-06 9:03 ` guoren 2021-06-06 9:03 ` [PATCH V5 1/3] riscv: " guoren 2021-06-06 9:03 ` guoren 2021-06-06 9:03 ` [PATCH V5 2/3] riscv: Add ASID-based tlbflushing methods guoren 2021-06-06 9:03 ` guoren 2021-06-06 14:38 ` Christoph Hellwig 2021-06-06 14:38 ` Christoph Hellwig 2021-06-06 9:03 ` guoren [this message] 2021-06-06 9:03 ` [RFC PATCH v2 02/11] riscv: asid: " guoren 2021-06-06 9:04 ` [RFC PATCH v2 03/11] riscv: asid: Optimize tlbflush coding convention guoren 2021-06-06 9:04 ` guoren 2021-06-06 9:04 ` [PATCH V5 3/3] riscv: tlbflush: Optimize " guoren 2021-06-06 9:04 ` guoren 2021-06-06 9:04 ` [RFC PATCH v2 04/11] riscv: pgtable: Fixup _PAGE_CHG_MASK usage guoren 2021-06-06 9:04 ` guoren 2021-06-06 9:04 ` [RFC PATCH v2 05/11] riscv: pgtable: Add custom protection_map init guoren 2021-06-06 9:04 ` guoren 2021-06-06 9:04 ` [RFC PATCH v2 06/11] riscv: pgtable: Add DMA_COHERENT with custom PTE attributes guoren 2021-06-06 9:04 ` guoren 2021-06-06 14:39 ` Christoph Hellwig 2021-06-06 14:39 ` Christoph Hellwig 2021-06-06 15:08 ` Guo Ren 2021-06-06 15:08 ` Guo Ren 2021-06-06 17:22 ` Nick Kossifidis 2021-06-06 17:22 ` Nick Kossifidis 2021-06-07 6:19 ` Christoph Hellwig 2021-06-07 6:19 ` Christoph Hellwig 2021-06-06 9:04 ` [RFC PATCH v2 07/11] riscv: cmo: Add dma-noncoherency support guoren 2021-06-06 9:04 ` guoren 2021-10-17 9:28 ` twd2 2021-10-17 9:28 ` twd2 2021-10-20 8:11 ` Guo Ren 2021-10-20 8:11 ` Guo Ren 2021-06-06 9:04 ` [RFC PATCH v2 08/11] riscv: cmo: Add vendor custom icache sync guoren 2021-06-06 9:04 ` guoren 2021-06-06 9:04 ` [RFC PATCH v2 09/11] riscv: soc: Initial DTS for Allwinner D1 NeZha board guoren 2021-06-06 9:04 ` guoren 2021-06-06 16:26 ` Jernej Škrabec 2021-06-06 16:26 ` Jernej Škrabec 2021-06-06 17:05 ` Guo Ren 2021-06-06 17:05 ` Guo Ren 2021-06-07 3:44 ` Guo Ren 2021-06-07 3:44 ` Guo Ren 2021-06-07 7:27 ` Maxime Ripard 2021-06-07 7:27 ` Maxime Ripard 2021-06-07 7:53 ` Guo Ren 2021-06-07 7:53 ` Guo Ren 2021-06-07 7:24 ` Maxime Ripard 2021-06-07 7:24 ` Maxime Ripard 2021-06-07 8:07 ` Guo Ren 2021-06-07 8:07 ` Guo Ren 2021-06-14 15:33 ` Maxime Ripard 2021-06-14 15:33 ` Maxime Ripard 2021-06-14 16:28 ` Guo Ren 2021-06-14 16:28 ` Guo Ren 2021-06-14 16:31 ` Jernej Škrabec 2021-06-14 16:31 ` Jernej Škrabec 2021-06-06 9:04 ` [RFC PATCH v2 10/11] riscv: soc: Add Allwinner SoC kconfig option guoren 2021-06-06 9:04 ` guoren 2021-06-07 7:19 ` Maxime Ripard 2021-06-07 7:19 ` Maxime Ripard 2021-06-07 7:27 ` Arnd Bergmann 2021-06-07 7:27 ` Arnd Bergmann 2021-06-07 7:45 ` Guo Ren 2021-06-07 7:45 ` Guo Ren 2021-06-07 7:43 ` Guo Ren 2021-06-07 7:43 ` Guo Ren 2021-06-07 12:12 ` Maxime Ripard 2021-06-07 12:12 ` Maxime Ripard 2021-06-07 12:39 ` Guo Ren 2021-06-07 12:39 ` Guo Ren 2021-06-06 9:04 ` [RFC PATCH v2 11/11] riscv: soc: Allwinner D1 GMAC driver only for temp use guoren 2021-06-06 9:04 ` guoren 2021-06-06 10:50 ` Andre Przywara 2021-06-06 10:50 ` Andre Przywara 2021-06-06 15:32 ` Guo Ren 2021-06-06 15:32 ` Guo Ren 2021-06-06 15:39 ` Jernej Škrabec 2021-06-06 15:39 ` Jernej Škrabec 2021-06-06 15:41 ` Guo Ren 2021-06-06 15:41 ` Guo Ren 2021-06-06 16:16 ` Arnd Bergmann 2021-06-06 16:16 ` Arnd Bergmann 2021-06-06 16:32 ` Jernej Škrabec 2021-06-06 16:32 ` Jernej Škrabec 2021-06-06 16:53 ` Guo Ren 2021-06-06 16:53 ` Guo Ren 2021-06-06 16:53 ` Guo Ren 2021-06-06 16:53 ` Guo Ren 2021-06-06 16:29 ` [RFC PATCH v2 00/11] riscv: Add DMA_COHERENT support for Allwinner D1 Jernej Škrabec 2021-06-06 16:29 ` Jernej Škrabec 2021-06-06 16:54 ` Guo Ren 2021-06-06 16:54 ` Guo Ren 2021-06-06 17:14 ` Jernej Škrabec 2021-06-06 17:14 ` Jernej Škrabec 2021-06-06 23:42 ` Guo Ren 2021-06-06 23:42 ` Guo Ren 2021-06-07 3:44 ` Anup Patel 2021-06-07 3:44 ` Anup Patel 2021-06-07 4:36 ` Guo Ren 2021-06-07 4:36 ` Guo Ren
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1622970249-50770-5-git-send-email-guoren@kernel.org \ --to=guoren@kernel.org \ --cc=anup.patel@wdc.com \ --cc=arnd@arndb.de \ --cc=drew@beagleboard.org \ --cc=guoren@linux.alibaba.com \ --cc=hch@lst.de \ --cc=lazyparser@gmail.com \ --cc=linux-arch@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-riscv@lists.infradead.org \ --cc=linux-sunxi@lists.linux.dev \ --cc=liush@allwinnertech.com \ --cc=maxime@cerno.tech \ --cc=palmerdabbelt@google.com \ --cc=wefu@redhat.com \ --cc=wens@csie.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.