From: Alexandre Ghiti <alexghiti@rivosinc.com>
To: Ryan Roberts <ryan.roberts@arm.com>,
Catalin Marinas <catalin.marinas@arm.com>,
Will Deacon <will@kernel.org>,
Alexander Potapenko <glider@google.com>,
Marco Elver <elver@google.com>,
Dmitry Vyukov <dvyukov@google.com>,
Paul Walmsley <paul.walmsley@sifive.com>,
Palmer Dabbelt <palmer@dabbelt.com>,
Albert Ou <aou@eecs.berkeley.edu>,
Ard Biesheuvel <ardb@kernel.org>,
Anup Patel <anup@brainfault.org>,
Atish Patra <atishp@atishpatra.org>,
Andrey Ryabinin <ryabinin.a.a@gmail.com>,
Andrey Konovalov <andreyknvl@gmail.com>,
Vincenzo Frascino <vincenzo.frascino@arm.com>,
Andrew Morton <akpm@linux-foundation.org>,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, kasan-dev@googlegroups.com,
linux-riscv@lists.infradead.org, linux-efi@vger.kernel.org,
kvm@vger.kernel.org, kvm-riscv@lists.infradead.org,
linux-mm@kvack.org
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Subject: [PATCH 04/12] mm, riscv, arm64: Use common ptep_get_lockless() function
Date: Wed, 8 May 2024 21:19:23 +0200 [thread overview]
Message-ID: <20240508191931.46060-5-alexghiti@rivosinc.com> (raw)
In-Reply-To: <20240508191931.46060-1-alexghiti@rivosinc.com>
Make riscv use the contpte aware ptep_get_lockless() function from arm64.
Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com>
---
arch/arm64/include/asm/pgtable.h | 11 +----
arch/arm64/mm/contpte.c | 57 --------------------------
arch/riscv/include/asm/pgtable.h | 2 +
include/linux/contpte.h | 1 +
mm/contpte.c | 69 ++++++++++++++++++++++++++++++++
5 files changed, 73 insertions(+), 67 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index e85b3a052a02..8a0603257436 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -1384,7 +1384,6 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
* where it is possible and makes sense to do so. The PTE_CONT bit is considered
* a private implementation detail of the public ptep API (see below).
*/
-extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);
extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned int nr, int full);
extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
@@ -1430,16 +1429,8 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
extern pte_t ptep_get(pte_t *ptep);
#define ptep_get ptep_get
+extern pte_t ptep_get_lockless(pte_t *ptep);
#define ptep_get_lockless ptep_get_lockless
-static inline pte_t ptep_get_lockless(pte_t *ptep)
-{
- pte_t pte = __ptep_get(ptep);
-
- if (likely(!pte_valid_cont(pte)))
- return pte;
-
- return contpte_ptep_get_lockless(ptep);
-}
static inline void set_pte(pte_t *ptep, pte_t pte)
{
diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c
index e225e458856e..5e9e40145085 100644
--- a/arch/arm64/mm/contpte.c
+++ b/arch/arm64/mm/contpte.c
@@ -28,63 +28,6 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr,
}
}
-pte_t contpte_ptep_get_lockless(pte_t *orig_ptep)
-{
- /*
- * The ptep_get_lockless() API requires us to read and return *orig_ptep
- * so that it is self-consistent, without the PTL held, so we may be
- * racing with other threads modifying the pte. Usually a READ_ONCE()
- * would suffice, but for the contpte case, we also need to gather the
- * access and dirty bits from across all ptes in the contiguous block,
- * and we can't read all of those neighbouring ptes atomically, so any
- * contiguous range may be unfolded/modified/refolded under our feet.
- * Therefore we ensure we read a _consistent_ contpte range by checking
- * that all ptes in the range are valid and have CONT_PTE set, that all
- * pfns are contiguous and that all pgprots are the same (ignoring
- * access/dirty). If we find a pte that is not consistent, then we must
- * be racing with an update so start again. If the target pte does not
- * have CONT_PTE set then that is considered consistent on its own
- * because it is not part of a contpte range.
- */
-
- pgprot_t orig_prot;
- unsigned long pfn;
- pte_t orig_pte;
- pgprot_t prot;
- pte_t *ptep;
- pte_t pte;
- int i;
-
-retry:
- orig_pte = __ptep_get(orig_ptep);
-
- if (!pte_valid_cont(orig_pte))
- return orig_pte;
-
- orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte)));
- ptep = arch_contpte_align_down(orig_ptep);
- pfn = pte_pfn(orig_pte) - (orig_ptep - ptep);
-
- for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) {
- pte = __ptep_get(ptep);
- prot = pte_pgprot(pte_mkold(pte_mkclean(pte)));
-
- if (!pte_valid_cont(pte) ||
- pte_pfn(pte) != pfn ||
- pgprot_val(prot) != pgprot_val(orig_prot))
- goto retry;
-
- if (pte_dirty(pte))
- orig_pte = pte_mkdirty(orig_pte);
-
- if (pte_young(pte))
- orig_pte = pte_mkyoung(orig_pte);
- }
-
- return orig_pte;
-}
-EXPORT_SYMBOL_GPL(contpte_ptep_get_lockless);
-
void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned int nr, int full)
{
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index ebfe6b16529e..62cad1b974f1 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -785,6 +785,8 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
extern pte_t ptep_get(pte_t *ptep);
#define ptep_get ptep_get
+extern pte_t ptep_get_lockless(pte_t *ptep);
+#define ptep_get_lockless ptep_get_lockless
extern void set_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pteval, unsigned int nr);
#define set_ptes set_ptes
diff --git a/include/linux/contpte.h b/include/linux/contpte.h
index 54d10204e9af..01da4bfc3af6 100644
--- a/include/linux/contpte.h
+++ b/include/linux/contpte.h
@@ -8,6 +8,7 @@
* a private implementation detail of the public ptep API (see below).
*/
pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);
+pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);
void __contpte_try_fold(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte);
void contpte_try_fold(struct mm_struct *mm, unsigned long addr,
diff --git a/mm/contpte.c b/mm/contpte.c
index 566745d7842f..060e0bc1a2a3 100644
--- a/mm/contpte.c
+++ b/mm/contpte.c
@@ -42,6 +42,7 @@
* - huge_ptep_clear_flush()
* - ptep_get()
* - set_ptes()
+ * - ptep_get_lockless()
*/
pte_t huge_ptep_get(pte_t *ptep)
@@ -589,4 +590,72 @@ __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr,
contpte_set_ptes(mm, addr, ptep, pte, nr);
}
}
+
+pte_t contpte_ptep_get_lockless(pte_t *orig_ptep)
+{
+ /*
+ * The ptep_get_lockless() API requires us to read and return *orig_ptep
+ * so that it is self-consistent, without the PTL held, so we may be
+ * racing with other threads modifying the pte. Usually a READ_ONCE()
+ * would suffice, but for the contpte case, we also need to gather the
+ * access and dirty bits from across all ptes in the contiguous block,
+ * and we can't read all of those neighbouring ptes atomically, so any
+ * contiguous range may be unfolded/modified/refolded under our feet.
+ * Therefore we ensure we read a _consistent_ contpte range by checking
+ * that all ptes in the range are valid and have CONT_PTE set, that all
+ * pfns are contiguous and that all pgprots are the same (ignoring
+ * access/dirty). If we find a pte that is not consistent, then we must
+ * be racing with an update so start again. If the target pte does not
+ * have CONT_PTE set then that is considered consistent on its own
+ * because it is not part of a contpte range.
+ */
+
+ pgprot_t orig_prot;
+ unsigned long pfn;
+ pte_t orig_pte;
+ pgprot_t prot;
+ pte_t *ptep;
+ pte_t pte;
+ int i, ncontig;
+
+retry:
+ orig_pte = __ptep_get(orig_ptep);
+
+ if (!pte_valid_cont(orig_pte))
+ return orig_pte;
+
+ orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte)));
+ ptep = arch_contpte_align_down(orig_ptep);
+ ncontig = arch_contpte_get_num_contig(NULL, 0, ptep, 0, NULL);
+ pfn = pte_pfn(orig_pte) - (orig_ptep - ptep);
+
+ for (i = 0; i < ncontig; i++, ptep++, pfn++) {
+ pte = __ptep_get(ptep);
+ prot = pte_pgprot(pte_mkold(pte_mkclean(pte)));
+
+ if (!pte_valid_cont(pte) ||
+ pte_pfn(pte) != pfn ||
+ pgprot_val(prot) != pgprot_val(orig_prot))
+ goto retry;
+
+ if (pte_dirty(pte))
+ orig_pte = pte_mkdirty(orig_pte);
+
+ if (pte_young(pte))
+ orig_pte = pte_mkyoung(orig_pte);
+ }
+
+ return orig_pte;
+}
+EXPORT_SYMBOL_GPL(contpte_ptep_get_lockless);
+
+__always_inline pte_t ptep_get_lockless(pte_t *ptep)
+{
+ pte_t pte = __ptep_get(ptep);
+
+ if (likely(!pte_valid_cont(pte)))
+ return pte;
+
+ return contpte_ptep_get_lockless(ptep);
+}
#endif /* CONFIG_THP_CONTPTE */
--
2.39.2
next prev parent reply other threads:[~2024-05-08 19:23 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-05-08 19:19 [PATCH 00/12] Make riscv use THP contpte support for arm64 Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 01/12] mm, arm64: Rename ARM64_CONTPTE to THP_CONTPTE Alexandre Ghiti
2024-05-09 0:46 ` Barry Song
2024-05-13 13:09 ` Alexandre Ghiti
2024-05-14 9:30 ` Barry Song
2024-05-08 19:19 ` [PATCH 02/12] mm, riscv, arm64: Use common ptep_get() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 03/12] mm, riscv, arm64: Use common set_ptes() function Alexandre Ghiti
2024-05-08 19:19 ` Alexandre Ghiti [this message]
2024-05-08 19:19 ` [PATCH 05/12] mm, riscv, arm64: Use common set_pte() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 06/12] mm, riscv, arm64: Use common pte_clear() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 07/12] mm, riscv, arm64: Use common ptep_get_and_clear() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 08/12] mm, riscv, arm64: Use common ptep_test_and_clear_young() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 09/12] mm, riscv, arm64: Use common ptep_clear_flush_young() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 10/12] mm, riscv, arm64: Use common ptep_set_access_flags() function Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 11/12] mm, riscv, arm64: Use common ptep_set_wrprotect()/wrprotect_ptes() functions Alexandre Ghiti
2024-05-08 19:19 ` [PATCH 12/12] mm, riscv, arm64: Use common get_and_clear_full_ptes()/clear_full_ptes() functions Alexandre Ghiti
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240508191931.46060-5-alexghiti@rivosinc.com \
--to=alexghiti@rivosinc.com \
--cc=akpm@linux-foundation.org \
--cc=andreyknvl@gmail.com \
--cc=anup@brainfault.org \
--cc=aou@eecs.berkeley.edu \
--cc=ardb@kernel.org \
--cc=atishp@atishpatra.org \
--cc=catalin.marinas@arm.com \
--cc=dvyukov@google.com \
--cc=elver@google.com \
--cc=glider@google.com \
--cc=kasan-dev@googlegroups.com \
--cc=kvm-riscv@lists.infradead.org \
--cc=kvm@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-efi@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-riscv@lists.infradead.org \
--cc=palmer@dabbelt.com \
--cc=paul.walmsley@sifive.com \
--cc=ryabinin.a.a@gmail.com \
--cc=ryan.roberts@arm.com \
--cc=vincenzo.frascino@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).