From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linux-mm@kvack.org, akpm@linux-foundation.org
Cc: mpe@ellerman.id.au, linuxppc-dev@lists.ozlabs.org,
kaleshsingh@google.com, npiggin@gmail.com,
joel@joelfernandes.org,
Christophe Leroy <christophe.leroy@csgroup.eu>,
Linus Torvalds <torvalds@linux-foundation.org>,
"Kirill A . Shutemov" <kirill@shutemov.name>,
"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Subject: [PATCH v7 02/11] mm/mremap: Fix race between MOVE_PUD mremap and pageout
Date: Mon, 7 Jun 2021 11:21:22 +0530 [thread overview]
Message-ID: <20210607055131.156184-3-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20210607055131.156184-1-aneesh.kumar@linux.ibm.com>
CPU 1 CPU 2 CPU 3
mremap(old_addr, new_addr) page_shrinker/try_to_unmap_one
mmap_write_lock_killable()
addr = old_addr
lock(pte_ptl)
lock(pud_ptl)
pud = *old_pud
pud_clear(old_pud)
flush_tlb_range(old_addr)
*new_pud = pud
*new_addr = 10; and fills
TLB with new addr
and old pfn
unlock(pud_ptl)
ptep_clear_flush()
old pfn is free.
Stale TLB entry
Fix this race by holding pud lock in pageout.
Fixes: c49dd3401802 ("mm: speedup mremap on 1GB or larger regions")
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
include/linux/rmap.h | 4 ++++
mm/page_vma_mapped.c | 13 ++++++++++---
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 272ab0c2b60b..491c65ce1d46 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -209,6 +209,7 @@ struct page_vma_mapped_walk {
pte_t *pte;
spinlock_t *pte_ptl;
spinlock_t *pmd_ptl;
+ spinlock_t *pud_ptl;
unsigned int flags;
};
@@ -221,6 +222,9 @@ static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
spin_unlock(pvmw->pte_ptl);
if (pvmw->pmd_ptl)
spin_unlock(pvmw->pmd_ptl);
+ if (pvmw->pud_ptl)
+ spin_unlock(pvmw->pud_ptl);
+
}
bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw);
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 87a2c94c7e27..c913bc34b1d3 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -180,8 +180,11 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
pud = pud_offset(p4d, pvmw->address);
if (!pud_present(*pud))
return false;
+
+ pvmw->pud_ptl = pud_lock(mm, pud);
pvmw->pmd = pmd_offset(pud, pvmw->address);
- pvmw->pmd_ptl = pmd_lock(mm, pvmw->pmd);
+ if (USE_SPLIT_PMD_PTLOCKS)
+ pvmw->pmd_ptl = pmd_lock(mm, pvmw->pmd);
/*
* Make sure the pmd value isn't cached in a register by the
* compiler and used as a stale value after we've observed a
@@ -235,8 +238,12 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
spin_unlock(pvmw->pte_ptl);
pvmw->pte_ptl = NULL;
}
- spin_unlock(pvmw->pmd_ptl);
- pvmw->pmd_ptl = NULL;
+ if (pvmw->pmd_ptl) {
+ spin_unlock(pvmw->pmd_ptl);
+ pvmw->pmd_ptl = NULL;
+ }
+ spin_unlock(pvmw->pud_ptl);
+ pvmw->pud_ptl = NULL;
goto restart;
} else {
pvmw->pte++;
--
2.31.1
WARNING: multiple messages have this Message-ID (diff)
From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linux-mm@kvack.org, akpm@linux-foundation.org
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>,
Linus Torvalds <torvalds@linux-foundation.org>,
npiggin@gmail.com, kaleshsingh@google.com,
joel@joelfernandes.org,
"Kirill A . Shutemov" <kirill@shutemov.name>,
linuxppc-dev@lists.ozlabs.org
Subject: [PATCH v7 02/11] mm/mremap: Fix race between MOVE_PUD mremap and pageout
Date: Mon, 7 Jun 2021 11:21:22 +0530 [thread overview]
Message-ID: <20210607055131.156184-3-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20210607055131.156184-1-aneesh.kumar@linux.ibm.com>
CPU 1 CPU 2 CPU 3
mremap(old_addr, new_addr) page_shrinker/try_to_unmap_one
mmap_write_lock_killable()
addr = old_addr
lock(pte_ptl)
lock(pud_ptl)
pud = *old_pud
pud_clear(old_pud)
flush_tlb_range(old_addr)
*new_pud = pud
*new_addr = 10; and fills
TLB with new addr
and old pfn
unlock(pud_ptl)
ptep_clear_flush()
old pfn is free.
Stale TLB entry
Fix this race by holding pud lock in pageout.
Fixes: c49dd3401802 ("mm: speedup mremap on 1GB or larger regions")
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
include/linux/rmap.h | 4 ++++
mm/page_vma_mapped.c | 13 ++++++++++---
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 272ab0c2b60b..491c65ce1d46 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -209,6 +209,7 @@ struct page_vma_mapped_walk {
pte_t *pte;
spinlock_t *pte_ptl;
spinlock_t *pmd_ptl;
+ spinlock_t *pud_ptl;
unsigned int flags;
};
@@ -221,6 +222,9 @@ static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
spin_unlock(pvmw->pte_ptl);
if (pvmw->pmd_ptl)
spin_unlock(pvmw->pmd_ptl);
+ if (pvmw->pud_ptl)
+ spin_unlock(pvmw->pud_ptl);
+
}
bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw);
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 87a2c94c7e27..c913bc34b1d3 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -180,8 +180,11 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
pud = pud_offset(p4d, pvmw->address);
if (!pud_present(*pud))
return false;
+
+ pvmw->pud_ptl = pud_lock(mm, pud);
pvmw->pmd = pmd_offset(pud, pvmw->address);
- pvmw->pmd_ptl = pmd_lock(mm, pvmw->pmd);
+ if (USE_SPLIT_PMD_PTLOCKS)
+ pvmw->pmd_ptl = pmd_lock(mm, pvmw->pmd);
/*
* Make sure the pmd value isn't cached in a register by the
* compiler and used as a stale value after we've observed a
@@ -235,8 +238,12 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
spin_unlock(pvmw->pte_ptl);
pvmw->pte_ptl = NULL;
}
- spin_unlock(pvmw->pmd_ptl);
- pvmw->pmd_ptl = NULL;
+ if (pvmw->pmd_ptl) {
+ spin_unlock(pvmw->pmd_ptl);
+ pvmw->pmd_ptl = NULL;
+ }
+ spin_unlock(pvmw->pud_ptl);
+ pvmw->pud_ptl = NULL;
goto restart;
} else {
pvmw->pte++;
--
2.31.1
next prev parent reply other threads:[~2021-06-07 5:52 UTC|newest]
Thread overview: 58+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-07 5:51 [PATCH v7 00/11] Speedup mremap on ppc64 Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 01/11] mm/mremap: Fix race between MOVE_PMD mremap and pageout Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-08 0:06 ` Hugh Dickins
2021-06-08 0:06 ` Hugh Dickins
2021-06-08 7:52 ` Aneesh Kumar K.V
2021-06-08 7:52 ` Aneesh Kumar K.V
2021-06-08 9:42 ` Kirill A. Shutemov
2021-06-08 9:42 ` Kirill A. Shutemov
2021-06-08 11:17 ` Aneesh Kumar K.V
2021-06-08 11:17 ` Aneesh Kumar K.V
2021-06-08 12:05 ` Kirill A. Shutemov
2021-06-08 12:05 ` Kirill A. Shutemov
2021-06-08 20:39 ` Hugh Dickins
2021-06-08 20:39 ` Hugh Dickins
2021-06-07 5:51 ` Aneesh Kumar K.V [this message]
2021-06-07 5:51 ` [PATCH v7 02/11] mm/mremap: Fix race between MOVE_PUD " Aneesh Kumar K.V
2021-06-14 14:55 ` [mm/mremap] ecf8443e51: vm-scalability.throughput -29.4% regression kernel test robot
2021-06-14 14:55 ` kernel test robot
2021-06-14 14:55 ` kernel test robot
2021-06-14 14:58 ` Linus Torvalds
2021-06-14 14:58 ` Linus Torvalds
2021-06-14 14:58 ` Linus Torvalds
2021-06-14 16:08 ` Aneesh Kumar K.V
2021-06-14 16:08 ` Aneesh Kumar K.V
2021-06-14 16:08 ` Aneesh Kumar K.V
2021-06-17 2:38 ` [LKP] " Liu, Yujie
2021-06-17 2:38 ` Liu, Yujie
2021-06-17 2:38 ` [LKP] " Liu, Yujie
2021-06-07 5:51 ` [PATCH v7 03/11] selftest/mremap_test: Update the test to handle pagesize other than 4K Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 04/11] selftest/mremap_test: Avoid crash with static build Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 05/11] mm/mremap: Convert huge PUD move to separate helper Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 06/11] mm/mremap: Don't enable optimized PUD move if page table levels is 2 Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 07/11] mm/mremap: Use pmd/pud_poplulate to update page table entries Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 08/11] powerpc/mm/book3s64: Fix possible build error Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 09/11] mm/mremap: Allow arch runtime override Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 10/11] powerpc/book3s64/mm: Update flush_tlb_range to flush page walk cache Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 5:51 ` [PATCH v7 11/11] powerpc/mm: Enable HAVE_MOVE_PMD support Aneesh Kumar K.V
2021-06-07 5:51 ` Aneesh Kumar K.V
2021-06-07 10:10 ` [PATCH v7 00/11] Speedup mremap on ppc64 Nick Piggin
2021-06-07 10:10 ` Nick Piggin
2021-06-08 4:39 ` Aneesh Kumar K.V
2021-06-08 4:39 ` Aneesh Kumar K.V
2021-06-08 5:03 ` Nicholas Piggin
2021-06-08 5:03 ` Nicholas Piggin
2021-06-08 17:10 ` Linus Torvalds
2021-06-08 17:10 ` Linus Torvalds
2021-06-16 1:44 ` Nicholas Piggin
2021-06-16 1:44 ` Nicholas Piggin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210607055131.156184-3-aneesh.kumar@linux.ibm.com \
--to=aneesh.kumar@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=christophe.leroy@csgroup.eu \
--cc=joel@joelfernandes.org \
--cc=kaleshsingh@google.com \
--cc=kirill@shutemov.name \
--cc=linux-mm@kvack.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=mpe@ellerman.id.au \
--cc=npiggin@gmail.com \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.