From: Byungchul Park <byungchul@sk.com>
To: linux-kernel@vger.kernel.org, linux-mm@kvack.org
Cc: kernel_team@skhynix.com, akpm@linux-foundation.org,
ying.huang@intel.com, vernhao@tencent.com,
mgorman@techsingularity.net, hughd@google.com,
willy@infradead.org, david@redhat.com, peterz@infradead.org,
luto@kernel.org, tglx@linutronix.de, mingo@redhat.com,
bp@alien8.de, dave.hansen@linux.intel.com, rjgolo@gmail.com
Subject: [PATCH v10 11/12] mm, migrate: apply luf mechanism to unmapping during migration
Date: Fri, 10 May 2024 15:52:05 +0900 [thread overview]
Message-ID: <20240510065206.76078-12-byungchul@sk.com> (raw)
In-Reply-To: <20240510065206.76078-1-byungchul@sk.com>
A new mechanism, LUF(Lazy Unmap Flush), defers tlb flush until folios
that have been unmapped and freed, eventually get allocated again. It's
safe for folios that had been mapped read only and were unmapped, since
the contents of the folios don't change while staying in pcp or buddy
so we can still read the data through the stale tlb entries.
Applied the mechanism to unmapping during migration.
Signed-off-by: Byungchul Park <byungchul@sk.com>
---
include/linux/rmap.h | 2 +-
mm/migrate.c | 56 ++++++++++++++++++++++++++++++++------------
mm/rmap.c | 9 ++++---
3 files changed, 48 insertions(+), 19 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 0f906dc6d280..1898a2c1c087 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -657,7 +657,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio,
int folio_referenced(struct folio *, int is_locked,
struct mem_cgroup *memcg, unsigned long *vm_flags);
-void try_to_migrate(struct folio *folio, enum ttu_flags flags);
+bool try_to_migrate(struct folio *folio, enum ttu_flags flags);
void try_to_unmap(struct folio *, enum ttu_flags flags);
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
diff --git a/mm/migrate.c b/mm/migrate.c
index f9ed7a2b8720..c8b0e5203e9a 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1090,7 +1090,8 @@ static void migrate_folio_undo_dst(struct folio *dst, bool locked,
/* Cleanup src folio upon migration success */
static void migrate_folio_done(struct folio *src,
- enum migrate_reason reason)
+ enum migrate_reason reason,
+ unsigned short int ugen)
{
/*
* Compaction can migrate also non-LRU pages which are
@@ -1101,8 +1102,12 @@ static void migrate_folio_done(struct folio *src,
mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON +
folio_is_file_lru(src), -folio_nr_pages(src));
- if (reason != MR_MEMORY_FAILURE)
- /* We release the page in page_handle_poison. */
+ /* We release the page in page_handle_poison. */
+ if (reason == MR_MEMORY_FAILURE)
+ check_luf_flush(ugen);
+ else if (ugen)
+ folio_put_ugen(src, ugen);
+ else
folio_put(src);
}
@@ -1110,7 +1115,8 @@ static void migrate_folio_done(struct folio *src,
static int migrate_folio_unmap(new_folio_t get_new_folio,
free_folio_t put_new_folio, unsigned long private,
struct folio *src, struct folio **dstp, enum migrate_mode mode,
- enum migrate_reason reason, struct list_head *ret)
+ enum migrate_reason reason, struct list_head *ret,
+ bool *can_luf)
{
struct folio *dst;
int rc = -EAGAIN;
@@ -1126,7 +1132,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
folio_clear_unevictable(src);
/* free_pages_prepare() will clear PG_isolated. */
list_del(&src->lru);
- migrate_folio_done(src, reason);
+ migrate_folio_done(src, reason, 0);
return MIGRATEPAGE_SUCCESS;
}
@@ -1244,7 +1250,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
/* Establish migration ptes */
VM_BUG_ON_FOLIO(folio_test_anon(src) &&
!folio_test_ksm(src) && !anon_vma, src);
- try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
+ *can_luf = try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
old_page_state |= PAGE_WAS_MAPPED;
}
@@ -1272,7 +1278,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
struct folio *src, struct folio *dst,
enum migrate_mode mode, enum migrate_reason reason,
- struct list_head *ret)
+ struct list_head *ret, unsigned short int ugen)
{
int rc;
int old_page_state = 0;
@@ -1326,7 +1332,7 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
if (anon_vma)
put_anon_vma(anon_vma);
folio_unlock(src);
- migrate_folio_done(src, reason);
+ migrate_folio_done(src, reason, ugen);
return rc;
out:
@@ -1616,7 +1622,7 @@ static void migrate_folios_move(struct list_head *src_folios,
struct list_head *ret_folios,
struct migrate_pages_stats *stats,
int *retry, int *thp_retry, int *nr_failed,
- int *nr_retry_pages)
+ int *nr_retry_pages, unsigned short int ugen)
{
struct folio *folio, *folio2, *dst, *dst2;
bool is_thp;
@@ -1633,7 +1639,7 @@ static void migrate_folios_move(struct list_head *src_folios,
rc = migrate_folio_move(put_new_folio, private,
folio, dst, mode,
- reason, ret_folios);
+ reason, ret_folios, ugen);
/*
* The rules are:
* Success: folio will be freed
@@ -1710,7 +1716,11 @@ static int migrate_pages_batch(struct list_head *from,
int rc, rc_saved = 0, nr_pages;
LIST_HEAD(unmap_folios);
LIST_HEAD(dst_folios);
+ LIST_HEAD(unmap_folios_luf);
+ LIST_HEAD(dst_folios_luf);
bool nosplit = (reason == MR_NUMA_MISPLACED);
+ unsigned short int ugen;
+ bool can_luf;
VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
!list_empty(from) && !list_is_singular(from));
@@ -1773,9 +1783,11 @@ static int migrate_pages_batch(struct list_head *from,
continue;
}
+ can_luf = false;
rc = migrate_folio_unmap(get_new_folio, put_new_folio,
private, folio, &dst, mode, reason,
- ret_folios);
+ ret_folios, &can_luf);
+
/*
* The rules are:
* Success: folio will be freed
@@ -1821,7 +1833,8 @@ static int migrate_pages_batch(struct list_head *from,
/* nr_failed isn't updated for not used */
stats->nr_thp_failed += thp_retry;
rc_saved = rc;
- if (list_empty(&unmap_folios))
+ if (list_empty(&unmap_folios) &&
+ list_empty(&unmap_folios_luf))
goto out;
else
goto move;
@@ -1835,8 +1848,13 @@ static int migrate_pages_batch(struct list_head *from,
stats->nr_thp_succeeded += is_thp;
break;
case MIGRATEPAGE_UNMAP:
- list_move_tail(&folio->lru, &unmap_folios);
- list_add_tail(&dst->lru, &dst_folios);
+ if (can_luf) {
+ list_move_tail(&folio->lru, &unmap_folios_luf);
+ list_add_tail(&dst->lru, &dst_folios_luf);
+ } else {
+ list_move_tail(&folio->lru, &unmap_folios);
+ list_add_tail(&dst->lru, &dst_folios);
+ }
break;
default:
/*
@@ -1856,6 +1874,8 @@ static int migrate_pages_batch(struct list_head *from,
stats->nr_thp_failed += thp_retry;
stats->nr_failed_pages += nr_retry_pages;
move:
+ /* Should be before try_to_unmap_flush() */
+ ugen = try_to_unmap_luf();
/* Flush TLBs for all unmapped folios */
try_to_unmap_flush();
@@ -1869,7 +1889,11 @@ static int migrate_pages_batch(struct list_head *from,
migrate_folios_move(&unmap_folios, &dst_folios,
put_new_folio, private, mode, reason,
ret_folios, stats, &retry, &thp_retry,
- &nr_failed, &nr_retry_pages);
+ &nr_failed, &nr_retry_pages, 0);
+ migrate_folios_move(&unmap_folios_luf, &dst_folios_luf,
+ put_new_folio, private, mode, reason,
+ ret_folios, stats, &retry, &thp_retry,
+ &nr_failed, &nr_retry_pages, ugen);
}
nr_failed += retry;
stats->nr_thp_failed += thp_retry;
@@ -1880,6 +1904,8 @@ static int migrate_pages_batch(struct list_head *from,
/* Cleanup remaining folios */
migrate_folios_undo(&unmap_folios, &dst_folios,
put_new_folio, private, ret_folios);
+ migrate_folios_undo(&unmap_folios_luf, &dst_folios_luf,
+ put_new_folio, private, ret_folios);
return rc;
}
diff --git a/mm/rmap.c b/mm/rmap.c
index e42783c02114..d25ae20a47b5 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2600,8 +2600,9 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
*
* Tries to remove all the page table entries which are mapping this folio and
* replace them with special swap entries. Caller must hold the folio lock.
+ * Return true if all the mappings are read-only, otherwise false.
*/
-void try_to_migrate(struct folio *folio, enum ttu_flags flags)
+bool try_to_migrate(struct folio *folio, enum ttu_flags flags)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_migrate_one,
@@ -2620,11 +2621,11 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags)
*/
if (WARN_ON_ONCE(flags & ~(TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
TTU_SYNC | TTU_BATCH_FLUSH)))
- return;
+ return false;
if (folio_is_zone_device(folio) &&
(!folio_is_device_private(folio) && !folio_is_device_coherent(folio)))
- return;
+ return false;
/*
* During exec, a temporary VMA is setup and later moved.
@@ -2649,6 +2650,8 @@ void try_to_migrate(struct folio *folio, enum ttu_flags flags)
fold_ubc(tlb_ubc_luf, tlb_ubc_ro);
else
fold_ubc(tlb_ubc, tlb_ubc_ro);
+
+ return can_luf;
}
#ifdef CONFIG_DEVICE_PRIVATE
--
2.17.1
next prev parent reply other threads:[~2024-05-10 6:52 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-05-10 6:51 [PATCH v10 00/12] LUF(Lazy Unmap Flush) reducing tlb numbers over 90% Byungchul Park
2024-05-10 6:51 ` [PATCH v10 01/12] x86/tlb: add APIs manipulating tlb batch's arch data Byungchul Park
2024-05-10 6:51 ` [PATCH v10 02/12] arm64: tlbflush: " Byungchul Park
2024-05-10 6:51 ` [PATCH v10 03/12] riscv, tlb: " Byungchul Park
2024-05-10 6:51 ` [PATCH v10 04/12] x86/tlb, riscv/tlb, mm/rmap: separate arch_tlbbatch_clear() out of arch_tlbbatch_flush() Byungchul Park
2024-05-10 6:51 ` [PATCH v10 05/12] mm: buddy: make room for a new variable, ugen, in struct page Byungchul Park
2024-05-10 6:52 ` [PATCH v10 06/12] mm: add folio_put_ugen() to deliver unmap generation number to pcp or buddy Byungchul Park
2024-05-10 6:52 ` [PATCH v10 07/12] mm: add a parameter, unmap generation number, to free_unref_folios() Byungchul Park
2024-05-10 6:52 ` [PATCH v10 08/12] mm/rmap: recognize read-only tlb entries during batched tlb flush Byungchul Park
2024-05-10 6:52 ` [PATCH v10 09/12] mm: implement LUF(Lazy Unmap Flush) defering tlb flush when folios get unmapped Byungchul Park
2024-05-10 6:52 ` [PATCH v10 10/12] mm: separate move/undo parts from migrate_pages_batch() Byungchul Park
2024-05-10 6:52 ` Byungchul Park [this message]
2024-05-10 6:52 ` [PATCH v10 12/12] mm, vmscan: apply luf mechanism to unmapping during folio reclaim Byungchul Park
2024-05-11 6:54 ` [PATCH v10 00/12] LUF(Lazy Unmap Flush) reducing tlb numbers over 90% Huang, Ying
2024-05-13 1:41 ` Byungchul Park
2024-05-11 7:15 ` Huang, Ying
2024-05-13 1:44 ` Byungchul Park
2024-05-22 2:16 ` Byungchul Park
2024-05-22 7:38 ` Huang, Ying
2024-05-22 10:27 ` Byungchul Park
2024-05-22 14:15 ` Byungchul Park
2024-05-24 17:16 ` Dave Hansen
2024-05-27 1:57 ` Byungchul Park
2024-05-27 2:43 ` Dave Hansen
2024-05-27 3:46 ` Byungchul Park
2024-05-27 4:19 ` Byungchul Park
2024-05-27 4:25 ` Byungchul Park
2024-05-27 22:58 ` Byungchul Park
2024-05-29 2:16 ` Huang, Ying
2024-05-30 1:02 ` Byungchul Park
2024-05-27 3:10 ` Huang, Ying
2024-05-27 3:56 ` Byungchul Park
2024-05-28 15:14 ` Dave Hansen
2024-05-29 5:00 ` Byungchul Park
2024-05-29 16:41 ` Dave Hansen
2024-05-30 0:50 ` Byungchul Park
2024-05-30 0:59 ` Byungchul Park
2024-05-30 1:11 ` Huang, Ying
2024-05-30 1:33 ` Byungchul Park
2024-05-30 7:18 ` Byungchul Park
2024-05-30 8:24 ` Huang, Ying
2024-05-30 8:41 ` Byungchul Park
2024-05-30 13:50 ` Dave Hansen
2024-05-31 2:06 ` Byungchul Park
2024-05-30 9:33 ` Byungchul Park
2024-05-31 1:45 ` Huang, Ying
2024-05-31 2:20 ` Byungchul Park
2024-05-28 8:41 ` David Hildenbrand
2024-05-29 4:39 ` Byungchul Park
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240510065206.76078-12-byungchul@sk.com \
--to=byungchul@sk.com \
--cc=akpm@linux-foundation.org \
--cc=bp@alien8.de \
--cc=dave.hansen@linux.intel.com \
--cc=david@redhat.com \
--cc=hughd@google.com \
--cc=kernel_team@skhynix.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=luto@kernel.org \
--cc=mgorman@techsingularity.net \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=rjgolo@gmail.com \
--cc=tglx@linutronix.de \
--cc=vernhao@tencent.com \
--cc=willy@infradead.org \
--cc=ying.huang@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).