From: Alexander Potapenko <glider@google.com>
To: glider@google.com
Cc: Alexander Viro <viro@zeniv.linux.org.uk>,
Andrew Morton <akpm@linux-foundation.org>,
Andrey Konovalov <andreyknvl@google.com>,
Andy Lutomirski <luto@kernel.org>,
Ard Biesheuvel <ard.biesheuvel@linaro.org>,
Arnd Bergmann <arnd@arndb.de>, Borislav Petkov <bp@alien8.de>,
Christoph Hellwig <hch@lst.de>, Christoph Lameter <cl@linux.com>,
David Rientjes <rientjes@google.com>,
Dmitry Vyukov <dvyukov@google.com>,
Eric Dumazet <edumazet@google.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
Herbert Xu <herbert@gondor.apana.org.au>,
Ilya Leoshkevich <iii@linux.ibm.com>,
Ingo Molnar <mingo@redhat.com>, Jens Axboe <axboe@kernel.dk>,
Joonsoo Kim <iamjoonsoo.kim@lge.com>,
Kees Cook <keescook@chromium.org>, Marco Elver <elver@google.com>,
Matthew Wilcox <willy@infradead.org>,
"Michael S. Tsirkin" <mst@redhat.com>,
Pekka Enberg <penberg@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Petr Mladek <pmladek@suse.com>,
Steven Rostedt <rostedt@goodmis.org>,
Thomas Gleixner <tglx@linutronix.de>,
Vasily Gorbik <gor@linux.ibm.com>,
Vegard Nossum <vegard.nossum@oracle.com>,
Vlastimil Babka <vbabka@suse.cz>,
linux-mm@kvack.org, linux-arch@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH 15/43] kmsan: mm: maintain KMSAN metadata for page operations
Date: Tue, 14 Dec 2021 17:20:22 +0100 [thread overview]
Message-ID: <20211214162050.660953-16-glider@google.com> (raw)
In-Reply-To: <20211214162050.660953-1-glider@google.com>
Insert KMSAN hooks that make the necessary bookkeeping changes:
- poison page shadow and origins in alloc_pages()/free_page();
- clear page shadow and origins in clear_page(), copy_user_highpage();
- copy page metadata in copy_highpage(), wp_page_copy();
- handle vmap()/vunmap()/iounmap();
Signed-off-by: Alexander Potapenko <glider@google.com>
---
Link: https://linux-review.googlesource.com/id/I6d4f53a0e7eab46fa29f0348f3095d9f2e326850
---
arch/x86/include/asm/page_64.h | 13 +++++++++++++
arch/x86/mm/ioremap.c | 3 +++
include/linux/highmem.h | 3 +++
mm/memory.c | 2 ++
mm/page_alloc.c | 14 ++++++++++++++
mm/vmalloc.c | 20 ++++++++++++++++++--
6 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 4bde0dc66100c..c10547510f1f4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -44,14 +44,27 @@ void clear_page_orig(void *page);
void clear_page_rep(void *page);
void clear_page_erms(void *page);
+/* This is an assembly header, avoid including too much of kmsan.h */
+#ifdef CONFIG_KMSAN
+void kmsan_unpoison_memory(const void *addr, size_t size);
+#endif
+__no_sanitize_memory
static inline void clear_page(void *page)
{
+#ifdef CONFIG_KMSAN
+ /* alternative_call_2() changes @page. */
+ void *page_copy = page;
+#endif
alternative_call_2(clear_page_orig,
clear_page_rep, X86_FEATURE_REP_GOOD,
clear_page_erms, X86_FEATURE_ERMS,
"=D" (page),
"0" (page)
: "cc", "memory", "rax", "rcx");
+#ifdef CONFIG_KMSAN
+ /* Clear KMSAN shadow for the pages that have it. */
+ kmsan_unpoison_memory(page_copy, PAGE_SIZE);
+#endif
}
void copy_page(void *to, void *from);
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 026031b3b7829..4d0349ecc7cd7 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -17,6 +17,7 @@
#include <linux/cc_platform.h>
#include <linux/efi.h>
#include <linux/pgtable.h>
+#include <linux/kmsan.h>
#include <asm/set_memory.h>
#include <asm/e820/api.h>
@@ -474,6 +475,8 @@ void iounmap(volatile void __iomem *addr)
return;
}
+ kmsan_iounmap_page_range((unsigned long)addr,
+ (unsigned long)addr + get_vm_area_size(p));
memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p));
/* Finally remove it */
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 39bb9b47fa9cd..3e1898a44d7e3 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -6,6 +6,7 @@
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/cacheflush.h>
+#include <linux/kmsan.h>
#include <linux/mm.h>
#include <linux/uaccess.h>
#include <linux/hardirq.h>
@@ -277,6 +278,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_user_page(vto, vfrom, vaddr, to);
+ kmsan_unpoison_memory(page_address(to), PAGE_SIZE);
kunmap_local(vto);
kunmap_local(vfrom);
}
@@ -292,6 +294,7 @@ static inline void copy_highpage(struct page *to, struct page *from)
vfrom = kmap_local_page(from);
vto = kmap_local_page(to);
copy_page(vto, vfrom);
+ kmsan_copy_page_meta(to, from);
kunmap_local(vto);
kunmap_local(vfrom);
}
diff --git a/mm/memory.c b/mm/memory.c
index 8f1de811a1dcb..ea9e48daadb15 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -51,6 +51,7 @@
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
+#include <linux/kmsan.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
@@ -3003,6 +3004,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
put_page(old_page);
return 0;
}
+ kmsan_copy_page_meta(new_page, old_page);
}
if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c5952749ad40b..fa8029b714a81 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -26,6 +26,7 @@
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/kasan.h>
+#include <linux/kmsan.h>
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
@@ -1288,6 +1289,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
VM_BUG_ON_PAGE(PageTail(page), page);
trace_mm_page_free(page, order);
+ kmsan_free_page(page, order);
if (unlikely(PageHWPoison(page)) && !order) {
/*
@@ -1734,6 +1736,9 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn,
{
if (early_page_uninitialised(pfn))
return;
+ if (!kmsan_memblock_free_pages(page, order))
+ /* KMSAN will take care of these pages. */
+ return;
__free_pages_core(page, order);
}
@@ -3663,6 +3668,14 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
/*
* Allocate a page from the given zone. Use pcplists for order-0 allocations.
*/
+
+/*
+ * Do not instrument rmqueue() with KMSAN. This function may call
+ * __msan_poison_alloca() through a call to set_pfnblock_flags_mask().
+ * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
+ * may call rmqueue() again, which will result in a deadlock.
+ */
+__no_sanitize_memory
static inline
struct page *rmqueue(struct zone *preferred_zone,
struct zone *zone, unsigned int order,
@@ -5389,6 +5402,7 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
}
trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
+ kmsan_alloc_page(page, order, alloc_gfp);
return page;
}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index d2a00ad4e1dd1..333de26b3c56e 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -319,6 +319,9 @@ int ioremap_page_range(unsigned long addr, unsigned long end,
err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
ioremap_max_page_shift);
flush_cache_vmap(addr, end);
+ if (!err)
+ kmsan_ioremap_page_range(addr, end, phys_addr, prot,
+ ioremap_max_page_shift);
return err;
}
@@ -418,7 +421,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
*
* This is an internal function only. Do not use outside mm/.
*/
-void vunmap_range_noflush(unsigned long start, unsigned long end)
+void __vunmap_range_noflush(unsigned long start, unsigned long end)
{
unsigned long next;
pgd_t *pgd;
@@ -440,6 +443,12 @@ void vunmap_range_noflush(unsigned long start, unsigned long end)
arch_sync_kernel_mappings(start, end);
}
+void vunmap_range_noflush(unsigned long start, unsigned long end)
+{
+ kmsan_vunmap_range_noflush(start, end);
+ __vunmap_range_noflush(start, end);
+}
+
/**
* vunmap_range - unmap kernel virtual addresses
* @addr: start of the VM area to unmap
@@ -574,7 +583,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
*
* This is an internal function only. Do not use outside mm/.
*/
-int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
pgprot_t prot, struct page **pages, unsigned int page_shift)
{
unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
@@ -600,6 +609,13 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
return 0;
}
+int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+ pgprot_t prot, struct page **pages, unsigned int page_shift)
+{
+ kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+ return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift);
+}
+
/**
* vmap_pages_range - map pages to a kernel virtual address
* @addr: start of the VM area to map
--
2.34.1.173.g76aa8bc2d0-goog
next prev parent reply other threads:[~2021-12-14 16:22 UTC|newest]
Thread overview: 75+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-12-14 16:20 [PATCH 00/43] Add KernelMemorySanitizer infrastructure Alexander Potapenko
2021-12-14 16:20 ` [PATCH 01/43] arch/x86: add missing include to sparsemem.h Alexander Potapenko
2021-12-14 16:20 ` [PATCH 02/43] stackdepot: reserve 5 extra bits in depot_stack_handle_t Alexander Potapenko
2021-12-14 16:20 ` [PATCH 03/43] kasan: common: adapt to the new prototype of __stack_depot_save() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 04/43] instrumented.h: allow instrumenting both sides of copy_from_user() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 05/43] asm: x86: instrument usercopy in get_user() and __put_user_size() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 06/43] asm-generic: instrument usercopy in cacheflush.h Alexander Potapenko
2021-12-14 16:20 ` [PATCH 07/43] compiler_attributes.h: add __disable_sanitizer_instrumentation Alexander Potapenko
2021-12-15 13:24 ` Mark Rutland
2021-12-15 13:33 ` Marco Elver
2021-12-14 16:20 ` [PATCH 08/43] kmsan: add ReST documentation Alexander Potapenko
2021-12-14 16:20 ` [PATCH 09/43] kmsan: introduce __no_sanitize_memory and __no_kmsan_checks Alexander Potapenko
2021-12-15 13:27 ` Mark Rutland
2021-12-14 16:20 ` [PATCH 10/43] kmsan: pgtable: reduce vmalloc space Alexander Potapenko
2021-12-15 13:36 ` Mark Rutland
2022-03-18 14:14 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 11/43] libnvdimm/pfn_dev: increase MAX_STRUCT_PAGE_SIZE Alexander Potapenko
2021-12-14 16:20 ` [PATCH 12/43] kcsan: clang: retire CONFIG_KCSAN_KCOV_BROKEN Alexander Potapenko
2021-12-15 13:33 ` Mark Rutland
2021-12-15 13:39 ` Marco Elver
2021-12-15 14:43 ` Mark Rutland
2022-03-18 14:34 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 13/43] kmsan: add KMSAN runtime core Alexander Potapenko
2021-12-14 16:34 ` Greg Kroah-Hartman
2021-12-16 10:33 ` Alexander Potapenko
2021-12-17 16:22 ` Greg Kroah-Hartman
2022-03-21 14:12 ` Alexander Potapenko
[not found] ` <CACT4Y+Y_torRwzh0eDMn+pJh=YT26hGrAPA499WqN1dV+4bDHA@mail.gmail.com>
2022-03-21 13:17 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 14/43] MAINTAINERS: add entry for KMSAN Alexander Potapenko
2021-12-14 16:20 ` Alexander Potapenko [this message]
2021-12-14 16:20 ` [PATCH 16/43] kmsan: mm: call KMSAN hooks from SLUB code Alexander Potapenko
2022-01-07 17:22 ` Vlastimil Babka
2022-03-25 13:15 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 17/43] kmsan: handle task creation and exiting Alexander Potapenko
2021-12-14 16:20 ` [PATCH 18/43] kmsan: unpoison @tlb in arch_tlb_gather_mmu() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 19/43] kmsan: init: call KMSAN initialization routines Alexander Potapenko
2021-12-14 16:20 ` [PATCH 20/43] instrumented.h: add KMSAN support Alexander Potapenko
2021-12-14 16:20 ` [PATCH 21/43] kmsan: mark noinstr as __no_sanitize_memory Alexander Potapenko
2021-12-15 13:49 ` Mark Rutland
2021-12-14 16:20 ` [PATCH 22/43] kmsan: initialize the output of READ_ONCE_NOCHECK() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 23/43] kmsan: make READ_ONCE_TASK_STACK() return initialized values Alexander Potapenko
2021-12-14 16:20 ` [PATCH 24/43] kmsan: disable KMSAN instrumentation for certain kernel parts Alexander Potapenko
2021-12-15 13:53 ` Mark Rutland
2021-12-14 16:20 ` [PATCH 25/43] kmsan: skip shadow checks in files doing context switches Alexander Potapenko
2021-12-15 14:13 ` Mark Rutland
2021-12-15 16:28 ` Alexander Potapenko
2021-12-15 17:22 ` Mark Rutland
2021-12-14 16:20 ` [PATCH 26/43] kmsan: virtio: check/unpoison scatterlist in vring_map_one_sg() Alexander Potapenko
2022-01-06 12:46 ` Michael S. Tsirkin
2021-12-14 16:20 ` [PATCH 27/43] x86: kmsan: add iomem support Alexander Potapenko
2021-12-14 16:20 ` [PATCH 28/43] kmsan: dma: unpoison DMA mappings Alexander Potapenko
2021-12-14 16:20 ` [PATCH 29/43] kmsan: handle memory sent to/from USB Alexander Potapenko
2021-12-14 16:20 ` [PATCH 30/43] kmsan: add tests for KMSAN Alexander Potapenko
2021-12-14 16:20 ` [PATCH 31/43] kmsan: disable strscpy() optimization under KMSAN Alexander Potapenko
2021-12-14 16:20 ` [PATCH 32/43] crypto: kmsan: disable accelerated configs " Alexander Potapenko
2021-12-14 16:20 ` [PATCH 33/43] kmsan: disable physical page merging in biovec Alexander Potapenko
2021-12-15 14:17 ` Mark Rutland
2021-12-15 16:30 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 34/43] kmsan: block: skip bio block merging logic for KMSAN Alexander Potapenko
2021-12-14 16:20 ` [PATCH 35/43] x86: kmsan: use __msan_ string functions where possible Alexander Potapenko
2021-12-14 16:20 ` [PATCH 36/43] x86: kmsan: sync metadata pages on page fault Alexander Potapenko
2021-12-14 16:20 ` [PATCH 37/43] x86: kasan: kmsan: support CONFIG_GENERIC_CSUM on x86, enable it for KASAN/KMSAN Alexander Potapenko
2021-12-14 16:20 ` [PATCH 38/43] x86: fs: kmsan: disable CONFIG_DCACHE_WORD_ACCESS Alexander Potapenko
2021-12-14 16:20 ` [PATCH 39/43] x86: kmsan: handle register passing from uninstrumented code Alexander Potapenko
2021-12-17 21:51 ` Thomas Gleixner
2021-12-20 14:35 ` Alexander Potapenko
2021-12-14 16:20 ` [PATCH 40/43] kmsan: kcov: unpoison area->list in kcov_remote_area_put() Alexander Potapenko
2021-12-14 16:20 ` [PATCH 41/43] security: kmsan: fix interoperability with auto-initialization Alexander Potapenko
2021-12-14 16:38 ` Greg Kroah-Hartman
2021-12-14 17:00 ` Alexander Potapenko
2021-12-14 17:33 ` Greg Kroah-Hartman
2021-12-14 16:20 ` [PATCH 42/43] objtool: kmsan: list KMSAN API functions as uaccess-safe Alexander Potapenko
2021-12-14 16:20 ` [PATCH 43/43] x86: kmsan: enable KMSAN builds for x86 Alexander Potapenko
2021-12-14 16:36 ` [PATCH 00/43] Add KernelMemorySanitizer infrastructure Greg Kroah-Hartman
2021-12-16 10:12 ` Alexander Potapenko
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211214162050.660953-16-glider@google.com \
--to=glider@google.com \
--cc=akpm@linux-foundation.org \
--cc=andreyknvl@google.com \
--cc=ard.biesheuvel@linaro.org \
--cc=arnd@arndb.de \
--cc=axboe@kernel.dk \
--cc=bp@alien8.de \
--cc=cl@linux.com \
--cc=dvyukov@google.com \
--cc=edumazet@google.com \
--cc=elver@google.com \
--cc=gor@linux.ibm.com \
--cc=gregkh@linuxfoundation.org \
--cc=hch@lst.de \
--cc=herbert@gondor.apana.org.au \
--cc=iamjoonsoo.kim@lge.com \
--cc=iii@linux.ibm.com \
--cc=keescook@chromium.org \
--cc=linux-arch@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=luto@kernel.org \
--cc=mingo@redhat.com \
--cc=mst@redhat.com \
--cc=penberg@kernel.org \
--cc=peterz@infradead.org \
--cc=pmladek@suse.com \
--cc=rientjes@google.com \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
--cc=vbabka@suse.cz \
--cc=vegard.nossum@oracle.com \
--cc=viro@zeniv.linux.org.uk \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).