Skip to content

Commit 19a809a

Browse files
aagittorvalds
aagit
authored and
torvalds
committed
userfaultfd: teach vma_merge to merge across vma->vm_userfaultfd_ctx
vma->vm_userfaultfd_ctx is yet another vma parameter that vma_merge must be aware about so that we can merge vmas back like they were originally before arming the userfaultfd on some memory range. Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Acked-by: Pavel Emelyanov <xemul@parallels.com> Cc: Sanidhya Kashyap <sanidhya.gatech@gmail.com> Cc: zhang.zhanghailiang@huawei.com Cc: "Kirill A. Shutemov" <kirill@shutemov.name> Cc: Andres Lagar-Cavilla <andreslc@google.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Hugh Dickins <hughd@google.com> Cc: Peter Feiner <pfeiner@google.com> Cc: "Dr. David Alan Gilbert" <dgilbert@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: "Huangpeng (Peter)" <peter.huangpeng@huawei.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 6b251fc commit 19a809a

File tree

6 files changed

+36
-19
lines changed

6 files changed

+36
-19
lines changed

‎include/linux/mm.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1835,7 +1835,7 @@ extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
18351835
extern struct vm_area_struct *vma_merge(struct mm_struct *,
18361836
struct vm_area_struct *prev, unsigned long addr, unsigned long end,
18371837
unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1838-
struct mempolicy *);
1838+
struct mempolicy *, struct vm_userfaultfd_ctx);
18391839
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
18401840
extern int split_vma(struct mm_struct *,
18411841
struct vm_area_struct *, unsigned long addr, int new_below);

‎mm/madvise.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ static long madvise_behavior(struct vm_area_struct *vma,
103103

104104
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
105105
*prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
106-
vma->vm_file, pgoff, vma_policy(vma));
106+
vma->vm_file, pgoff, vma_policy(vma),
107+
vma->vm_userfaultfd_ctx);
107108
if (*prev) {
108109
vma = *prev;
109110
goto success;

‎mm/mempolicy.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -722,8 +722,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
722722
pgoff = vma->vm_pgoff +
723723
((vmstart - vma->vm_start) >> PAGE_SHIFT);
724724
prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
725-
vma->anon_vma, vma->vm_file, pgoff,
726-
new_pol);
725+
vma->anon_vma, vma->vm_file, pgoff,
726+
new_pol, vma->vm_userfaultfd_ctx);
727727
if (prev) {
728728
vma = prev;
729729
next = vma->vm_next;

‎mm/mlock.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -510,7 +510,8 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
510510

511511
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
512512
*prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
513-
vma->vm_file, pgoff, vma_policy(vma));
513+
vma->vm_file, pgoff, vma_policy(vma),
514+
vma->vm_userfaultfd_ctx);
514515
if (*prev) {
515516
vma = *prev;
516517
goto success;

‎mm/mmap.c‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#include <linux/notifier.h>
4242
#include <linux/memory.h>
4343
#include <linux/printk.h>
44+
#include <linux/userfaultfd_k.h>
4445

4546
#include <asm/uaccess.h>
4647
#include <asm/cacheflush.h>
@@ -919,7 +920,8 @@ again: remove_next = 1 + (end > next->vm_end);
919920
* per-vma resources, so we don't attempt to merge those.
920921
*/
921922
static inline int is_mergeable_vma(struct vm_area_struct *vma,
922-
struct file *file, unsigned long vm_flags)
923+
struct file *file, unsigned long vm_flags,
924+
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
923925
{
924926
/*
925927
* VM_SOFTDIRTY should not prevent from VMA merging, if we
@@ -935,6 +937,8 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma,
935937
return 0;
936938
if (vma->vm_ops && vma->vm_ops->close)
937939
return 0;
940+
if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
941+
return 0;
938942
return 1;
939943
}
940944

@@ -965,9 +969,11 @@ static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
965969
*/
966970
static int
967971
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
968-
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
972+
struct anon_vma *anon_vma, struct file *file,
973+
pgoff_t vm_pgoff,
974+
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
969975
{
970-
if (is_mergeable_vma(vma, file, vm_flags) &&
976+
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
971977
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
972978
if (vma->vm_pgoff == vm_pgoff)
973979
return 1;
@@ -984,9 +990,11 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
984990
*/
985991
static int
986992
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
987-
struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
993+
struct anon_vma *anon_vma, struct file *file,
994+
pgoff_t vm_pgoff,
995+
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
988996
{
989-
if (is_mergeable_vma(vma, file, vm_flags) &&
997+
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
990998
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
991999
pgoff_t vm_pglen;
9921000
vm_pglen = vma_pages(vma);
@@ -1029,7 +1037,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
10291037
struct vm_area_struct *prev, unsigned long addr,
10301038
unsigned long end, unsigned long vm_flags,
10311039
struct anon_vma *anon_vma, struct file *file,
1032-
pgoff_t pgoff, struct mempolicy *policy)
1040+
pgoff_t pgoff, struct mempolicy *policy,
1041+
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
10331042
{
10341043
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
10351044
struct vm_area_struct *area, *next;
@@ -1056,14 +1065,17 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
10561065
if (prev && prev->vm_end == addr &&
10571066
mpol_equal(vma_policy(prev), policy) &&
10581067
can_vma_merge_after(prev, vm_flags,
1059-
anon_vma, file, pgoff)) {
1068+
anon_vma, file, pgoff,
1069+
vm_userfaultfd_ctx)) {
10601070
/*
10611071
* OK, it can. Can we now merge in the successor as well?
10621072
*/
10631073
if (next && end == next->vm_start &&
10641074
mpol_equal(policy, vma_policy(next)) &&
10651075
can_vma_merge_before(next, vm_flags,
1066-
anon_vma, file, pgoff+pglen) &&
1076+
anon_vma, file,
1077+
pgoff+pglen,
1078+
vm_userfaultfd_ctx) &&
10671079
is_mergeable_anon_vma(prev->anon_vma,
10681080
next->anon_vma, NULL)) {
10691081
/* cases 1, 6 */
@@ -1084,7 +1096,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
10841096
if (next && end == next->vm_start &&
10851097
mpol_equal(policy, vma_policy(next)) &&
10861098
can_vma_merge_before(next, vm_flags,
1087-
anon_vma, file, pgoff+pglen)) {
1099+
anon_vma, file, pgoff+pglen,
1100+
vm_userfaultfd_ctx)) {
10881101
if (prev && addr < prev->vm_end) /* case 4 */
10891102
err = vma_adjust(prev, prev->vm_start,
10901103
addr, prev->vm_pgoff, NULL);
@@ -1570,8 +1583,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
15701583
/*
15711584
* Can we just expand an old mapping?
15721585
*/
1573-
vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff,
1574-
NULL);
1586+
vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
1587+
NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
15751588
if (vma)
15761589
goto out;
15771590

@@ -2757,7 +2770,7 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
27572770

27582771
/* Can we just expand an old private anonymous mapping? */
27592772
vma = vma_merge(mm, prev, addr, addr + len, flags,
2760-
NULL, NULL, pgoff, NULL);
2773+
NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
27612774
if (vma)
27622775
goto out;
27632776

@@ -2913,7 +2926,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
29132926
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
29142927
return NULL; /* should never get here */
29152928
new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2916-
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
2929+
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
2930+
vma->vm_userfaultfd_ctx);
29172931
if (new_vma) {
29182932
/*
29192933
* Source vma may have been merged into new_vma

‎mm/mprotect.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,8 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
292292
*/
293293
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
294294
*pprev = vma_merge(mm, *pprev, start, end, newflags,
295-
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
295+
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
296+
vma->vm_userfaultfd_ctx);
296297
if (*pprev) {
297298
vma = *pprev;
298299
goto success;

0 commit comments

Comments
 (0)