Skip to content

Commit

Permalink
migrate_pages: batch _unmap and _move
Browse files Browse the repository at this point in the history 
In this patch the _unmap and _move stage of the folio migration is
batched.  That for, previously, it is,

  for each folio
    _unmap()
    _move()

Now, it is,

  for each folio
    _unmap()
  for each folio
    _move()

Based on this, we can batch the TLB flushing and use some hardware
accelerator to copy folios between batched _unmap and batched _move
stages.

Link: https://lkml.kernel.org/r/20230213123444.155149-6-ying.huang@intel.com
Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Bharata B Rao <bharata@amd.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Xin Hao <xhao@linux.alibaba.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
  • Loading branch information
@yhuang-intel @akpm00
yhuang-intel authored and akpm00 committed Feb 17, 2023
1 parent 64c8902 commit 5dfab10
Showing 1 changed file with 189 additions and 25 deletions.
214 changes: 189 additions & 25 deletions mm/migrate.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1051,6 +1051,33 @@ static void __migrate_folio_extract(struct folio *dst,
dst->private = NULL;
}

/* Restore the source folio to the original state upon failure */
static void migrate_folio_undo_src(struct folio *src,
int page_was_mapped,
struct anon_vma *anon_vma,
struct list_head *ret)
{
if (page_was_mapped)
remove_migration_ptes(src, src, false);
/* Drop an anon_vma reference if we took one */
if (anon_vma)
put_anon_vma(anon_vma);
folio_unlock(src);
list_move_tail(&src->lru, ret);
}

/* Restore the destination folio to the original state upon failure */
static void migrate_folio_undo_dst(struct folio *dst,
free_page_t put_new_page,
unsigned long private)
{
folio_unlock(dst);
if (put_new_page)
put_new_page(&dst->page, private);
else
folio_put(dst);
}

/* Cleanup src folio upon migration success */
static void migrate_folio_done(struct folio *src,
enum migrate_reason reason)
Expand All @@ -1069,8 +1096,8 @@ static void migrate_folio_done(struct folio *src,
folio_put(src);
}

static int __migrate_folio_unmap(struct folio *src, struct folio *dst,
int force, enum migrate_mode mode)
static int __migrate_folio_unmap(struct folio *src, struct folio *dst, int force,
bool avoid_force_lock, enum migrate_mode mode)
{
int rc = -EAGAIN;
int page_was_mapped = 0;
Expand All @@ -1097,6 +1124,17 @@ static int __migrate_folio_unmap(struct folio *src, struct folio *dst,
if (current->flags & PF_MEMALLOC)
goto out;

/*
* We have locked some folios and are going to wait to lock
* this folio. To avoid a potential deadlock, let's bail
* out and not do that. The locked folios will be moved and
* unlocked, then we can wait to lock this folio.
*/
if (avoid_force_lock) {
rc = -EDEADLOCK;
goto out;
}

folio_lock(src);
}

Expand Down Expand Up @@ -1205,10 +1243,20 @@ static int __migrate_folio_move(struct folio *src, struct folio *dst,
int page_was_mapped = 0;
struct anon_vma *anon_vma = NULL;
bool is_lru = !__PageMovable(&src->page);
struct list_head *prev;

__migrate_folio_extract(dst, &page_was_mapped, &anon_vma);
prev = dst->lru.prev;
list_del(&dst->lru);

rc = move_to_new_folio(dst, src, mode);

if (rc == -EAGAIN) {
list_add(&dst->lru, prev);
__migrate_folio_record(dst, page_was_mapped, anon_vma);
return rc;
}

if (unlikely(!is_lru))
goto out_unlock_both;

Expand Down Expand Up @@ -1251,7 +1299,7 @@ static int __migrate_folio_move(struct folio *src, struct folio *dst,
/* Obtain the lock on page, remove all ptes. */
static int migrate_folio_unmap(new_page_t get_new_page, free_page_t put_new_page,
unsigned long private, struct folio *src,
struct folio **dstp, int force,
struct folio **dstp, int force, bool avoid_force_lock,
enum migrate_mode mode, enum migrate_reason reason,
struct list_head *ret)
{
Expand Down Expand Up @@ -1279,15 +1327,15 @@ static int migrate_folio_unmap(new_page_t get_new_page, free_page_t put_new_page
*dstp = dst;

dst->private = NULL;
rc = __migrate_folio_unmap(src, dst, force, mode);
rc = __migrate_folio_unmap(src, dst, force, avoid_force_lock, mode);
if (rc == MIGRATEPAGE_UNMAP)
return rc;

/*
* A folio that has not been unmapped will be restored to
* right list unless we want to retry.
*/
if (rc != -EAGAIN)
if (rc != -EAGAIN && rc != -EDEADLOCK)
list_move_tail(&src->lru, ret);

if (put_new_page)
Expand Down Expand Up @@ -1326,9 +1374,8 @@ static int migrate_folio_move(free_page_t put_new_page, unsigned long private,
*/
if (rc == MIGRATEPAGE_SUCCESS) {
migrate_folio_done(src, reason);
} else {
if (rc != -EAGAIN)
list_add_tail(&src->lru, ret);
} else if (rc != -EAGAIN) {
list_add_tail(&src->lru, ret);

if (put_new_page)
put_new_page(&dst->page, private);
Expand Down Expand Up @@ -1603,12 +1650,16 @@ static int migrate_hugetlbs(struct list_head *from, new_page_t get_new_page,
return nr_failed;
}

/*
* migrate_pages_batch() first unmaps folios in the from list as many as
* possible, then move the unmapped folios.
*/
static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
free_page_t put_new_page, unsigned long private,
enum migrate_mode mode, int reason, struct list_head *ret_folios,
struct migrate_pages_stats *stats)
{
int retry = 1;
int retry;
int large_retry = 1;
int thp_retry = 1;
int nr_failed = 0;
Expand All @@ -1617,13 +1668,19 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
int pass = 0;
bool is_large = false;
bool is_thp = false;
struct folio *folio, *folio2, *dst = NULL;
int rc, nr_pages;
struct folio *folio, *folio2, *dst = NULL, *dst2;
int rc, rc_saved, nr_pages;
LIST_HEAD(split_folios);
LIST_HEAD(unmap_folios);
LIST_HEAD(dst_folios);
bool nosplit = (reason == MR_NUMA_MISPLACED);
bool no_split_folio_counting = false;
bool avoid_force_lock;

split_folio_migration:
retry:
rc_saved = 0;
avoid_force_lock = false;
retry = 1;
for (pass = 0;
pass < NR_MAX_MIGRATE_PAGES_RETRY && (retry || large_retry);
pass++) {
Expand All @@ -1645,16 +1702,15 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
cond_resched();

rc = migrate_folio_unmap(get_new_page, put_new_page, private,
folio, &dst, pass > 2, mode,
reason, ret_folios);
if (rc == MIGRATEPAGE_UNMAP)
rc = migrate_folio_move(put_new_page, private,
folio, dst, mode,
reason, ret_folios);
folio, &dst, pass > 2, avoid_force_lock,
mode, reason, ret_folios);
/*
* The rules are:
* Success: folio will be freed
* Unmap: folio will be put on unmap_folios list,
* dst folio put on dst_folios list
* -EAGAIN: stay on the from list
* -EDEADLOCK: stay on the from list
* -ENOMEM: stay on the from list
* -ENOSYS: stay on the from list
* Other errno: put on ret_folios list
Expand Down Expand Up @@ -1689,7 +1745,7 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
case -ENOMEM:
/*
* When memory is low, don't bother to try to migrate
* other folios, just exit.
* other folios, move unmapped folios, then exit.
*/
if (is_large) {
nr_large_failed++;
Expand Down Expand Up @@ -1728,7 +1784,19 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
/* nr_failed isn't updated for not used */
nr_large_failed += large_retry;
stats->nr_thp_failed += thp_retry;
goto out;
rc_saved = rc;
if (list_empty(&unmap_folios))
goto out;
else
goto move;
case -EDEADLOCK:
/*
* The folio cannot be locked for potential deadlock.
* Go move (and unlock) all locked folios. Then we can
* try again.
*/
rc_saved = rc;
goto move;
case -EAGAIN:
if (is_large) {
large_retry++;
Expand All @@ -1742,6 +1810,15 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
stats->nr_succeeded += nr_pages;
stats->nr_thp_succeeded += is_thp;
break;
case MIGRATEPAGE_UNMAP:
/*
* We have locked some folios, don't force lock
* to avoid deadlock.
*/
avoid_force_lock = true;
list_move_tail(&folio->lru, &unmap_folios);
list_add_tail(&dst->lru, &dst_folios);
break;
default:
/*
* Permanent failure (-EBUSY, etc.):
Expand All @@ -1765,25 +1842,112 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
nr_large_failed += large_retry;
stats->nr_thp_failed += thp_retry;
stats->nr_failed_pages += nr_retry_pages;
move:
retry = 1;
for (pass = 0;
pass < NR_MAX_MIGRATE_PAGES_RETRY && (retry || large_retry);
pass++) {
retry = 0;
large_retry = 0;
thp_retry = 0;
nr_retry_pages = 0;

dst = list_first_entry(&dst_folios, struct folio, lru);
dst2 = list_next_entry(dst, lru);
list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
is_large = folio_test_large(folio);
is_thp = is_large && folio_test_pmd_mappable(folio);
nr_pages = folio_nr_pages(folio);

cond_resched();

rc = migrate_folio_move(put_new_page, private,
folio, dst, mode,
reason, ret_folios);
/*
* The rules are:
* Success: folio will be freed
* -EAGAIN: stay on the unmap_folios list
* Other errno: put on ret_folios list
*/
switch(rc) {
case -EAGAIN:
if (is_large) {
large_retry++;
thp_retry += is_thp;
} else if (!no_split_folio_counting) {
retry++;
}
nr_retry_pages += nr_pages;
break;
case MIGRATEPAGE_SUCCESS:
stats->nr_succeeded += nr_pages;
stats->nr_thp_succeeded += is_thp;
break;
default:
if (is_large) {
nr_large_failed++;
stats->nr_thp_failed += is_thp;
} else if (!no_split_folio_counting) {
nr_failed++;
}

stats->nr_failed_pages += nr_pages;
break;
}
dst = dst2;
dst2 = list_next_entry(dst, lru);
}
}
nr_failed += retry;
nr_large_failed += large_retry;
stats->nr_thp_failed += thp_retry;
stats->nr_failed_pages += nr_retry_pages;

if (rc_saved)
rc = rc_saved;
else
rc = nr_failed + nr_large_failed;
out:
/* Cleanup remaining folios */
dst = list_first_entry(&dst_folios, struct folio, lru);
dst2 = list_next_entry(dst, lru);
list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
int page_was_mapped = 0;
struct anon_vma *anon_vma = NULL;

__migrate_folio_extract(dst, &page_was_mapped, &anon_vma);
migrate_folio_undo_src(folio, page_was_mapped, anon_vma,
ret_folios);
list_del(&dst->lru);
migrate_folio_undo_dst(dst, put_new_page, private);
dst = dst2;
dst2 = list_next_entry(dst, lru);
}

/*
* Try to migrate split folios of fail-to-migrate large folios, no
* nr_failed counting in this round, since all split folios of a
* large folio is counted as 1 failure in the first round.
*/
if (!list_empty(&split_folios)) {
if (rc >= 0 && !list_empty(&split_folios)) {
/*
* Move non-migrated folios (after NR_MAX_MIGRATE_PAGES_RETRY
* retries) to ret_folios to avoid migrating them again.
*/
list_splice_init(from, ret_folios);
list_splice_init(&split_folios, from);
no_split_folio_counting = true;
retry = 1;
goto split_folio_migration;
goto retry;
}

rc = nr_failed + nr_large_failed;
out:
/*
* We have unlocked all locked folios, so we can force lock now, let's
* try again.
*/
if (rc == -EDEADLOCK)
goto retry;

return rc;
}

Expand Down

0 comments on commit 5dfab10

Please sign in to comment.