Skip to content

Commit 0b20fce

Browse files
chaseyuJaegeuk Kim
chaseyu
authored and
Jaegeuk Kim
committed
f2fs: cache global IPU bio
In commit 8648de2 ("f2fs: add bio cache for IPU"), we added f2fs_submit_ipu_bio() in __write_data_page() as below: __write_data_page() if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode)) { f2fs_submit_ipu_bio(sbi, bio, page); .... } in order to avoid below deadlock: Thread A Thread B - __write_data_page (inode x, page y) - f2fs_do_write_data_page - set_page_writeback ---- set writeback flag in page y - f2fs_inplace_write_data - f2fs_balance_fs - lock gc_mutex - lock gc_mutex - f2fs_gc - do_garbage_collect - gc_data_segment - move_data_page - f2fs_wait_on_page_writeback - wait_on_page_writeback --- wait writeback of page y However, the bio submission breaks the merge of IPU IOs. So in this patch let's add a global bio cache for merged IPU pages, then f2fs_wait_on_page_writeback() is able to submit bio if a writebacked page is cached in global bio cache. Signed-off-by: Chao Yu <yuchao0@huawei.com> Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
1 parent 688078e commit 0b20fce

File tree

4 files changed

+169
-32
lines changed

4 files changed

+169
-32
lines changed

‎fs/f2fs/data.c‎

Lines changed: 147 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#define NUM_PREALLOC_POST_READ_CTXS 128
3030

3131
static struct kmem_cache *bio_post_read_ctx_cache;
32+
static struct kmem_cache *bio_entry_slab;
3233
static mempool_t *bio_post_read_ctx_pool;
3334

3435
static bool __is_cp_guaranteed(struct page *page)
@@ -543,6 +544,126 @@ static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
543544
return io_type_is_mergeable(io, fio);
544545
}
545546

547+
static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
548+
struct page *page, enum temp_type temp)
549+
{
550+
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
551+
struct bio_entry *be;
552+
553+
be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS);
554+
be->bio = bio;
555+
bio_get(bio);
556+
557+
if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
558+
f2fs_bug_on(sbi, 1);
559+
560+
down_write(&io->bio_list_lock);
561+
list_add_tail(&be->list, &io->bio_list);
562+
up_write(&io->bio_list_lock);
563+
}
564+
565+
static void del_bio_entry(struct bio_entry *be)
566+
{
567+
list_del(&be->list);
568+
kmem_cache_free(bio_entry_slab, be);
569+
}
570+
571+
static int add_ipu_page(struct f2fs_sb_info *sbi, struct bio **bio,
572+
struct page *page)
573+
{
574+
enum temp_type temp;
575+
bool found = false;
576+
int ret = -EAGAIN;
577+
578+
for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
579+
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
580+
struct list_head *head = &io->bio_list;
581+
struct bio_entry *be;
582+
583+
down_write(&io->bio_list_lock);
584+
list_for_each_entry(be, head, list) {
585+
if (be->bio != *bio)
586+
continue;
587+
588+
found = true;
589+
590+
if (bio_add_page(*bio, page, PAGE_SIZE, 0) == PAGE_SIZE) {
591+
ret = 0;
592+
break;
593+
}
594+
595+
/* bio is full */
596+
del_bio_entry(be);
597+
__submit_bio(sbi, *bio, DATA);
598+
break;
599+
}
600+
up_write(&io->bio_list_lock);
601+
}
602+
603+
if (ret) {
604+
bio_put(*bio);
605+
*bio = NULL;
606+
}
607+
608+
return ret;
609+
}
610+
611+
void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
612+
struct bio **bio, struct page *page)
613+
{
614+
enum temp_type temp;
615+
bool found = false;
616+
struct bio *target = bio ? *bio : NULL;
617+
618+
for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
619+
struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
620+
struct list_head *head = &io->bio_list;
621+
struct bio_entry *be;
622+
623+
if (list_empty(head))
624+
continue;
625+
626+
down_read(&io->bio_list_lock);
627+
list_for_each_entry(be, head, list) {
628+
if (target)
629+
found = (target == be->bio);
630+
else
631+
found = __has_merged_page(be->bio, NULL,
632+
page, 0);
633+
if (found)
634+
break;
635+
}
636+
up_read(&io->bio_list_lock);
637+
638+
if (!found)
639+
continue;
640+
641+
found = false;
642+
643+
down_write(&io->bio_list_lock);
644+
list_for_each_entry(be, head, list) {
645+
if (target)
646+
found = (target == be->bio);
647+
else
648+
found = __has_merged_page(be->bio, NULL,
649+
page, 0);
650+
if (found) {
651+
target = be->bio;
652+
del_bio_entry(be);
653+
break;
654+
}
655+
}
656+
up_write(&io->bio_list_lock);
657+
}
658+
659+
if (found)
660+
__submit_bio(sbi, target, DATA);
661+
if (bio && *bio) {
662+
bio_put(*bio);
663+
*bio = NULL;
664+
}
665+
}
666+
546667
int f2fs_merge_page_bio(struct f2fs_io_info *fio)
547668
{
548669
struct bio *bio = *fio->bio;
@@ -557,20 +678,17 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
557678
f2fs_trace_ios(fio, 0);
558679

559680
if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
560-
fio->new_blkaddr)) {
561-
__submit_bio(fio->sbi, bio, fio->type);
562-
bio = NULL;
563-
}
681+
fio->new_blkaddr))
682+
f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
564683
alloc_new:
565684
if (!bio) {
566685
bio = __bio_alloc(fio, BIO_MAX_PAGES);
567686
bio_set_op_attrs(bio, fio->op, fio->op_flags);
568-
}
569687

570-
if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
571-
__submit_bio(fio->sbi, bio, fio->type);
572-
bio = NULL;
573-
goto alloc_new;
688+
add_bio_entry(fio->sbi, bio, page, fio->temp);
689+
} else {
690+
if (add_ipu_page(fio->sbi, &bio, page))
691+
goto alloc_new;
574692
}
575693

576694
if (fio->io_wbc)
@@ -584,19 +702,6 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
584702
return 0;
585703
}
586704

587-
static void f2fs_submit_ipu_bio(struct f2fs_sb_info *sbi, struct bio **bio,
588-
struct page *page)
589-
{
590-
if (!bio)
591-
return;
592-
593-
if (!__has_merged_page(*bio, NULL, page, 0))
594-
return;
595-
596-
__submit_bio(sbi, *bio, DATA);
597-
*bio = NULL;
598-
}
599-
600705
void f2fs_submit_page_write(struct f2fs_io_info *fio)
601706
{
602707
struct f2fs_sb_info *sbi = fio->sbi;
@@ -2215,14 +2320,12 @@ static int __write_data_page(struct page *page, bool *submitted,
22152320

22162321
unlock_page(page);
22172322
if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2218-
!F2FS_I(inode)->cp_task) {
2219-
f2fs_submit_ipu_bio(sbi, bio, page);
2323+
!F2FS_I(inode)->cp_task)
22202324
f2fs_balance_fs(sbi, need_balance_fs);
2221-
}
22222325

22232326
if (unlikely(f2fs_cp_error(sbi))) {
2224-
f2fs_submit_ipu_bio(sbi, bio, page);
22252327
f2fs_submit_merged_write(sbi, DATA);
2328+
f2fs_submit_merged_ipu_write(sbi, bio, NULL);
22262329
submitted = NULL;
22272330
}
22282331

@@ -2342,13 +2445,11 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
23422445
}
23432446

23442447
if (PageWriteback(page)) {
2345-
if (wbc->sync_mode != WB_SYNC_NONE) {
2448+
if (wbc->sync_mode != WB_SYNC_NONE)
23462449
f2fs_wait_on_page_writeback(page,
23472450
DATA, true, true);
2348-
f2fs_submit_ipu_bio(sbi, &bio, page);
2349-
} else {
2451+
else
23502452
goto continue_unlock;
2351-
}
23522453
}
23532454

23542455
if (!clear_page_dirty_for_io(page))
@@ -2406,7 +2507,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
24062507
NULL, 0, DATA);
24072508
/* submit cached bio of IPU write */
24082509
if (bio)
2409-
__submit_bio(sbi, bio, DATA);
2510+
f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
24102511

24112512
return ret;
24122513
}
@@ -3211,8 +3312,22 @@ int __init f2fs_init_post_read_processing(void)
32113312
return -ENOMEM;
32123313
}
32133314

3214-
void __exit f2fs_destroy_post_read_processing(void)
3315+
void f2fs_destroy_post_read_processing(void)
32153316
{
32163317
mempool_destroy(bio_post_read_ctx_pool);
32173318
kmem_cache_destroy(bio_post_read_ctx_cache);
32183319
}
3320+
3321+
int __init f2fs_init_bio_entry_cache(void)
3322+
{
3323+
bio_entry_slab = f2fs_kmem_cache_create("bio_entry_slab",
3324+
sizeof(struct bio_entry));
3325+
if (!bio_entry_slab)
3326+
return -ENOMEM;
3327+
return 0;
3328+
}
3329+
3330+
void __exit f2fs_destroy_bio_entry_cache(void)
3331+
{
3332+
kmem_cache_destroy(bio_entry_slab);
3333+
}

‎fs/f2fs/f2fs.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1068,6 +1068,11 @@ struct f2fs_io_info {
10681068
unsigned char version; /* version of the node */
10691069
};
10701070

1071+
struct bio_entry {
1072+
struct bio *bio;
1073+
struct list_head list;
1074+
};
1075+
10711076
#define is_read_io(rw) ((rw) == READ)
10721077
struct f2fs_bio_info {
10731078
struct f2fs_sb_info *sbi; /* f2fs superblock */
@@ -1077,6 +1082,8 @@ struct f2fs_bio_info {
10771082
struct rw_semaphore io_rwsem; /* blocking op for bio */
10781083
spinlock_t io_lock; /* serialize DATA/NODE IOs */
10791084
struct list_head io_list; /* track fios */
1085+
struct list_head bio_list; /* bio entry list head */
1086+
struct rw_semaphore bio_list_lock; /* lock to protect bio entry list */
10801087
};
10811088

10821089
#define FDEV(i) (sbi->devs[i])
@@ -3195,10 +3202,14 @@ void f2fs_destroy_checkpoint_caches(void);
31953202
*/
31963203
int f2fs_init_post_read_processing(void);
31973204
void f2fs_destroy_post_read_processing(void);
3205+
int f2fs_init_bio_entry_cache(void);
3206+
void f2fs_destroy_bio_entry_cache(void);
31983207
void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
31993208
void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
32003209
struct inode *inode, struct page *page,
32013210
nid_t ino, enum page_type type);
3211+
void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
3212+
struct bio **bio, struct page *page);
32023213
void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
32033214
int f2fs_submit_page_bio(struct f2fs_io_info *fio);
32043215
int f2fs_merge_page_bio(struct f2fs_io_info *fio);

‎fs/f2fs/segment.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3379,7 +3379,10 @@ void f2fs_wait_on_page_writeback(struct page *page,
33793379
if (PageWriteback(page)) {
33803380
struct f2fs_sb_info *sbi = F2FS_P_SB(page);
33813381

3382+
/* submit cached LFS IO */
33823383
f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3384+
/* sbumit cached IPU IO */
3385+
f2fs_submit_merged_ipu_write(sbi, NULL, page);
33833386
if (ordered) {
33843387
wait_on_page_writeback(page);
33853388
f2fs_bug_on(sbi, locked && PageWriteback(page));

‎fs/f2fs/super.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3342,6 +3342,8 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
33423342
sbi->write_io[i][j].bio = NULL;
33433343
spin_lock_init(&sbi->write_io[i][j].io_lock);
33443344
INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);
3345+
INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list);
3346+
init_rwsem(&sbi->write_io[i][j].bio_list_lock);
33453347
}
33463348
}
33473349

@@ -3753,8 +3755,13 @@ static int __init init_f2fs_fs(void)
37533755
err = f2fs_init_post_read_processing();
37543756
if (err)
37553757
goto free_root_stats;
3758+
err = f2fs_init_bio_entry_cache();
3759+
if (err)
3760+
goto free_post_read;
37563761
return 0;
37573762

3763+
free_post_read:
3764+
f2fs_destroy_post_read_processing();
37583765
free_root_stats:
37593766
f2fs_destroy_root_stats();
37603767
unregister_filesystem(&f2fs_fs_type);
@@ -3778,6 +3785,7 @@ static int __init init_f2fs_fs(void)
37783785

37793786
static void __exit exit_f2fs_fs(void)
37803787
{
3788+
f2fs_destroy_bio_entry_cache();
37813789
f2fs_destroy_post_read_processing();
37823790
f2fs_destroy_root_stats();
37833791
unregister_filesystem(&f2fs_fs_type);

0 commit comments

Comments
 (0)