Skip to content

Commit

Permalink
bio: add allocation cache abstraction
Browse files Browse the repository at this point in the history 
Add a per-cpu bio_set cache for bio allocations, enabling us to quickly
recycle them instead of going through the slab allocator. This cache
isn't IRQ safe, and hence is only really suitable for polled IO.

Very simple - keeps a count of bio's in the cache, and maintains a max
of 512 with a slack of 64. If we get above max + slack, we drop slack
number of bio's.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
@axboe
axboe committed Aug 23, 2021
1 parent 6c7ef54 commit be4d234
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 14 deletions.
134 changes: 120 additions & 14 deletions block/bio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@
#include "blk.h"
#include "blk-rq-qos.h"

struct bio_alloc_cache {
struct bio_list free_list;
unsigned int nr;
};

static struct biovec_slab {
int nr_vecs;
char *name;
Expand Down Expand Up @@ -619,6 +624,53 @@ void guard_bio_eod(struct bio *bio)
bio_truncate(bio, maxsector << 9);
}

#define ALLOC_CACHE_MAX 512
#define ALLOC_CACHE_SLACK 64

static void bio_alloc_cache_prune(struct bio_alloc_cache *cache,
unsigned int nr)
{
unsigned int i = 0;
struct bio *bio;

while ((bio = bio_list_pop(&cache->free_list)) != NULL) {
cache->nr--;
bio_free(bio);
if (++i == nr)
break;
}
}

static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node)
{
struct bio_set *bs;

bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead);
if (bs->cache) {
struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);

bio_alloc_cache_prune(cache, -1U);
}
return 0;
}

static void bio_alloc_cache_destroy(struct bio_set *bs)
{
int cpu;

if (!bs->cache)
return;

cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
for_each_possible_cpu(cpu) {
struct bio_alloc_cache *cache;

cache = per_cpu_ptr(bs->cache, cpu);
bio_alloc_cache_prune(cache, -1U);
}
free_percpu(bs->cache);
}

/**
* bio_put - release a reference to a bio
* @bio: bio to release reference to
Expand All @@ -629,16 +681,23 @@ void guard_bio_eod(struct bio *bio)
**/
void bio_put(struct bio *bio)
{
if (!bio_flagged(bio, BIO_REFFED))
bio_free(bio);
else {
if (unlikely(bio_flagged(bio, BIO_REFFED))) {
BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
if (!atomic_dec_and_test(&bio->__bi_cnt))
return;
}

/*
* last put frees it
*/
if (atomic_dec_and_test(&bio->__bi_cnt))
bio_free(bio);
if (bio_flagged(bio, BIO_PERCPU_CACHE)) {
struct bio_alloc_cache *cache;

bio_uninit(bio);
cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu());
bio_list_add_head(&cache->free_list, bio);
if (++cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK)
bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK);
put_cpu();
} else {
bio_free(bio);
}
}
EXPORT_SYMBOL(bio_put);
Expand Down Expand Up @@ -1530,6 +1589,7 @@ int biovec_init_pool(mempool_t *pool, int pool_entries)
*/
void bioset_exit(struct bio_set *bs)
{
bio_alloc_cache_destroy(bs);
if (bs->rescue_workqueue)
destroy_workqueue(bs->rescue_workqueue);
bs->rescue_workqueue = NULL;
Expand Down Expand Up @@ -1591,12 +1651,18 @@ int bioset_init(struct bio_set *bs,
biovec_init_pool(&bs->bvec_pool, pool_size))
goto bad;

if (!(flags & BIOSET_NEED_RESCUER))
return 0;

bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
if (!bs->rescue_workqueue)
goto bad;
if (flags & BIOSET_NEED_RESCUER) {
bs->rescue_workqueue = alloc_workqueue("bioset",
WQ_MEM_RECLAIM, 0);
if (!bs->rescue_workqueue)
goto bad;
}
if (flags & BIOSET_PERCPU_CACHE) {
bs->cache = alloc_percpu(struct bio_alloc_cache);
if (!bs->cache)
goto bad;
cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead);
}

return 0;
bad:
Expand All @@ -1623,6 +1689,43 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
}
EXPORT_SYMBOL(bioset_init_from_src);

/**
* bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb
* @kiocb: kiocb describing the IO
* @bs: bio_set to allocate from
*
* Description:
* Like @bio_alloc_bioset, but pass in the kiocb. The kiocb is only
* used to check if we should dip into the per-cpu bio_set allocation
* cache. The allocation uses GFP_KERNEL internally.
*
*/
struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs,
struct bio_set *bs)
{
struct bio_alloc_cache *cache;
struct bio *bio;

if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS)
return bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs);

cache = per_cpu_ptr(bs->cache, get_cpu());
bio = bio_list_pop(&cache->free_list);
if (bio) {
cache->nr--;
put_cpu();
bio_init(bio, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs);
bio->bi_pool = bs;
bio_set_flag(bio, BIO_PERCPU_CACHE);
return bio;
}
put_cpu();
bio = bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs);
bio_set_flag(bio, BIO_PERCPU_CACHE);
return bio;
}
EXPORT_SYMBOL_GPL(bio_alloc_kiocb);

static int __init init_bio(void)
{
int i;
Expand All @@ -1637,6 +1740,9 @@ static int __init init_bio(void)
SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
}

cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL,
bio_cpu_dead);

if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
panic("bio: can't allocate bios\n");

Expand Down
13 changes: 13 additions & 0 deletions include/linux/bio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,7 @@ static inline struct bio *bio_next_split(struct bio *bio, int sectors,
enum {
BIOSET_NEED_BVECS = BIT(0),
BIOSET_NEED_RESCUER = BIT(1),
BIOSET_PERCPU_CACHE = BIT(2),
};
extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
extern void bioset_exit(struct bio_set *);
Expand All @@ -409,6 +410,8 @@ extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);

struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs,
struct bio_set *bs);
struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs,
struct bio_set *bs);
struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
extern void bio_put(struct bio *);

Expand Down Expand Up @@ -699,6 +702,11 @@ struct bio_set {
struct kmem_cache *bio_slab;
unsigned int front_pad;

/*
* per-cpu bio alloc cache
*/
struct bio_alloc_cache __percpu *cache;

mempool_t bio_pool;
mempool_t bvec_pool;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
Expand All @@ -715,6 +723,11 @@ struct bio_set {
struct bio_list rescue_list;
struct work_struct rescue_work;
struct workqueue_struct *rescue_workqueue;

/*
* Hot un-plug notifier for the per-cpu cache, if used
*/
struct hlist_node cpuhp_dead;
};

static inline bool bioset_initialized(struct bio_set *bs)
Expand Down
1 change: 1 addition & 0 deletions include/linux/blk_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ enum {
BIO_TRACKED, /* set if bio goes through the rq_qos path */
BIO_REMAPPED,
BIO_ZONE_WRITE_LOCKED, /* Owns a zoned device zone write lock */
BIO_PERCPU_CACHE, /* can participate in per-cpu alloc cache */
BIO_FLAG_LAST
};

Expand Down
1 change: 1 addition & 0 deletions include/linux/cpuhotplug.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ enum cpuhp_state {
CPUHP_ARM_OMAP_WAKE_DEAD,
CPUHP_IRQ_POLL_DEAD,
CPUHP_BLOCK_SOFTIRQ_DEAD,
CPUHP_BIO_DEAD,
CPUHP_ACPI_CPUDRV_DEAD,
CPUHP_S390_PFAULT_DEAD,
CPUHP_BLK_MQ_DEAD,
Expand Down

0 comments on commit be4d234

Please sign in to comment.