Skip to content

Commit

Permalink
busdma: better handling of small segment bouncing
Browse files Browse the repository at this point in the history 
Typically, when a DMA transaction requires bouncing, we will break up
the request into segments that are, at maximum, page-sized.

However, in the atypical case of a driver whose maximum segment size is
smaller than PAGE_SIZE, we end up inefficiently assigning each segment
its own bounce page. For example, the dwmmc driver has a maximum segment
size of 2048 (PAGE_SIZE / 2); a 4-page transfer ends up requiring 8
bounce pages in the current scheme.

We should attempt to batch segments into bounce pages more efficiently.
This is achieved by pushing all considerations of the maximum segment
size into the new _bus_dmamap_addsegs() function, which wraps
_bus_dmamap_addseg(). Thus we allocate the minimal number of bounce
pages required to complete the entire transfer, while still performing
the transfer with smaller-sized transactions.

For most drivers with a segment size >= PAGE_SIZE, this will have no
impact. For drivers like dwmmc mentioned above, this improves the memory
and performance efficiency when bouncing a large transfer.

Co-authored-by:	jhb
Reviewed by:	jhb
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D45048
  • Loading branch information
@mhorne
mhorne committed May 7, 2024
1 parent 5604069 commit a77e1f0
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 91 deletions.
21 changes: 7 additions & 14 deletions sys/arm/arm/busdma_machdep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,7 @@ _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
*/
curaddr = buf;
while (buflen != 0) {
sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = buflen;
if (must_bounce(dmat, map, curaddr, sgsize) != 0) {
sgsize = MIN(sgsize,
PAGE_SIZE - (curaddr & PAGE_MASK));
Expand Down Expand Up @@ -833,7 +833,6 @@ _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, bus_dmamap_t map,
while (vaddr < vendaddr) {
sg_len = MIN(vendaddr - vaddr,
(PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)));
sg_len = MIN(sg_len, dmat->maxsegsz);
if (__predict_true(pmap == kernel_pmap))
paddr = pmap_kextract(vaddr);
else
Expand Down Expand Up @@ -884,7 +883,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,

while (buflen > 0) {
curaddr = buf;
sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = buflen;
if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
sgsize)) {
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
Expand All @@ -908,9 +907,8 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
buf += sgsize;
buflen -= sgsize;
Expand Down Expand Up @@ -1000,11 +998,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
/*
* Compute the segment size, and adjust counts.
*/
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
if (sgsize > dmat->maxsegsz)
sgsize = dmat->maxsegsz;
if (buflen < sgsize)
sgsize = buflen;
sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));

if (map->pagesneeded != 0 && must_bounce(dmat, map, curaddr,
sgsize)) {
Expand Down Expand Up @@ -1037,9 +1031,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
vaddr += sgsize;
buflen -= sgsize;
Expand Down
24 changes: 10 additions & 14 deletions sys/arm64/arm64/busdma_bounce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,7 +643,7 @@ _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
count = 0;
curaddr = buf;
while (buflen != 0) {
sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = buflen;
if (must_bounce(dmat, map, curaddr, sgsize)) {
sgsize = MIN(sgsize,
PAGE_SIZE - (curaddr & PAGE_MASK));
Expand Down Expand Up @@ -696,15 +696,13 @@ _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
vendaddr = (vm_offset_t)buf + buflen;

while (vaddr < vendaddr) {
sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
sg_len = MIN(sg_len, dmat->common.maxsegsz);
sg_len = MIN(vendaddr - vaddr,
PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
if (pmap == kernel_pmap)
paddr = pmap_kextract(vaddr);
else
paddr = pmap_extract(pmap, vaddr);
if (must_bounce(dmat, map, paddr,
min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr &
PAGE_MASK)))) != 0) {
if (must_bounce(dmat, map, paddr, sg_len) != 0) {
sg_len = roundup2(sg_len,
dmat->common.alignment);
map->pagesneeded++;
Expand Down Expand Up @@ -746,7 +744,7 @@ bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,

while (buflen > 0) {
curaddr = buf;
sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = buflen;
if (map->pagesneeded != 0 &&
must_bounce(dmat, map, curaddr, sgsize)) {
/*
Expand Down Expand Up @@ -780,9 +778,8 @@ bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
buf += sgsize;
buflen -= sgsize;
Expand Down Expand Up @@ -858,7 +855,7 @@ bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
/*
* Compute the segment size, and adjust counts.
*/
sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = buflen;
if ((map->flags & DMAMAP_FROM_DMAMEM) == 0)
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));

Expand Down Expand Up @@ -897,9 +894,8 @@ bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
vaddr += sgsize;
buflen -= sgsize;
Expand Down
22 changes: 22 additions & 0 deletions sys/kern/subr_busdma_bounce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,28 @@ _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
return (sgsize);
}

/*
* Add a contiguous physical range to the segment list, respecting the tag's
* maximum segment size and splitting it into multiple segments as necessary.
*/
static bool
_bus_dmamap_addsegs(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
{
bus_size_t done, todo;

while (sgsize > 0) {
todo = MIN(sgsize, dmat_maxsegsz(dmat));
done = _bus_dmamap_addseg(dmat, map, curaddr, todo, segs,
segp);
if (done == 0)
return (false);
curaddr += done;
sgsize -= done;
}
return (true);
}

static void
busdma_thread(void *dummy __unused)
{
Expand Down
26 changes: 9 additions & 17 deletions sys/powerpc/powerpc/busdma_machdep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,7 +487,7 @@ _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
*/
curaddr = buf;
while (buflen != 0) {
sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = buflen;
if (must_bounce(dmat, curaddr)) {
sgsize = MIN(sgsize,
PAGE_SIZE - (curaddr & PAGE_MASK));
Expand Down Expand Up @@ -523,8 +523,8 @@ _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
while (vaddr < vendaddr) {
bus_size_t sg_len;

sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
sg_len = MIN(sg_len, dmat->maxsegsz);
sg_len = MIN(vendaddr - vaddr,
PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
if (pmap == kernel_pmap)
paddr = pmap_kextract(vaddr);
else
Expand Down Expand Up @@ -569,15 +569,14 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,

while (buflen > 0) {
curaddr = buf;
sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = buflen;
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize);
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
buf += sgsize;
buflen -= sgsize;
Expand Down Expand Up @@ -632,8 +631,6 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
vaddr = (vm_offset_t)buf;

while (buflen > 0) {
bus_size_t max_sgsize;

/*
* Get the physical address for this segment.
*/
Expand All @@ -648,20 +645,15 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
/*
* Compute the segment size, and adjust counts.
*/
max_sgsize = MIN(buflen, dmat->maxsegsz);
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));
if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
sgsize = roundup2(sgsize, dmat->alignment);
sgsize = MIN(sgsize, max_sgsize);
curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize);
} else {
sgsize = MIN(sgsize, max_sgsize);
}

sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
vaddr += sgsize;
buflen -= sgsize;
Expand Down
27 changes: 10 additions & 17 deletions sys/riscv/riscv/busdma_bounce.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
*/
curaddr = buf;
while (buflen != 0) {
sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = buflen;
if (addr_needs_bounce(dmat, curaddr)) {
sgsize = MIN(sgsize,
PAGE_SIZE - (curaddr & PAGE_MASK));
Expand Down Expand Up @@ -534,8 +534,8 @@ _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
vendaddr = (vm_offset_t)buf + buflen;

while (vaddr < vendaddr) {
sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
sg_len = MIN(sg_len, dmat->common.maxsegsz);
sg_len = MIN(vendaddr - vaddr,
PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
if (pmap == kernel_pmap)
paddr = pmap_kextract(vaddr);
else
Expand Down Expand Up @@ -582,7 +582,7 @@ bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,

while (buflen > 0) {
curaddr = buf;
sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = buflen;
if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) &&
map->pagesneeded != 0 &&
addr_needs_bounce(dmat, curaddr)) {
Expand All @@ -607,9 +607,8 @@ bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
buf += sgsize;
buflen -= sgsize;
Expand All @@ -631,7 +630,7 @@ bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
int *segp)
{
struct sync_list *sl;
bus_size_t sgsize, max_sgsize;
bus_size_t sgsize;
bus_addr_t curaddr, sl_pend;
vm_offset_t kvaddr, vaddr, sl_vend;
int error;
Expand Down Expand Up @@ -668,17 +667,14 @@ bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
/*
* Compute the segment size, and adjust counts.
*/
max_sgsize = MIN(buflen, dmat->common.maxsegsz);
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));
if (((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) &&
map->pagesneeded != 0 &&
addr_needs_bounce(dmat, curaddr)) {
sgsize = roundup2(sgsize, dmat->common.alignment);
sgsize = MIN(sgsize, max_sgsize);
curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
sgsize);
} else if ((dmat->bounce_flags & BF_COHERENT) == 0) {
sgsize = MIN(sgsize, max_sgsize);
if (map->sync_count > 0) {
sl_pend = sl->paddr + sl->datacount;
sl_vend = sl->vaddr + sl->datacount;
Expand All @@ -704,12 +700,9 @@ bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
sl->datacount = sgsize;
} else
sl->datacount += sgsize;
} else {
sgsize = MIN(sgsize, max_sgsize);
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
if (sgsize == 0)
if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
segp))
break;
vaddr += sgsize;
buflen -= sgsize;
Expand Down
Loading

0 comments on commit a77e1f0

Please sign in to comment.