Skip to content

Commit b5c58b2

Browse files
rleonChristoph Hellwig
rleon
authored and
Christoph Hellwig
committed
dma-mapping: direct calls for dma-iommu
Directly call into dma-iommu just like we have been doing for dma-direct for a while. This avoids the indirect call overhead for IOMMU ops and removes the need to have DMA ops entirely for many common configurations. Signed-off-by: Leon Romanovsky <leonro@nvidia.com> Signed-off-by: Leon Romanovsky <leon@kernel.org> Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Acked-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Christoph Hellwig <hch@lst.de>
1 parent f69e342 commit b5c58b2

File tree

10 files changed

+269
-93
lines changed

10 files changed

+269
-93
lines changed

Diff for: ‎MAINTAINERS

+1
Original file line numberDiff line numberDiff line change
@@ -11722,6 +11722,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
1172211722
F: drivers/iommu/dma-iommu.c
1172311723
F: drivers/iommu/dma-iommu.h
1172411724
F: drivers/iommu/iova.c
11725+
F: include/linux/iommu-dma.h
1172511726
F: include/linux/iova.h
1172611727

1172711728
IOMMU SUBSYSTEM

Diff for: ‎drivers/iommu/Kconfig

+1-1
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ config OF_IOMMU
151151
# IOMMU-agnostic DMA-mapping layer
152152
config IOMMU_DMA
153153
def_bool ARM64 || X86 || S390
154-
select DMA_OPS
154+
select DMA_OPS_HELPERS
155155
select IOMMU_API
156156
select IOMMU_IOVA
157157
select IRQ_MSI_IOMMU

Diff for: ‎drivers/iommu/dma-iommu.c

+36-68
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <linux/gfp.h>
1818
#include <linux/huge_mm.h>
1919
#include <linux/iommu.h>
20+
#include <linux/iommu-dma.h>
2021
#include <linux/iova.h>
2122
#include <linux/irq.h>
2223
#include <linux/list_sort.h>
@@ -1037,9 +1038,8 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
10371038
return NULL;
10381039
}
10391040

1040-
static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev,
1041-
size_t size, enum dma_data_direction dir, gfp_t gfp,
1042-
unsigned long attrs)
1041+
struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev, size_t size,
1042+
enum dma_data_direction dir, gfp_t gfp, unsigned long attrs)
10431043
{
10441044
struct dma_sgt_handle *sh;
10451045

@@ -1055,7 +1055,7 @@ static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev,
10551055
return &sh->sgt;
10561056
}
10571057

1058-
static void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
1058+
void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
10591059
struct sg_table *sgt, enum dma_data_direction dir)
10601060
{
10611061
struct dma_sgt_handle *sh = sgt_handle(sgt);
@@ -1066,8 +1066,8 @@ static void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
10661066
kfree(sh);
10671067
}
10681068

1069-
static void iommu_dma_sync_single_for_cpu(struct device *dev,
1070-
dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
1069+
void iommu_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1070+
size_t size, enum dma_data_direction dir)
10711071
{
10721072
phys_addr_t phys;
10731073

@@ -1081,8 +1081,8 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev,
10811081
swiotlb_sync_single_for_cpu(dev, phys, size, dir);
10821082
}
10831083

1084-
static void iommu_dma_sync_single_for_device(struct device *dev,
1085-
dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
1084+
void iommu_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
1085+
size_t size, enum dma_data_direction dir)
10861086
{
10871087
phys_addr_t phys;
10881088

@@ -1096,9 +1096,8 @@ static void iommu_dma_sync_single_for_device(struct device *dev,
10961096
arch_sync_dma_for_device(phys, size, dir);
10971097
}
10981098

1099-
static void iommu_dma_sync_sg_for_cpu(struct device *dev,
1100-
struct scatterlist *sgl, int nelems,
1101-
enum dma_data_direction dir)
1099+
void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
1100+
int nelems, enum dma_data_direction dir)
11021101
{
11031102
struct scatterlist *sg;
11041103
int i;
@@ -1112,9 +1111,8 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev,
11121111
arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
11131112
}
11141113

1115-
static void iommu_dma_sync_sg_for_device(struct device *dev,
1116-
struct scatterlist *sgl, int nelems,
1117-
enum dma_data_direction dir)
1114+
void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
1115+
int nelems, enum dma_data_direction dir)
11181116
{
11191117
struct scatterlist *sg;
11201118
int i;
@@ -1129,9 +1127,9 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,
11291127
arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
11301128
}
11311129

1132-
static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
1133-
unsigned long offset, size_t size, enum dma_data_direction dir,
1134-
unsigned long attrs)
1130+
dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
1131+
unsigned long offset, size_t size, enum dma_data_direction dir,
1132+
unsigned long attrs)
11351133
{
11361134
phys_addr_t phys = page_to_phys(page) + offset;
11371135
bool coherent = dev_is_dma_coherent(dev);
@@ -1189,7 +1187,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
11891187
return iova;
11901188
}
11911189

1192-
static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
1190+
void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
11931191
size_t size, enum dma_data_direction dir, unsigned long attrs)
11941192
{
11951193
struct iommu_domain *domain = iommu_get_dma_domain(dev);
@@ -1342,8 +1340,8 @@ static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg,
13421340
* impedance-matching, to be able to hand off a suitably-aligned list,
13431341
* but still preserve the original offsets and sizes for the caller.
13441342
*/
1345-
static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
1346-
int nents, enum dma_data_direction dir, unsigned long attrs)
1343+
int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1344+
enum dma_data_direction dir, unsigned long attrs)
13471345
{
13481346
struct iommu_domain *domain = iommu_get_dma_domain(dev);
13491347
struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -1462,8 +1460,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
14621460
return ret;
14631461
}
14641462

1465-
static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
1466-
int nents, enum dma_data_direction dir, unsigned long attrs)
1463+
void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1464+
enum dma_data_direction dir, unsigned long attrs)
14671465
{
14681466
dma_addr_t end = 0, start;
14691467
struct scatterlist *tmp;
@@ -1512,15 +1510,15 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
15121510
__iommu_dma_unmap(dev, start, end - start);
15131511
}
15141512

1515-
static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
1513+
dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
15161514
size_t size, enum dma_data_direction dir, unsigned long attrs)
15171515
{
15181516
return __iommu_dma_map(dev, phys, size,
15191517
dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
15201518
dma_get_mask(dev));
15211519
}
15221520

1523-
static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
1521+
void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
15241522
size_t size, enum dma_data_direction dir, unsigned long attrs)
15251523
{
15261524
__iommu_dma_unmap(dev, handle, size);
@@ -1557,7 +1555,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
15571555
dma_free_contiguous(dev, page, alloc_size);
15581556
}
15591557

1560-
static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
1558+
void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
15611559
dma_addr_t handle, unsigned long attrs)
15621560
{
15631561
__iommu_dma_unmap(dev, handle, size);
@@ -1601,8 +1599,8 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
16011599
return NULL;
16021600
}
16031601

1604-
static void *iommu_dma_alloc(struct device *dev, size_t size,
1605-
dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1602+
void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
1603+
gfp_t gfp, unsigned long attrs)
16061604
{
16071605
bool coherent = dev_is_dma_coherent(dev);
16081606
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
@@ -1635,7 +1633,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
16351633
return cpu_addr;
16361634
}
16371635

1638-
static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
1636+
int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
16391637
void *cpu_addr, dma_addr_t dma_addr, size_t size,
16401638
unsigned long attrs)
16411639
{
@@ -1666,7 +1664,7 @@ static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
16661664
vma->vm_page_prot);
16671665
}
16681666

1669-
static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
1667+
int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
16701668
void *cpu_addr, dma_addr_t dma_addr, size_t size,
16711669
unsigned long attrs)
16721670
{
@@ -1693,72 +1691,42 @@ static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
16931691
return ret;
16941692
}
16951693

1696-
static unsigned long iommu_dma_get_merge_boundary(struct device *dev)
1694+
unsigned long iommu_dma_get_merge_boundary(struct device *dev)
16971695
{
16981696
struct iommu_domain *domain = iommu_get_dma_domain(dev);
16991697

17001698
return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
17011699
}
17021700

1703-
static size_t iommu_dma_opt_mapping_size(void)
1701+
size_t iommu_dma_opt_mapping_size(void)
17041702
{
17051703
return iova_rcache_range();
17061704
}
17071705

1708-
static size_t iommu_dma_max_mapping_size(struct device *dev)
1706+
size_t iommu_dma_max_mapping_size(struct device *dev)
17091707
{
17101708
if (dev_is_untrusted(dev))
17111709
return swiotlb_max_mapping_size(dev);
17121710

17131711
return SIZE_MAX;
17141712
}
17151713

1716-
static const struct dma_map_ops iommu_dma_ops = {
1717-
.flags = DMA_F_PCI_P2PDMA_SUPPORTED |
1718-
DMA_F_CAN_SKIP_SYNC,
1719-
.alloc = iommu_dma_alloc,
1720-
.free = iommu_dma_free,
1721-
.alloc_pages_op = dma_common_alloc_pages,
1722-
.free_pages = dma_common_free_pages,
1723-
.alloc_noncontiguous = iommu_dma_alloc_noncontiguous,
1724-
.free_noncontiguous = iommu_dma_free_noncontiguous,
1725-
.mmap = iommu_dma_mmap,
1726-
.get_sgtable = iommu_dma_get_sgtable,
1727-
.map_page = iommu_dma_map_page,
1728-
.unmap_page = iommu_dma_unmap_page,
1729-
.map_sg = iommu_dma_map_sg,
1730-
.unmap_sg = iommu_dma_unmap_sg,
1731-
.sync_single_for_cpu = iommu_dma_sync_single_for_cpu,
1732-
.sync_single_for_device = iommu_dma_sync_single_for_device,
1733-
.sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu,
1734-
.sync_sg_for_device = iommu_dma_sync_sg_for_device,
1735-
.map_resource = iommu_dma_map_resource,
1736-
.unmap_resource = iommu_dma_unmap_resource,
1737-
.get_merge_boundary = iommu_dma_get_merge_boundary,
1738-
.opt_mapping_size = iommu_dma_opt_mapping_size,
1739-
.max_mapping_size = iommu_dma_max_mapping_size,
1740-
};
1741-
17421714
void iommu_setup_dma_ops(struct device *dev)
17431715
{
17441716
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
17451717

17461718
if (dev_is_pci(dev))
17471719
dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;
17481720

1749-
if (iommu_is_dma_domain(domain)) {
1750-
if (iommu_dma_init_domain(domain, dev))
1751-
goto out_err;
1752-
dev->dma_ops = &iommu_dma_ops;
1753-
} else if (dev->dma_ops == &iommu_dma_ops) {
1754-
/* Clean up if we've switched *from* a DMA domain */
1755-
dev->dma_ops = NULL;
1756-
}
1721+
dev->dma_iommu = iommu_is_dma_domain(domain);
1722+
if (dev->dma_iommu && iommu_dma_init_domain(domain, dev))
1723+
goto out_err;
17571724

17581725
return;
17591726
out_err:
1760-
pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
1761-
dev_name(dev));
1727+
pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
1728+
dev_name(dev));
1729+
dev->dma_iommu = false;
17621730
}
17631731

17641732
static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,

Diff for: ‎drivers/iommu/intel/Kconfig

-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ config DMAR_DEBUG
1212
config INTEL_IOMMU
1313
bool "Support for Intel IOMMU using DMA Remapping Devices"
1414
depends on PCI_MSI && ACPI && X86
15-
select DMA_OPS
1615
select IOMMU_API
1716
select IOMMU_IOVA
1817
select IOMMUFD_DRIVER if IOMMUFD

Diff for: ‎include/linux/device.h

+5
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,8 @@ struct device_physical_location {
707707
* for dma allocations. This flag is managed by the dma ops
708708
* instance from ->dma_supported.
709709
* @dma_skip_sync: DMA sync operations can be skipped for coherent buffers.
710+
* @dma_iommu: Device is using default IOMMU implementation for DMA and
711+
* doesn't rely on dma_ops structure.
710712
*
711713
* At the lowest level, every device in a Linux system is represented by an
712714
* instance of struct device. The device structure contains the information
@@ -822,6 +824,9 @@ struct device {
822824
#ifdef CONFIG_DMA_NEED_SYNC
823825
bool dma_skip_sync:1;
824826
#endif
827+
#ifdef CONFIG_IOMMU_DMA
828+
bool dma_iommu:1;
829+
#endif
825830
};
826831

827832
/**

Diff for: ‎include/linux/dma-map-ops.h

-13
Original file line numberDiff line numberDiff line change
@@ -13,20 +13,7 @@
1313
struct cma;
1414
struct iommu_ops;
1515

16-
/*
17-
* Values for struct dma_map_ops.flags:
18-
*
19-
* DMA_F_PCI_P2PDMA_SUPPORTED: Indicates the dma_map_ops implementation can
20-
* handle PCI P2PDMA pages in the map_sg/unmap_sg operation.
21-
* DMA_F_CAN_SKIP_SYNC: DMA sync operations can be skipped if the device is
22-
* coherent and it's not an SWIOTLB buffer.
23-
*/
24-
#define DMA_F_PCI_P2PDMA_SUPPORTED (1 << 0)
25-
#define DMA_F_CAN_SKIP_SYNC (1 << 1)
26-
2716
struct dma_map_ops {
28-
unsigned int flags;
29-
3017
void *(*alloc)(struct device *dev, size_t size,
3118
dma_addr_t *dma_handle, gfp_t gfp,
3219
unsigned long attrs);

0 commit comments

Comments
 (0)