From 1cca91aeb8ae8f531f9c3b9c59d83630e9941a5e Mon Sep 17 00:00:00 2001 From: Florian Freudiger <25648113+FlorianFreudiger@users.noreply.github.com> Date: Tue, 15 Aug 2023 13:24:23 +0200 Subject: [PATCH] Allow adding MSI capability via vfu_pci_add_capability (#758) Signed-off-by: Florian Freudiger <25648113+FlorianFreudiger@users.noreply.github.com> --- include/libvfio-user.h | 1 + include/pci_caps/msi.h | 33 +++++++++++--------- lib/pci_caps.c | 67 ++++++++++++++++++++++++++++++++++++++++ test/py/libvfio_user.py | 7 +++++ test/py/test_pci_caps.py | 58 ++++++++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 14 deletions(-) diff --git a/include/libvfio-user.h b/include/libvfio-user.h index 7cd28b59..72369b62 100644 --- a/include/libvfio-user.h +++ b/include/libvfio-user.h @@ -981,6 +981,7 @@ vfu_pci_get_config_space(vfu_ctx_t *vfu_ctx); * Certain standard capabilities are handled entirely within the library: * * PCI_CAP_ID_EXP (pxcap) + * PCI_CAP_ID_MSI (msicap) * PCI_CAP_ID_MSIX (msixcap) * PCI_CAP_ID_PM (pmcap) * diff --git a/include/pci_caps/msi.h b/include/pci_caps/msi.h index c148c9e5..481abeea 100644 --- a/include/pci_caps/msi.h +++ b/include/pci_caps/msi.h @@ -39,33 +39,38 @@ extern "C" { #endif +/* Message Control for MSI */ struct mc { - unsigned int msie:1; - unsigned int mmc:3; - unsigned int mme:3; - unsigned int c64:1; - unsigned int pvm:1; - unsigned int res1:7; + unsigned int msie:1; /* RW */ + unsigned int mmc:3; /* RO */ + unsigned int mme:3; /* RW */ + unsigned int c64:1; /* RO */ + unsigned int pvm:1; /* RO */ + unsigned int res1:7; /* not implemented, extended message data control */ } __attribute__ ((packed)); _Static_assert(sizeof(struct mc) == 0x2, "bad MC size"); +/* Message Address for MSI */ struct ma { - unsigned int res1:2; - unsigned int addr:30; + unsigned int res1:2; /* read must return 0, write has no effect */ + unsigned int addr:30; /* RW */ } __attribute__ ((packed)); _Static_assert(sizeof(struct ma) == 0x4, "bad MA size"); +#define VFIO_USER_PCI_CAP_MSI_SIZEOF (0x18) + struct msicap { struct cap_hdr hdr; struct mc mc; struct ma ma; - uint32_t mua; - uint16_t md; - uint16_t padding; - uint32_t mmask; - uint32_t mpend; + uint32_t mua; /* RW */ + uint16_t md; /* RW */ + uint16_t padding; /* not implemented, extended message data */ + uint32_t mmask; /* RW */ + uint32_t mpend; /* RO */ } __attribute__ ((packed)); -_Static_assert(sizeof(struct msicap) == 0x18, "bad MSICAP size"); +_Static_assert(sizeof(struct msicap) == VFIO_USER_PCI_CAP_MSI_SIZEOF, + "bad MSICAP size"); _Static_assert(offsetof(struct msicap, hdr) == 0, "bad offset"); #ifdef __cplusplus diff --git a/lib/pci_caps.c b/lib/pci_caps.c index 39acec8d..a6f400f5 100644 --- a/lib/pci_caps.c +++ b/lib/pci_caps.c @@ -99,6 +99,8 @@ cap_size(vfu_ctx_t *vfu_ctx, void *data, bool extended) return PCI_PM_SIZEOF; case PCI_CAP_ID_EXP: return VFIO_USER_PCI_CAP_EXP_SIZEOF; + case PCI_CAP_ID_MSI: + return VFIO_USER_PCI_CAP_MSI_SIZEOF; case PCI_CAP_ID_MSIX: return PCI_CAP_MSIX_SIZEOF; case PCI_CAP_ID_VNDR: @@ -166,6 +168,67 @@ cap_write_pm(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char * buf, return ERROR_INT(EINVAL); } +static ssize_t +cap_write_msi(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf, + size_t count, loff_t offset) +{ + struct msicap *msi = cap_data(vfu_ctx, cap); + struct msicap new_msi = *msi; + + memcpy((char *)&new_msi + offset - cap->off, buf, count); + + if (msi->mc.msie != new_msi.mc.msie) { + msi->mc.msie = new_msi.mc.msie; + vfu_log(vfu_ctx, LOG_DEBUG, "%s MSI", + msi->mc.msie ? "enable" : "disable"); + } + + if (msi->mc.mme != new_msi.mc.mme) { + if (new_msi.mc.mme > 5) { + vfu_log(vfu_ctx, LOG_ERR, + "MSI cannot have more than 32 interrupt vectors"); + return ERROR_INT(EINVAL); + } + + if (new_msi.mc.mme > msi->mc.mmc) { + vfu_log(vfu_ctx, LOG_ERR, + "MSI cannot have more interrupt vectors" + " in MME than defined in MMC"); + return ERROR_INT(EINVAL); + } + msi->mc.mme = new_msi.mc.mme; + + vfu_log(vfu_ctx, LOG_DEBUG, + "MSI Updated Multiple Message Enable count"); + } + + if (msi->ma.addr != new_msi.ma.addr) { + msi->ma.addr = new_msi.ma.addr; + vfu_log(vfu_ctx, LOG_DEBUG, + "MSI Message Address set to %x", msi->ma.addr << 2); + } + + if (msi->mua != new_msi.mua) { + msi->mua = new_msi.mua; + vfu_log(vfu_ctx, LOG_DEBUG, + "MSI Message Upper Address set to %x", msi->mua); + } + + if (msi->md != new_msi.md) { + msi->md = new_msi.md; + vfu_log(vfu_ctx, LOG_DEBUG, + "MSI Message Data set to %x", msi->md); + } + + if (msi->mmask != new_msi.mmask) { + msi->mmask = new_msi.mmask; + vfu_log(vfu_ctx, LOG_DEBUG, + "MSI Mask Bits set to %x", msi->mmask); + } + + return count; +} + static ssize_t cap_write_msix(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf, size_t count, loff_t offset) @@ -682,6 +745,10 @@ vfu_pci_add_capability(vfu_ctx_t *vfu_ctx, size_t pos, int flags, void *data) cap.name = "PCI Express"; cap.cb = cap_write_px; break; + case PCI_CAP_ID_MSI: + cap.name = "MSI"; + cap.cb = cap_write_msi; + break; case PCI_CAP_ID_MSIX: cap.name = "MSI-X"; cap.cb = cap_write_msix; diff --git a/test/py/libvfio_user.py b/test/py/libvfio_user.py index 8848dbff..6d60798d 100644 --- a/test/py/libvfio_user.py +++ b/test/py/libvfio_user.py @@ -70,6 +70,7 @@ PCI_CAP_ID_PM = 0x1 PCI_CAP_ID_VNDR = 0x9 +PCI_CAP_ID_MSI = 0x5 PCI_CAP_ID_MSIX = 0x11 PCI_CAP_ID_EXP = 0x10 @@ -83,6 +84,12 @@ PCI_EXT_CAP_VNDR_HDR_SIZEOF = 8 +# MSI registers +PCI_MSI_FLAGS = 2 # Message Control offset +PCI_MSI_ADDRESS_LO = 4 # Message Address offset +PCI_MSI_FLAGS_ENABLE = 0x0001 # MSI enable +PCI_CAP_MSI_SIZEOF = 24 # size of MSI registers + # MSI-X registers PCI_MSIX_FLAGS = 2 # Message Control PCI_MSIX_TABLE = 4 # Table offset diff --git a/test/py/test_pci_caps.py b/test/py/test_pci_caps.py index edd16833..b7ad07b8 100644 --- a/test/py/test_pci_caps.py +++ b/test/py/test_pci_caps.py @@ -342,6 +342,64 @@ def test_pci_cap_write_px(mock_quiesce, mock_reset): expect=errno.EINVAL) +def test_pci_cap_write_msi(): + setup_pci_dev(realize=True) + sock = connect_client(ctx) + + # Set MMC to 100b (16 interrupt vectors) + mmc = 0b00001000 + + # Bad MME with 101b (32 interrupt vectors), over MMC + mme_bad = 0b01010000 + # Test MME with 100b (16 interrupt vectors) + mme_good = 0b01000000 + + # Test if capability is placed at right offset + pos = vfu_pci_add_capability(ctx, pos=0, flags=0, + data=struct.pack("ccHIIIII", + to_byte(PCI_CAP_ID_MSI), + b'\0', mmc, 0, 0, 0, 0, 0)) + assert pos == cap_offsets[0] + + offset = vfu_pci_find_capability(ctx, False, PCI_CAP_ID_MSI) + + # Test if write fails as expected + # as MME is out of bounds, 111b is over the max of 101b (32 vectors) + data = b'\xff\xff' + write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, + offset=offset + PCI_MSI_FLAGS, + count=len(data), data=data, expect=errno.EINVAL) + + # Test if write fails as expected + # as MME is over MMC, 101b (32 vectors) > 100b (16 vectors) + data = to_bytes_le(mme_bad, 2) + write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, + offset=offset + PCI_MSI_FLAGS, + count=len(data), data=data, expect=errno.EINVAL) + + # Test good write, MSI Enable + good MME + data = to_bytes_le(PCI_MSI_FLAGS_ENABLE | mme_good, 2) + write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, + offset=offset + PCI_MSI_FLAGS, + count=len(data), data=data) + + size_before_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_FLAGS + size_after_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_ADDRESS_LO + + # reset + data = size_before_flags * b'\x00' + write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, + offset=offset + PCI_MSI_FLAGS, + count=len(data), data=data) + + # Check if MMC is still present after reset (since it is RO) + expected = (to_bytes_le(PCI_CAP_ID_MSI) + b'\x00' + + to_bytes_le(mmc, 2) + (size_after_flags * b'\x00')) + payload = read_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, offset=offset, + count=len(expected)) + assert expected == payload + + def test_pci_cap_write_msix(): setup_pci_dev(realize=True) sock = connect_client(ctx)