Allow adding MSI capability via vfu_pci_add_capability (#758)

Signed-off-by: Florian Freudiger <25648113+FlorianFreudiger@users.noreply.github.com>

include/libvfio-user.h

@@ -981,6 +981,7 @@ vfu_pci_get_config_space(vfu_ctx_t *vfu_ctx);
  * Certain standard capabilities are handled entirely within the library:
  *
  * PCI_CAP_ID_EXP (pxcap)
+ * PCI_CAP_ID_MSI (msicap)
  * PCI_CAP_ID_MSIX (msixcap)
  * PCI_CAP_ID_PM (pmcap)
  *

include/pci_caps/msi.h

@@ -39,33 +39,38 @@
 extern "C" {
 #endif
 
+/* Message Control for MSI */
 struct mc {
- unsigned int msie:1;
- unsigned int mmc:3;
- unsigned int mme:3;
- unsigned int c64:1;
- unsigned int pvm:1;
- unsigned int res1:7;
+ unsigned int msie:1; /* RW */
+ unsigned int mmc:3; /* RO */
+ unsigned int mme:3; /* RW */
+ unsigned int c64:1; /* RO */
+ unsigned int pvm:1; /* RO */
+ unsigned int res1:7; /* not implemented, extended message data control */
 } __attribute__ ((packed));
 _Static_assert(sizeof(struct mc) == 0x2, "bad MC size");
 
+/* Message Address for MSI */
 struct ma {
- unsigned int res1:2;
- unsigned int addr:30;
+ unsigned int res1:2; /* read must return 0, write has no effect */
+ unsigned int addr:30; /* RW */
 } __attribute__ ((packed));
 _Static_assert(sizeof(struct ma) == 0x4, "bad MA size");
 
+#define VFIO_USER_PCI_CAP_MSI_SIZEOF (0x18)
+
 struct msicap {
  struct cap_hdr hdr;
  struct mc mc;
  struct ma ma;
- uint32_t mua;
- uint16_t md;
- uint16_t padding;
- uint32_t mmask;
- uint32_t mpend;
+ uint32_t mua; /* RW */
+ uint16_t md; /* RW */
+ uint16_t padding; /* not implemented, extended message data */
+ uint32_t mmask; /* RW */
+ uint32_t mpend; /* RO */
 } __attribute__ ((packed));
-_Static_assert(sizeof(struct msicap) == 0x18, "bad MSICAP size");
+_Static_assert(sizeof(struct msicap) == VFIO_USER_PCI_CAP_MSI_SIZEOF,
+ "bad MSICAP size");
 _Static_assert(offsetof(struct msicap, hdr) == 0, "bad offset");
 
 #ifdef __cplusplus

lib/pci_caps.c

@@ -99,6 +99,8 @@ cap_size(vfu_ctx_t *vfu_ctx, void *data, bool extended)
  return PCI_PM_SIZEOF;
  case PCI_CAP_ID_EXP:
  return VFIO_USER_PCI_CAP_EXP_SIZEOF;
+ case PCI_CAP_ID_MSI:
+ return VFIO_USER_PCI_CAP_MSI_SIZEOF;
  case PCI_CAP_ID_MSIX:
  return PCI_CAP_MSIX_SIZEOF;
  case PCI_CAP_ID_VNDR:
@@ -166,6 +168,67 @@ cap_write_pm(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char * buf,
  return ERROR_INT(EINVAL);
 }
 
+static ssize_t
+cap_write_msi(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf,
+ size_t count, loff_t offset)
+{
+ struct msicap *msi = cap_data(vfu_ctx, cap);
+ struct msicap new_msi = *msi;
+
+ memcpy((char *)&new_msi + offset - cap->off, buf, count);
+
+ if (msi->mc.msie != new_msi.mc.msie) {
+ msi->mc.msie = new_msi.mc.msie;
+ vfu_log(vfu_ctx, LOG_DEBUG, "%s MSI",
+ msi->mc.msie ? "enable" : "disable");
+ }
+
+ if (msi->mc.mme != new_msi.mc.mme) {
+ if (new_msi.mc.mme > 5) {
+ vfu_log(vfu_ctx, LOG_ERR,
+ "MSI cannot have more than 32 interrupt vectors");
+ return ERROR_INT(EINVAL);
+ }
+
+ if (new_msi.mc.mme > msi->mc.mmc) {
+ vfu_log(vfu_ctx, LOG_ERR,
+ "MSI cannot have more interrupt vectors"
+ " in MME than defined in MMC");
+ return ERROR_INT(EINVAL);
+ }
+ msi->mc.mme = new_msi.mc.mme;
+
+ vfu_log(vfu_ctx, LOG_DEBUG,
+ "MSI Updated Multiple Message Enable count");
+ }
+
+ if (msi->ma.addr != new_msi.ma.addr) {
+ msi->ma.addr = new_msi.ma.addr;
+ vfu_log(vfu_ctx, LOG_DEBUG,
+ "MSI Message Address set to %x", msi->ma.addr << 2);
+ }
+
+ if (msi->mua != new_msi.mua) {
+ msi->mua = new_msi.mua;
+ vfu_log(vfu_ctx, LOG_DEBUG,
+ "MSI Message Upper Address set to %x", msi->mua);
+ }
+
+ if (msi->md != new_msi.md) {
+ msi->md = new_msi.md;
+ vfu_log(vfu_ctx, LOG_DEBUG,
+ "MSI Message Data set to %x", msi->md);
+ }
+
+ if (msi->mmask != new_msi.mmask) {
+ msi->mmask = new_msi.mmask;
+ vfu_log(vfu_ctx, LOG_DEBUG,
+ "MSI Mask Bits set to %x", msi->mmask);
+ }
+
+ return count;
+}
+
 static ssize_t
 cap_write_msix(vfu_ctx_t *vfu_ctx, struct pci_cap *cap, char *buf,
  size_t count, loff_t offset)
@@ -682,6 +745,10 @@ vfu_pci_add_capability(vfu_ctx_t *vfu_ctx, size_t pos, int flags, void *data)
  cap.name = "PCI Express";
  cap.cb = cap_write_px;
  break;
+ case PCI_CAP_ID_MSI:
+ cap.name = "MSI";
+ cap.cb = cap_write_msi;
+ break;
  case PCI_CAP_ID_MSIX:
  cap.name = "MSI-X";
  cap.cb = cap_write_msix;

test/py/libvfio_user.py

@@ -70,6 +70,7 @@
 
 PCI_CAP_ID_PM = 0x1
 PCI_CAP_ID_VNDR = 0x9
+PCI_CAP_ID_MSI = 0x5
 PCI_CAP_ID_MSIX = 0x11
 PCI_CAP_ID_EXP = 0x10
 
@@ -83,6 +84,12 @@
 
 PCI_EXT_CAP_VNDR_HDR_SIZEOF = 8
 
+# MSI registers
+PCI_MSI_FLAGS = 2 # Message Control offset
+PCI_MSI_ADDRESS_LO = 4 # Message Address offset
+PCI_MSI_FLAGS_ENABLE = 0x0001 # MSI enable
+PCI_CAP_MSI_SIZEOF = 24 # size of MSI registers
+
 # MSI-X registers
 PCI_MSIX_FLAGS = 2 # Message Control
 PCI_MSIX_TABLE = 4 # Table offset

test/py/test_pci_caps.py

@@ -342,6 +342,64 @@ def test_pci_cap_write_px(mock_quiesce, mock_reset):
  expect=errno.EINVAL)
 
 
+def test_pci_cap_write_msi():
+ setup_pci_dev(realize=True)
+ sock = connect_client(ctx)
+
+ # Set MMC to 100b (16 interrupt vectors)
+ mmc = 0b00001000
+
+ # Bad MME with 101b (32 interrupt vectors), over MMC
+ mme_bad = 0b01010000
+ # Test MME with 100b (16 interrupt vectors)
+ mme_good = 0b01000000
+
+ # Test if capability is placed at right offset
+ pos = vfu_pci_add_capability(ctx, pos=0, flags=0,
+ data=struct.pack("ccHIIIII",
+ to_byte(PCI_CAP_ID_MSI),
+ b'\0', mmc, 0, 0, 0, 0, 0))
+ assert pos == cap_offsets[0]
+
+ offset = vfu_pci_find_capability(ctx, False, PCI_CAP_ID_MSI)
+
+ # Test if write fails as expected
+ # as MME is out of bounds, 111b is over the max of 101b (32 vectors)
+ data = b'\xff\xff'
+ write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+ offset=offset + PCI_MSI_FLAGS,
+ count=len(data), data=data, expect=errno.EINVAL)
+
+ # Test if write fails as expected
+ # as MME is over MMC, 101b (32 vectors) > 100b (16 vectors)
+ data = to_bytes_le(mme_bad, 2)
+ write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+ offset=offset + PCI_MSI_FLAGS,
+ count=len(data), data=data, expect=errno.EINVAL)
+
+ # Test good write, MSI Enable + good MME
+ data = to_bytes_le(PCI_MSI_FLAGS_ENABLE | mme_good, 2)
+ write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+ offset=offset + PCI_MSI_FLAGS,
+ count=len(data), data=data)
+
+ size_before_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_FLAGS
+ size_after_flags = PCI_CAP_MSI_SIZEOF - PCI_MSI_ADDRESS_LO
+
+ # reset
+ data = size_before_flags * b'\x00'
+ write_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX,
+ offset=offset + PCI_MSI_FLAGS,
+ count=len(data), data=data)
+
+ # Check if MMC is still present after reset (since it is RO)
+ expected = (to_bytes_le(PCI_CAP_ID_MSI) + b'\x00' +
+ to_bytes_le(mmc, 2) + (size_after_flags * b'\x00'))
+ payload = read_region(ctx, sock, VFU_PCI_DEV_CFG_REGION_IDX, offset=offset,
+ count=len(expected))
+ assert expected == payload
+
+
 def test_pci_cap_write_msix():
  setup_pci_dev(realize=True)
  sock = connect_client(ctx)