Feat virtio blk support

README.md

@@ -35,8 +35,9 @@ The following variables are used throughout this document:
 | BRIDGE_IP   | opi-intel-bridge gRPC listening IP address e.g. 10.10.10.10 or localhost                                                                           |
 | BRIDGE_PORT | opi-intel-bridge gRPC listening port e.g. 50051                                                                                                    |
 | BRIDGE_ADDR | BRIDGE_IP:BRIDGE_PORT                                                                                                                              |
-| PF_BDF      | physical function PCI address e.g. 0000:3b:00.1                                                                                                    |
-| VF_BDF      | virtual function PCI address e.g. 0000:40:00.0 can be found in pf's virtfn\<X\> where X equals to virtual_function in CreateNvmeController minus 1 |
+| NVME_PF_BDF | physical function PCI address e.g. 0000:3b:00.1 for Nvme                                                                                           |
+| NVME_VF_BDF | virtual function PCI address e.g. 0000:40:00.0 can be found in pf's virtfn\<X\> where X equals to virtual_function in CreateNvmeController minus 1 |
+| BLK_PF_BDF  | physical function PCI address e.g. 0000:af:01.0 for virtio-blk                                                                                     |
 | TARGET_IP   | storage target ip address                                                                                                                          |
 | TARGET_PORT | storage target port                                                                                                                                |
 
@@ -93,15 +94,15 @@ opi_api.storage.v1.NullDebugService
 or specify commands manually
 
 ```bash
-# PF creation
+# Nvme PF creation
 grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeSubsystem "{nvme_subsystem : {spec : {nqn: 'nqn.2022-09.io.spdk:opitest2', serial_number: 'myserial2', model_number: 'mymodel2', max_namespaces: 11} }, nvme_subsystem_id : 'subsystem2' }"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR ListNvmeSubsystems "{parent : 'todo'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmeSubsystem "{name : '//storage.opiproject.org/volumes/subsystem2'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{nvme_controller : {spec : {nvme_controller_id: 2, subsystem_name_ref : '//storage.opiproject.org/volumes/subsystem2', pcie_id : {physical_function : 0, virtual_function : 0, port_id: 0}, max_nsq:5, max_ncq:5 } }, nvme_controller_id : 'controller1'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR ListNvmeControllers "{parent : '//storage.opiproject.org/volumes/subsystem2'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmeController "{name : '//storage.opiproject.org/volumes/controller1'}"
 
-# VF creation on PF0
+# Nvme VF creation on PF0
 grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeSubsystem "{nvme_subsystem : {spec : {nqn: 'nqn.2022-09.io.spdk:opitest3', serial_number: 'mev-opi-serial', model_number: 'mev-opi-model', max_namespaces: 11} }, nvme_subsystem_id : 'subsystem03' }"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmeController "{nvme_controller : {spec : {nvme_controller_id: 2, subsystem_name_ref : '//storage.opiproject.org/volumes/subsystem03', pcie_id : {physical_function : 0, virtual_function : 3}, max_nsq:5, max_ncq:5 } }, nvme_controller_id : 'controller3'}"
 
@@ -113,6 +114,9 @@ grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateNvmePath "{nvme_path
 grpc_cli call --json_input --json_output $BRIDGE_ADDR ListNvmePaths "{parent : 'todo'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR GetNvmePath "{name: '//storage.opiproject.org/volumes/nvmetcp12path0'}"
 
+# Virtio-blk PF creation (virtio-blk requires a volume, that's why it is created after connection to storage-target)
+grpc_cli --json_input --json_output call $BRIDGE_ADDR CreateVirtioBlk "{virtio_blk_id: 'virtioblk0', virtio_blk : { volume_name_ref: 'nvmetcp12n0', pcie_id: { physical_function: '0', virtual_function: '0', port_id: '0'}}}"
+
 # Create QoS volume
 grpc_cli call --json_input --json_output $BRIDGE_ADDR CreateQosVolume "{'qos_volume' : {'volume_name_ref' :'nvmetcp12n1', 'max_limit' : { 'rw_iops_kiops': 3 } }, 'qos_volume_id' : 'qosnvmetcp12n1' }"
 
@@ -134,29 +138,32 @@ grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteEncryptedVolume "{'n
 # Delete QoS volume
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteQosVolume "{name : '//storage.opiproject.org/volumes/qosnvmetcp12n1'}"
 
+# Delete virtio-blk PF
+grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteVirtioBlk "{ name: '//storage.opiproject.org/volumes/virtioblk0'}"
+
 # Disconnect from storage-target
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmePath "{name: '//storage.opiproject.org/volumes/nvmetcp12path0'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeRemoteController "{name: '//storage.opiproject.org/volumes/nvmetcp12'}"
 
-# Delete VF
+# Delete Nvme VF
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeController "{name : '//storage.opiproject.org/volumes/controller3'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeSubsystem "{name : '//storage.opiproject.org/volumes/subsystem03'}"
 
-# Delete PF
+# Delete Nvme PF
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeController "{name : '//storage.opiproject.org/volumes/controller1'}"
 grpc_cli call --json_input --json_output $BRIDGE_ADDR DeleteNvmeSubsystem "{name : '//storage.opiproject.org/volumes/subsystem2'}"
 ```
 
-To observe devices on host:
+To observe Nvme devices on host:
 
 After PF is created
 
 ```bash
 # Bind driver to PF
 modprobe nvme
-cd /sys/bus/pci/devices/$PF_BDF
+cd /sys/bus/pci/devices/$NVME_PF_BDF
 echo 'nvme' > ./driver_override
-echo $PF_BDF > /sys/bus/pci/drivers/nvme/bind
+echo $NVME_PF_BDF > /sys/bus/pci/drivers/nvme/bind
 
 # Allocate resources and prepare for VF creation
 echo 0 > ./sriov_drivers_autoprobe
@@ -166,27 +173,36 @@ echo 4 > ./sriov_numvfs
 After VF is created
 
 ```bash
-cd /sys/bus/pci/devices/$PF_BDF
+cd /sys/bus/pci/devices/$NVME_PF_BDF
 echo 'nvme' > ./virtfn0/driver_override
-echo $VF_BDF > /sys/bus/pci/drivers/nvme/bind
+echo $NVME_VF_BDF > /sys/bus/pci/drivers/nvme/bind
 ```
 
 Before VF is deleted
 
 ```bash
-cd /sys/bus/pci/devices/$PF_BDF
-echo $VF_BDF > /sys/bus/pci/drivers/nvme/unbind
+cd /sys/bus/pci/devices/$NVME_PF_BDF
+echo $NVME_VF_BDF > /sys/bus/pci/drivers/nvme/unbind
 echo '(null)' > ./virtfn2/driver_override
 ```
 
 Before PF is deleted
 
 ```bash
-cd /sys/bus/pci/devices/$PF_BDF
-echo $PF_BDF > /sys/bus/pci/drivers/nvme/unbind
+cd /sys/bus/pci/devices/$NVME_PF_BDF
+echo $NVME_PF_BDF > /sys/bus/pci/drivers/nvme/unbind
 echo '(null)' > ./driver_override
 ```
 
+To observe Virtio-blk devices on host:
+
+After PF is created
+
+```bash
+echo 1 > /sys/bus/pci/devices/$BLK_PF_BDF/remove
+echo 1 > /sys/bus/pci/rescan
+```
+
 ### Mutual TLS setup
 
 In order to pass configuration data to the xPU securely and only by authenticated/allowed clients it is recommended to secure the gRPC port with mutual TLS.

cmd/main.go

@@ -70,7 +70,7 @@ func main() {
 	middleendOpiIntelServer := me.NewServer(jsonRPC)
 
 	pb.RegisterFrontendNvmeServiceServer(s, frontendOpiIntelServer)
-	pb.RegisterFrontendVirtioBlkServiceServer(s, frontendOpiSpdkServer)
+	pb.RegisterFrontendVirtioBlkServiceServer(s, frontendOpiIntelServer)
 	pb.RegisterFrontendVirtioScsiServiceServer(s, frontendOpiSpdkServer)
 	pb.RegisterNvmeRemoteControllerServiceServer(s, backendOpiSpdkServer)
 	pb.RegisterNullVolumeServiceServer(s, backendOpiSpdkServer)

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/opiproject/gospdk v0.0.0-20230812114418-14a6e1aa7495
 	github.com/opiproject/opi-api v0.0.0-20230826011814-48b273e85a4f
 	github.com/opiproject/opi-smbios-bridge v0.1.3-0.20230826031720-37c30351653d
-	github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230826100942-02082cbd7c2f
+	github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230828145122-166c2bf0724a
 	github.com/opiproject/opi-strongswan-bridge v0.1.1
 	go.einride.tech/aip v0.62.0
 	google.golang.org/grpc v1.57.0

go.sum

@@ -69,6 +69,8 @@ github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230819062743-325cfd95e0e8 h1:e3
 github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230819062743-325cfd95e0e8/go.mod h1:a2Pg0biAqVzFim/nYBcoAsXnJO6F75EA8G+fSbRFKcQ=
 github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230826100942-02082cbd7c2f h1:TT2oyclzRM7NpddO3m4w9Dwx0ecMcKs4R08mTxc7KTQ=
 github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230826100942-02082cbd7c2f/go.mod h1:jnl+YPbFoFfFk9uw0YSMfJ3paOKOtWwQIejkCcqCbiQ=
+github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230828145122-166c2bf0724a h1:zBG+PkDVkMFxZLkgdk9BCDY9ihvdlO2ZdSGm+2d4bpA=
+github.com/opiproject/opi-spdk-bridge v0.1.2-0.20230828145122-166c2bf0724a/go.mod h1:jnl+YPbFoFfFk9uw0YSMfJ3paOKOtWwQIejkCcqCbiQ=
 github.com/opiproject/opi-strongswan-bridge v0.1.1 h1:Mz/8AtA0DD8O/H9jCsDw7wuVoWNiqgXICLsD10XWJ+g=
 github.com/opiproject/opi-strongswan-bridge v0.1.1/go.mod h1:ek3r3zLa9nOfb2a7ybdMJvb5BSGU9I17Xo38UMXfE+k=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

pkg/frontend/blk.go

@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2023 Intel Corporation
+
+// Package frontend implements the FrontEnd APIs (host facing) of the storage Server
+package frontend
+
+import (
+	"fmt"
+
+	"github.com/opiproject/gospdk/spdk"
+	pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
+	"github.com/opiproject/opi-intel-bridge/pkg/models"
+	"github.com/opiproject/opi-spdk-bridge/pkg/frontend"
+)
+
+const (
+	blkTransport = "mev_blk_transport"
+)
+
+type mevBlkTransport struct{}
+
+func (v mevBlkTransport) CreateParams(virtioBlk *pb.VirtioBlk) (any, error) {
+	ctrlr, err := v.getCtrlr(virtioBlk.PcieId)
+	if err != nil {
+		return nil, err
+	}
+
+	vqCount := 1
+	if virtioBlk.GetMaxIoQps() != 0 {
+		vqCount = int(virtioBlk.GetMaxIoQps())
+	}
+
+	return models.MevVhostCreateBlkControllerParams{
+		Ctrlr:     ctrlr,
+		DevName:   virtioBlk.VolumeNameRef,
+		Transport: blkTransport,
+		VqCount:   vqCount,
+	}, nil
+}
+
+func (v mevBlkTransport) DeleteParams(virtioBlk *pb.VirtioBlk) (any, error) {
+	ctrlr, err := v.getCtrlr(virtioBlk.PcieId)
+	if err != nil {
+		return nil, err
+	}
+
+	return spdk.VhostDeleteControllerParams{Ctrlr: ctrlr}, nil
+}
+
+func (v mevBlkTransport) getCtrlr(pci *pb.PciEndpoint) (string, error) {
+	if pci.PortId.Value != 0 {
+		return "", fmt.Errorf("only port 0 is supported")
+	}
+
+	if pci.VirtualFunction.Value != 0 {
+		return "", fmt.Errorf("virtual functions are not supported")
+	}
+
+	return fmt.Sprintf("h0-pf%v-vf0-PF", pci.PhysicalFunction.Value), nil
+}
+
+// NewMevBlkTransport creates an isntance of mevBlkTransport
+func NewMevBlkTransport() frontend.VirtioBlkTransport {
+	return &mevBlkTransport{}
+}

pkg/frontend/blk_test.go

@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2023 Intel Corporation
+
+// Package frontend implements the FrontEnd APIs (host facing) of the storage Server
+package frontend
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/opiproject/gospdk/spdk"
+	pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
+	"github.com/opiproject/opi-intel-bridge/pkg/models"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+func TestFrontEnd_CreateBlkParams(t *testing.T) {
+	tests := map[string]struct {
+		in        *pb.VirtioBlk
+		out       any
+		expectErr bool
+	}{
+		"fail on virtual function": {
+			in: &pb.VirtioBlk{
+				PcieId: &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(0), VirtualFunction: wrapperspb.Int32(1), PortId: wrapperspb.Int32(0)},
+			},
+			out:       nil,
+			expectErr: true,
+		},
+		"fail on non zero port": {
+			in: &pb.VirtioBlk{
+				PcieId: &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(0), VirtualFunction: wrapperspb.Int32(0), PortId: wrapperspb.Int32(1)},
+			},
+			out:       nil,
+			expectErr: true,
+		},
+		"valid pf": {
+			in: &pb.VirtioBlk{
+				PcieId:        &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(1), VirtualFunction: wrapperspb.Int32(0), PortId: wrapperspb.Int32(0)},
+				VolumeNameRef: "volume42",
+				MaxIoQps:      5,
+			},
+			out: models.MevVhostCreateBlkControllerParams{
+				Ctrlr:     "h0-pf1-vf0-PF",
+				DevName:   "volume42",
+				Transport: blkTransport,
+				VqCount:   5,
+			},
+			expectErr: false,
+		},
+		"empty max_io_qps": {
+			in: &pb.VirtioBlk{
+				PcieId:        &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(3), VirtualFunction: wrapperspb.Int32(0), PortId: wrapperspb.Int32(0)},
+				VolumeNameRef: "volume42",
+			},
+			out: models.MevVhostCreateBlkControllerParams{
+				Ctrlr:     "h0-pf3-vf0-PF",
+				DevName:   "volume42",
+				Transport: blkTransport,
+				VqCount:   1,
+			},
+			expectErr: false,
+		},
+	}
+	for testName, tt := range tests {
+		t.Run(testName, func(t *testing.T) {
+			transport := NewMevBlkTransport()
+
+			params, err := transport.CreateParams(tt.in)
+
+			if (err != nil) != tt.expectErr {
+				t.Errorf("Expected error: %v, received: %v", tt.expectErr, err)
+			}
+			if !reflect.DeepEqual(params, tt.out) {
+				t.Errorf("Expected params: %v, received %v", tt.out, params)
+			}
+		})
+	}
+}
+
+func TestFrontEnd_DeleteBlkParams(t *testing.T) {
+	tests := map[string]struct {
+		in        *pb.VirtioBlk
+		out       any
+		expectErr bool
+	}{
+		"fail on virtual function": {
+			in: &pb.VirtioBlk{
+				PcieId: &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(0), VirtualFunction: wrapperspb.Int32(1), PortId: wrapperspb.Int32(0)},
+			},
+			out:       nil,
+			expectErr: true,
+		},
+		"fail on non zero port": {
+			in: &pb.VirtioBlk{
+				PcieId: &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(0), VirtualFunction: wrapperspb.Int32(0), PortId: wrapperspb.Int32(1)},
+			},
+			out:       nil,
+			expectErr: true,
+		},
+		"valid pf": {
+			in: &pb.VirtioBlk{
+				PcieId:        &pb.PciEndpoint{PhysicalFunction: wrapperspb.Int32(1), VirtualFunction: wrapperspb.Int32(0), PortId: wrapperspb.Int32(0)},
+				VolumeNameRef: "volume42",
+			},
+			out: spdk.VhostDeleteControllerParams{
+				Ctrlr: "h0-pf1-vf0-PF",
+			},
+			expectErr: false,
+		},
+	}
+	for testName, tt := range tests {
+		t.Run(testName, func(t *testing.T) {
+			transport := NewMevBlkTransport()
+
+			params, err := transport.DeleteParams(tt.in)
+
+			if (err != nil) != tt.expectErr {
+				t.Errorf("Expected error: %v, received: %v", tt.expectErr, err)
+			}
+			if !reflect.DeepEqual(params, tt.out) {
+				t.Errorf("Expected params: %v, received %v", tt.out, params)
+			}
+		})
+	}
+}

pkg/frontend/frontend.go

@@ -14,15 +14,18 @@ import (
 // Server contains frontend related OPI services
 type Server struct {
 	pb.FrontendNvmeServiceServer
+	pb.FrontendVirtioBlkServiceServer
 
 	nvme *frontend.NvmeParameters
 	rpc  spdk.JSONRPC
 }
 
 // NewServer creates initialized instance of Nvme server
 func NewServer(jsonRPC spdk.JSONRPC) *Server {
-	opiSpdkServer := frontend.NewServerWithSubsystemListener(jsonRPC, NewSubsystemListener())
+	opiSpdkServer := frontend.NewCustomizedServer(
+		jsonRPC, NewSubsystemListener(), NewMevBlkTransport())
 	return &Server{
+		opiSpdkServer,
 		opiSpdkServer,
 		&opiSpdkServer.Nvme,
 		jsonRPC,

pkg/models/spdk.go

@@ -37,3 +37,11 @@ type NpiQosBwIopsLimitParams struct {
 
 // NpiQosBwIopsLimitResult is the result of setting QoS limits
 type NpiQosBwIopsLimitResult bool
+
+// MevVhostCreateBlkControllerParams holds parameters to create a virtio-blk device
+type MevVhostCreateBlkControllerParams struct {
+	Ctrlr     string `json:"ctrlr"`
+	DevName   string `json:"dev_name"`
+	Transport string `json:"transport"`
+	VqCount   int    `json:"vq_count,omitempty"`
+}