Skip to content

Commit

Permalink
Merge pull request #212 from lstocchi/i129
Browse files Browse the repository at this point in the history
feat: add support for NBD
  • Loading branch information
openshift-merge-bot[bot] authored Nov 25, 2024
2 parents 2f144cf + 86ed52a commit 4af22e9
Show file tree
Hide file tree
Showing 9 changed files with 362 additions and 10 deletions.
5 changes: 5 additions & 0 deletions cmd/vfkit/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,11 @@ func runVirtualMachine(vmConfig *config.VirtualMachine, vm *vf.VirtualMachine) e
defer closer.Close()
}

if err := vf.ListenNetworkBlockDevices(vm); err != nil {
log.Debugf("%v", err)
return err
}

if err := setupGuestTimeSync(vm, vmConfig.TimeSync()); err != nil {
log.Warnf("Error configuring guest time synchronization")
log.Debugf("%v", err)
Expand Down
22 changes: 22 additions & 0 deletions doc/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,28 @@ This adds a read only USB mass storage device to the VM which will be backed by
--device usb-mass-storage,path=/Users/virtuser/distro.iso,readonly
```

### Network Block Device

#### Description

The `--device nbd` option allows to connect to a remote NBD server, effectively accessing a remote block device over the network as if it were a local disk.

The NBD client running on the VM is informed in case the connection drops and it tries to reconnect automatically to the server.

#### Arguments
- `uri`: the URI that refers to the NBD server to which the NBD client will connect, e.g. `nbd://10.10.2.8:10000/export`. More info at https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md
- `deviceId`: `/dev/disk/by-id/virtio-` identifier to use for this device.
- `sync`: the mode in which the NBD client synchronizes data with the NBD server. It can be `full`or `none`, more info at https://developer.apple.com/documentation/virtualization/vzdisksynchronizationmode?language=objc
- `timeout`: the timeout value in milliseconds for the connection between the client and server
- `readonly`: if specified the device will be read only.

#### Example

This allows to connect to the export of the remote NBD server:
```
--device nbd,uri=nbd://192.168.64.4:11111/export,deviceId=nbd1,timeout=3000
```


### Networking

Expand Down
10 changes: 10 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,16 @@ func (vm *VirtualMachine) VirtioVsockDevices() []*VirtioVsock {
return vsockDevs
}

func (vm *VirtualMachine) NetworkBlockDevice(deviceID string) *NetworkBlockDevice {
for _, dev := range vm.Devices {
if nbdDev, isNbdDev := dev.(*NetworkBlockDevice); isNbdDev && nbdDev.DeviceIdentifier == deviceID {
return nbdDev
}
}

return nil
}

// AddDevice adds a dev to vm. This device can be created with one of the
// VirtioXXXNew methods.
func (vm *VirtualMachine) AddDevice(dev VirtioDevice) error {
Expand Down
23 changes: 23 additions & 0 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,26 @@ func TestAddIgnitionFile_OneOption(t *testing.T) {
assert.Equal(t, uint32(ignitionVsockPort), vm.Devices[0].(*VirtioVsock).Port)
assert.Equal(t, "file1", vm.Ignition.ConfigPath)
}

func TestNetworkBlockDevice(t *testing.T) {
vm := &VirtualMachine{}
gpu, _ := VirtioGPUNew()
vm.Devices = append(vm.Devices, gpu)
nbd, _ := NetworkBlockDeviceNew("uri", 1000, SynchronizationFullMode)
nbd.DeviceIdentifier = "nbd1"
vm.Devices = append(vm.Devices, nbd)
nbd2, _ := NetworkBlockDeviceNew("uri2", 1000, SynchronizationFullMode)
nbd2.DeviceIdentifier = "nbd2"
vm.Devices = append(vm.Devices, nbd2)

nbdItem := vm.NetworkBlockDevice("nbd2")
assert.Equal(t, "nbd2", nbdItem.DeviceIdentifier)
assert.Equal(t, "uri2", nbdItem.URI)
}

func TestNetworkBlockDevice_NoDevice(t *testing.T) {
vm := &VirtualMachine{}

nbdItem := vm.NetworkBlockDevice("nbd2")
require.Nil(t, nbdItem)
}
16 changes: 16 additions & 0 deletions pkg/config/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
nvme vmComponentKind = "nvme"
rosetta vmComponentKind = "rosetta"
ignition vmComponentKind = "ignition"
vfNbd vmComponentKind = "nbd"
)

type jsonKind struct {
Expand Down Expand Up @@ -174,6 +175,10 @@ func unmarshalDevice(rawMsg json.RawMessage) (VirtioDevice, error) {
var newDevice USBMassStorage
err = json.Unmarshal(rawMsg, &newDevice)
dev = &newDevice
case vfNbd:
var newDevice NetworkBlockDevice
err = json.Unmarshal(rawMsg, &newDevice)
dev = &newDevice
default:
err = fmt.Errorf("unknown 'kind' field: '%s'", kind)
}
Expand Down Expand Up @@ -395,3 +400,14 @@ func (ign *Ignition) MarshalJSON() ([]byte, error) {
Ignition: *ign,
})
}

func (dev *NetworkBlockDevice) MarshalJSON() ([]byte, error) {
type devWithKind struct {
jsonKind
NetworkBlockDevice
}
return json.Marshal(devWithKind{
jsonKind: kind(vfNbd),
NetworkBlockDevice: *dev,
})
}
22 changes: 17 additions & 5 deletions pkg/config/json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,12 +170,15 @@ var jsonTests = map[string]jsonTest{
// rosetta
rosetta, err := RosettaShareNew("vz-rosetta")
require.NoError(t, err)
err = vm.AddDevices(fs, usb, rosetta)
// NBD
nbd, err := NetworkBlockDeviceNew("uri", 1, SynchronizationFullMode)
require.NoError(t, err)
err = vm.AddDevices(fs, usb, rosetta, nbd)
require.NoError(t, err)

return vm
},
expectedJSON: `{"vcpus":3,"memoryBytes":4194304000,"bootloader":{"kind":"linuxBootloader","vmlinuzPath":"/vmlinuz","initrdPath":"/initrd","kernelCmdLine":"console=hvc0"},"devices":[{"kind":"virtioserial","logFile":"/virtioserial"},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","nat":true,"macAddress":"00:11:22:33:44:55"},{"kind":"virtiorng"},{"kind":"virtioblk","devName":"virtio-blk","imagePath":"/virtioblk"},{"kind":"virtiosock","port":1234,"socketURL":"/virtiovsock"},{"kind":"virtiofs","mountTag":"tag","sharedDir":"/virtiofs"},{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"/usbmassstorage","readOnly":true},{"kind":"rosetta","mountTag":"vz-rosetta","installRosetta":false,"ignoreIfMissing":false}]}`,
expectedJSON: `{"vcpus":3,"memoryBytes":4194304000,"bootloader":{"kind":"linuxBootloader","vmlinuzPath":"/vmlinuz","initrdPath":"/initrd","kernelCmdLine":"console=hvc0"},"devices":[{"kind":"virtioserial","logFile":"/virtioserial"},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","nat":true,"macAddress":"00:11:22:33:44:55"},{"kind":"virtiorng"},{"kind":"virtioblk","devName":"virtio-blk","imagePath":"/virtioblk"},{"kind":"virtiosock","port":1234,"socketURL":"/virtiovsock"},{"kind":"virtiofs","mountTag":"tag","sharedDir":"/virtiofs"},{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"/usbmassstorage","readOnly":true},{"kind":"rosetta","mountTag":"vz-rosetta","installRosetta":false,"ignoreIfMissing":false},{"kind":"nbd", "devName":"nbd", "uri":"uri", "SynchronizationMode":"full","Timeout":1000000}]}`,
},
}

Expand Down Expand Up @@ -274,7 +277,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{
return blk
},

skipFields: []string{"DevName"},
skipFields: []string{"DevName", "URI"},
expectedJSON: `{"kind":"virtioblk","devName":"virtio-blk","imagePath":"ImagePath","readOnly":true,"deviceIdentifier":"DeviceIdentifier"}`,
},
"USBMassStorage": {
Expand All @@ -283,7 +286,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{
require.NoError(t, err)
return usb
},
skipFields: []string{"DevName"},
skipFields: []string{"DevName", "URI"},
expectedJSON: `{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"ImagePath","readOnly":true}`,
},
"NVMExpressController": {
Expand All @@ -292,7 +295,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{
require.NoError(t, err)
return nvme
},
skipFields: []string{"DevName"},
skipFields: []string{"DevName", "URI"},
expectedJSON: `{"kind":"nvme","devName":"nvme","imagePath":"ImagePath","readOnly":true}`,
},
"LinuxBootloader": {
Expand All @@ -307,6 +310,15 @@ var jsonStabilityTests = map[string]jsonStabilityTest{
obj: &TimeSync{},
expectedJSON: `{"vsockPort":3}`,
},
"NetworkBlockDevice": {
newObjectFunc: func(t *testing.T) any {
nbd, err := NetworkBlockDeviceNew("uri", 1000, SynchronizationFullMode)
require.NoError(t, err)
return nbd
},
skipFields: []string{"DevName", "ImagePath"},
expectedJSON: `{"kind":"nbd","deviceIdentifier":"DeviceIdentifier","devName":"nbd","uri":"URI","readOnly":true,"SynchronizationMode":"SynchronizationMode","Timeout":2}`,
},
}

type jsonStabilityTest struct {
Expand Down
110 changes: 106 additions & 4 deletions pkg/config/virtio.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"strconv"
"strings"
"time"
)

// The VirtioDevice interface is an interface which is implemented by all virtio devices.
Expand Down Expand Up @@ -108,6 +109,19 @@ type VirtioSerial struct {
UsesStdio bool `json:"usesStdio,omitempty"`
}

type NBDSynchronizationMode string

const (
SynchronizationFullMode NBDSynchronizationMode = "full"
SynchronizationNoneMode NBDSynchronizationMode = "none"
)

type NetworkBlockDevice struct {
VirtioBlk
Timeout time.Duration
SynchronizationMode NBDSynchronizationMode
}

// TODO: Add VirtioBalloon
// https://github.com/Code-Hex/vz/blob/master/memory_balloon.go

Expand Down Expand Up @@ -170,6 +184,8 @@ func deviceFromCmdLine(deviceOpts string) (VirtioDevice, error) {
dev = &VirtioInput{}
case "virtio-gpu":
dev = &VirtioGPU{}
case "nbd":
dev = networkBlockDeviceNewEmpty()
default:
return nil, fmt.Errorf("unknown device type: %s", opts[0])
}
Expand Down Expand Up @@ -663,6 +679,79 @@ func (dev *RosettaShare) FromOptions(options []option) error {
return nil
}

func networkBlockDeviceNewEmpty() *NetworkBlockDevice {
return &NetworkBlockDevice{
VirtioBlk: VirtioBlk{
StorageConfig: StorageConfig{
DevName: "nbd",
},
DeviceIdentifier: "",
},
Timeout: time.Duration(15000 * time.Millisecond), // set a default timeout to 15s
SynchronizationMode: SynchronizationFullMode, // default mode to full
}
}

// NetworkBlockDeviceNew creates a new disk by connecting to a remote Network Block Device (NBD) server.
// The provided uri must be in the format <scheme>://<address>/<export-name>
// where scheme could have any of these value: nbd, nbds, nbd+unix and nbds+unix.
// More info can be found at https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md
// This allows the virtual machine to access and use the remote storage as if it were a local disk.
func NetworkBlockDeviceNew(uri string, timeout uint32, synchronization NBDSynchronizationMode) (*NetworkBlockDevice, error) {
nbd := networkBlockDeviceNewEmpty()
nbd.URI = uri
nbd.Timeout = time.Duration(timeout) * time.Millisecond
nbd.SynchronizationMode = synchronization

return nbd, nil
}

func (nbd *NetworkBlockDevice) ToCmdLine() ([]string, error) {
cmdLine, err := nbd.VirtioBlk.ToCmdLine()
if err != nil {
return []string{}, err
}
if len(cmdLine) != 2 {
return []string{}, fmt.Errorf("unexpected storage config commandline")
}

if nbd.Timeout.Milliseconds() > 0 {
cmdLine[1] = fmt.Sprintf("%s,timeout=%d", cmdLine[1], nbd.Timeout.Milliseconds())
}
if nbd.SynchronizationMode == "none" || nbd.SynchronizationMode == "full" {
cmdLine[1] = fmt.Sprintf("%s,sync=%s", cmdLine[1], nbd.SynchronizationMode)
}

return cmdLine, nil
}

func (nbd *NetworkBlockDevice) FromOptions(options []option) error {
unhandledOpts := []option{}
for _, option := range options {
switch option.key {
case "timeout":
timeoutMS, err := strconv.ParseInt(option.value, 10, 32)
if err != nil {
return err
}
nbd.Timeout = time.Duration(timeoutMS) * time.Millisecond
case "sync":
switch option.value {
case string(SynchronizationFullMode):
nbd.SynchronizationMode = SynchronizationFullMode
case string(SynchronizationNoneMode):
nbd.SynchronizationMode = SynchronizationNoneMode
default:
return fmt.Errorf("invalid sync mode: %s, must be 'full' or 'none'", option.value)
}
default:
unhandledOpts = append(unhandledOpts, option)
}
}

return nbd.VirtioBlk.FromOptions(unhandledOpts)
}

type USBMassStorage struct {
StorageConfig
}
Expand Down Expand Up @@ -691,15 +780,26 @@ func (dev *USBMassStorage) SetReadOnly(readOnly bool) {
// StorageConfig configures a disk device.
type StorageConfig struct {
DevName string `json:"devName"`
ImagePath string `json:"imagePath"`
ImagePath string `json:"imagePath,omitempty"`
URI string `json:"uri,omitempty"`
ReadOnly bool `json:"readOnly,omitempty"`
}

func (config *StorageConfig) ToCmdLine() ([]string, error) {
if config.ImagePath == "" {
return nil, fmt.Errorf("%s devices need the path to a disk image", config.DevName)
if config.ImagePath != "" && config.URI != "" {
return nil, fmt.Errorf("%s devices cannot have both path to a disk image and a uri to a remote block device", config.DevName)
}
if config.ImagePath == "" && config.URI == "" {
return nil, fmt.Errorf("%s devices need a path to a disk image or a uri to a remote block device", config.DevName)
}
var value string
if config.ImagePath != "" {
value = fmt.Sprintf("%s,path=%s", config.DevName, config.ImagePath)
}
value := fmt.Sprintf("%s,path=%s", config.DevName, config.ImagePath)
if config.URI != "" {
value = fmt.Sprintf("%s,uri=%s", config.DevName, config.URI)
}

if config.ReadOnly {
value += ",readonly"
}
Expand All @@ -711,6 +811,8 @@ func (config *StorageConfig) FromOptions(options []option) error {
switch option.key {
case "path":
config.ImagePath = option.value
case "uri":
config.URI = option.value
case "readonly":
if option.value != "" {
return fmt.Errorf("unexpected value for virtio-blk 'readonly' option: %s", option.value)
Expand Down
Loading

0 comments on commit 4af22e9

Please sign in to comment.