diff --git a/cmd/vfkit/main.go b/cmd/vfkit/main.go index 9fbee45b..3dbb6524 100644 --- a/cmd/vfkit/main.go +++ b/cmd/vfkit/main.go @@ -189,6 +189,11 @@ func runVirtualMachine(vmConfig *config.VirtualMachine, vm *vf.VirtualMachine) e defer closer.Close() } + if err := vf.ListenNetworkBlockDevices(vm); err != nil { + log.Debugf("%v", err) + return err + } + if err := setupGuestTimeSync(vm, vmConfig.TimeSync()); err != nil { log.Warnf("Error configuring guest time synchronization") log.Debugf("%v", err) diff --git a/doc/usage.md b/doc/usage.md index 0f62b2fe..acb4d711 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -223,6 +223,28 @@ This adds a read only USB mass storage device to the VM which will be backed by --device usb-mass-storage,path=/Users/virtuser/distro.iso,readonly ``` +### Network Block Device + +#### Description + +The `--device nbd` option allows to connect to a remote NBD server, effectively accessing a remote block device over the network as if it were a local disk. + +The NBD client running on the VM is informed in case the connection drops and it tries to reconnect automatically to the server. + +#### Arguments +- `uri`: the URI that refers to the NBD server to which the NBD client will connect, e.g. `nbd://10.10.2.8:10000/export`. More info at https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md +- `deviceId`: `/dev/disk/by-id/virtio-` identifier to use for this device. +- `sync`: the mode in which the NBD client synchronizes data with the NBD server. It can be `full`or `none`, more info at https://developer.apple.com/documentation/virtualization/vzdisksynchronizationmode?language=objc +- `timeout`: the timeout value in milliseconds for the connection between the client and server +- `readonly`: if specified the device will be read only. + +#### Example + +This allows to connect to the export of the remote NBD server: +``` +--device nbd,uri=nbd://192.168.64.4:11111/export,deviceId=nbd1,timeout=3000 +``` + ### Networking diff --git a/pkg/config/config.go b/pkg/config/config.go index 483a9fb8..a72c9d9a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -178,6 +178,16 @@ func (vm *VirtualMachine) VirtioVsockDevices() []*VirtioVsock { return vsockDevs } +func (vm *VirtualMachine) NetworkBlockDevice(deviceID string) *NetworkBlockDevice { + for _, dev := range vm.Devices { + if nbdDev, isNbdDev := dev.(*NetworkBlockDevice); isNbdDev && nbdDev.DeviceIdentifier == deviceID { + return nbdDev + } + } + + return nil +} + // AddDevice adds a dev to vm. This device can be created with one of the // VirtioXXXNew methods. func (vm *VirtualMachine) AddDevice(dev VirtioDevice) error { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 75c8d175..4b5763c1 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -21,3 +21,26 @@ func TestAddIgnitionFile_OneOption(t *testing.T) { assert.Equal(t, uint32(ignitionVsockPort), vm.Devices[0].(*VirtioVsock).Port) assert.Equal(t, "file1", vm.Ignition.ConfigPath) } + +func TestNetworkBlockDevice(t *testing.T) { + vm := &VirtualMachine{} + gpu, _ := VirtioGPUNew() + vm.Devices = append(vm.Devices, gpu) + nbd, _ := NetworkBlockDeviceNew("uri", 1000, SynchronizationFullMode) + nbd.DeviceIdentifier = "nbd1" + vm.Devices = append(vm.Devices, nbd) + nbd2, _ := NetworkBlockDeviceNew("uri2", 1000, SynchronizationFullMode) + nbd2.DeviceIdentifier = "nbd2" + vm.Devices = append(vm.Devices, nbd2) + + nbdItem := vm.NetworkBlockDevice("nbd2") + assert.Equal(t, "nbd2", nbdItem.DeviceIdentifier) + assert.Equal(t, "uri2", nbdItem.URI) +} + +func TestNetworkBlockDevice_NoDevice(t *testing.T) { + vm := &VirtualMachine{} + + nbdItem := vm.NetworkBlockDevice("nbd2") + require.Nil(t, nbdItem) +} diff --git a/pkg/config/json.go b/pkg/config/json.go index a46667f9..e4f0b4a3 100644 --- a/pkg/config/json.go +++ b/pkg/config/json.go @@ -29,6 +29,7 @@ const ( nvme vmComponentKind = "nvme" rosetta vmComponentKind = "rosetta" ignition vmComponentKind = "ignition" + vfNbd vmComponentKind = "nbd" ) type jsonKind struct { @@ -174,6 +175,10 @@ func unmarshalDevice(rawMsg json.RawMessage) (VirtioDevice, error) { var newDevice USBMassStorage err = json.Unmarshal(rawMsg, &newDevice) dev = &newDevice + case vfNbd: + var newDevice NetworkBlockDevice + err = json.Unmarshal(rawMsg, &newDevice) + dev = &newDevice default: err = fmt.Errorf("unknown 'kind' field: '%s'", kind) } @@ -395,3 +400,14 @@ func (ign *Ignition) MarshalJSON() ([]byte, error) { Ignition: *ign, }) } + +func (dev *NetworkBlockDevice) MarshalJSON() ([]byte, error) { + type devWithKind struct { + jsonKind + NetworkBlockDevice + } + return json.Marshal(devWithKind{ + jsonKind: kind(vfNbd), + NetworkBlockDevice: *dev, + }) +} diff --git a/pkg/config/json_test.go b/pkg/config/json_test.go index c06ee37d..3df9fccc 100644 --- a/pkg/config/json_test.go +++ b/pkg/config/json_test.go @@ -170,12 +170,15 @@ var jsonTests = map[string]jsonTest{ // rosetta rosetta, err := RosettaShareNew("vz-rosetta") require.NoError(t, err) - err = vm.AddDevices(fs, usb, rosetta) + // NBD + nbd, err := NetworkBlockDeviceNew("uri", 1, SynchronizationFullMode) + require.NoError(t, err) + err = vm.AddDevices(fs, usb, rosetta, nbd) require.NoError(t, err) return vm }, - expectedJSON: `{"vcpus":3,"memoryBytes":4194304000,"bootloader":{"kind":"linuxBootloader","vmlinuzPath":"/vmlinuz","initrdPath":"/initrd","kernelCmdLine":"console=hvc0"},"devices":[{"kind":"virtioserial","logFile":"/virtioserial"},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","nat":true,"macAddress":"00:11:22:33:44:55"},{"kind":"virtiorng"},{"kind":"virtioblk","devName":"virtio-blk","imagePath":"/virtioblk"},{"kind":"virtiosock","port":1234,"socketURL":"/virtiovsock"},{"kind":"virtiofs","mountTag":"tag","sharedDir":"/virtiofs"},{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"/usbmassstorage","readOnly":true},{"kind":"rosetta","mountTag":"vz-rosetta","installRosetta":false,"ignoreIfMissing":false}]}`, + expectedJSON: `{"vcpus":3,"memoryBytes":4194304000,"bootloader":{"kind":"linuxBootloader","vmlinuzPath":"/vmlinuz","initrdPath":"/initrd","kernelCmdLine":"console=hvc0"},"devices":[{"kind":"virtioserial","logFile":"/virtioserial"},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","nat":true,"macAddress":"00:11:22:33:44:55"},{"kind":"virtiorng"},{"kind":"virtioblk","devName":"virtio-blk","imagePath":"/virtioblk"},{"kind":"virtiosock","port":1234,"socketURL":"/virtiovsock"},{"kind":"virtiofs","mountTag":"tag","sharedDir":"/virtiofs"},{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"/usbmassstorage","readOnly":true},{"kind":"rosetta","mountTag":"vz-rosetta","installRosetta":false,"ignoreIfMissing":false},{"kind":"nbd", "devName":"nbd", "uri":"uri", "SynchronizationMode":"full","Timeout":1000000}]}`, }, } @@ -274,7 +277,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{ return blk }, - skipFields: []string{"DevName"}, + skipFields: []string{"DevName", "URI"}, expectedJSON: `{"kind":"virtioblk","devName":"virtio-blk","imagePath":"ImagePath","readOnly":true,"deviceIdentifier":"DeviceIdentifier"}`, }, "USBMassStorage": { @@ -283,7 +286,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{ require.NoError(t, err) return usb }, - skipFields: []string{"DevName"}, + skipFields: []string{"DevName", "URI"}, expectedJSON: `{"kind":"usbmassstorage","devName":"usb-mass-storage","imagePath":"ImagePath","readOnly":true}`, }, "NVMExpressController": { @@ -292,7 +295,7 @@ var jsonStabilityTests = map[string]jsonStabilityTest{ require.NoError(t, err) return nvme }, - skipFields: []string{"DevName"}, + skipFields: []string{"DevName", "URI"}, expectedJSON: `{"kind":"nvme","devName":"nvme","imagePath":"ImagePath","readOnly":true}`, }, "LinuxBootloader": { @@ -307,6 +310,15 @@ var jsonStabilityTests = map[string]jsonStabilityTest{ obj: &TimeSync{}, expectedJSON: `{"vsockPort":3}`, }, + "NetworkBlockDevice": { + newObjectFunc: func(t *testing.T) any { + nbd, err := NetworkBlockDeviceNew("uri", 1000, SynchronizationFullMode) + require.NoError(t, err) + return nbd + }, + skipFields: []string{"DevName", "ImagePath"}, + expectedJSON: `{"kind":"nbd","deviceIdentifier":"DeviceIdentifier","devName":"nbd","uri":"URI","readOnly":true,"SynchronizationMode":"SynchronizationMode","Timeout":2}`, + }, } type jsonStabilityTest struct { diff --git a/pkg/config/virtio.go b/pkg/config/virtio.go index 8fa85599..1278cad0 100644 --- a/pkg/config/virtio.go +++ b/pkg/config/virtio.go @@ -7,6 +7,7 @@ import ( "os" "strconv" "strings" + "time" ) // The VirtioDevice interface is an interface which is implemented by all virtio devices. @@ -108,6 +109,19 @@ type VirtioSerial struct { UsesStdio bool `json:"usesStdio,omitempty"` } +type NBDSynchronizationMode string + +const ( + SynchronizationFullMode NBDSynchronizationMode = "full" + SynchronizationNoneMode NBDSynchronizationMode = "none" +) + +type NetworkBlockDevice struct { + VirtioBlk + Timeout time.Duration + SynchronizationMode NBDSynchronizationMode +} + // TODO: Add VirtioBalloon // https://github.com/Code-Hex/vz/blob/master/memory_balloon.go @@ -170,6 +184,8 @@ func deviceFromCmdLine(deviceOpts string) (VirtioDevice, error) { dev = &VirtioInput{} case "virtio-gpu": dev = &VirtioGPU{} + case "nbd": + dev = networkBlockDeviceNewEmpty() default: return nil, fmt.Errorf("unknown device type: %s", opts[0]) } @@ -663,6 +679,79 @@ func (dev *RosettaShare) FromOptions(options []option) error { return nil } +func networkBlockDeviceNewEmpty() *NetworkBlockDevice { + return &NetworkBlockDevice{ + VirtioBlk: VirtioBlk{ + StorageConfig: StorageConfig{ + DevName: "nbd", + }, + DeviceIdentifier: "", + }, + Timeout: time.Duration(15000 * time.Millisecond), // set a default timeout to 15s + SynchronizationMode: SynchronizationFullMode, // default mode to full + } +} + +// NetworkBlockDeviceNew creates a new disk by connecting to a remote Network Block Device (NBD) server. +// The provided uri must be in the format ://
/ +// where scheme could have any of these value: nbd, nbds, nbd+unix and nbds+unix. +// More info can be found at https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md +// This allows the virtual machine to access and use the remote storage as if it were a local disk. +func NetworkBlockDeviceNew(uri string, timeout uint32, synchronization NBDSynchronizationMode) (*NetworkBlockDevice, error) { + nbd := networkBlockDeviceNewEmpty() + nbd.URI = uri + nbd.Timeout = time.Duration(timeout) * time.Millisecond + nbd.SynchronizationMode = synchronization + + return nbd, nil +} + +func (nbd *NetworkBlockDevice) ToCmdLine() ([]string, error) { + cmdLine, err := nbd.VirtioBlk.ToCmdLine() + if err != nil { + return []string{}, err + } + if len(cmdLine) != 2 { + return []string{}, fmt.Errorf("unexpected storage config commandline") + } + + if nbd.Timeout.Milliseconds() > 0 { + cmdLine[1] = fmt.Sprintf("%s,timeout=%d", cmdLine[1], nbd.Timeout.Milliseconds()) + } + if nbd.SynchronizationMode == "none" || nbd.SynchronizationMode == "full" { + cmdLine[1] = fmt.Sprintf("%s,sync=%s", cmdLine[1], nbd.SynchronizationMode) + } + + return cmdLine, nil +} + +func (nbd *NetworkBlockDevice) FromOptions(options []option) error { + unhandledOpts := []option{} + for _, option := range options { + switch option.key { + case "timeout": + timeoutMS, err := strconv.ParseInt(option.value, 10, 32) + if err != nil { + return err + } + nbd.Timeout = time.Duration(timeoutMS) * time.Millisecond + case "sync": + switch option.value { + case string(SynchronizationFullMode): + nbd.SynchronizationMode = SynchronizationFullMode + case string(SynchronizationNoneMode): + nbd.SynchronizationMode = SynchronizationNoneMode + default: + return fmt.Errorf("invalid sync mode: %s, must be 'full' or 'none'", option.value) + } + default: + unhandledOpts = append(unhandledOpts, option) + } + } + + return nbd.VirtioBlk.FromOptions(unhandledOpts) +} + type USBMassStorage struct { StorageConfig } @@ -691,15 +780,26 @@ func (dev *USBMassStorage) SetReadOnly(readOnly bool) { // StorageConfig configures a disk device. type StorageConfig struct { DevName string `json:"devName"` - ImagePath string `json:"imagePath"` + ImagePath string `json:"imagePath,omitempty"` + URI string `json:"uri,omitempty"` ReadOnly bool `json:"readOnly,omitempty"` } func (config *StorageConfig) ToCmdLine() ([]string, error) { - if config.ImagePath == "" { - return nil, fmt.Errorf("%s devices need the path to a disk image", config.DevName) + if config.ImagePath != "" && config.URI != "" { + return nil, fmt.Errorf("%s devices cannot have both path to a disk image and a uri to a remote block device", config.DevName) + } + if config.ImagePath == "" && config.URI == "" { + return nil, fmt.Errorf("%s devices need a path to a disk image or a uri to a remote block device", config.DevName) + } + var value string + if config.ImagePath != "" { + value = fmt.Sprintf("%s,path=%s", config.DevName, config.ImagePath) } - value := fmt.Sprintf("%s,path=%s", config.DevName, config.ImagePath) + if config.URI != "" { + value = fmt.Sprintf("%s,uri=%s", config.DevName, config.URI) + } + if config.ReadOnly { value += ",readonly" } @@ -711,6 +811,8 @@ func (config *StorageConfig) FromOptions(options []option) error { switch option.key { case "path": config.ImagePath = option.value + case "uri": + config.URI = option.value case "readonly": if option.value != "" { return fmt.Errorf("unexpected value for virtio-blk 'readonly' option: %s", option.value) diff --git a/pkg/config/virtio_test.go b/pkg/config/virtio_test.go index d592edd5..5829c6e9 100644 --- a/pkg/config/virtio_test.go +++ b/pkg/config/virtio_test.go @@ -2,6 +2,7 @@ package config import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -12,6 +13,7 @@ type virtioDevTest struct { expectedDev VirtioDevice expectedCmdLine []string alternateCmdLine []string + errorMsg string } var virtioDevTests = map[string]virtioDevTest{ @@ -216,6 +218,41 @@ var virtioDevTests = map[string]virtioDevTest{ }, expectedCmdLine: []string{"--device", "virtio-gpu,width=1920,height=1080"}, }, + "NetworkBlockDeviceNew": { + newDev: func() (VirtioDevice, error) { + return NetworkBlockDeviceNew("nbd://1.1.1.1:10000", 1000, SynchronizationNoneMode) + }, + expectedDev: &NetworkBlockDevice{ + VirtioBlk: VirtioBlk{ + StorageConfig: StorageConfig{ + DevName: "nbd", + URI: "nbd://1.1.1.1:10000", + }, + DeviceIdentifier: "", + }, + Timeout: time.Duration(1000 * time.Millisecond), + SynchronizationMode: SynchronizationNoneMode, + }, + expectedCmdLine: []string{"--device", "nbd,uri=nbd://1.1.1.1:10000,timeout=1000,sync=none"}, + }, + "StorageConfigErrorImageUri": { + newDev: func() (VirtioDevice, error) { + return &StorageConfig{ + DevName: "dev", + ImagePath: "path", + URI: "uri", + }, nil + }, + errorMsg: "dev devices cannot have both path to a disk image and a uri to a remote block device", + }, + "StorageConfigErrorNoImageOrUri": { + newDev: func() (VirtioDevice, error) { + return &StorageConfig{ + DevName: "dev", + }, nil + }, + errorMsg: "dev devices need a path to a disk image or a uri to a remote block device", + }, } func testVirtioDev(t *testing.T, test *virtioDevTest) { @@ -245,12 +282,25 @@ func testVirtioDev(t *testing.T, test *virtioDevTest) { } +func testErrorVirtioDev(t *testing.T, test *virtioDevTest) { + dev, err := test.newDev() + require.NoError(t, err) + + _, err = dev.ToCmdLine() + require.Error(t, err) + require.EqualError(t, err, test.errorMsg) +} + func TestVirtioDevices(t *testing.T) { t.Run("virtio-devices", func(t *testing.T) { for name := range virtioDevTests { t.Run(name, func(t *testing.T) { test := virtioDevTests[name] - testVirtioDev(t, &test) + if test.errorMsg != "" { + testErrorVirtioDev(t, &test) + } else { + testVirtioDev(t, &test) + } }) } diff --git a/pkg/vf/virtio.go b/pkg/vf/virtio.go index 641ecd16..500f8eef 100644 --- a/pkg/vf/virtio.go +++ b/pkg/vf/virtio.go @@ -2,8 +2,10 @@ package vf import ( "fmt" + "net/url" "os" "path/filepath" + "strings" "syscall" "github.com/crc-org/vfkit/pkg/config" @@ -25,6 +27,12 @@ type VirtioSerial config.VirtioSerial type VirtioVsock config.VirtioVsock type VirtioInput config.VirtioInput type VirtioGPU config.VirtioGPU +type NetworkBlockDevice config.NetworkBlockDevice + +type vzNetworkBlockDevice struct { + *vz.VirtioBlockDeviceConfiguration + config *NetworkBlockDevice +} func (dev *NVMExpressController) toVz() (vz.StorageDeviceConfiguration, error) { var storageConfig StorageConfig = StorageConfig(dev.StorageConfig) @@ -289,6 +297,108 @@ func (dev *VirtioVsock) AddToVirtualMachineConfig(vmConfig *VirtualMachineConfig return nil } +func (dev *NetworkBlockDevice) toVz() (vz.StorageDeviceConfiguration, error) { + if err := dev.validateNbdURI(dev.URI); err != nil { + return nil, fmt.Errorf("invalid NBD device 'uri': %s", err.Error()) + } + + if err := dev.validateNbdDeviceIdentifier(dev.DeviceIdentifier); err != nil { + return nil, fmt.Errorf("invalid NBD device 'deviceId': %s", err.Error()) + } + + attachment, err := vz.NewNetworkBlockDeviceStorageDeviceAttachment(dev.URI, dev.Timeout, dev.ReadOnly, dev.SynchronizationModeVZ()) + if err != nil { + return nil, err + } + + vzdev, err := vz.NewVirtioBlockDeviceConfiguration(attachment) + if err != nil { + return nil, err + } + err = vzdev.SetBlockDeviceIdentifier(dev.DeviceIdentifier) + if err != nil { + return nil, err + } + + return vzNetworkBlockDevice{VirtioBlockDeviceConfiguration: vzdev, config: dev}, nil +} + +func (dev *NetworkBlockDevice) validateNbdURI(uri string) error { + if uri == "" { + return fmt.Errorf("'uri' must be specified") + } + + parsed, err := url.Parse(uri) + if err != nil { + return fmt.Errorf("error: %w", err) + } + + // The format specified by https://github.com/NetworkBlockDevice/nbd/blob/master/doc/uri.md + if parsed.Scheme != "nbd" && parsed.Scheme != "nbds" && parsed.Scheme != "nbd+unix" && parsed.Scheme != "nbds+unix" { + return fmt.Errorf("invalid scheme: %s. Expected one of: 'nbd', 'nbds', 'nbd+unix', or 'nbds+unix'", parsed.Scheme) + } + + return nil +} + +func (dev *NetworkBlockDevice) validateNbdDeviceIdentifier(deviceID string) error { + if deviceID == "" { + return fmt.Errorf("'deviceId' must be specified") + } + + if strings.Contains(deviceID, "/") { + return fmt.Errorf("invalid 'deviceId': it cannot contain any forward slash") + } + + if len(deviceID) > 255 { + return fmt.Errorf("invalid 'deviceId': exceeds maximum length") + } + + return nil +} + +func (dev *NetworkBlockDevice) SynchronizationModeVZ() vz.DiskSynchronizationMode { + if dev.SynchronizationMode == config.SynchronizationNoneMode { + return vz.DiskSynchronizationModeNone + } + return vz.DiskSynchronizationModeFull +} + +func (dev *NetworkBlockDevice) AddToVirtualMachineConfig(vmConfig *VirtualMachineConfiguration) error { + storageDeviceConfig, err := dev.toVz() + if err != nil { + return err + } + log.Infof("Adding NBD device (uri: %s, deviceId: %s)", dev.URI, dev.DeviceIdentifier) + vmConfig.storageDevicesConfiguration = append(vmConfig.storageDevicesConfiguration, storageDeviceConfig) + + return nil +} + +func ListenNetworkBlockDevices(vm *VirtualMachine) error { + for _, dev := range vm.vfConfig.storageDevicesConfiguration { + if nbdDev, isNbdDev := dev.(vzNetworkBlockDevice); isNbdDev { + nbdAttachment, isNbdAttachment := dev.Attachment().(*vz.NetworkBlockDeviceStorageDeviceAttachment) + if !isNbdAttachment { + log.Info("Found NBD device with no NBD attachment. Please file a vfkit bug.") + return fmt.Errorf("NetworkBlockDevice must use a NBD attachment") + } + nbdConfig := nbdDev.config + go func() { + for { + select { + case err := <-nbdAttachment.DidEncounterError(): + log.Infof("Disconnected from NBD server %s. Error %v", nbdConfig.URI, err.Error()) + case <-nbdAttachment.Connected(): + log.Infof("Successfully connected to NBD server %s.", nbdConfig.URI) + } + } + }() + } + } + return nil +} + func AddToVirtualMachineConfig(vmConfig *VirtualMachineConfiguration, dev config.VirtioDevice) error { switch d := dev.(type) { case *config.USBMassStorage: @@ -314,6 +424,8 @@ func AddToVirtualMachineConfig(vmConfig *VirtualMachineConfiguration, dev config return (*VirtioInput)(d).AddToVirtualMachineConfig(vmConfig) case *config.VirtioGPU: return (*VirtioGPU)(d).AddToVirtualMachineConfig(vmConfig) + case *config.NetworkBlockDevice: + return (*NetworkBlockDevice)(d).AddToVirtualMachineConfig(vmConfig) default: return fmt.Errorf("Unexpected virtio device type: %T", d) }