Skip to content

Commit

Permalink
Add remotevm-util
Browse files Browse the repository at this point in the history
Signed-off-by: Daniel Canter <dcanter@microsoft.com>
  • Loading branch information
dcantah committed May 10, 2022
1 parent 18f4761 commit 2940aa7
Show file tree
Hide file tree
Showing 15 changed files with 375 additions and 98 deletions.
154 changes: 154 additions & 0 deletions cmd/remotevm-util/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package main

import (
"context"
"fmt"
"io"
"log"
"net"
"os"
"os/signal"
"syscall"

"github.com/Microsoft/hcsshim/internal/vm"
"github.com/Microsoft/hcsshim/internal/vm/remotevm"
"github.com/urfave/cli"
)

const usage = `remotevm-util is a simple tool to test out remotevm implementations`

func main() {
app := cli.NewApp()
app.Name = "remotevm-util"
app.Commands = []cli.Command{
launchVMCommand,
}
app.Usage = usage

if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}

const (
kernelPath = "kernel"
initrdPath = "initrd"
binPath = "binpath"
ttrpcAddr = "ttrpc"
vmName = "name"
)

func listenVsock(ctx context.Context, port uint32, uvm vm.UVM) (net.Listener, error) {
vmsocket, ok := uvm.(vm.VMSocketManager)
if !ok {
return nil, fmt.Errorf("stopping vm socket configuration: %w", vm.ErrNotSupported)
}
return vmsocket.VMSocketListen(ctx, vm.VSock, port)
}

var launchVMCommand = cli.Command{
Name: "launch",
Usage: "Launches a VM using the specified kernel and initrd",
Flags: []cli.Flag{
cli.StringFlag{
Name: kernelPath,
Usage: "Specifies path to the kernel to boot off of",
Required: true,
},
cli.StringFlag{
Name: initrdPath,
Usage: "Specifies path of the initrd to use as the rootfs",
Required: true,
},
cli.StringFlag{
Name: binPath,
Usage: "Path to the binary implementing the vmservice interface",
Required: true,
},
cli.StringFlag{
Name: vmName,
Usage: "Specifies the name to use for the VM",
Required: true,
},
},
Action: func(clictx *cli.Context) error {
// Precomputed default kernel args that are used for LCOW. We can use the same.
kernelArgs := `panic=-1 root=/dev/foo debug noapic pci=off`
ctx := context.Background()
builder, err := remotevm.NewUVMBuilder(
ctx,
vmName,
os.Args[0],
clictx.String(binPath),
clictx.String(ttrpcAddr),
vm.Linux,
)
if err != nil {
return err
}

boot := builder.(vm.BootManager)
if err := boot.SetLinuxKernelDirectBoot(
clictx.String(kernelPath),
clictx.String(initrdPath),
kernelArgs,
); err != nil {
return fmt.Errorf("failed to set Linux kernel direct boot: %w", err)
}

proc := builder.(vm.ProcessorManager)
if err := proc.SetProcessorCount(2); err != nil {
return err
}

mem := builder.(vm.MemoryManager)
if err := mem.SetMemoryLimit(ctx, 2048); err != nil {
return err
}

opts := []vm.CreateOpt{remotevm.WithIgnoreSupported()}
vm, err := builder.Create(ctx, opts)
if err != nil {
return err
}

if err := vm.Start(ctx); err != nil {
return err
}
defer vm.Close()

fmt.Printf("Started VM %s\n", vm.ID())

errCh := make(chan error)
go func() {
fmt.Printf("Waiting on VM %s\n", vm.ID())
if err := vm.Wait(); err != nil {
errCh <- err
}
}()

sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
defer signal.Stop(sigChan)

select {
case <-sigChan:
fmt.Println("Received signal, exiting.")
if err := vm.Stop(ctx); err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
os.Exit(0)
case err := <-errCh:
if err != nil {
return err
}
}

return vm.ExitError()
},
}

func relayIO(r io.Reader, w io.Writer) error {
return nil
}
Binary file added cmd/rvm
Binary file not shown.
18 changes: 16 additions & 2 deletions internal/vm/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,15 @@ import (
"context"
)

// CreateOpt can be used to apply virtstack specific settings during creation time.
type CreateOpt func(ctx context.Context, uvmb UVMBuilder) error

type UVMBuilder interface {
// Create will create the Utility VM in a paused/powered off state with whatever is present in the implementation
// of the interfaces config at the time of the call.
Create(ctx context.Context) (UVM, error)
//
// `opts` can be used to set virtstack specific configurations for the Utility VM.
Create(ctx context.Context, opts []CreateOpt) (UVM, error)
}

type MemoryBackingType uint8
Expand All @@ -29,18 +34,27 @@ type MemoryConfig struct {
// MemoryManager handles setting and managing memory configurations for the Utility VM.
type MemoryManager interface {
// SetMemoryLimit sets the amount of memory in megabytes that the Utility VM will be assigned.
SetMemoryLimit(memoryMB uint64) error
SetMemoryLimit(ctx context.Context, memoryMB uint64) error
// SetMemoryConfig sets an array of different memory configuration options available. This includes things like the
// type of memory to back the VM (virtual/physical).
SetMemoryConfig(config *MemoryConfig) error
// SetMMIOConfig sets memory mapped IO configurations for the Utility VM.
SetMMIOConfig(lowGapMB uint64, highBaseMB uint64, highGapMB uint64) error
}

// ProcessorLimits is used when modifying processor scheduling limits of a virtual machine.
type ProcessorLimits struct {
// Maximum amount of host CPU resources that the virtual machine can use.
Limit uint64
// Value describing the relative priority of this virtual machine compared to other virtual machines.
Weight uint64
}

// ProcessorManager handles setting and managing processor configurations for the Utility VM.
type ProcessorManager interface {
// SetProcessorCount sets the number of virtual processors that will be assigned to the Utility VM.
SetProcessorCount(count uint32) error
SetProcessorLimits(ctx context.Context, limits *ProcessorLimits) error
}

// SerialManager manages setting up serial consoles for the Utility VM.
Expand Down
2 changes: 0 additions & 2 deletions internal/vm/remotevm/boot.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build windows

package remotevm

import (
Expand Down
87 changes: 49 additions & 38 deletions internal/vm/remotevm/builder.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
//go:build windows

package remotevm

import (
"context"
"io"
"io/ioutil"
"net"
"os"
"os/exec"

"github.com/Microsoft/hcsshim/internal/jobobject"
"github.com/Microsoft/hcsshim/internal/log"
"github.com/Microsoft/hcsshim/internal/logfields"
"github.com/Microsoft/hcsshim/internal/vm"
"github.com/Microsoft/hcsshim/internal/vmservice"
"github.com/containerd/ttrpc"
Expand All @@ -22,30 +21,30 @@ import (
var _ vm.UVMBuilder = &utilityVMBuilder{}

type utilityVMBuilder struct {
id string
guestOS vm.GuestOS
job *jobobject.JobObject
config *vmservice.VMConfig
client vmservice.VMService
id, binpath, addr string
guestOS vm.GuestOS
ignoreSupported bool
config *vmservice.VMConfig
client vmservice.VMService
}

func NewUVMBuilder(ctx context.Context, id, owner, binPath, addr string, guestOS vm.GuestOS) (vm.UVMBuilder, error) {
var job *jobobject.JobObject
func NewUVMBuilder(ctx context.Context, id, owner, binPath, addr string, guestOS vm.GuestOS) (_ vm.UVMBuilder, err error) {
if binPath != "" {
log.G(ctx).WithFields(logrus.Fields{
"binary": binPath,
"address": addr,
}).Debug("starting remotevm server process")

opts := &jobobject.Options{
Name: id,
}
job, err := jobobject.Create(ctx, opts)
if err != nil {
return nil, errors.Wrap(err, "failed to create job object for remotevm process")
// If no address passed, just generate a random one.
if addr == "" {
addr, err = randomUnixSockAddr()
if err != nil {
return nil, err
}
}

cmd := exec.Command(binPath, "--ttrpc", addr)
cmd.Stderr = os.Stderr
p, err := cmd.StdoutPipe()
if err != nil {
return nil, errors.Wrap(err, "failed to create stdout pipe")
Expand All @@ -55,14 +54,6 @@ func NewUVMBuilder(ctx context.Context, id, owner, binPath, addr string, guestOS
return nil, errors.Wrap(err, "failed to start remotevm server process")
}

if err := job.Assign(uint32(cmd.Process.Pid)); err != nil {
return nil, errors.Wrap(err, "failed to assign remotevm process to job")
}

if err := job.SetTerminateOnLastHandleClose(); err != nil {
return nil, errors.Wrap(err, "failed to set terminate on last handle closed for remotevm job object")
}

// Wait for stdout to close. This is our signal that the server is successfully up and running.
_, _ = io.Copy(ioutil.Discard, p)
}
Expand All @@ -85,27 +76,47 @@ func NewUVMBuilder(ctx context.Context, id, owner, binPath, addr string, guestOS
SerialConfig: &vmservice.SerialConfig{},
ExtraData: make(map[string]string),
},
job: job,
client: vmClient,
}, nil
}

func (uvmb *utilityVMBuilder) Create(ctx context.Context) (vm.UVM, error) {
// Grab what capabilities the virtstack supports up front.
capabilities, err := uvmb.client.CapabilitiesVM(ctx, &ptypes.Empty{})
if err != nil {
return nil, errors.Wrap(err, "failed to get virtstack capabilities from vmservice")
func (uvmb *utilityVMBuilder) Create(ctx context.Context, opts []vm.CreateOpt) (_ vm.UVM, err error) {
// Apply any opts
for _, o := range opts {
if err := o(ctx, uvmb); err != nil {
return nil, errors.Wrap(err, "failed applying create options for Utility VM")
}
}

if _, err := uvmb.client.CreateVM(ctx, &vmservice.CreateVMRequest{Config: uvmb.config, LogID: uvmb.id}); err != nil {
var capabilities *vmservice.CapabilitiesVMResponse
if !uvmb.ignoreSupported {
// Grab what capabilities the virtstack supports up front.
capabilities, err = uvmb.client.CapabilitiesVM(ctx, &ptypes.Empty{})
if err != nil {
return nil, errors.Wrap(err, "failed to get virtstack capabilities from vmservice")
}
}

_, err = uvmb.client.CreateVM(ctx, &vmservice.CreateVMRequest{Config: uvmb.config, LogID: uvmb.id})
if err != nil {
return nil, errors.Wrap(err, "failed to create remote VM")
}

return &utilityVM{
id: uvmb.id,
job: uvmb.job,
config: uvmb.config,
client: uvmb.client,
capabilities: capabilities,
}, nil
log.G(ctx).WithFields(logrus.Fields{
logfields.UVMID: uvmb.id,
"vmservice-address": uvmb.addr,
"vmservice-binary-path": uvmb.binpath,
}).Debug("created utility VM")

uvm := &utilityVM{
id: uvmb.id,
waitBlock: make(chan struct{}),
ignoreSupported: uvmb.ignoreSupported,
config: uvmb.config,
client: uvmb.client,
capabilities: capabilities,
}

go uvm.waitBackground()
return uvm, nil
}
6 changes: 3 additions & 3 deletions internal/vm/remotevm/memory.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
//go:build windows

package remotevm

import (
"context"

"github.com/Microsoft/hcsshim/internal/vm"
"github.com/Microsoft/hcsshim/internal/vmservice"
)

func (uvmb *utilityVMBuilder) SetMemoryLimit(memoryMB uint64) error {
func (uvmb *utilityVMBuilder) SetMemoryLimit(ctx context.Context, memoryMB uint64) error {
if uvmb.config.MemoryConfig == nil {
uvmb.config.MemoryConfig = &vmservice.MemoryConfig{}
}
Expand Down
20 changes: 20 additions & 0 deletions internal/vm/remotevm/opts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package remotevm

import (
"context"

"github.com/pkg/errors"

"github.com/Microsoft/hcsshim/internal/vm"
)

func WithIgnoreSupported() vm.CreateOpt {
return func(ctx context.Context, uvmb vm.UVMBuilder) error {
builder, ok := uvmb.(*utilityVMBuilder)
if !ok {
return errors.New("object is not a remotevm UVMBuilder")
}
builder.ignoreSupported = true
return nil
}
}
Loading

0 comments on commit 2940aa7

Please sign in to comment.