Merge pull request #139 from weaveworks/startup-check

Verify `ignite-spawn` has started before returning from `start`

cmd/ignite-spawn/ignite-spawn.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path"
 
+	"github.com/weaveworks/ignite/pkg/constants"
+
 	log "github.com/sirupsen/logrus"
 	api "github.com/weaveworks/ignite/pkg/apis/ignite/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/container"
@@ -49,9 +51,7 @@ func StartVM(co *options) error {
 	}
 
 	// Serve metrics over an unix socket in the VM's own directory
-	// TODO: when restarting a VM using `start`, we get a panic:
-	// panic: listen unix /var/lib/firecracker/vm/6a2b6ebafcb0e75c/prometheus.sock: bind: address already in use
-	metricsSocket := path.Join(co.vm.ObjectPath(), "prometheus.sock")
+	metricsSocket := path.Join(co.vm.ObjectPath(), constants.PROMETHEUS_SOCKET)
 	go prometheus.ServeMetrics(metricsSocket)
 
 	// VM state handling

pkg/constants/vm.go

@@ -15,4 +15,7 @@ const (
 
 	// TODO: remove this when the old dm code is removed
 	OVERLAY_FILE = "overlay.dm"
+
+	// Prometheus socket filename
+	PROMETHEUS_SOCKET = "prometheus.sock"
 )

pkg/operations/start.go

@@ -5,8 +5,10 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	api "github.com/weaveworks/ignite/pkg/apis/ignite/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/constants"
@@ -82,13 +84,16 @@ func StartVM(vm *vmmd.VM, debug bool) error {
 		if err := setupCNINetworking(containerID); err != nil {
 			return err
 		}
+
 		log.Printf("Networking is now handled by CNI")
 	}
 
 	log.Printf("Started Firecracker VM %q in a container with ID %q", vm.GetUID(), containerID)
+
 	// TODO: Follow-up the container here with a defer, or dedicated goroutine. We should output
 	// if it started successfully or not
-	return nil
+	// TODO: This is temporary until we have proper communication to the container
+	return waitForSpawn(vm)
 }
 
 func setupCNINetworking(containerID string) error {
@@ -139,3 +144,21 @@ func verifyPulled(image string) error {
 
 	return nil
 }
+
+// TODO: This check for the Prometheus socket file is temporary
+// until we get a proper ignite <-> ignite-spawn communication channel
+func waitForSpawn(vm *vmmd.VM) error {
+	const timeout = 10 * time.Second
+	const checkInterval = 100 * time.Millisecond
+
+	startTime := time.Now()
+	for time.Now().Sub(startTime) < timeout {
+		time.Sleep(checkInterval)
+
+		if util.FileExists(path.Join(vm.ObjectPath(), constants.PROMETHEUS_SOCKET)) {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("timeout waiting for ignite-spawn startup")
+}