firmware: fetch EDK2 firmware instead of Rust Hypervisor Firmware

go.mod

@@ -27,6 +27,7 @@ require (
 	golang.org/x/term v0.14.0
 	gvisor.dev/gvisor v0.0.0-20230926030033-4af66e670562
 	inet.af/tcpproxy v0.0.0-20221017015627-91f861402626
+	pault.ag/go/debian v0.16.0
 )
 
 require (
@@ -38,6 +39,8 @@ require (
 	github.com/google/btree v1.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
+	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
@@ -53,9 +56,12 @@ require (
 	github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.0 // indirect
+	pault.ag/go/topsort v0.1.1 // indirect
 )

go.sum

@@ -38,6 +38,10 @@ github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/ra
 github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
+github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKMYMj0FJRCF6tQubUuQ02GM=
+github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/oui v0.0.0-20150225163751-35b4deb627f8 h1:8vTSNy6M0xiuAOmKh271gD8sr6mM+5RzXAiqIUL0KmE=
 github.com/klauspost/oui v0.0.0-20150225163751-35b4deb627f8/go.mod h1:iaF36Fc2UmrXJ7AGL+fEZU9WWuZiB+4dp9tQtADeZ6A=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
@@ -110,6 +114,10 @@ github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 h1:8mhqcHPqT
 github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
@@ -141,3 +149,7 @@ gvisor.dev/gvisor v0.0.0-20230926030033-4af66e670562 h1:ucLWTFM679XhXAK5se5JoHV7
 gvisor.dev/gvisor v0.0.0-20230926030033-4af66e670562/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY=
 inet.af/tcpproxy v0.0.0-20221017015627-91f861402626 h1:2dMP3Ox/Wh5BiItwOt4jxRsfzkgyBrHzx2nW28Yg6nc=
 inet.af/tcpproxy v0.0.0-20221017015627-91f861402626/go.mod h1:Tojt5kmHpDIR2jMojxzZK2w2ZR7OILODmUo2gaSwjrk=
+pault.ag/go/debian v0.16.0 h1:fivXn/IO9rn2nzTGndflDhOkNU703Axs/StWihOeU2g=
+pault.ag/go/debian v0.16.0/go.mod h1:JFl0XWRCv9hWBrB5MDDZjA5GSEs1X3zcFK/9kCNIUmE=
+pault.ag/go/topsort v0.1.1 h1:L0QnhUly6LmTv0e3DEzbN2q6/FGgAcQvaEw65S53Bg4=
+pault.ag/go/topsort v0.1.1/go.mod h1:r1kc/L0/FZ3HhjezBIPaNVhkqv8L0UJ9bxRuHRVZ0q4=

internal/binaryfetcher/binaryfetcher.go

@@ -2,15 +2,15 @@ package binaryfetcher
 
 import (
 	"context"
-	"fmt"
 	"github.com/cirruslabs/vetu/internal/homedir"
 	"io"
-	"net/http"
 	"os"
 	"path/filepath"
 )
 
-func Fetch(ctx context.Context, downloadURL string, binaryName string, executable bool) (string, error) {
+type FetchFunc func(ctx context.Context, binaryFile io.Writer) error
+
+func GetOrFetch(ctx context.Context, fetchFunc FetchFunc, binaryName string, executable bool) (string, error) {
 	// Determine the binary path
 	binaryPath, err := binaryPath(binaryName)
 	if err != nil {
@@ -22,32 +22,15 @@ func Fetch(ctx context.Context, downloadURL string, binaryName string, executabl
 		return binaryPath, nil
 	}
 
-	// Download and cache the binary if not available in the cache
-	client := http.Client{}
-
-	request, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
-	if err != nil {
-		return "", err
-	}
-
-	resp, err := client.Do(request)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to fetch %q binary from %s: HTTP %d",
-			binaryName, downloadURL, resp.StatusCode)
-	}
-
+	// Run the user-provided function to fetch the binary file
+	// if not available in the cache
 	binaryFile, err := os.Create(binaryPath)
 	if err != nil {
 		return "", err
 	}
 	defer binaryFile.Close()
 
-	if _, err := io.Copy(binaryFile, resp.Body); err != nil {
+	if err := fetchFunc(ctx, binaryFile); err != nil {
 		return "", err
 	}
 

internal/binaryfetcher/fetchurl.go

@@ -0,0 +1,48 @@
+package binaryfetcher
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+)
+
+func FetchURLToFile(ctx context.Context, url string) (string, error) {
+	resp, err := FetchURL(ctx, url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	tempFile, err := os.CreateTemp("", "")
+	if err != nil {
+		return "", err
+	}
+
+	if _, err := io.Copy(tempFile, resp.Body); err != nil {
+		return "", err
+	}
+
+	return tempFile.Name(), tempFile.Close()
+}
+
+func FetchURL(ctx context.Context, url string) (*http.Response, error) {
+	client := http.Client{}
+
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch %s: HTTP %d", url, resp.StatusCode)
+	}
+
+	return resp, nil
+}

internal/externalbinary/firmware/firmware.go

@@ -1,44 +1,114 @@
 package firmware
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"github.com/cirruslabs/vetu/internal/binaryfetcher"
+	"github.com/samber/lo"
+	"io"
 	"os"
+	"path"
+	"pault.ag/go/debian/control"
+	"pault.ag/go/debian/deb"
 	"runtime"
 )
 
 const (
-	edk2BinaryPath = "/usr/share/cloud-hypervisor/CLOUDHV_EFI.fd"
-	baseURL        = "https://github.com/cirruslabs/rust-hypervisor-firmware/releases/latest/download/"
-)
+	systemEDKPath = "/usr/share/cloud-hypervisor/CLOUDHV_EFI.fd"
 
-var goarchToDownloadURL = map[string]string{
-	"amd64": baseURL + "hypervisor-fw",
-	"arm64": baseURL + "hypervisor-fw-aarch64",
-}
+	debRepositoryURL  = "https://download.opensuse.org/repositories/home:/cloud-hypervisor/xUbuntu_22.04"
+	debTargetPackage  = "edk2-cloud-hypervisor"
+	debTargetFilename = "CLOUDHV_EFI.fd"
+)
 
 func Firmware(ctx context.Context) (string, string, error) {
 	// Always prefer the EDK2 firmware installed on the system
-	_, err := os.Stat(edk2BinaryPath)
+	_, err := os.Stat(systemEDKPath)
 	if err == nil {
-		return edk2BinaryPath, "EDK2 firmware", nil
+		return systemEDKPath, "EDK2 firmware", nil
+	}
+
+	// Fall back to downloading the EDK2 firmware from a .deb-repository
+	fmt.Printf("no EDK2 firmware installed on the system, downloading it from %s...\n",
+		debRepositoryURL)
+
+	binaryPath, err := binaryfetcher.GetOrFetch(ctx, func(ctx context.Context, binaryFile io.Writer) error {
+		// Fetch the Packages file to determine the appropriate .deb
+		// that'll run on runtime.GOARCH
+		debURL, err := determineDebURL(ctx)
+		if err != nil {
+			return err
+		}
+
+		// Fetch the .deb file and extract the firmware contents to binaryFile
+		return downloadAndExtractDeb(ctx, debURL, binaryFile)
+	}, debTargetFilename, true)
+	if err != nil {
+		return "", "", err
+	}
+
+	return binaryPath, "downloaded EDK2 firmware", nil
+}
+
+func determineDebURL(ctx context.Context) (string, error) {
+	// Fetch the Packages file and parse it
+	resp, err := binaryfetcher.FetchURL(ctx, debRepositoryURL+"/Packages")
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	sources, err := control.ParseBinaryIndex(bufio.NewReader(resp.Body))
+	if err != nil {
+		return "", err
 	}
 
-	// Fall back to downloading the Rust Hypervisor Firmware from GitHub
-	downloadURL, ok := goarchToDownloadURL[runtime.GOARCH]
+	// Find the package that contains EDK2 firmware for the current architecture
+	edk2Source, ok := lo.Find(sources, func(source control.BinaryIndex) bool {
+		return source.Package == debTargetPackage && source.Architecture.CPU == runtime.GOARCH
+	})
 	if !ok {
-		return "", "", fmt.Errorf("no EDK2 firmware installed on the system "+
-			"and architecture %q is not available in Rust Hypervisor Firmware's GitHub releases", runtime.GOARCH)
+		return "", fmt.Errorf("cannot find edk2-cloud-hypervisor package for %v in the repository",
+			runtime.GOARCH)
 	}
 
-	fmt.Printf("no EDK2 firmware installed on the system, downloading Rust Hypervisor Firmware "+
-		"from %s...\n", downloadURL)
+	return debRepositoryURL + "/" + edk2Source.Filename, nil
+}
 
-	binaryPath, err := binaryfetcher.Fetch(ctx, downloadURL, "hypervisor-fw", true)
+func downloadAndExtractDeb(ctx context.Context, debURL string, binaryFile io.Writer) error {
+	// Fetch the .deb package and parse it
+	debPath, err := binaryfetcher.FetchURLToFile(ctx, debURL)
 	if err != nil {
-		return "", "", err
+		return err
 	}
+	defer os.Remove(debPath)
 
-	return binaryPath, "Rust Hypervisor Firmware", nil
+	parsedDeb, debCloser, err := deb.LoadFile(debPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = debCloser()
+	}()
+
+	// Iterate over .deb package data files and look for EDK2 firmware
+	for {
+		next, err := parsedDeb.Data.Next()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return fmt.Errorf("cannot find %s file in the %s package", debTargetFilename,
+					debURL)
+			}
+
+			return err
+		}
+
+		if path.Base(next.Name) == debTargetFilename {
+			_, err := io.Copy(binaryFile, parsedDeb.Data)
+
+			return err
+		}
+	}
 }

internal/externalcommand/cloudhypervisor/cloudhypervisor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/cirruslabs/vetu/internal/binaryfetcher"
+	"io"
 	"os/exec"
 	"runtime"
 )
@@ -31,7 +32,19 @@ func CloudHypervisor(ctx context.Context, args ...string) (*exec.Cmd, error) {
 
 		fmt.Printf("no %q binary found in PATH, downloading it from %s...\n", binaryName, downloadURL)
 
-		binaryPath, err = binaryfetcher.Fetch(ctx, downloadURL, binaryName, true)
+		binaryPath, err = binaryfetcher.GetOrFetch(ctx, func(ctx context.Context, binaryFile io.Writer) error {
+			// Download the Cloud Hypervisor binary if not available in the cache
+			resp, err := binaryfetcher.FetchURL(ctx, downloadURL)
+			if err != nil {
+				return err
+			}
+
+			if _, err := io.Copy(binaryFile, resp.Body); err != nil {
+				return err
+			}
+
+			return nil
+		}, binaryName, true)
 		if err != nil {
 			return nil, err
 		}