Check if a signed URL is specified and use it download tools (#953)

* Wrap v2 tools install function inside tools.Download

* Download tools defaulting to the replace behaviour

* Improve archive renamer and fix failing tests

* Find the correct tool and system when `version=latest` is specified

* Reintroduce caching option when downloading tools

Author: MatteoPologruto

tools/download.go

@@ -16,43 +16,18 @@
 package tools
 
 import (
-	"bytes"
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 
+	"github.com/arduino/arduino-create-agent/gen/tools"
+	"github.com/arduino/arduino-create-agent/utilities"
 	"github.com/arduino/arduino-create-agent/v2/pkgs"
-	"github.com/arduino/go-paths-helper"
-	"github.com/blang/semver"
-	"github.com/codeclysm/extract/v3"
 )
 
-// public vars to allow override in the tests
-var (
-	OS   = runtime.GOOS
-	Arch = runtime.GOARCH
-)
-
-func pathExists(path string) bool {
-	_, err := os.Stat(path)
-	if err == nil {
-		return true
-	}
-	if os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
-
 // Download will parse the index at the indexURL for the tool to download.
 // It will extract it in a folder in .arduino-create, and it will update the
 // Installed map.
@@ -70,97 +45,21 @@ func pathExists(path string) bool {
 // if it already exists.
 func (t *Tools) Download(pack, name, version, behaviour string) error {
 
-	body, err := t.index.Read()
-	if err != nil {
-		return err
-	}
-
-	var data pkgs.Index
-	json.Unmarshal(body, &data)
-
-	// Find the tool by name
-	correctTool, correctSystem := findTool(pack, name, version, data)
-
-	if correctTool.Name == "" || correctSystem.URL == "" {
-		t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
-		return nil
-	}
-
-	key := correctTool.Name + "-" + correctTool.Version
-
-	// Check if it already exists
-	if behaviour == "keep" {
-		location, ok := t.getMapValue(key)
-		if ok && pathExists(location) {
-			// overwrite the default tool with this one
-			t.setMapValue(correctTool.Name, location)
-			t.logger("The tool is already present on the system")
-			return t.writeMap()
-		}
-	}
-
-	// Download the tool
-	t.logger("Downloading tool " + name + " from " + correctSystem.URL)
-	resp, err := http.Get(correctSystem.URL)
+	tool := pkgs.New(t.index, t.directory.String(), behaviour)
+	_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-
-	// Read the body
-	body, err = io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	// Checksum
-	checksum := sha256.Sum256(body)
-	checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])
-
-	if checkSumString != correctSystem.Checksum {
-		return errors.New("checksum doesn't match")
-	}
-
-	tempPath := paths.TempDir()
-	// Create a temporary dir to extract package
-	if err := tempPath.MkdirAll(); err != nil {
-		return fmt.Errorf("creating temp dir for extraction: %s", err)
-	}
-	tempDir, err := tempPath.MkTempDir("package-")
-	if err != nil {
-		return fmt.Errorf("creating temp dir for extraction: %s", err)
-	}
-	defer tempDir.RemoveAll()
 
-	t.logger("Unpacking tool " + name)
-	ctx := context.Background()
-	reader := bytes.NewReader(body)
-	// Extract into temp directory
-	if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
-		return fmt.Errorf("extracting archive: %s", err)
-	}
-
-	location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
-	err = location.RemoveAll()
+	path := filepath.Join(pack, name, version)
+	safePath, err := utilities.SafeJoin(t.directory.String(), path)
 	if err != nil {
 		return err
 	}
 
-	// Check package content and find package root dir
-	root, err := findPackageRoot(tempDir)
-	if err != nil {
-		return fmt.Errorf("searching package root dir: %s", err)
-	}
-
-	if err := root.Rename(location); err != nil {
-		if err := root.CopyDirTo(location); err != nil {
-			return fmt.Errorf("moving extracted archive to destination dir: %s", err)
-		}
-	}
-
 	// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
 	// AFAIK this is only the case for the windows-driver tool
-	err = t.installDrivers(location.String())
+	err = t.installDrivers(safePath)
 	if err != nil {
 		return err
 	}
@@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
 	t.logger("Ensure that the files are executable")
 
 	// Update the tool map
-	t.logger("Updating map with location " + location.String())
-
-	t.setMapValue(name, location.String())
-	t.setMapValue(name+"-"+correctTool.Version, location.String())
-	return t.writeMap()
-}
+	t.logger("Updating map with location " + safePath)
 
-func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
-	files, err := parent.ReadDir()
-	if err != nil {
-		return nil, fmt.Errorf("reading package root dir: %s", err)
-	}
-	files.FilterOutPrefix("__MACOSX")
+	t.setMapValue(name, safePath)
+	t.setMapValue(name+"-"+version, safePath)
 
-	// if there is only one dir, it is the root dir
-	if len(files) == 1 && files[0].IsDir() {
-		return files[0], nil
-	}
-	return parent, nil
-}
-
-func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
-	var correctTool pkgs.Tool
-	correctTool.Version = "0.0"
-
-	for _, p := range data.Packages {
-		if p.Name != pack {
-			continue
-		}
-		for _, t := range p.Tools {
-			if version != "latest" {
-				if t.Name == name && t.Version == version {
-					correctTool = t
-				}
-			} else {
-				// Find latest
-				v1, _ := semver.Make(t.Version)
-				v2, _ := semver.Make(correctTool.Version)
-				if t.Name == name && v1.Compare(v2) > 0 {
-					correctTool = t
-				}
-			}
-		}
-	}
-
-	// Find the url based on system
-	correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
-
-	return correctTool, correctSystem
+	return nil
 }
 
 func (t *Tools) installDrivers(location string) error {
 	OkPressed := 6
 	extension := ".bat"
 	// add .\ to force locality
 	preamble := ".\\"
-	if OS != "windows" {
+	if runtime.GOOS != "windows" {
 		extension = ".sh"
 		// add ./ to force locality
 		preamble = "./"
@@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
 			os.Chdir(location)
 			t.logger(preamble + "post_install" + extension)
 			oscmd := exec.Command(preamble + "post_install" + extension)
-			if OS != "linux" {
+			if runtime.GOOS != "linux" {
 				// spawning a shell could be the only way to let the user type his password
 				TellCommandNotToSpawnShell(oscmd)
 			}

tools/download_test.go

@@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
 		{"linux", "arm", "arm-linux-gnueabihf"},
 	}
 	defer func() {
-		OS = runtime.GOOS     // restore `runtime.OS`
-		Arch = runtime.GOARCH // restore `runtime.ARCH`
+		pkgs.OS = runtime.GOOS     // restore `runtime.OS`
+		pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
 	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
@@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
 	require.NoError(t, err)
 	for _, tc := range testCases {
 		t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
-			OS = tc.hostOS     // override `runtime.OS` for testing purposes
-			Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
+			pkgs.OS = tc.hostOS     // override `runtime.OS` for testing purposes
+			pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
 			// Find the tool by name
-			correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
+			correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
+			require.True(t, found)
 			require.NotNil(t, correctTool)
 			require.NotNil(t, correctSystem)
 			require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 		{"windows", "amd64", "i686-mingw32"},
 	}
 	defer func() {
-		OS = runtime.GOOS     // restore `runtime.OS`
-		Arch = runtime.GOARCH // restore `runtime.ARCH`
+		pkgs.OS = runtime.GOOS     // restore `runtime.OS`
+		pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
 	}()
 	testIndex := paths.New("testdata", "test_tool_index.json")
 	buf, err := testIndex.ReadFile()
@@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
 	require.NoError(t, err)
 	for _, tc := range testCases {
 		t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
-			OS = tc.hostOS     // override `runtime.OS` for testing purposes
-			Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
+			pkgs.OS = tc.hostOS     // override `runtime.OS` for testing purposes
+			pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
 			// Find the tool by name
-			correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
+			correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
+			require.True(t, found)
 			require.NotNil(t, correctTool)
 			require.NotNil(t, correctSystem)
 			require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
 				if filePath.IsDir() {
 					require.DirExists(t, filePath.String())
 				} else {
-					if OS == "windows" {
+					if runtime.GOOS == "windows" {
 						require.FileExists(t, filePath.String()+".exe")
 					} else {
 						require.FileExists(t, filePath.String())

tools/tools.go

@@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
 	return value, ok
 }
 
-// writeMap() writes installed map to the json file "installed.json"
-func (t *Tools) writeMap() error {
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	b, err := json.Marshal(t.installed)
-	if err != nil {
-		return err
-	}
-	filePath := t.directory.Join("installed.json")
-	return filePath.WriteFile(b)
-}
-
 // readMap() reads the installed map from json file "installed.json"
 func (t *Tools) readMap() error {
 	t.mutex.Lock()

v2/http.go

@@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
 	logAdapter := LogAdapter{Logger: logger}
 
 	// Mount tools
-	toolsSvc := pkgs.New(index, directory)
+	toolsSvc := pkgs.New(index, directory, "replace")
 	toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
 	toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
 	toolssvr.Mount(mux, toolsServer)