Fix agent not able to install esptool after 1.6.0 (#986)

umbynos · MatteoPologruto · web-flow · commit 6287df1e7a16 · 2024-08-05T16:01:59.000+02:00
* fix link * drop test on old tool not used anymore, and introduce again `arduino-fwuploader` * add tests to install esptool to verify #980 * implement properly rename function to fix extraction issue with esptool * fix test * Update v2/pkgs/tools_test.go Co-authored-by: MatteoPologruto <109663225+MatteoPologruto@users.noreply.github.com> --------- Co-authored-by: MatteoPologruto <109663225+MatteoPologruto@users.noreply.github.com>
diff --git a/main_test.go b/main_test.go
@@ -116,6 +116,18 @@ func TestInstallToolV2(t *testing.T) {
 		Signature: &bossacSignature,
 	}
 
+	esptoolURL := "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/x86_64-linux-gnu.esptool-f80ae31.tar.gz"
+	esptoolChecksum := "SHA-256:bded1dca953377838b6086a9bcd40a1dc5286ba5f69f2372c22a1d1819baad24"
+	esptoolSignature := "852b58871419ce5e5633ecfaa72c0f0fa890ceb51164b362b8133bc0e3e003a21cec48935b8cdc078f4031219cbf17fb7edd9d7c9ca8ed85492911c9ca6353c9aa4691eb91fda99563a6bd49aeca0d9981fb05ec76e45c6024f8a6822862ad1e34ddc652fbbf4fa909887a255d4f087398ec386577efcec523c21203be3d10fc9e9b0f990a7536875a77dc2bc5cbffea7734b62238e31719111b718bacccebffc9be689545540e81d23b81caa66214376f58a0d6a45cf7efc5d3af62ab932b371628162fffe403906f41d5534921e5be081c5ac2ecc9db5caec03a105cc44b00ce19a95ad079843501eb8182e0717ce327867380c0e39d2b48698547fc1d0d66"
+	esptoolInstallURLOK := tools.ToolPayload{
+		Name:      "esptool",
+		Version:   "2.5.0-3-20ed2b9",
+		Packager:  "esp8266",
+		URL:       &esptoolURL,
+		Checksum:  &esptoolChecksum,
+		Signature: &esptoolSignature,
+	}
+
 	wrongSignature := "wr0ngs1gn4tur3"
 	bossacInstallWrongSig := tools.ToolPayload{
 		Name:      "bossac",
@@ -147,6 +159,7 @@ func TestInstallToolV2(t *testing.T) {
 		{bossacInstallWrongSig, http.StatusInternalServerError, "verification error"},
 		{bossacInstallWrongCheck, http.StatusInternalServerError, "checksum of downloaded file doesn't match"},
 		{bossacInstallNoURL, http.StatusOK, "ok"},
+		{esptoolInstallURLOK, http.StatusOK, "ok"},
 	}
 
 	for _, test := range tests {
diff --git a/tools/download_test.go b/tools/download_test.go
@@ -120,7 +120,6 @@ func TestDownload(t *testing.T) {
 		{"rp2040tools", "1.0.6", []string{"elf2uf2", "picotool", "pioasm", "rp2040load"}},
 		{"esptool_py", "4.5.1", []string{"esptool"}},
 		{"arduino-fwuploader", "2.2.2", []string{"arduino-fwuploader"}},
-		{"fwupdater", "0.1.12", []string{"firmwares", "FirmwareUploader"}}, // old legacy tool
 	}
 	// prepare the test environment
 	tempDir := t.TempDir()
diff --git a/tools/testdata/test_tool_index.json b/tools/testdata/test_tool_index.json
@@ -514,61 +514,6 @@
               "size": "6829396"
             }
           ]
-        },
-        {
-          "name": "fwupdater",
-          "version": "0.1.12",
-          "systems": [
-            {
-              "host": "i686-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
-              "checksum": "SHA-256:2fec2bdfd20ad4950bc9ba37108dc2a7c152f569174279c0697efe1f5a0db781",
-              "size": "26097546"
-            },
-            {
-              "host": "x86_64-pc-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
-              "checksum": "SHA-256:ce57d0afef30cb7d3513f5da326346c99d6bf4923bbc2200634086811f3fb31e",
-              "size": "26073327"
-            },
-            {
-              "host": "i686-mingw32",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_32bit.zip",
-              "archiveFileName": "FirmwareUploader_0.1.12_Windows_32bit.zip",
-              "checksum": "SHA-256:558568b453caa1c821def8cc6d34555d0c910eb7e7e871de3ae1c39ae6f01bdd",
-              "size": "25743641"
-            },
-            {
-              "host": "x86_64-mingw32",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_64bit.zip",
-              "archiveFileName": "FirmwareUploader_0.1.12_Windows_64bit.zip",
-              "checksum": "SHA-256:ec16de33620985434280c92c3c322257b89bb67adf8fd4d5dd5f9467ea1e9e40",
-              "size": "25851428"
-            },
-            {
-              "host": "i386-apple-darwin11",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
-              "checksum": "SHA-256:a470361b57f86ddfcaecd274d844af51ee1d23a71cd6c26e30fcef2152d1a03f",
-              "size": "25792860"
-            },
-            {
-              "host": "arm-linux-gnueabihf",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
-              "checksum": "SHA-256:855fa0a9b942c3ee18906efc510bdfe30bf3334ff28ffbb476e648ff30033847",
-              "size": "25936245"
-            },
-            {
-              "host": "aarch64-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
-              "checksum": "SHA-256:691461e64fe075e9a79801347c2bd895fb72f8f2c45a7cd49056c6ad9efe8fc4",
-              "size": "25967430"
-            }
-          ]
         }
       ]
     }
diff --git a/v2/pkgs/pkgs.go b/v2/pkgs/pkgs.go
@@ -50,7 +50,7 @@ type System struct {
 	Checksum string `json:"checksum"`
 }
 
-// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L129-L142
+// Source: https://github.com/arduino/arduino-cli/blob/master/internal/arduino/cores/tools.go#L129-L142
 var (
 	regexpLinuxArm   = regexp.MustCompile("arm.*-linux-gnueabihf")
 	regexpLinuxArm64 = regexp.MustCompile("(aarch64|arm64)-linux-gnu")
@@ -66,7 +66,7 @@ var (
 	regexpFreeBSD64  = regexp.MustCompile("amd64-freebsd[0-9]*")
 )
 
-// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L144-L176
+// Source: https://github.com/arduino/arduino-cli/blob/master/internal/arduino/cores/tools.go#L144-L176
 func (s *System) isExactMatchWith(osName, osArch string) bool {
 	if s.Host == "all" {
 		return true
@@ -101,7 +101,7 @@ func (s *System) isExactMatchWith(osName, osArch string) bool {
 	return false
 }
 
-// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L178-L198
+// Source: https://github.com/arduino/arduino-cli/blob/master/internal/arduino/cores/tools.go#L178-L198
 func (s *System) isCompatibleWith(osName, osArch string) (bool, int) {
 	if s.isExactMatchWith(osName, osArch) {
 		return true, 1000
@@ -125,7 +125,7 @@ func (s *System) isCompatibleWith(osName, osArch string) (bool, int) {
 }
 
 // GetFlavourCompatibleWith returns the downloadable resource (System) compatible with the specified OS/Arch
-// Source: https://github.com/arduino/arduino-cli/blob/master/arduino/cores/tools.go#L206-L216
+// Source: https://github.com/arduino/arduino-cli/blob/master/internal/arduino/cores/tools.go#L206-L216
 func (t *Tool) GetFlavourCompatibleWith(osName, osArch string) System {
 	var correctSystem System
 	maxSimilarity := -1
diff --git a/v2/pkgs/testdata/test_tool_index.json b/v2/pkgs/testdata/test_tool_index.json
@@ -514,61 +514,6 @@
               "size": "6829396"
             }
           ]
-        },
-        {
-          "name": "fwupdater",
-          "version": "0.1.12",
-          "systems": [
-            {
-              "host": "i686-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_32bit.tar.bz2",
-              "checksum": "SHA-256:2fec2bdfd20ad4950bc9ba37108dc2a7c152f569174279c0697efe1f5a0db781",
-              "size": "26097546"
-            },
-            {
-              "host": "x86_64-pc-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_64bit.tar.bz2",
-              "checksum": "SHA-256:ce57d0afef30cb7d3513f5da326346c99d6bf4923bbc2200634086811f3fb31e",
-              "size": "26073327"
-            },
-            {
-              "host": "i686-mingw32",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_32bit.zip",
-              "archiveFileName": "FirmwareUploader_0.1.12_Windows_32bit.zip",
-              "checksum": "SHA-256:558568b453caa1c821def8cc6d34555d0c910eb7e7e871de3ae1c39ae6f01bdd",
-              "size": "25743641"
-            },
-            {
-              "host": "x86_64-mingw32",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Windows_64bit.zip",
-              "archiveFileName": "FirmwareUploader_0.1.12_Windows_64bit.zip",
-              "checksum": "SHA-256:ec16de33620985434280c92c3c322257b89bb67adf8fd4d5dd5f9467ea1e9e40",
-              "size": "25851428"
-            },
-            {
-              "host": "i386-apple-darwin11",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_macOS_64bit.tar.bz2",
-              "checksum": "SHA-256:a470361b57f86ddfcaecd274d844af51ee1d23a71cd6c26e30fcef2152d1a03f",
-              "size": "25792860"
-            },
-            {
-              "host": "arm-linux-gnueabihf",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM.tar.bz2",
-              "checksum": "SHA-256:855fa0a9b942c3ee18906efc510bdfe30bf3334ff28ffbb476e648ff30033847",
-              "size": "25936245"
-            },
-            {
-              "host": "aarch64-linux-gnu",
-              "url": "http://downloads.arduino.cc/tools/FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
-              "archiveFileName": "FirmwareUploader_0.1.12_Linux_ARM64.tar.bz2",
-              "checksum": "SHA-256:691461e64fe075e9a79801347c2bd895fb72f8f2c45a7cd49056c6ad9efe8fc4",
-              "size": "25967430"
-            }
-          ]
         }
       ]
     }
diff --git a/v2/pkgs/tools.go b/v2/pkgs/tools.go
@@ -269,15 +269,19 @@ func (t *Tools) Remove(ctx context.Context, payload *tools.ToolPayload) (*tools.
 	return &tools.Operation{Status: "ok"}, nil
 }
 
+// rename function is used to rename the path of the extracted files
 func rename(base string) extract.Renamer {
+	// "Rename" the given path adding the "base" and removing the root folder in "path" (if present).
 	return func(path string) string {
 		parts := strings.Split(filepath.ToSlash(path), "/")
-		newPath := strings.Join(parts[1:], "/")
-		if newPath == "" {
-			newPath = filepath.Join(newPath, path)
+		if len(parts) <= 1 {
+			// The path does not contain a root folder. This might happen for tool packages (zip files)
+			// that have an invalid structure. Do not try to remove the root folder in these cases.
+			return filepath.Join(base, path)
 		}
-		path = filepath.Join(base, newPath)
-		return path
+		// Removes the first part of the path (the root folder).
+		path = strings.Join(parts[1:], "/")
+		return filepath.Join(base, path)
 	}
 }
 
diff --git a/v2/pkgs/tools_test.go b/v2/pkgs/tools_test.go
@@ -230,9 +230,9 @@ func TestInstall(t *testing.T) {
 		{Name: "dfu-util", Version: "0.10.0-arduino1", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
 		{Name: "rp2040tools", Version: "1.0.6", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
 		{Name: "esptool_py", Version: "4.5.1", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
-		// At the moment we don't install these ones because they are packaged in a different way: they do not have a top level dir, causing the rename funcion to behave incorrectly
-		// {Name: "fwupdater", Version: "0.1.12", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
-		// {Name: "arduino-fwuploader", Version: "2.2.2", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
+		{Name: "arduino-fwuploader", Version: "2.2.2", Packager: "arduino-test", URL: nil, Checksum: nil, Signature: nil},
+		// test download of a tool not present in index. the same archive is downloaded on linux/win/mac See https://github.com/arduino/arduino-create-agent/issues/980
+		{Name: "esptool", Version: "2.5.0-3-20ed2b9", Packager: "esp8266", URL: strpoint("https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/x86_64-linux-gnu.esptool-f80ae31.tar.gz"), Checksum: strpoint("SHA-256:bded1dca953377838b6086a9bcd40a1dc5286ba5f69f2372c22a1d1819baad24"), Signature: strpoint("852b58871419ce5e5633ecfaa72c0f0fa890ceb51164b362b8133bc0e3e003a21cec48935b8cdc078f4031219cbf17fb7edd9d7c9ca8ed85492911c9ca6353c9aa4691eb91fda99563a6bd49aeca0d9981fb05ec76e45c6024f8a6822862ad1e34ddc652fbbf4fa909887a255d4f087398ec386577efcec523c21203be3d10fc9e9b0f990a7536875a77dc2bc5cbffea7734b62238e31719111b718bacccebffc9be689545540e81d23b81caa66214376f58a0d6a45cf7efc5d3af62ab932b371628162fffe403906f41d5534921e5be081c5ac2ecc9db5caec03a105cc44b00ce19a95ad079843501eb8182e0717ce327867380c0e39d2b48698547fc1d0d66")},
 	}
 
 	expectedFiles := map[string][]string{
@@ -244,8 +244,8 @@ func TestInstall(t *testing.T) {
 		"dfu-util-0.10.0-arduino1": {"dfu-prefix", "dfu-suffix", "dfu-util"},
 		"rp2040tools-1.0.6":        {"elf2uf2", "picotool", "pioasm", "rp2040load"},
 		"esptool_py-4.5.1":         {"esptool"},
-		// "fwupdater-0.1.12":         {"firmwares", "FirmwareUploader"}, // old legacy tool
-		// "arduino-fwuploader-2.2.2": {"arduino-fwuploader"},
+		"arduino-fwuploader-2.2.2": {"arduino-fwuploader"},
+		// "esptool-2.5.0-3-20ed2b9":  {"esptool"}, // we don't check if there is esptool in the archive because it's the same archive even on windows (no extension)
 	}
 	for _, tc := range testCases {
 		t.Run(tc.Name+"-"+tc.Version, func(t *testing.T) {
@@ -254,7 +254,7 @@ func TestInstall(t *testing.T) {
 			require.NoError(t, err)
 
 			// Check that the tool has been downloaded
-			toolDir := paths.New(tmp).Join("arduino-test", tc.Name, tc.Version)
+			toolDir := paths.New(tmp).Join(tc.Packager, tc.Name, tc.Version)
 			require.DirExists(t, toolDir.String())
 
 			// Check that the files have been created

Original file line number	Diff line number	Diff line change
`@@ -120,7 +120,6 @@ func TestDownload(t *testing.T) {`
`120`	`120`	`{"rp2040tools", "1.0.6", []string{"elf2uf2", "picotool", "pioasm", "rp2040load"}},`
`121`	`121`	`{"esptool_py", "4.5.1", []string{"esptool"}},`
`122`	`122`	`{"arduino-fwuploader", "2.2.2", []string{"arduino-fwuploader"}},`
`123`		`- {"fwupdater", "0.1.12", []string{"firmwares", "FirmwareUploader"}}, // old legacy tool`
`124`	`123`	`}`
`125`	`124`	`// prepare the test environment`
`126`	`125`	`tempDir := t.TempDir()`
Original file line number	Diff line number	Diff line change
`@@ -269,15 +269,19 @@ func (t Tools) Remove(ctx context.Context, payload tools.ToolPayload) (*tools.`
`269`	`269`	`return &tools.Operation{Status: "ok"}, nil`
`270`	`270`	`}`
`271`	`271`
	`272`	`+// rename function is used to rename the path of the extracted files`
`272`	`273`	`func rename(base string) extract.Renamer {`
	`274`	`+ // "Rename" the given path adding the "base" and removing the root folder in "path" (if present).`
`273`	`275`	`return func(path string) string {`
`274`	`276`	`parts := strings.Split(filepath.ToSlash(path), "/")`
`275`		`- newPath := strings.Join(parts[1:], "/")`
`276`		`- if newPath == "" {`
`277`		`- newPath = filepath.Join(newPath, path)`
	`277`	`+ if len(parts) <= 1 {`
	`278`	`+ // The path does not contain a root folder. This might happen for tool packages (zip files)`
	`279`	`+ // that have an invalid structure. Do not try to remove the root folder in these cases.`
	`280`	`+ return filepath.Join(base, path)`
`278`	`281`	`}`
`279`		`- path = filepath.Join(base, newPath)`
`280`		`- return path`
	`282`	`+ // Removes the first part of the path (the root folder).`
	`283`	`+ path = strings.Join(parts[1:], "/")`
	`284`	`+ return filepath.Join(base, path)`
`281`	`285`	`}`
`282`	`286`	`}`
`283`	`287`