[ws-daemon] Improve download speed of snapshots

Author: aledbf

components/ws-daemon/debug.Dockerfile

@@ -27,6 +27,7 @@ RUN apt update \
       apt-transport-https \
       python3-crcmod \
       gnupg \
+      aria22 \
   && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
   && curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
   && apt update && apt install -y --no-install-recommends  google-cloud-sdk=${CLOUD_SDK_VERSION}-0 \

components/ws-daemon/leeway.Dockerfile

@@ -26,6 +26,7 @@ RUN apt update \
       git git-lfs openssh-client lz4 e2fsprogs coreutils tar strace xfsprogs curl ca-certificates \
       apt-transport-https \
       python3-crcmod \
+      aria22 \
   && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
   && curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
   && apt update && apt install -y --no-install-recommends  google-cloud-sdk=${CLOUD_SDK_VERSION}-0 \

components/ws-daemon/pkg/content/initializer.go

@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"errors"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -390,13 +389,28 @@ func (rs *remoteContentStorage) Download(ctx context.Context, destination string
 	}
 
 	span.SetTag("URL", info.URL)
-	resp, err := http.Get(info.URL)
+
+	tempFile, err := os.CreateTemp("", "remote-content-")
+	if err != nil {
+		return true, xerrors.Errorf("cannot create temporal file: %w", err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	args := []string{
+		"-x16", "-j12",
+		info.URL,
+		"-o", tempFile.Name(),
+	}
+	cmd := exec.Command("aria2c", args...)
+	var out []byte
+	out, err = cmd.CombinedOutput()
 	if err != nil {
-		return true, err
+		log.WithError(err).WithField("out", string(out)).Error("unexpected error downloading file")
+		return true, xerrors.Errorf("unexpected error downloading file")
 	}
-	defer resp.Body.Close()
 
-	err = archive.ExtractTarbal(ctx, resp.Body, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
+	err = archive.ExtractTarbal(ctx, tempFile, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
 	if err != nil {
 		return true, xerrors.Errorf("tar %s: %s", destination, err.Error())
 	}