Clean up

Cargo.toml

@@ -75,7 +75,6 @@ lto = "off"
 [workspace.dependencies]
 # Keep consistent with preset_env_base through swc_core
 browserslist-rs = { version = "0.12.2" }
-<<<<<<< HEAD
 mdxjs = { version = "0.1.13" }
 modularize_imports = { version = "0.31.0" }
 styled_components = { version = "0.58.0" }
@@ -84,15 +83,6 @@ swc_core = { version = "0.78.24" }
 swc_emotion = { version = "0.34.0" }
 swc_relay = { version = "0.6.0" }
 testing = { version = "0.33.14" }
-=======
-mdxjs = { version = "0.1.12" }
-modularize_imports = { version = "0.30.0" }
-styled_components = { version = "0.57.0" }
-styled_jsx = { version = "0.34.0" }
-swc_core = { version = "0.76.46" }
-swc_emotion = { version = "0.33.0" }
-swc_relay = { version = "0.5.0" }
-testing = { version = "0.33.13" }
 
 auto-hash-map = { path = "crates/turbo-tasks-auto-hash-map" }
 node-file-trace = { path = "crates/node-file-trace", default-features = false }
@@ -138,10 +128,9 @@ turbopack-test-utils = { path = "crates/turbopack-test-utils" }
 turbopack-tests = { path = "crates/turbopack-tests" }
 turbopath = { path = "crates/turborepo-paths" }
 turborepo = { path = "crates/turborepo" }
-turborepo-cache = { path = "crates/turborepo-cache" }
-turborepo-env = { path = "crates/turborepo-env" }
 turborepo-api-client = { path = "crates/turborepo-api-client" }
 turborepo-cache = { path = "crates/turborepo-cache" }
+turborepo-env = { path = "crates/turborepo-env" }
 turborepo-ffi = { path = "crates/turborepo-ffi" }
 turborepo-fs = { path = "crates/turborepo-fs" }
 turborepo-lib = { path = "crates/turborepo-lib" }

cli/internal/cache/cache.go

@@ -7,7 +7,6 @@ package cache
 
 import (
 	"errors"
-	"github.com/vercel/turbo/cli/internal/client"
 	"sync"
 
 	"github.com/vercel/turbo/cli/internal/analytics"
@@ -120,7 +119,7 @@ var _remoteOnlyHelp = `Ignore the local filesystem cache for all tasks. Only
 allow reading and caching artifacts using the remote cache.`
 
 // New creates a new cache
-func New(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client *client.APIClient, recorder analytics.Recorder, onCacheRemoved OnCacheRemoved) (Cache, error) {
+func New(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client client, recorder analytics.Recorder, onCacheRemoved OnCacheRemoved) (Cache, error) {
 	c, err := newSyncCache(opts, repoRoot, client, recorder, onCacheRemoved)
 	if err != nil && !errors.Is(err, ErrNoCachesEnabled) {
 		return nil, err
@@ -132,7 +131,7 @@ func New(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client *client.APICli
 }
 
 // newSyncCache can return an error with a usable noopCache.
-func newSyncCache(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client *client.APIClient, recorder analytics.Recorder, onCacheRemoved OnCacheRemoved) (Cache, error) {
+func newSyncCache(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client client, recorder analytics.Recorder, onCacheRemoved OnCacheRemoved) (Cache, error) {
 	// Check to see if the user has turned off particular cache implementations.
 	useFsCache := !opts.SkipFilesystem
 	useHTTPCache := !opts.SkipRemote
@@ -141,8 +140,8 @@ func newSyncCache(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client *clie
 	// yourself out of having a cache. We should tell you about it but we shouldn't fail
 	// your build for that reason.
 	//
-	// Further, since the HttpCache can be removed at runtime, we need to insert a noopCache
-	// as a backup if you are configured to have *just* an HttpCache.
+	// Further, since the httpCache can be removed at runtime, we need to insert a noopCache
+	// as a backup if you are configured to have *just* an httpCache.
 	//
 	// This is reduced from (!useFsCache && !useHTTPCache) || (!useFsCache & useHTTPCache)
 	useNoopCache := !useFsCache
@@ -169,7 +168,7 @@ func newSyncCache(opts Opts, repoRoot turbopath.AbsoluteSystemPath, client *clie
 	}
 
 	// Precisely two cache implementations:
-	// fsCache and HttpCache OR HttpCache and noopCache
+	// fsCache and httpCache OR httpCache and noopCache
 	useMultiplexer := len(cacheImplementations) > 1
 	if useMultiplexer {
 		// We have early-returned any possible errors for this scenario.

cli/internal/cache/cache_http.go

@@ -4,6 +4,8 @@
 package cache
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -15,16 +17,16 @@ import (
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
 
-type cacheAPIClient interface {
+type client interface {
 	PutArtifact(hash string, body []byte, duration int, tag string) error
 	FetchArtifact(hash string) (*http.Response, error)
 	ArtifactExists(hash string) (*http.Response, error)
 	GetTeamID() string
 }
 
-type HttpCache struct {
+type httpCache struct {
 	writable       bool
-	client         cacheAPIClient
+	client         client
 	requestLimiter limiter
 	recorder       analytics.Recorder
 	signerVerifier *ArtifactSignatureAuthentication
@@ -40,18 +42,8 @@ func (l limiter) acquire() {
 func (l limiter) release() {
 	<-l
 }
-func (cache *HttpCache) GetAPIClient() cacheAPIClient {
-	return cache.client
-}
-func (cache *HttpCache) GetRepoRoot() turbopath.AbsoluteSystemPath {
-	return cache.repoRoot
-}
-
-func (cache *HttpCache) GetAuthenticator() *ArtifactSignatureAuthentication {
-	return cache.signerVerifier
-}
 
-func (cache *HttpCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration int, files []turbopath.AnchoredSystemPath) error {
+func (cache *httpCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration int, files []turbopath.AnchoredSystemPath) error {
 	// if cache.writable {
 	cache.requestLimiter.acquire()
 	defer cache.requestLimiter.release()
@@ -85,7 +77,7 @@ func (cache *HttpCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, du
 }
 
 // write writes a series of files into the given Writer.
-func (cache *HttpCache) write(w io.WriteCloser, anchor turbopath.AbsoluteSystemPath, files []turbopath.AnchoredSystemPath, cacheErrorChan chan error) {
+func (cache *httpCache) write(w io.WriteCloser, anchor turbopath.AbsoluteSystemPath, files []turbopath.AnchoredSystemPath, cacheErrorChan chan error) {
 	cacheItem := cacheitem.CreateWriter(w)
 
 	for _, file := range files {
@@ -100,7 +92,7 @@ func (cache *HttpCache) write(w io.WriteCloser, anchor turbopath.AbsoluteSystemP
 	cacheErrorChan <- cacheItem.Close()
 }
 
-func (cache *HttpCache) Fetch(_ turbopath.AbsoluteSystemPath, key string, _ []string) (ItemStatus, []turbopath.AnchoredSystemPath, error) {
+func (cache *httpCache) Fetch(_ turbopath.AbsoluteSystemPath, key string, _ []string) (ItemStatus, []turbopath.AnchoredSystemPath, error) {
 	cache.requestLimiter.acquire()
 	defer cache.requestLimiter.release()
 	hit, files, duration, err := cache.retrieve(key)
@@ -112,7 +104,7 @@ func (cache *HttpCache) Fetch(_ turbopath.AbsoluteSystemPath, key string, _ []st
 	return newRemoteTaskCacheStatus(hit, duration), files, err
 }
 
-func (cache *HttpCache) Exists(key string) ItemStatus {
+func (cache *httpCache) Exists(key string) ItemStatus {
 	cache.requestLimiter.acquire()
 	defer cache.requestLimiter.release()
 	hit, timeSaved, err := cache.exists(key)
@@ -122,7 +114,7 @@ func (cache *HttpCache) Exists(key string) ItemStatus {
 	return newRemoteTaskCacheStatus(hit, timeSaved)
 }
 
-func (cache *HttpCache) logFetch(hit bool, hash string, duration int) {
+func (cache *httpCache) logFetch(hit bool, hash string, duration int) {
 	var event string
 	if hit {
 		event = CacheEventHit
@@ -160,6 +152,57 @@ func (cache *httpCache) exists(hash string) (bool, int, error) {
 	return true, duration, err
 }
 
+func (cache *httpCache) retrieve(hash string) (bool, []turbopath.AnchoredSystemPath, int, error) {
+	resp, err := cache.client.FetchArtifact(hash)
+	if err != nil {
+		return false, nil, 0, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		return false, nil, 0, nil // doesn't exist - not an error
+	} else if resp.StatusCode != http.StatusOK {
+		b, _ := ioutil.ReadAll(resp.Body)
+		return false, nil, 0, fmt.Errorf("%s", string(b))
+	}
+
+	duration, err := getDurationFromResponse(resp)
+	if err != nil {
+		return false, nil, 0, err
+	}
+
+	var tarReader io.Reader
+
+	defer func() { _ = resp.Body.Close() }()
+	if cache.signerVerifier.isEnabled() {
+		expectedTag := resp.Header.Get("x-artifact-tag")
+		if expectedTag == "" {
+			// If the verifier is enabled all incoming artifact downloads must have a signature
+			return false, nil, 0, errors.New("artifact verification failed: Downloaded artifact is missing required x-artifact-tag header")
+		}
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return false, nil, 0, fmt.Errorf("artifact verification failed: %w", err)
+		}
+		isValid, err := cache.signerVerifier.validate(hash, b, expectedTag)
+		if err != nil {
+			return false, nil, 0, fmt.Errorf("artifact verification failed: %w", err)
+		}
+		if !isValid {
+			err = fmt.Errorf("artifact verification failed: artifact tag does not match expected tag %s", expectedTag)
+			return false, nil, 0, err
+		}
+		// The artifact has been verified and the body can be read and untarred
+		tarReader = bytes.NewReader(b)
+	} else {
+		tarReader = resp.Body
+	}
+	files, err := restoreTar(cache.repoRoot, tarReader)
+	if err != nil {
+		return false, nil, 0, err
+	}
+	return true, files, duration, nil
+}
+
 // getDurationFromResponse extracts the duration from the response header
 func getDurationFromResponse(resp *http.Response) (int, error) {
 	duration := 0
@@ -175,18 +218,23 @@ func getDurationFromResponse(resp *http.Response) (int, error) {
 	return duration, nil
 }
 
-func (cache *HttpCache) Clean(_ turbopath.AbsoluteSystemPath) {
+func restoreTar(root turbopath.AbsoluteSystemPath, reader io.Reader) ([]turbopath.AnchoredSystemPath, error) {
+	cache := cacheitem.FromReader(reader, true)
+	return cache.Restore(root)
+}
+
+func (cache *httpCache) Clean(_ turbopath.AbsoluteSystemPath) {
 	// Not possible; this implementation can only clean for a hash.
 }
 
-func (cache *HttpCache) CleanAll() {
+func (cache *httpCache) CleanAll() {
 	// Also not possible.
 }
 
-func (cache *HttpCache) Shutdown() {}
+func (cache *httpCache) Shutdown() {}
 
-func newHTTPCache(opts Opts, client cacheAPIClient, recorder analytics.Recorder, repoRoot turbopath.AbsoluteSystemPath) *HttpCache {
-	return &HttpCache{
+func newHTTPCache(opts Opts, client client, recorder analytics.Recorder, repoRoot turbopath.AbsoluteSystemPath) *httpCache {
+	return &httpCache{
 		writable:       true,
 		client:         client,
 		requestLimiter: make(limiter, 20),
@@ -195,7 +243,7 @@ func newHTTPCache(opts Opts, client cacheAPIClient, recorder analytics.Recorder,
 		signerVerifier: &ArtifactSignatureAuthentication{
 			// TODO(Gaspar): this should use RemoteCacheOptions.TeamId once we start
 			// enforcing team restrictions for repositories.
-			teamId:  client.GetTeamID(),
+			teamID:  client.GetTeamID(),
 			enabled: opts.RemoteCacheOpts.Signature,
 		},
 	}

cli/internal/cache/cache_http_test.go

@@ -54,7 +54,7 @@ func TestRemoteCachingDisabled(t *testing.T) {
 		Message: "Remote Caching has been disabled for this team. A team owner can enable it here: $URL",
 	}
 	client := &errorResp{err: clientErr}
-	cache := &HttpCache{
+	cache := &httpCache{
 		client:         client,
 		requestLimiter: make(limiter, 20),
 	}