Use default MaxDownloadParallelism based on number of CPU cores (#2047)

* Use default value based on number of CPU cores * renames
GoogleCloudPlatform · Jun 27, 2024 · ef2179c · ef2179c
1 parent dd1542d
commit ef2179c
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 3 deletions.
diff --git a/internal/config/config_util.go b/internal/config/config_util.go
@@ -17,6 +17,7 @@ package config
 import (
 	"fmt"
 	"math"
+	"runtime"
 	"time"
 )
 
@@ -112,3 +113,7 @@ func ListCacheTtlSecsToDuration(secs int64) time.Duration {
 
 	return time.Duration(secs * int64(time.Second))
 }
+
+func DefaultMaxParallelDownloads() int {
+	return max(16, 2*runtime.NumCPU())
+}
diff --git a/internal/config/config_util_test.go b/internal/config/config_util_test.go
@@ -267,3 +267,7 @@ func Test_ListCacheTtlSecsToDuration_InvalidCall(t *testing.T) {
 	// Calling with invalid argument to trigger panic.
 	ListCacheTtlSecsToDuration(-3)
 }
+
+func Test_DefaultMaxParallelDownloads(t *testing.T) {
+	assert.GreaterOrEqual(t, DefaultMaxParallelDownloads(), 16)
+}
diff --git a/internal/config/mount_config.go b/internal/config/mount_config.go
@@ -63,7 +63,6 @@ const (
 	DefaultEnableParallelDownloads  = false
 	DefaultDownloadChunkSizeMB      = 25
 	DefaultParallelDownloadsPerFile = 10
-	DefaultMaxParallelDownloads     = -1
 )
 
 type WriteConfig struct {
@@ -188,7 +187,7 @@ func NewMountConfig() *MountConfig {
 		MaxSizeMB:                DefaultFileCacheMaxSizeMB,
 		EnableParallelDownloads:  DefaultEnableParallelDownloads,
 		ParallelDownloadsPerFile: DefaultParallelDownloadsPerFile,
-		MaxParallelDownloads:     DefaultMaxParallelDownloads,
+		MaxParallelDownloads:     DefaultMaxParallelDownloads(),
 		DownloadChunkSizeMB:      DefaultDownloadChunkSizeMB,
 		EnableCRC:                DefaultEnableCRC,
 	}

diff --git a/internal/config/yaml_parser_test.go b/internal/config/yaml_parser_test.go
@@ -44,7 +44,7 @@ func validateDefaultConfig(t *testing.T, mountConfig *MountConfig) {
 	assert.False(t, mountConfig.FileCacheConfig.CacheFileForRangeRead)
 	assert.False(t, mountConfig.FileCacheConfig.EnableParallelDownloads)
 	assert.Equal(t, 10, mountConfig.FileCacheConfig.ParallelDownloadsPerFile)
-	assert.Equal(t, -1, mountConfig.FileCacheConfig.MaxParallelDownloads)
+	assert.GreaterOrEqual(t, mountConfig.FileCacheConfig.MaxParallelDownloads, 16)
 	assert.Equal(t, 25, mountConfig.FileCacheConfig.DownloadChunkSizeMB)
 	assert.True(t, mountConfig.FileCacheConfig.EnableCRC)
 	assert.Equal(t, 1, mountConfig.GCSConnection.GRPCConnPoolSize)