feat: Add support to Redis cluster mode (#2113)

* Store Redis credentials via dedicated fields: for security reasons, as discussed in #2083
* Deprecate `url` usage for Redis config

docs/Configuration.md

@@ -63,7 +63,17 @@ expiration = 3600
 key_prefix = "/custom/prefix/if/need"
 
 [cache.redis]
+# Deprecated, use `endpoint` instead
 url = "redis://user:passwd@1.2.3.4:6379/?db=1"
+## Refer to the `opendal` documentation for more information about Redis endpoint
+# Single-node endpoint. Mutually exclusive with `cluster_endpoints`
+endpoint = "redis://127.0.0.1:6379"
+# Multiple-node list of endpoints (cluster mode). Mutually exclusive with `endpoint`
+cluster_endpoints = "redis://10.0.0.1:6379,redis://10.0.0.2:6379"
+username = "user"
+password = "passwd"
+# Database number to use. Default is 0
+db = 1
 # Entry expiration time in seconds. Default is 0 (never expire)
 expiration = 3600
 key_prefix = "/custom/prefix/if/need"

src/cache/cache.rs

@@ -612,13 +612,49 @@ pub fn storage_from_config(
             }
             #[cfg(feature = "redis")]
             CacheType::Redis(config::RedisCacheConfig {
+                ref endpoint,
+                ref cluster_endpoints,
+                ref username,
+                ref password,
+                ref db,
                 ref url,
                 ref ttl,
                 ref key_prefix,
             }) => {
-                debug!("Init redis cache with url {url}");
-                let storage = RedisCache::build(url, key_prefix, *ttl)
-                    .map_err(|err| anyhow!("create redis cache failed: {err:?}"))?;
+                let storage = match (endpoint, cluster_endpoints, url) {
+                    (Some(url), None, None) => {
+                        debug!("Init redis single-node cache with url {url}");
+                        RedisCache::build_single(
+                            url,
+                            username.as_deref(),
+                            password.as_deref(),
+                            *db,
+                            key_prefix,
+                            *ttl,
+                        )
+                    }
+                    (None, Some(urls), None) => {
+                        debug!("Init redis cluster cache with urls {urls}");
+                        RedisCache::build_cluster(
+                            urls,
+                            username.as_deref(),
+                            password.as_deref(),
+                            *db,
+                            key_prefix,
+                            *ttl,
+                        )
+                    }
+                    (None, None, Some(url)) => {
+                        warn!("Init redis single-node cache from deprecated API with url {url}");
+                        if username.is_some() || password.is_some() || *db != crate::config::DEFAULT_REDIS_DB {
+                            bail!("`username`, `password` and `db` has no effect when `url` is set. Please use `endpoint` or `cluster_endpoints` for new API accessing");
+                        }
+
+                        RedisCache::build_from_url(url, key_prefix, *ttl)
+                    }
+                    _ => bail!("Only one of `endpoint`, `cluster_endpoints`, `url` must be set"),
+                }
+                .map_err(|err| anyhow!("create redis cache failed: {err:?}"))?;
                 return Ok(Arc::new(storage));
             }
             #[cfg(feature = "s3")]

src/cache/redis.rs

@@ -25,8 +25,8 @@ use url::Url;
 pub struct RedisCache;
 
 impl RedisCache {
-    /// Create a new `RedisCache`.
-    pub fn build(url: &str, key_prefix: &str, ttl: u64) -> Result<Operator> {
+    /// Create a new `RedisCache` for the given URL.
+    pub fn build_from_url(url: &str, key_prefix: &str, ttl: u64) -> Result<Operator> {
         let parsed = Url::parse(url)?;
 
         let mut builder = Redis::default();
@@ -52,4 +52,56 @@ impl RedisCache {
             .finish();
         Ok(op)
     }
+
+    /// Create a new `RedisCache` for the given single instance.
+    pub fn build_single(
+        endpoint: &str,
+        username: Option<&str>,
+        password: Option<&str>,
+        db: u32,
+        key_prefix: &str,
+        ttl: u64,
+    ) -> Result<Operator> {
+        let mut builder = Redis::default();
+        builder.endpoint(endpoint);
+
+        Self::build_common(builder, username, password, db, key_prefix, ttl)
+    }
+
+    /// Create a new `RedisCache` for the given cluster.
+    pub fn build_cluster(
+        endpoints: &str,
+        username: Option<&str>,
+        password: Option<&str>,
+        db: u32,
+        key_prefix: &str,
+        ttl: u64,
+    ) -> Result<Operator> {
+        let mut builder = Redis::default();
+        builder.cluster_endpoints(endpoints);
+
+        Self::build_common(builder, username, password, db, key_prefix, ttl)
+    }
+
+    fn build_common(
+        mut builder: Redis,
+        username: Option<&str>,
+        password: Option<&str>,
+        db: u32,
+        key_prefix: &str,
+        ttl: u64,
+    ) -> Result<Operator> {
+        builder.username(username.unwrap_or_default());
+        builder.password(password.unwrap_or_default());
+        builder.root(key_prefix);
+        if ttl != 0 {
+            builder.default_ttl(Duration::from_secs(ttl));
+        }
+        builder.db(db.into());
+
+        let op = Operator::new(builder)?
+            .layer(LoggingLayer::default())
+            .finish();
+        Ok(op)
+    }
 }

src/compiler/gcc.rs

@@ -154,7 +154,7 @@ use self::ArgData::*;
 
 const ARCH_FLAG: &str = "-arch";
 
-// Mostly taken from https://github.com/ccache/ccache/blob/master/src/compopt.c#L32-L84
+// Mostly taken from https://github.com/ccache/ccache/blob/master/src/compopt.cpp#L52-L172
 counted_array!(pub static ARGS: [ArgInfo<ArgData>; _] = [
     flag!("-", TooHardFlag),
     flag!("--coverage", Coverage),

src/config.rs

@@ -250,10 +250,33 @@ pub struct MemcachedCacheConfig {
 ///
 /// Please change this value freely if we have a better choice.
 const DEFAULT_REDIS_CACHE_TTL: u64 = 0;
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub const DEFAULT_REDIS_DB: u32 = 0;
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
 #[serde(deny_unknown_fields)]
 pub struct RedisCacheConfig {
-    pub url: String,
+    /// The single-node redis endpoint.
+    /// Mutually exclusive with `cluster_endpoints`.
+    pub endpoint: Option<String>,
+
+    /// The redis cluster endpoints.
+    /// Mutually exclusive with `endpoint`.
+    pub cluster_endpoints: Option<String>,
+
+    /// Username to authenticate with.
+    pub username: Option<String>,
+
+    /// Password to authenticate with.
+    pub password: Option<String>,
+
+    /// The redis URL.
+    /// Deprecated in favor of `endpoint`.
+    pub url: Option<String>,
+
+    /// the db number to use
+    ///
+    /// Default to 0
+    #[serde(default)]
+    pub db: u32,
 
     /// the ttl (expiration) time in seconds.
     ///
@@ -642,10 +665,12 @@ fn config_from_env() -> Result<EnvConfig> {
 
         let key_prefix = key_prefix_from_env_var("SCCACHE_REDIS_KEY_PREFIX");
 
+        // TODO: add possibility to load new values from env
         Some(RedisCacheConfig {
-            url,
+            url: Some(url),
             ttl,
             key_prefix,
+            ..Default::default()
         })
     } else {
         None
@@ -1186,9 +1211,13 @@ fn config_overrides() {
                 rw_mode: CacheModeConfig::ReadWrite,
             }),
             redis: Some(RedisCacheConfig {
-                url: "myotherredisurl".to_owned(),
+                endpoint: Some("myotherredisurl".to_owned()),
                 ttl: 24 * 3600,
                 key_prefix: "/redis/prefix".into(),
+                db: 10,
+                username: Some("user".to_owned()),
+                password: Some("secret".to_owned()),
+                ..Default::default()
             }),
             ..Default::default()
         },
@@ -1208,9 +1237,10 @@ fn config_overrides() {
                 key_prefix: String::new(),
             }),
             redis: Some(RedisCacheConfig {
-                url: "myredisurl".to_owned(),
+                url: Some("myredisurl".to_owned()),
                 ttl: 25 * 3600,
                 key_prefix: String::new(),
+                ..Default::default()
             }),
             ..Default::default()
         },
@@ -1222,9 +1252,13 @@ fn config_overrides() {
         Config::from_env_and_file_configs(env_conf, file_conf),
         Config {
             cache: Some(CacheType::Redis(RedisCacheConfig {
-                url: "myotherredisurl".to_owned(),
+                endpoint: Some("myotherredisurl".to_owned()),
                 ttl: 24 * 3600,
                 key_prefix: "/redis/prefix".into(),
+                db: 10,
+                username: Some("user".to_owned()),
+                password: Some("secret".to_owned()),
+                ..Default::default()
             }),),
             fallback_cache: DiskCacheConfig {
                 dir: "/env-cache".into(),
@@ -1390,7 +1424,12 @@ expiration = 90000
 key_prefix = "/custom/prefix/if/need"
 
 [cache.redis]
-url = "redis://user:passwd@1.2.3.4:6379/1"
+url = "redis://user:passwd@1.2.3.4:6379/?db=1"
+endpoint = "redis://127.0.0.1:6379"
+cluster_endpoints = "tcp://10.0.0.1:6379,redis://10.0.0.2:6379"
+username = "another_user"
+password = "new_passwd"
+db = 12
 expiration = 86400
 key_prefix = "/my/redis/cache"
 
@@ -1442,7 +1481,12 @@ no_credentials = true
                     version: "sccache".to_string()
                 }),
                 redis: Some(RedisCacheConfig {
-                    url: "redis://user:passwd@1.2.3.4:6379/1".to_owned(),
+                    url: Some("redis://user:passwd@1.2.3.4:6379/?db=1".to_owned()),
+                    endpoint: Some("redis://127.0.0.1:6379".to_owned()),
+                    cluster_endpoints: Some("tcp://10.0.0.1:6379,redis://10.0.0.2:6379".to_owned()),
+                    username: Some("another_user".to_owned()),
+                    password: Some("new_passwd".to_owned()),
+                    db: 12,
                     ttl: 24 * 3600,
                     key_prefix: "/my/redis/cache".into(),
                 }),