Change default max-concurrent-requests value to 1024 (#748)

docs/reference/launcher.md

@@ -78,7 +78,7 @@ Options:
           The maximum amount of concurrent requests for this particular deployment. Having a low limit will refuse clients requests instead of having them wait for too long and is usually good to handle backpressure correctly
 
           [env: MAX_CONCURRENT_REQUESTS=]
-          [default: 128]
+          [default: 1024]
 
       --max-best-of <MAX_BEST_OF>
           This is the maximum allowed value for clients to set `best_of`. Best of makes `n` generations at the same time, and return the best in terms of overall log probability over the entire generated sequence

launcher/src/main.rs

@@ -397,7 +397,7 @@ struct Args {
     /// The maximum amount of concurrent requests for this particular deployment.
     /// Having a low limit will refuse clients requests instead of having them
     /// wait for too long and is usually good to handle backpressure correctly.
-    #[clap(default_value = "128", long, env)]
+    #[clap(default_value = "1024", long, env)]
     max_concurrent_requests: usize,
 
     /// This is the maximum allowed value for clients to set `best_of`.

router/src/main.rs

@@ -30,7 +30,7 @@ use tracing_subscriber::{EnvFilter, Layer};
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 struct Args {
-    #[clap(default_value = "128", long, env)]
+    #[clap(default_value = "1024", long, env)]
     max_concurrent_requests: usize,
     #[clap(default_value = "2", long, env)]
     max_best_of: usize,