feat(frontend): support setting HTTP host via CLI (--http-host) (#2523)

Author: suzusuzu

components/frontend/src/dynamo/frontend/main.py

@@ -87,6 +87,12 @@ def parse_args():
     parser.add_argument(
         "--kv-cache-block-size", type=int, help="KV cache block size (u32)."
     )
+    parser.add_argument(
+        "--http-host",
+        type=str,
+        default=os.environ.get("DYN_HTTP_HOST", "0.0.0.0"),
+        help="HTTP host for the engine (str). Can be set via DYN_HTTP_HOST env var.",
+    )
     parser.add_argument(
         "--http-port",
         type=int,
@@ -209,6 +215,7 @@ async def async_main():
         kv_router_config = None
 
     kwargs = {
+        "http_host": flags.http_host,
         "http_port": flags.http_port,
         "kv_cache_block_size": flags.kv_cache_block_size,
         "router_config": RouterConfig(

lib/bindings/python/rust/llm/entrypoint.rs

@@ -102,6 +102,7 @@ pub(crate) struct EntrypointArgs {
     template_file: Option<PathBuf>,
     router_config: Option<RouterConfig>,
     kv_cache_block_size: Option<u32>,
+    http_host: Option<String>,
     http_port: u16,
     tls_cert_path: Option<PathBuf>,
     tls_key_path: Option<PathBuf>,
@@ -112,7 +113,7 @@ pub(crate) struct EntrypointArgs {
 impl EntrypointArgs {
     #[allow(clippy::too_many_arguments)]
     #[new]
-    #[pyo3(signature = (engine_type, model_path=None, model_name=None, model_config=None, endpoint_id=None, context_length=None, template_file=None, router_config=None, kv_cache_block_size=None, http_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None))]
+    #[pyo3(signature = (engine_type, model_path=None, model_name=None, model_config=None, endpoint_id=None, context_length=None, template_file=None, router_config=None, kv_cache_block_size=None, http_host=None, http_port=None, tls_cert_path=None, tls_key_path=None, extra_engine_args=None))]
     pub fn new(
         engine_type: EngineType,
         model_path: Option<PathBuf>,
@@ -123,6 +124,7 @@ impl EntrypointArgs {
         template_file: Option<PathBuf>,
         router_config: Option<RouterConfig>,
         kv_cache_block_size: Option<u32>,
+        http_host: Option<String>,
         http_port: Option<u16>,
         tls_cert_path: Option<PathBuf>,
         tls_key_path: Option<PathBuf>,
@@ -153,6 +155,7 @@ impl EntrypointArgs {
             template_file,
             router_config,
             kv_cache_block_size,
+            http_host,
             http_port: http_port.unwrap_or(DEFAULT_HTTP_PORT),
             tls_cert_path,
             tls_key_path,
@@ -184,6 +187,7 @@ pub fn make_engine<'p>(
         .request_template(args.template_file.clone())
         .kv_cache_block_size(args.kv_cache_block_size)
         .router_config(args.router_config.clone().map(|rc| rc.into()))
+        .http_host(args.http_host.clone())
         .http_port(args.http_port)
         .tls_cert_path(args.tls_cert_path.clone())
         .tls_key_path(args.tls_key_path.clone())

lib/llm/src/entrypoint/input/http.rs

@@ -45,6 +45,9 @@ pub async fn run(runtime: Runtime, engine_config: EngineConfig) -> anyhow::Resul
             );
         }
     };
+    if let Some(http_host) = local_model.http_host() {
+        http_service_builder = http_service_builder.host(http_host);
+    }
     http_service_builder =
         http_service_builder.with_request_template(engine_config.local_model().request_template());
 

lib/llm/src/local_model.rs

@@ -50,6 +50,7 @@ pub struct LocalModelBuilder {
     template_file: Option<PathBuf>,
     router_config: Option<RouterConfig>,
     kv_cache_block_size: u32,
+    http_host: Option<String>,
     http_port: u16,
     tls_cert_path: Option<PathBuf>,
     tls_key_path: Option<PathBuf>,
@@ -64,6 +65,7 @@ impl Default for LocalModelBuilder {
     fn default() -> Self {
         LocalModelBuilder {
             kv_cache_block_size: DEFAULT_KV_CACHE_BLOCK_SIZE,
+            http_host: Default::default(),
             http_port: DEFAULT_HTTP_PORT,
             tls_cert_path: Default::default(),
             tls_key_path: Default::default(),
@@ -115,6 +117,11 @@ impl LocalModelBuilder {
         self
     }
 
+    pub fn http_host(&mut self, host: Option<String>) -> &mut Self {
+        self.http_host = host;
+        self
+    }
+
     pub fn http_port(&mut self, port: u16) -> &mut Self {
         self.http_port = port;
         self
@@ -200,6 +207,7 @@ impl LocalModelBuilder {
                 full_path: PathBuf::new(),
                 endpoint_id,
                 template,
+                http_host: self.http_host.take(),
                 http_port: self.http_port,
                 tls_cert_path: self.tls_cert_path.take(),
                 tls_key_path: self.tls_key_path.take(),
@@ -275,6 +283,7 @@ impl LocalModelBuilder {
             full_path,
             endpoint_id,
             template,
+            http_host: self.http_host.take(),
             http_port: self.http_port,
             tls_cert_path: self.tls_cert_path.take(),
             tls_key_path: self.tls_key_path.take(),
@@ -290,6 +299,7 @@ pub struct LocalModel {
     card: ModelDeploymentCard,
     endpoint_id: EndpointId,
     template: Option<RequestTemplate>,
+    http_host: Option<String>,
     http_port: u16,
     tls_cert_path: Option<PathBuf>,
     tls_key_path: Option<PathBuf>,
@@ -321,6 +331,10 @@ impl LocalModel {
         self.template.clone()
     }
 
+    pub fn http_host(&self) -> Option<String> {
+        self.http_host.clone()
+    }
+
     pub fn http_port(&self) -> u16 {
         self.http_port
     }