feat(http): TLS support (#2492)

Signed-off-by: Hannah Zhang <hannahz@nvidia.com>

Author: grahamking

Cargo.lock

@@ -118,8 +118,9 @@ Dynamo provides a simple way to spin up a local set of inference components incl
 - **Workers** – Set of pre-configured LLM serving engines.
 
 ```
-# Start an OpenAI compatible HTTP server, a pre-processor (prompt templating and tokenization) and a router:
-python -m dynamo.frontend --http-port 8080
+# Start an OpenAI compatible HTTP server, a pre-processor (prompt templating and tokenization) and a router.
+# Pass the TLS certificate and key paths to use HTTPS instead of HTTP.
+python -m dynamo.frontend --http-port 8080 [--tls-cert-path cert.pem] [--tls-key-path key.pem]
 
 # Start the SGLang engine, connecting to NATS and etcd to receive requests. You can run several of these,
 # both for the same model and for multiple models. The frontend node will discover them.

README.md

@@ -16,10 +16,15 @@
 # Worker example:
 # - cd lib/bindings/python/examples/hello_world
 # - python server_sglang_static.py
+#
+# For TLS:
+# - python -m dynamo.frontend --http-port 8443 --tls-cert-path cert.pem --tls-key-path key.pem
+#
 
 import argparse
 import asyncio
 import os
+import pathlib
 import re
 
 import uvloop
@@ -85,6 +90,18 @@ def parse_args():
     parser.add_argument(
         "--http-port", type=int, default=8080, help="HTTP port for the engine (u16)."
     )
+    parser.add_argument(
+        "--tls-cert-path",
+        type=pathlib.Path,
+        default=None,
+        help="TLS certificate path, PEM format.",
+    )
+    parser.add_argument(
+        "--tls-key-path",
+        type=pathlib.Path,
+        default=None,
+        help="TLS certificate key path, PEM format.",
+    )
     parser.add_argument(
         "--router-mode",
         type=str,
@@ -149,6 +166,8 @@ def parse_args():
 
     if flags.static_endpoint and (not flags.model_name or not flags.model_path):
         parser.error("--static-endpoint requires both --model-name and --model-path")
+    if bool(flags.tls_cert_path) ^ bool(flags.tls_key_path):  # ^ is XOR
+        parser.error("--tls-cert-path and --tls-key-path must be provided together")
 
     return flags
 
@@ -192,6 +211,10 @@ async def async_main():
         kwargs["model_name"] = flags.model_name
     if flags.model_path:
         kwargs["model_path"] = flags.model_path
+    if flags.tls_cert_path:
+        kwargs["tls_cert_path"] = flags.tls_cert_path
+    if flags.tls_key_path:
+        kwargs["tls_key_path"] = flags.tls_key_path
 
     if is_static:
         # out=dyn://<static_endpoint>

components/frontend/src/dynamo/frontend/main.py

@@ -45,9 +45,18 @@ pub struct Flags {
     pub model_path_flag: Option<PathBuf>,
 
     /// HTTP port. `in=http` only
+    /// If tls_cert_path and tls_key_path are provided, this will be TLS/HTTPS.
     #[arg(long, default_value = "8080")]
     pub http_port: u16,
 
+    /// TLS certificate file
+    #[arg(long, requires = "tls_key_path")]
+    pub tls_cert_path: Option<PathBuf>,
+
+    /// TLS certificate key file
+    #[arg(long, requires = "tls_cert_path")]
+    pub tls_key_path: Option<PathBuf>,
+
     /// The name of the model we are serving
     #[arg(long)]
     pub model_name: Option<String>,

launch/dynamo-run/src/flags.rs

@@ -20,7 +20,7 @@ pub async fn run(
     runtime: Runtime,
     in_opt: Input,
     out_opt: Option<Output>,
-    flags: Flags,
+    mut flags: Flags,
 ) -> anyhow::Result<()> {
     //
     // Configure
@@ -39,7 +39,9 @@ pub async fn run(
         .kv_cache_block_size(flags.kv_cache_block_size)
         // Only set if user provides. Usually loaded from tokenizer_config.json
         .context_length(flags.context_length)
-        .http_port(Some(flags.http_port))
+        .http_port(flags.http_port)
+        .tls_cert_path(flags.tls_cert_path.take())
+        .tls_key_path(flags.tls_key_path.take())
         .router_config(Some(flags.router_config()))
         .request_template(flags.request_template.clone())
         .migration_limit(flags.migration_limit)