feat: Add frontend support for min_tokens and ignore_eos (outside of nvext) and Structured Output / Guided Decoding (#2380)

Signed-off-by: KrishnanPrash <140860868+KrishnanPrash@users.noreply.github.com>
Co-authored-by: Ryan McCormick <rmccormick@nvidia.com>
Co-authored-by: Ayush Agarwal <ayushag@nvidia.com>
Signed-off-by: Hannah Zhang <hannahz@nvidia.com>

Author: 3 people

lib/llm/src/entrypoint/input/batch.rs

@@ -222,7 +222,11 @@ async fn evaluate(
         )
         .temperature(template.as_ref().map_or(0.7, |t| t.temperature))
         .build()?;
-    let req = NvCreateChatCompletionRequest { inner, nvext: None };
+    let req = NvCreateChatCompletionRequest {
+        inner,
+        common: Default::default(),
+        nvext: None,
+    };
     let mut stream = engine.generate(Context::new(req)).await?;
     let mut output = String::new();
     while let Some(item) = stream.next().await {

lib/llm/src/entrypoint/input/text.rs

@@ -118,6 +118,7 @@ async fn main_loop(
 
         let req = NvCreateChatCompletionRequest {
             inner,
+            common: Default::default(),
             nvext: Some(nvext),
         };
 

lib/llm/src/http/service/openai.rs

@@ -1237,6 +1237,7 @@ mod tests {
                 messages: vec![],
                 ..Default::default()
             },
+            common: Default::default(),
             nvext: None,
         };
         let result = validate_chat_completion_required_fields(&request);
@@ -1263,6 +1264,7 @@ mod tests {
                 )],
                 ..Default::default()
             },
+            common: Default::default(),
             nvext: None,
         };
         let result = validate_chat_completion_required_fields(&request);

lib/llm/src/protocols/openai.rs

@@ -20,8 +20,10 @@ use super::{
     common::{self, SamplingOptionsProvider, StopConditionsProvider},
     ContentProvider,
 };
+use crate::protocols::openai::common_ext::CommonExtProvider;
 
 pub mod chat_completions;
+pub mod common_ext;
 pub mod completions;
 pub mod embeddings;
 pub mod models;
@@ -61,9 +63,23 @@ trait OpenAIStopConditionsProvider {
     fn get_stop(&self) -> Option<Vec<String>>;
 
     fn nvext(&self) -> Option<&nvext::NvExt>;
+
+    /// Get ignore_eos from CommonExt if the type supports it.
+    /// Default returns None for types without CommonExt support.
+    fn get_common_ignore_eos(&self) -> Option<bool> {
+        None
+    }
+
+    /// Get the effective ignore_eos value, considering both CommonExt and NvExt.
+    /// CommonExt (root-level) takes precedence over NvExt.
+    fn get_ignore_eos(&self) -> Option<bool> {
+        // Check common first (takes precedence), then fall back to nvext
+        self.get_common_ignore_eos()
+            .or_else(|| self.nvext().and_then(|nv| nv.ignore_eos))
+    }
 }
 
-impl<T: OpenAISamplingOptionsProvider> SamplingOptionsProvider for T {
+impl<T: OpenAISamplingOptionsProvider + CommonExtProvider> SamplingOptionsProvider for T {
     fn extract_sampling_options(&self) -> Result<common::SamplingOptions> {
         // let result = self.validate();
         // if let Err(e) = result {
@@ -88,29 +104,26 @@ impl<T: OpenAISamplingOptionsProvider> SamplingOptionsProvider for T {
             }
         }
 
-        let mut guided_decoding = None;
-        if let Some(nvext) = self.nvext() {
-            let guided_decoding_backend = nvext.guided_decoding_backend.clone();
-            let guided_json = nvext.guided_json.clone();
-            let guided_regex = nvext.guided_regex.clone();
-            let guided_grammar = nvext.guided_grammar.clone();
-            let guided_choice = nvext.guided_choice.clone();
-
-            match common::GuidedDecodingOptions::from_optional(
-                guided_json,
-                guided_regex,
-                guided_choice,
-                guided_grammar,
-                guided_decoding_backend,
-            ) {
-                Ok(options) => guided_decoding = options,
-                Err(e) => {
-                    // Handle the validation error (log, return error, etc.)
-                    tracing::error!("Invalid guided decoding options: {}", e);
-                    return Err(e);
-                }
+        let guided_decoding_backend = self.get_guided_decoding_backend();
+        let guided_json = self.get_guided_json();
+        let guided_regex = self.get_guided_regex();
+        let guided_grammar = self.get_guided_grammar();
+        let guided_choice = self.get_guided_choice();
+
+        let guided_decoding = match common::GuidedDecodingOptions::from_optional(
+            guided_json.cloned(),
+            guided_regex,
+            guided_choice,
+            guided_grammar,
+            guided_decoding_backend,
+        ) {
+            Ok(options) => options,
+            Err(e) => {
+                // Handle the validation error (log, return error, etc.)
+                tracing::error!("Invalid guided decoding options: {:?}", e);
+                return Err(e);
             }
-        }
+        };
 
         Ok(common::SamplingOptions {
             n: None,
@@ -142,11 +155,8 @@ impl<T: OpenAIStopConditionsProvider> StopConditionsProvider for T {
             }
         }
 
-        let mut ignore_eos = None;
-
-        if let Some(nvext) = self.nvext() {
-            ignore_eos = nvext.ignore_eos;
-        }
+        // Use the trait method to get ignore_eos, which handles precedence
+        let ignore_eos = self.get_ignore_eos();
 
         Ok(common::StopConditions {
             max_tokens,

lib/llm/src/protocols/openai/chat_completions.rs

@@ -20,8 +20,10 @@ use validator::Validate;
 use crate::engines::ValidateRequest;
 
 use super::{
-    nvext::NvExt, nvext::NvExtProvider, validate, OpenAISamplingOptionsProvider,
-    OpenAIStopConditionsProvider,
+    common_ext::{CommonExt, CommonExtProvider},
+    nvext::NvExt,
+    nvext::NvExtProvider,
+    validate, OpenAISamplingOptionsProvider, OpenAIStopConditionsProvider,
 };
 
 mod aggregator;
@@ -31,17 +33,21 @@ pub use aggregator::DeltaAggregator;
 pub use delta::DeltaGenerator;
 
 /// A request structure for creating a chat completion, extending OpenAI's
-/// `CreateChatCompletionRequest` with [`NvExt`] extensions.
+/// `CreateChatCompletionRequest` with [`NvExt`] extensions and common fields.
 ///
 /// # Fields
 /// - `inner`: The base OpenAI chat completion request, embedded using `serde(flatten)`.
-/// - `nvext`: The optional NVIDIA extension field. See [`NvExt`] for
-///   more details.
+/// - `common`: Common extension fields (ignore_eos, min_tokens) at root level, embedded using `serde(flatten)`.
+/// - `nvext`: The optional NVIDIA extension field. See [`NvExt`] for more details.
+///   Note: If ignore_eos is specified in both common and nvext, the common (root-level) value takes precedence.
 #[derive(Serialize, Deserialize, Validate, Debug, Clone)]
 pub struct NvCreateChatCompletionRequest {
     #[serde(flatten)]
     pub inner: async_openai::types::CreateChatCompletionRequest,
 
+    #[serde(flatten, default)]
+    pub common: CommonExt,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub nvext: Option<NvExt>,
 }
@@ -139,6 +145,52 @@ impl OpenAISamplingOptionsProvider for NvCreateChatCompletionRequest {
     }
 }
 
+/// Implements `CommonExtProvider` for `NvCreateChatCompletionRequest`,
+/// providing access to common extension fields.
+impl CommonExtProvider for NvCreateChatCompletionRequest {
+    /// Returns a reference to the CommonExt struct.
+    fn common_ext(&self) -> Option<&CommonExt> {
+        Some(&self.common)
+    }
+
+    /// Guided Decoding Options
+    fn get_guided_json(&self) -> Option<&serde_json::Value> {
+        self.common
+            .guided_json
+            .as_ref()
+            .or_else(|| self.nvext.as_ref().and_then(|nv| nv.guided_json.as_ref()))
+    }
+
+    fn get_guided_regex(&self) -> Option<String> {
+        self.common
+            .guided_regex
+            .clone()
+            .or_else(|| self.nvext.as_ref().and_then(|nv| nv.guided_regex.clone()))
+    }
+
+    fn get_guided_grammar(&self) -> Option<String> {
+        self.common
+            .guided_grammar
+            .clone()
+            .or_else(|| self.nvext.as_ref().and_then(|nv| nv.guided_grammar.clone()))
+    }
+
+    fn get_guided_choice(&self) -> Option<Vec<String>> {
+        self.common
+            .guided_choice
+            .clone()
+            .or_else(|| self.nvext.as_ref().and_then(|nv| nv.guided_choice.clone()))
+    }
+
+    fn get_guided_decoding_backend(&self) -> Option<String> {
+        self.common.guided_decoding_backend.clone().or_else(|| {
+            self.nvext
+                .as_ref()
+                .and_then(|nv| nv.guided_decoding_backend.clone())
+        })
+    }
+}
+
 /// Implements `OpenAIStopConditionsProvider` for `NvCreateChatCompletionRequest`,
 /// providing access to stop conditions that control chat completion behavior.
 impl OpenAIStopConditionsProvider for NvCreateChatCompletionRequest {
@@ -149,12 +201,10 @@ impl OpenAIStopConditionsProvider for NvCreateChatCompletionRequest {
     }
 
     /// Retrieves the minimum number of tokens required in the response.
-    ///
-    /// # Note
-    /// This method is currently a placeholder and always returns `None`
-    /// since `min_tokens` is not an OpenAI-supported parameter.
+    /// Returns `min_tokens` Value
+    /// `min_tokens` is not an OpenAI-supported parameter.
     fn get_min_tokens(&self) -> Option<u32> {
-        None
+        self.common.min_tokens
     }
 
     /// Retrieves the stop conditions that terminate the chat completion response.
@@ -175,6 +225,11 @@ impl OpenAIStopConditionsProvider for NvCreateChatCompletionRequest {
     fn nvext(&self) -> Option<&NvExt> {
         self.nvext.as_ref()
     }
+
+    /// Get ignore_eos from CommonExt.
+    fn get_common_ignore_eos(&self) -> Option<bool> {
+        self.common.ignore_eos
+    }
 }
 
 /// Implements `ValidateRequest` for `NvCreateChatCompletionRequest`,