diff --git a/bindings/node/native/src/processors.rs b/bindings/node/native/src/processors.rs index 823a3d16d..399a57f3a 100644 --- a/bindings/node/native/src/processors.rs +++ b/bindings/node/native/src/processors.rs @@ -120,7 +120,7 @@ fn template_processing(mut cx: FunctionContext) -> JsResult { if let Some(pair) = pair { builder.try_pair(pair).map_err(Error)?; } - let processor = builder.build().map_err(Error)?; + let processor = builder.build().map_err(|e| Error(e.to_string()))?; let mut js_processor = JsPostProcessor::new::<_, JsPostProcessor, _>(&mut cx, vec![])?; let guard = cx.lock(); diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs index 641fe2298..442fa0340 100644 --- a/bindings/python/src/processors.rs +++ b/bindings/python/src/processors.rs @@ -405,7 +405,9 @@ impl PyTemplateProcessing { if let Some(sp) = special_tokens { builder.special_tokens(sp); } - let processor = builder.build().map_err(exceptions::PyValueError::new_err)?; + let processor = builder + .build() + .map_err(|e| exceptions::PyValueError::new_err(e.to_string()))?; Ok(( PyTemplateProcessing {}, diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml index 28ad83594..1fb257f37 100644 --- a/tokenizers/Cargo.toml +++ b/tokenizers/Cargo.toml @@ -55,7 +55,7 @@ unicode-segmentation = "1.6" indicatif = {version = "0.15", optional = true} itertools = "0.9" log = "0.4" -derive_builder = "0.9" +derive_builder = "0.12" spm_precompiled = "0.1" dirs = "3.0" reqwest = { version = "0.11", optional = true } diff --git a/tokenizers/src/processors/template.rs b/tokenizers/src/processors/template.rs index 313b4911a..17550e907 100644 --- a/tokenizers/src/processors/template.rs +++ b/tokenizers/src/processors/template.rs @@ -350,6 +350,18 @@ pub struct TemplateProcessing { special_tokens: Tokens, } +impl From<&str> for TemplateProcessingBuilderError { + fn from(e: &str) -> Self { + e.to_string().into() + } +} + +impl PartialEq for TemplateProcessingBuilderError { + fn eq(&self, other: &Self) -> bool { + self.to_string() == other.to_string() + } +} + /// We use this custom deserializer to provided the values for `added_single` /// and `added_pair` during deserialization, while not having to serialize them #[doc(hidden)] @@ -1074,4 +1086,18 @@ mod tests { Err("Template for `pair` must use both sequences".into()) ); } + + #[test] + fn expect_wrong_error_message() { + let processor = TemplateProcessing::builder() + .try_single("$0") + .unwrap() + .try_pair("$0 $1") + .unwrap() + .build(); + assert_ne!( + processor, + Err("Expect the left side error message to be different from the right side!".into()) + ); + } }