Merge pull request #174 from L-jasmine/chore/limit_ctx_size

[Backend] Limit ctx-size to a maximum of 8K.
moxin-org · Jul 25, 2024 · 1cc2945 · 1cc2945
2 parents c7a2c74 + c86e422
commit 1cc2945
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/moxin-backend/src/backend_impls/api_server.rs b/moxin-backend/src/backend_impls/api_server.rs
@@ -35,7 +35,7 @@ fn create_wasi(
     load_model: &LoadModelOptions,
 ) -> wasmedge_sdk::WasmEdgeResult<WasiModule> {
     // use model metadata context size
-    let ctx_size = Some(format!("{}", file.context_size));
+    let ctx_size = Some(format!("{}", file.context_size.min(8 * 1024)));
 
     let n_gpu_layers = match load_model.gpu_layers {
         moxin_protocol::protocol::GPULayers::Specific(n) => Some(n.to_string()),