fix: properly cleanup unuseable llm instance

fixes #453
tiddly-gittly · Jul 19, 2023 · 3ca373a · 3ca373a
1 parent 185eefd
commit 3ca373a
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 28 deletions.
diff --git a/src/services/languageModel/llmWorker/llamaCpp.ts b/src/services/languageModel/llmWorker/llamaCpp.ts
@@ -16,24 +16,29 @@ export async function loadLLama(
  // use dynamic import cjs version to fix https://github.com/andywer/threads.js/issues/478
  const { LLamaCpp } = await import('llama-node/dist/llm/llama-cpp.cjs');
  subscriber?.next({ message: 'library loaded, new LLM now', ...loggerCommonMeta });
- runnerInstance = new LLM(LLamaCpp);
- const loadConfig: LoadConfig = {
- enableLogging: true,
- nCtx: 1024,
- seed: 0,
- f16Kv: false,
- logitsAll: false,
- vocabOnly: false,
- useMlock: false,
- embedding: false,
- useMmap: true,
- nGpuLayers: 0,
- ...loadConfigOverwrite,
- };
- subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
- await runnerInstance.load(loadConfig);
- subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
- return runnerInstance;
+ try {
+ runnerInstance = new LLM(LLamaCpp);
+ const loadConfig: LoadConfig = {
+ enableLogging: true,
+ nCtx: 1024,
+ seed: 0,
+ f16Kv: false,
+ logitsAll: false,
+ vocabOnly: false,
+ useMlock: false,
+ embedding: false,
+ useMmap: true,
+ nGpuLayers: 0,
+ ...loadConfigOverwrite,
+ };
+ subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
+ await runnerInstance.load(loadConfig);
+ subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
+ return runnerInstance;
+ } catch (error) {
+ unloadLLama();
+ throw error;
+ }
 }
 export function unloadLLama() {
  runnerInstance = undefined;

diff --git a/src/services/languageModel/llmWorker/rwkvCpp.ts b/src/services/languageModel/llmWorker/rwkvCpp.ts
@@ -16,16 +16,21 @@ export async function loadRwkv(
  // use dynamic import cjs version to fix https://github.com/andywer/threads.js/issues/478
  const { RwkvCpp } = await import('llama-node/dist/llm/rwkv-cpp.cjs');
  subscriber?.next({ message: 'library loaded, new LLM now', ...loggerCommonMeta });
- runnerInstance = new LLM(RwkvCpp);
- const loadConfig: LoadConfig = {
- enableLogging: true,
- nThreads: 4,
- ...loadConfigOverwrite,
- };
- subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
- await runnerInstance.load(loadConfig);
- subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
- return runnerInstance;
+ try {
+ runnerInstance = new LLM(RwkvCpp);
+ const loadConfig: LoadConfig = {
+ enableLogging: true,
+ nThreads: 4,
+ ...loadConfigOverwrite,
+ };
+ subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
+ await runnerInstance.load(loadConfig);
+ subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
+ return runnerInstance;
+ } catch (error) {
+ unloadRwkv();
+ throw error;
+ }
 }
 export function unloadRwkv() {
  runnerInstance = undefined;