Use package:typed_isolate for easier isolate management

example/chat.dart

@@ -1,3 +1,5 @@
+// ignore_for_file: avoid_print
+
 import 'dart:io';
 
 // import 'package:flutter/foundation.dart';
@@ -23,7 +25,6 @@ void main() async {
     contextParams.threads = cores.length;
     contextParams.threadsBatch = cores.length;
     contextParams.context = 512 * 4;
-
     // /Users/adel/Workspace/llama.cpp/models/tinyllama-2-1b-miniguanaco.Q3_K_L.gguf
     // /Users/adel/Workspace/llama.cpp/models/openhermes-2.5-neural-chat-v3-3-slerp.Q5_K_M.gguf
     Llama llama = Llama(
@@ -68,7 +69,7 @@ Context: Teplizumab traces its roots to a New Jersey drug company called Ortho P
 
     llama.dispose();
   } catch (e) {
-    // print("Error: ${e.toString()}");
+    print("Error: ${e.toString()}");
   }
 }
 

lib/llama_cpp_dart.dart

@@ -5,5 +5,4 @@ export 'src/model_params.dart';
 export 'src/context_params.dart';
 export 'src/sampling_params.dart';
 export 'src/llama.dart';
-export 'src/llama_processor.dart';
 export 'src/prompt_format.dart';

lib/src/isolate_child.dart

@@ -0,0 +1,37 @@
+import 'package:typed_isolate/typed_isolate.dart';
+
+import "llama.dart";
+import "isolate_types.dart";
+
+class LlamaChild extends IsolateChild<LlamaResponse, LlamaCommand> {
+  LlamaChild() : super(id: 1);
+
+  // TODO: What's this used for?
+  bool shouldStop = false;
+  Llama? llama;
+
+  @override
+  void onData(LlamaCommand data) {
+    switch (data) {
+      case LlamaStop() || LlamaClear():
+        shouldStop = true;
+        llama?.clear();
+      case LlamaLoad(:final path, :final modelParams, :final contextParams, :final samplingParams):
+        llama = Llama(path, modelParams, contextParams, samplingParams);
+      case LlamaPrompt(:final prompt):
+        _sendPrompt(prompt);
+      case LlamaInit(:final libraryPath):
+        Llama.libraryPath = libraryPath;
+    }
+  }
+
+  void _sendPrompt(String prompt) {
+    llama?.setPrompt(prompt);
+    while (true) {
+      if (shouldStop) break;
+      final (text, isDone) = llama!.getNext();
+      sendToParent(LlamaResponse(text: text, isDone: isDone));
+      if (isDone) shouldStop = true;
+    }
+  }
+}

lib/src/isolate_parent.dart

@@ -0,0 +1,69 @@
+import 'dart:async';
+
+import 'package:typed_isolate/typed_isolate.dart';
+
+import "llama.dart";
+import 'alpaca_format.dart';
+import 'chatml_format.dart';
+import 'prompt_format.dart';
+
+import "isolate_child.dart";
+import "isolate_types.dart";
+
+extension on PromptFormatType {
+  PromptFormat? get formatter => switch (this) {
+    PromptFormatType.raw => null,
+    PromptFormatType.alpaca => AlpacaFormat(),
+    PromptFormatType.chatml => ChatMLFormat(),
+  };
+}
+
+class LlamaParent {
+  final _controller = StreamController<String>.broadcast();
+  final _parent = IsolateParent<LlamaCommand, LlamaResponse>();
+
+  StreamSubscription<LlamaResponse>? _subscription;
+
+  List<Map<String, dynamic>> messages = [];
+
+  final LlamaLoad loadCommand;
+  final PromptFormat? formatter;
+  LlamaParent(this.loadCommand) :
+    formatter = loadCommand.modelParams.format.formatter;
+
+  Stream<String> get stream => _controller.stream;
+
+  void _onData(LlamaResponse data) {
+    if (data.isDone) _controller.close();
+    _parseResponse(data.text);
+  }
+
+  void _parseResponse(String response) {
+    final processed = formatter == null ? response : formatter!.filterResponse(response);
+    if (processed != null) _controller.add(processed);
+  }
+
+  void init() async {
+    _parent.init();
+    _subscription = _parent.stream.listen(_onData);
+    await _parent.spawn(LlamaChild());
+    _parent.sendToChild(data: LlamaInit(Llama.libraryPath), id: 1);
+    _parent.sendToChild(data: loadCommand, id: 1);
+  }
+
+  void sendPrompt(String prompt) async {
+    final formattedPrompt = messages.isEmpty
+      ? formatter?.formatPrompt(prompt) ?? prompt
+      : formatter?.formatMessages(messages) ?? prompt;
+    _parent.sendToChild(id: 1, data: LlamaPrompt(formattedPrompt));
+  }
+
+  void stop() => _parent.sendToChild(id: 1, data: LlamaStop());
+
+  Future<void> dispose() async {
+    await _subscription?.cancel();
+    await _controller.close();
+    _parent.sendToChild(id: 1, data: LlamaClear());
+    _parent.dispose();
+  }
+}

lib/src/isolate_types.dart

@@ -0,0 +1,39 @@
+
+import "model_params.dart";
+import "context_params.dart";
+import "sampling_params.dart";
+
+sealed class LlamaCommand { }
+class LlamaStop extends LlamaCommand { }
+class LlamaClear extends LlamaCommand { }
+class LlamaInit extends LlamaCommand {
+  final String? libraryPath;
+  LlamaInit(this.libraryPath);
+}
+
+class LlamaPrompt extends LlamaCommand {
+  final String prompt;
+  LlamaPrompt(this.prompt);
+}
+
+class LlamaResponse {
+  final String text;
+  final bool isDone;
+  LlamaResponse({
+    required this.text,
+    required this.isDone,
+  });
+}
+
+class LlamaLoad extends LlamaCommand {
+  final String path;
+  final ModelParams modelParams;
+  final ContextParams contextParams;
+  final SamplingParams samplingParams;
+  LlamaLoad({
+    required this.path,
+    required this.modelParams,
+    required this.contextParams,
+    required this.samplingParams,
+  });
+}