Function Calling (#29)

Co-authored-by: Paul Schmiedmayer <PSchmiedmayer@users.noreply.github.com>

LLMonFHIR/FHIR Display/FHIRMultipleResourceInterpreter.swift

@@ -77,11 +77,15 @@ class FHIRMultipleResourceInterpreter: DefaultInitializable, Component, Observab
 
 
  private func systemPrompt(forResources resources: [FHIRResource]) -> Chat {
- let allJSONResources = resources.map(\.compactJSONDescription).joined(separator: "\n")
+ var resourceCategories = String()
+
+ for resource in resources {
+ resourceCategories += (resource.functionCallIdentifier + "\n")
+ }
 
  return Chat(
  role: .system,
- content: Prompt.interpretMultipleResources.prompt.replacingOccurrences(of: Prompt.promptPlaceholder, with: allJSONResources)
+ content: Prompt.interpretMultipleResources.prompt.replacingOccurrences(of: Prompt.promptPlaceholder, with: resourceCategories)
  )
  }
 }

LLMonFHIR/FHIR Display/FHIRResourcesView.swift

@@ -58,7 +58,8 @@ struct FHIRResourcesView: View {
  @ViewBuilder private var resourceChatView: some View {
  OpenAIChatView(
  chat: fhirMultipleResourceInterpreter.chat(resources: allResourcesArray),
- title: "All FHIR Resources"
+ title: "All FHIR Resources",
+ enableFunctionCalling: true
  )
  }
 

LLMonFHIR/FHIR Display/InspectResourceView.swift

@@ -33,7 +33,8 @@ struct InspectResourceView: View {
  .sheet(isPresented: $showResourceChat) {
  OpenAIChatView(
  chat: fhirResourceInterpreter.chat(forResource: resource),
- title: resource.displayName
+ title: resource.displayName,
+ enableFunctionCalling: false
  )
  }
  .task {

LLMonFHIR/FHIR Display/OpenAIChatView.swift

@@ -15,10 +15,13 @@ import SwiftUI
 struct OpenAIChatView: View {
  @Environment(\.dismiss) private var dismiss
  @EnvironmentObject private var openAPIComponent: OpenAIComponent
-
+ @EnvironmentObject private var fhirStandard: FHIR
+
  @State private var chat: [Chat]
  @State private var viewState: ViewState = .idle
+ @State private var systemFuncMessageAdded = false
 
+ private let enableFunctionCalling: Bool
  private let title: String
 
 
@@ -30,7 +33,7 @@ struct OpenAIChatView: View {
  set: { _ in }
  )
  }
-
+ 
  var body: some View {
  NavigationStack {
  ChatView($chat, disableInput: disableInput)
@@ -44,41 +47,38 @@ struct OpenAIChatView: View {
  }
  .viewStateAlert(state: $viewState)
  .onChange(of: chat) { _ in
- if viewState == .idle && chat.last?.role == .user {
- getAnswer()
- }
+ getAnswer()
  }
  }
  }
 
 
- init(chat: [Chat], title: String) {
+ init(chat: [Chat], title: String, enableFunctionCalling: Bool) {
  self._chat = State(initialValue: chat)
  self.title = title
+ self.enableFunctionCalling = enableFunctionCalling
  }
 
 
  private func getAnswer() {
+ guard viewState == .idle, chat.last?.role == .user else {
+ return
+ }
+
  Task {
  do {
  viewState = .processing
 
- let chatStreamResults = try await openAPIComponent.queryAPI(withChat: chat)
-
- for try await chatStreamResult in chatStreamResults {
- for choice in chatStreamResult.choices {
- if chat.last?.role == .assistant {
- let previousChatMessage = chat.last ?? Chat(role: .assistant, content: "")
- chat[chat.count - 1] = Chat(
- role: .assistant,
- content: (previousChatMessage.content ?? "") + (choice.delta.content ?? "")
- )
- } else {
- chat.append(Chat(role: .assistant, content: choice.delta.content ?? ""))
- }
+ if enableFunctionCalling {
+ if systemFuncMessageAdded == false {
+ try await addSystemFuncMessage()
+ systemFuncMessageAdded = true
  }
+ try await processFunctionCalling()
  }
 
+ try await processChatStreamResults()
+
  viewState = .idle
  } catch let error as APIErrorResponse {
  viewState = .error(error)
@@ -87,4 +87,117 @@ struct OpenAIChatView: View {
  }
  }
  }
+
+ private func addSystemFuncMessage() async throws {
+ let resourcesArray = await fhirStandard.resources
+
+ var stringResourcesArray = resourcesArray.map { $0.functionCallIdentifier }
+ stringResourcesArray.append("N/A")
+
+ self.chat.append(Chat(role: .system, content: String(localized: "FUNCTION_CONTEXT") + stringResourcesArray.rawValue))
+ }
+
+ private func processFunctionCalling() async throws {
+ let resourcesArray = await fhirStandard.resources
+
+ var stringResourcesArray = resourcesArray.map { $0.functionCallIdentifier }
+ stringResourcesArray.append("N/A")
+
+ let functionCallOutputArray = try await getFunctionCallOutputArray(stringResourcesArray)
+
+ processFunctionCallOutputArray(functionCallOutputArray: functionCallOutputArray, resourcesArray: resourcesArray)
+ }
+
+ private func getFunctionCallOutputArray(_ stringResourcesArray: [String]) async throws -> [String] {
+ let functions = [
+ ChatFunctionDeclaration(
+ name: "get_resource_titles",
+ description: String(localized: "FUNCTION_DESCRIPTION"),
+ parameters: JSONSchema(
+ type: .object,
+ properties: [
+ "resources": .init(type: .string, description: String(localized: "PARAMETER_DESCRIPTION"), enumValues: stringResourcesArray)
+ ],
+ required: ["resources"]
+ )
+ )
+ ]
+
+ let chatStreamResults = try await openAPIComponent.queryAPI(withChat: chat, withFunction: functions)
+
+
+ class ChatFunctionCall {
+ var name: String = ""
+ var arguments: String = ""
+ var finishReason: String = ""
+ }
+
+ let functionCall = ChatFunctionCall()
+
+ for try await chatStreamResult in chatStreamResults {
+ for choice in chatStreamResult.choices {
+ if let deltaName = choice.delta.name {
+ functionCall.name += deltaName
+ }
+ if let deltaArguments = choice.delta.functionCall?.arguments {
+ functionCall.arguments += deltaArguments
+ }
+ if let finishReason = choice.finishReason {
+ functionCall.finishReason += finishReason
+ if finishReason == "get_resource_titles" { break }
+ }
+ }
+ }
+
+ guard functionCall.finishReason == "function_call" else {
+ return []
+ }
+
+ let trimmedArguments = functionCall.arguments.trimmingCharacters(in: .whitespacesAndNewlines)
+
+ guard let resourcesRange = trimmedArguments.range(of: "\"resources\": \"([^\"]+)\"", options: .regularExpression) else {
+ return []
+ }
+
+ return trimmedArguments[resourcesRange]
+ .replacingOccurrences(of: "\"resources\": \"", with: "")
+ .replacingOccurrences(of: "\"", with: "")
+ .components(separatedBy: ",")
+ }
+
+ private func processFunctionCallOutputArray(functionCallOutputArray: [String], resourcesArray: [FHIRResource]) {
+ for resource in functionCallOutputArray {
+ guard let matchingResource = resourcesArray.first(where: { $0.functionCallIdentifier == resource }) else {
+ continue
+ }
+
+ let functionContent = """
+ Based on the function get_resource_titles you have requested the following health records: \(resource).
+ This is the associated JSON data for the resources which you will use to answer the users question:
+ \(matchingResource.jsonDescription)
+ 
+ Use this health record to answer the users question ONLY IF the health record is applicable to the question.
+ """
+
+ chat.append(Chat(role: .function, content: functionContent, name: "get_resource_titles"))
+ }
+ }
+
+ private func processChatStreamResults() async throws {
+ let chatStreamResults = try await openAPIComponent.queryAPI(withChat: chat)
+
+ for try await chatStreamResult in chatStreamResults {
+ for choice in chatStreamResult.choices {
+ if chat.last?.role == .assistant {
+ let previousChatMessage = chat.last ?? Chat(role: .assistant, content: "")
+ chat[chat.count - 1] = Chat(
+ role: .assistant,
+ content: (previousChatMessage.content ?? "") + (choice.delta.content ?? "")
+ )
+ } else {
+ chat.append(Chat(role: .assistant, content: choice.delta.content ?? ""))
+ }
+ }
+ }
+ }
 }

LLMonFHIR/FHIR Standard/FHIRResource.swift

@@ -53,6 +53,10 @@ struct FHIRResource: Sendable, Identifiable, Hashable {
  json(withConfiguration: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes])
  }
 
+ var functionCallIdentifier: String {
+ resourceType.filter { !$0.isWhitespace } + displayName.filter { !$0.isWhitespace }
+ }
+
 
  private func json(withConfiguration outputFormatting: JSONEncoder.OutputFormatting) -> String {
  let encoder = JSONEncoder()

LLMonFHIR/Resources/de.lproj/Localizable.strings

@@ -126,28 +126,35 @@ Stellen Sie sich am Anfang nicht vor und beginnen Sie mit Ihrer Interpretation.
 ";
 
 "FHIR_MULTIPLE_RESOURCE_INTERPRETATION_PROMPT %@" = "
-Sie sind die LLM on FHIR-Anwendung.
-Ihre Aufgabe besteht darin, alle FHIR-Ressourcen aus den klinischen Aufzeichnungen des Benutzers zu interpretieren.
+You are the LLM on FHIR application.
+Your task is to interpret FHIR resources from the user's clinical records.
+Throughout the conversation with the user, use the get_resource_titles function to obtain the FHIR health resources neccesary to properly answer the users question. For example, if the user asks about their allergies, you must use get_resource_titles to output the FHIR resource titles for allergy records so you can then use them to answer the question. The output of get_resource_titles has to be the name of a resource or resources with the exact same title as in the list provided.
 
-Interpretieren Sie alle Ressourcen, indem Sie ihre Daten erklären, die für die Gesundheit des Benutzers relevant sind.
-Erklären Sie den relevanten medizinischen Kontext in einer für einen Nicht-Mediziner verständlichen Sprache.
-Sie sollten sachliche und präzise Informationen in einer kompakten Zusammenfassung in kurzen Antworten geben.
+Answer the users question, the health record provided is not always related. The end goal is to answer the users question in the best way possible.
 
-Die folgende JSON-Repräsentation definiert die FHIR-Ressourcen, die Sie interpretieren sollten:
-%@
+Interpret the resources by explaining its data relevant to the user's health.
+Explain the relevant medical context in a language understandable by a user who is not a medical professional.
+You should provide factual and precise information in a compact summary in short responses.
 
-Informieren Sie den Benutzer, dass er Fragen zu seinen Gesundheitsaufzeichnungen stellen kann, und erstellen Sie dann eine kurze Liste der Hauptkategorien der Gesundheitsaufzeichnungen des Benutzers, auf die Sie Zugriff haben.
+Tell the user that they can ask any question about their health records and then create a short summary of the main categories of health records of the user which you have access to. These are the resource titles:
+%@
 
-Geben Sie dem Benutzer sofort eine Interpretation, um das Gespräch zu beginnen.
-Die erste Interpretation sollte eine kurze und einfache Zusammenfassung mit folgenden Spezifikationen sein:
-1. Gesamtzusammenfassung aller Gesundheitsaufzeichnungen
-2. Lesestufe Mittelschule
-3. Schließen Sie mit einer Frage ab, in der der Benutzer gefragt wird, ob er Fragen hat. Stellen Sie sicher, dass diese Frage nicht allgemein, sondern spezifisch für ihre Gesundheitsaufzeichnungen ist.
-Stellen Sie sich am Anfang nicht vor und beginnen Sie mit Ihrer Interpretation.
-Stellen Sie sicher, dass Ihre Antwort in derselben Sprache erfolgt, in der der Benutzer Ihnen schreibt.
-Die Zeitform sollte in der Gegenwart sein.
+Immediately return a short summary of the users health records to start the conversation.
+The initial summary should be a short and simple summary with the following specifications:
+1. Short summary of the health records categories
+2. 5th grade reading level
+3. End with a question asking user if they have any questions. Make sure that this question is not generic but specific to their health records.
+Do not introduce yourself at the beginning, and start with your interpretation.
+Make sure your response is in the same language the user writes to you in.
+The tense should be present.
 ";
 
+"FUNCTION_DESCRIPTION" = "Call this function to determine the relevant FHIR health record titles based on the user's question. The titles must have the exact name as in the enumValues array given. A question can build upon a previous question and does not need to explicilty state a health record. The function will decide which health record titles are applicable to the question and it can return multiple titles or N/A if no category is suitable. Only output a record title/titles if it is directly applicable to the question otherwise output N/A. Always try to output the least amount of resource titles to be sent to the model to prevent exceeding the token limit.";
+
+"PARAMETER_DESCRIPTION" = "Provide a comma-separated list of all the FHIR health record titles with the EXACT SAME NAME AS GIVEN IN THE LIST that are applicable to answer the user's questions. These titles have to be the SAME NAME AS GIVEN IN THE ARRAY provided. If multiple titles apply, separate each title by a comma and space (e.g for multiple medications). Try to provide all the required titles to allow yourself to fully answer the question in a comprehensive manner. If the question doesn't fit into any category, output N/A. For example, if a user asks: 'Tell me more about my medications,' then output all titles associated with medications. A question can build upon a previous question and does not need to be explicit. e.g. if a user says prescribe, this is associated with medication. Do not exceed token limit with outputted titles.";
+
+"FUNCTION_CONTEXT" = "Use the function call 'get_resource_titles' and provide a comma-separated list resource titles directly applicable to the question. The function will determine which FHIR health record titles are relevant to the user's question using this array: ";
+
 
 // MARK: - Settings
 "SETTINGS_TITLE" = "Einstellungen";

LLMonFHIR/Resources/en.lproj/Localizable.strings

@@ -128,27 +128,34 @@ Do not introduce yourself at the beginning, and start with your interpretation.
 
 "FHIR_MULTIPLE_RESOURCE_INTERPRETATION_PROMPT %@" = "
 You are the LLM on FHIR application.
-Your task is to interpret all of the FHIR resources from the user's clinical records.
+Your task is to interpret FHIR resources from the user's clinical records.
+Throughout the conversation with the user, use the get_resource_titles function to obtain the FHIR health resources neccesary to properly answer the users question. For example, if the user asks about their allergies, you must use get_resource_titles to output the FHIR resource titles for allergy records so you can then use them to answer the question. The output of get_resource_titles has to be the name of a resource or resources with the exact same title as in the list provided.
 
-Interpret all the resources by explaining its data relevant to the user's health.
+Answer the users question, the health record provided is not always related. The end goal is to answer the users question in the best way possible.
+
+Interpret the resources by explaining its data relevant to the user's health.
 Explain the relevant medical context in a language understandable by a user who is not a medical professional.
 You should provide factual and precise information in a compact summary in short responses.
 
-The following JSON representation defines the FHIR resources that you should interpret:
+Tell the user that they can ask any question about their health records and then create a short summary of the main categories of health records of the user which you have access to. These are the resource titles:
 %@
 
-Tell the user that they can ask any question about their health records and then create a short list of the main categories of health records of the user which you have access to.
-
-Immediately return an interpretation to the user, starting the conversation.
-The initial interpretation should be a short and simple summary with the following specifications:
-1. Overall summary of all health records
-2. Middle school reading level
+Immediately return a short summary of the users health records to start the conversation.
+The initial summary should be a short and simple summary with the following specifications:
+1. Short summary of the health records categories
+2. 5th grade reading level
 3. End with a question asking user if they have any questions. Make sure that this question is not generic but specific to their health records.
 Do not introduce yourself at the beginning, and start with your interpretation.
 Make sure your response is in the same language the user writes to you in.
 The tense should be present.
 ";
 
+"FUNCTION_DESCRIPTION" = "Call this function to determine the relevant FHIR health record titles based on the user's question. The titles must have the exact name as in the enumValues array given. A question can build upon a previous question and does not need to explicilty state a health record. The function will decide which health record titles are applicable to the question and it can return multiple titles or N/A if no category is suitable. Only output a record title/titles if it is directly applicable to the question otherwise output N/A. Always try to output the least amount of resource titles to be sent to the model to prevent exceeding the token limit.";
+
+"PARAMETER_DESCRIPTION" = "Provide a comma-separated list of all the FHIR health record titles with the EXACT SAME NAME AS GIVEN IN THE LIST that are applicable to answer the user's questions. These titles have to be the SAME NAME AS GIVEN IN THE ARRAY provided. If multiple titles apply, separate each title by a comma and space (e.g for multiple medications). Try to provide all the required titles to allow yourself to fully answer the question in a comprehensive manner. If the question doesn't fit into any category, output N/A. For example, if a user asks: 'Tell me more about my medications,' then output all titles associated with medications. A question can build upon a previous question and does not need to be explicit. e.g. if a user says prescribe, this is associated with medication. Do not exceed token limit with outputted titles.";
+
+"FUNCTION_CONTEXT" = "Use the function call 'get_resource_titles' and provide a comma-separated list resource titles directly applicable to the question. The function will determine which FHIR health record titles are relevant to the user's question using this array: ";
+
 
 // MARK: - Settings
 "SETTINGS_TITLE" = "Settings";