Merge pull request #1940 from quarkiverse/#1939

Refactor multi-modality handling

Author: geoand

core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java

@@ -1703,10 +1703,19 @@ private AiServiceMethodCreateInfo gatherMethodMetadata(
         boolean switchToWorkerThreadForToolExecution = detectIfToolExecutionRequiresAWorkerThread(method, tools,
                 methodToolClassInfo.keySet());
 
-        var methodReturnTypeSignature = returnTypeSignature(method.returnType(),
-                new TypeArgMapper(method.declaringClass(), index));
+        TypeArgMapper typeArgMapper = new TypeArgMapper(method.declaringClass(), index);
+        var methodReturnTypeSignature = typeSignature(method.returnType(), typeArgMapper);
 
-        return new AiServiceMethodCreateInfo(method.declaringClass().name().toString(), method.name(), systemMessageInfo,
+        List<AiServiceMethodCreateInfo.ParameterInfo> parameterInfoList = new ArrayList<>();
+        for (MethodParameterInfo p : method.parameters()) {
+            parameterInfoList.add(new AiServiceMethodCreateInfo.ParameterInfo(p.name(),
+                    typeSignature(p.type(), typeArgMapper),
+                    p.declaredAnnotations().stream().map(an -> an.name().toString()).collect(
+                            Collectors.toSet())));
+        }
+
+        return new AiServiceMethodCreateInfo(method.declaringClass().name().toString(), method.name(), parameterInfoList,
+                systemMessageInfo,
                 userMessageInfo, memoryIdParamPosition, requiresModeration, methodReturnTypeSignature,
                 overrideChatModelParamPosition, metricsTimedInfo, metricsCountedInfo, spanInfo, responseSchemaInfo,
                 methodToolClassInfo, methodMcpClientNames, switchToWorkerThreadForToolExecution,
@@ -1817,7 +1826,7 @@ private java.lang.reflect.Type javaLangReturnType(MethodInfo method) {
         }
     }
 
-    private String returnTypeSignature(Type returnType, TypeArgMapper typeArgMapper) {
+    private String typeSignature(Type returnType, TypeArgMapper typeArgMapper) {
         return AsmUtil.getSignature(returnType, typeArgMapper);
     }
 
@@ -1979,7 +1988,7 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
             return AiServiceMethodCreateInfo.UserMessageInfo.fromTemplate(
                     AiServiceMethodCreateInfo.TemplateInfo.fromText(userMessageTemplate,
                             TemplateParameterInfo.toNameToArgsPositionMap(templateParams)),
-                    userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
+                    userNameParamPosition);
         } else {
             Optional<AnnotationInstance> userMessageOnMethodParam = method.annotations(LangChain4jDotNames.USER_MESSAGE)
                     .stream()
@@ -1992,13 +2001,11 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
                                     Short.valueOf(userMessageOnMethodParam.get().target().asMethodParameter().position())
                                             .intValue(),
                                     TemplateParameterInfo.toNameToArgsPositionMap(templateParams)),
-                            userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition,
-                            videoParamPosition);
+                            userNameParamPosition);
                 } else {
                     return AiServiceMethodCreateInfo.UserMessageInfo.fromMethodParam(
                             userMessageOnMethodParam.get().target().asMethodParameter().position(),
-                            userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition,
-                            videoParamPosition);
+                            userNameParamPosition);
                 }
             } else {
                 Set<String> templateParamNames = Collections.EMPTY_SET;
@@ -2031,9 +2038,7 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
                     if (undefinedParams > 1) {
                         if (fallbackToDummyUserMesage.test(method)) {
                             return AiServiceMethodCreateInfo.UserMessageInfo.fromTemplate(
-                                    AiServiceMethodCreateInfo.TemplateInfo.fromText("", Map.of()), Optional.empty(),
-                                    Optional.empty(),
-                                    Optional.empty(), Optional.empty(), Optional.empty());
+                                    AiServiceMethodCreateInfo.TemplateInfo.fromText("", Map.of()), Optional.empty());
                         }
 
                         throw illegalConfigurationForMethod(
@@ -2044,12 +2049,10 @@ private AiServiceMethodCreateInfo.UserMessageInfo gatherUserMessageInfo(MethodIn
                 }
                 if (userMessageParamPosition == -1) {
                     // There is no user message
-                    return new AiServiceMethodCreateInfo.UserMessageInfo(Optional.empty(), Optional.empty(), Optional.empty(),
-                            Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
+                    return new AiServiceMethodCreateInfo.UserMessageInfo(Optional.empty(), Optional.empty(), Optional.empty());
                 } else {
                     return AiServiceMethodCreateInfo.UserMessageInfo.fromMethodParam(userMessageParamPosition,
-                            userNameParamPosition,
-                            imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
+                            userNameParamPosition);
 
                 }
             }

core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java

@@ -4,6 +4,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.Supplier;
@@ -29,6 +30,7 @@
 public final class AiServiceMethodCreateInfo {
     private final String interfaceName;
     private final String methodName;
+    private final List<ParameterInfo> parameterInfo;
     private final Optional<TemplateInfo> systemMessageInfo;
     private final UserMessageInfo userMessageInfo;
     private final Optional<Integer> memoryIdParamPosition;
@@ -65,7 +67,9 @@ public final class AiServiceMethodCreateInfo {
     private final boolean switchToWorkerThreadForToolExecution;
 
     @RecordableConstructor
-    public AiServiceMethodCreateInfo(String interfaceName, String methodName,
+    public AiServiceMethodCreateInfo(String interfaceName,
+            String methodName,
+            List<ParameterInfo> parameterInfo,
             Optional<TemplateInfo> systemMessageInfo,
             UserMessageInfo userMessageInfo,
             Optional<Integer> memoryIdParamPosition,
@@ -85,6 +89,7 @@ public AiServiceMethodCreateInfo(String interfaceName, String methodName,
             OutputGuardrailsLiteral outputGuardrails) {
         this.interfaceName = interfaceName;
         this.methodName = methodName;
+        this.parameterInfo = parameterInfo;
         this.systemMessageInfo = systemMessageInfo;
         this.userMessageInfo = userMessageInfo;
         this.memoryIdParamPosition = memoryIdParamPosition;
@@ -126,6 +131,10 @@ public String getMethodName() {
         return methodName;
     }
 
+    public List<ParameterInfo> getParameterInfo() {
+        return parameterInfo;
+    }
+
     public Optional<TemplateInfo> getSystemMessageInfo() {
         return systemMessageInfo;
     }
@@ -254,27 +263,15 @@ public void setResponseAugmenter(Class<? extends AiResponseAugmenter<?>> augment
 
     public record UserMessageInfo(Optional<TemplateInfo> template,
             Optional<Integer> paramPosition,
-            Optional<Integer> userNameParamPosition,
-            Optional<Integer> imageParamPosition,
-            Optional<Integer> audioParamPosition,
-            Optional<Integer> pdfParamPosition,
-            Optional<Integer> videoParamPosition) {
-
-        public static UserMessageInfo fromMethodParam(int paramPosition, Optional<Integer> userNameParamPosition,
-                Optional<Integer> imageParamPosition, Optional<Integer> audioParamPosition,
-                Optional<Integer> pdfParamPosition,
-                Optional<Integer> videoParamPosition) {
+            Optional<Integer> userNameParamPosition) {
+
+        public static UserMessageInfo fromMethodParam(int paramPosition, Optional<Integer> userNameParamPosition) {
             return new UserMessageInfo(Optional.empty(), Optional.of(paramPosition),
-                    userNameParamPosition, imageParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
+                    userNameParamPosition);
         }
 
-        public static UserMessageInfo fromTemplate(TemplateInfo templateInfo, Optional<Integer> userNameParamPosition,
-                Optional<Integer> imageUrlParamPosition,
-                Optional<Integer> audioParamPosition,
-                Optional<Integer> pdfParamPosition,
-                Optional<Integer> videoParamPosition) {
-            return new UserMessageInfo(Optional.of(templateInfo), Optional.empty(), userNameParamPosition,
-                    imageUrlParamPosition, audioParamPosition, pdfParamPosition, videoParamPosition);
+        public static UserMessageInfo fromTemplate(TemplateInfo templateInfo, Optional<Integer> userNameParamPosition) {
+            return new UserMessageInfo(Optional.of(templateInfo), Optional.empty(), userNameParamPosition);
         }
     }
 
@@ -405,4 +402,8 @@ public static ResponseSchemaInfo of(boolean enabled, Optional<TemplateInfo> syst
                     structuredOutputSchema);
         }
     }
+
+    public record ParameterInfo(String name, String typeDescriptor, Set<String> annotationTypes) {
+
+    }
 }