Image recognition, external URLs, content references

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "maven" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"

.gitignore

@@ -33,3 +33,4 @@ target
 .cgptdevbench/llmsearch.db
 .linklint
 .lycheecache
+build.log

TODO.txt

@@ -1,5 +1,7 @@
 # List of minor Todos
 
+Image description from URL?
+
 AEM for content fragments?
 URL as base text , inner links
 Append button for AEM
@@ -31,3 +33,14 @@ ignore: align, fileReference, target, style, element
 ## Check out Adobe Sensei GenAI
 
 https://business.adobe.com/summit/2023/sessions/opening-keynote-gs1.html at 1:20:00 or something
+
+## Images
+
+https://github.com/TheoKanning/openai-java/issues/397
+Alternative: https://github.com/namankhurpia/Easy-open-ai
+https://mvnrepository.com/artifact/io.github.namankhurpia/easyopenai -> many dependencies :-(
+
+##
+
+DTB Chat Completion Gen
+https://chat.openai.com/share/c095d1db-4e72-4abe-8794-c1fe9e01fbf7

aem/core/src/main/java/com/composum/ai/aem/core/impl/AemApproximateMarkdownServicePlugin.java

@@ -4,8 +4,14 @@
 import static com.day.cq.commons.jcr.JcrConstants.JCR_DESCRIPTION;
 import static com.day.cq.commons.jcr.JcrConstants.JCR_TITLE;
 
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.PrintWriter;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -14,7 +20,9 @@
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import javax.imageio.ImageIO;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.sling.api.SlingHttpServletRequest;
 import org.apache.sling.api.SlingHttpServletResponse;
@@ -57,6 +65,9 @@ public class AemApproximateMarkdownServicePlugin implements ApproximateMarkdownS
             @Nonnull Resource resource, @Nonnull PrintWriter out,
             @Nonnull ApproximateMarkdownService service,
             @Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) {
+        if (renderDamAssets(resource, out, response)) {
+            return PluginResult.HANDLED_ALL;
+        }
         if (resourceRendersAsComponentMatching(resource, FULLY_IGNORED_TYPES)) {
             return PluginResult.HANDLED_ALL;
         }
@@ -305,4 +316,80 @@ protected List<Resource> listModelResources(List<Resource> list, Resource traver
         return list;
     }
 
+    /**
+     * If the resource is a dam:Asset or a dam:AssetContent jcr:content then we return an image link
+     */
+    protected boolean renderDamAssets(Resource resource, PrintWriter out, SlingHttpServletResponse response) {
+        Resource assetNode = resource;
+        if (resource.isResourceType("dam:AssetContent")) {
+            assetNode = resource.getParent();
+        }
+        if (assetNode.isResourceType("dam:Asset")) {
+            String mimeType = assetNode.getValueMap().get("jcr:content/metadata/dc:format", String.class);
+            if (StringUtils.startsWith(mimeType, "image/")) {
+                String name = StringUtils.defaultString(assetNode.getValueMap().get("jcr:content/jcr:title", String.class), assetNode.getName());
+                out.println("![" + name + "](" + assetNode.getPath());
+                try {
+                    response.addHeader(ApproximateMarkdownService.HEADER_IMAGEPATH, resource.getParent().getPath());
+                } catch (RuntimeException e) {
+                    LOG.warn("Unable to set header " + ApproximateMarkdownService.HEADER_IMAGEPATH + " to " + resource.getParent().getPath(), e);
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Retrieves the imageURL in a way useable for ChatGPT - usually data:image/jpeg;base64,{base64_image}
+     */
+    @Nullable
+    @Override
+    public String getImageUrl(@Nullable Resource imageResource) {
+        Resource assetNode = imageResource;
+        if (imageResource.isResourceType("dam:AssetContent")) {
+            assetNode = imageResource.getParent();
+        }
+        if (assetNode.isResourceType("dam:Asset")) {
+            String mimeType = assetNode.getValueMap().get("jcr:content/metadata/dc:format", String.class);
+            Resource originalRendition = assetNode.getChild("jcr:content/renditions/original/jcr:content");
+            if (StringUtils.startsWith(mimeType, "image/") && originalRendition != null) {
+                try (InputStream is = originalRendition.adaptTo(InputStream.class)) {
+                    if (is == null) {
+                        LOG.warn("Unable to get InputStream from image resource {}", assetNode.getPath());
+                        return null;
+                    }
+                    byte[] data = IOUtils.toByteArray(is);
+                    data = resizeToMaxSize(data, mimeType, 512);
+                    return "data:" + mimeType + ";base64," + new String(Base64.getEncoder().encode(data));
+                } catch (IOException e) {
+                    LOG.warn("Unable to get InputStream from image resource {}", assetNode.getPath(), e);
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * We resize the image to a maximum width and height of maxSize, keeping the aspect ratio. If it's smaller, it's
+     * returned as is. It could be of types image/jpeg, image/png or image/gif .
+     */
+    protected byte[] resizeToMaxSize(@Nonnull byte[] imageData, String mimeType, int maxSize) throws IOException {
+        ByteArrayInputStream inputStream = new ByteArrayInputStream(imageData);
+        BufferedImage originalImage = ImageIO.read(inputStream);
+        int width = originalImage.getWidth();
+        int height = originalImage.getHeight();
+        if (width <= maxSize && height <= maxSize) {
+            return imageData;
+        }
+        double factor = maxSize * 1.0 / (Math.max(width, height) + 1);
+        int newWidth = (int) (width * factor);
+        int newHeight = (int) (height * factor);
+        BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, originalImage.getType());
+        resizedImage.createGraphics().drawImage(originalImage, 0, 0, newWidth, newHeight, null);
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        ImageIO.write(resizedImage, mimeType.substring("image/".length()), outputStream);
+        return outputStream.toByteArray();
+    }
+
 }

aem/core/src/main/java/com/composum/ai/aem/core/impl/ContentCreationSelectorsServlet.java

@@ -0,0 +1,104 @@
+package com.composum.ai.aem.core.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nonnull;
+import javax.servlet.Servlet;
+import javax.servlet.ServletException;
+
+import org.apache.sling.api.SlingHttpServletRequest;
+import org.apache.sling.api.SlingHttpServletResponse;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.api.resource.ResourceMetadata;
+import org.apache.sling.api.resource.ValueMap;
+import org.apache.sling.api.servlets.SlingSafeMethodsServlet;
+import org.apache.sling.api.wrappers.ValueMapDecorator;
+import org.osgi.framework.Constants;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+
+import com.adobe.granite.ui.components.ds.DataSource;
+import com.adobe.granite.ui.components.ds.SimpleDataSource;
+import com.adobe.granite.ui.components.ds.ValueMapResource;
+import com.composum.ai.backend.slingbase.ApproximateMarkdownService;
+import com.google.gson.Gson;
+
+/**
+ * Servlet that reads the content selectors from a JSON file, adds links in the content and provides that to the dialog.
+ */
+@Component(service = Servlet.class,
+        property = {
+                Constants.SERVICE_DESCRIPTION + "=Composum Pages Content Creation Selectors Servlet",
+                "sling.servlet.resourceTypes=composum-ai/servlets/contentcreationselectors",
+        })
+public class ContentCreationSelectorsServlet extends SlingSafeMethodsServlet {
+
+    private final Gson gson = new Gson();
+
+    /**
+     * JCR path to a JSON with the basic content selectors supported by the dialog.
+     */
+    public static final String PATH_CONTENTSELECTORS = "/conf/composum-ai/settings/dialogs/contentcreation/contentselectors.json";
+
+    @Reference
+    private ApproximateMarkdownService approximateMarkdownService;
+
+    @Override
+    protected void doGet(@Nonnull SlingHttpServletRequest request, @Nonnull SlingHttpServletResponse response) throws ServletException, IOException {
+        Map<String, String> contentSelectors = readPredefinedContentSelectors(request);
+        String path = request.getParameter("path");
+        Resource resource = request.getResourceResolver().getResource(path);
+        if (resource != null) {
+            addContentPaths(resource, contentSelectors);
+        }
+        DataSource dataSource = transformToDatasource(request, contentSelectors);
+        request.setAttribute(DataSource.class.getName(), dataSource);
+    }
+
+    /**
+     * We look for content paths in the component and it's parent. That seems more appropriate than the component itself
+     * in AEM - often interesting links are contained one level up, e.g. for text fields in teasers.
+     */
+    protected void addContentPaths(Resource resource, Map<String, String> contentSelectors) {
+        if (resource.getPath().contains("/jcr:content/")) {
+            resource = resource.getParent();
+        }
+        List<ApproximateMarkdownService.Link> componentLinks = approximateMarkdownService.getComponentLinks(resource);
+        for (ApproximateMarkdownService.Link link : componentLinks) {
+            contentSelectors.put(link.getPath(), link.getTitle() + " (" + link.getPath() + ")");
+        }
+    }
+
+    protected Map<String, String> readPredefinedContentSelectors(SlingHttpServletRequest request) throws IOException {
+        Resource resource = request.getResourceResolver().getResource(PATH_CONTENTSELECTORS);
+        Map<String, String> contentSelectors;
+        try (InputStream in = resource.adaptTo(InputStream.class);
+             Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) {
+            contentSelectors = gson.fromJson(reader, Map.class);
+        }
+        return contentSelectors;
+    }
+
+    protected static DataSource transformToDatasource(SlingHttpServletRequest request, Map<String, String> contentSelectors) {
+        List<Resource> resourceList = contentSelectors.entrySet().stream()
+                .map(entry -> {
+                    Map<String, Object> values = new HashMap<>();
+                    values.put("value", entry.getKey());
+                    values.put("text", entry.getValue());
+                    ValueMap valueMap = new ValueMapDecorator(values);
+                    return new ValueMapResource(request.getResourceResolver(), new ResourceMetadata(), "nt:unstructured", valueMap);
+                })
+                .collect(Collectors.toList());
+        DataSource dataSource = new SimpleDataSource(resourceList.iterator());
+        return dataSource;
+    }
+
+}

aem/pom.xml

@@ -269,7 +269,7 @@ Bundle-DocURL:
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.8.1</version>
+                    <version>3.12.1</version>
                     <configuration>
                         <encoding>${source.encoding}</encoding>
                         <source>${java.source}</source>

aem/ui.apps/src/main/content/jcr_root/apps/composum-ai/components/contentcreation/_cq_dialog/.content.xml

@@ -69,8 +69,8 @@
                                                      granite:class="composum-ai-content-selector">
                                         <datasource
                                                 jcr:primaryType="nt:unstructured"
-                                                sling:resourceType="cq/gui/components/common/wcm/datasources/childresources"
-                                                path="/conf/composum-ai/settings/dialogs/contentcreation/contentselectors"/>
+                                                sling:resourceType="composum-ai/servlets/contentcreationselectors"
+                                                additionalAttribute="17"/>
                                     </contentSelector>
                                     <url
                                             jcr:primaryType="nt:unstructured"
@@ -84,7 +84,8 @@
                                             sling:resourceType="granite/ui/components/coral/foundation/form/textarea"
                                             fieldDescription="The base text that is modified according to the prompt. Will be overwritten when the Content Selector is changed."
                                             fieldLabel="Source Content ('Data' for the instructions)" rows="10"
-                                            name="./sourcePlaintext" granite:class="composum-ai-source-plaintext">
+                                            name="./sourcePlaintext"
+                                            granite:class="composum-ai-source-plaintext composum-ai-source-container">
                                         <granite:rendercondition
                                                 jcr:primaryType="nt:unstructured"
                                                 sling:resourceType="granite/ui/components/coral/foundation/renderconditions/simple"
@@ -93,7 +94,7 @@
                                     </sourcePlaintext>
                                     <container
                                             jcr:primaryType="nt:unstructured"
-                                            granite:class="composum-ai-source-richtext"
+                                            granite:class="composum-ai-source-richtext composum-ai-source-container"
                                             sling:resourceType="granite/ui/components/coral/foundation/container">
                                         <items jcr:primaryType="nt:unstructured">
                                             <sourceRichtext
@@ -226,6 +227,21 @@
                                             </sourceRichtext>
                                         </items>
                                     </container>
+                                    <image-container
+                                            jcr:primaryType="nt:unstructured"
+                                            granite:class="composum-ai-source-image-container hidden"
+                                            sling:resourceType="granite/ui/components/coral/foundation/container">
+                                        <items jcr:primaryType="nt:unstructured">
+                                            <imagediv
+                                                    jcr:primaryType="nt:unstructured"
+                                                    sling:resourceType="granite/ui/components/coral/foundation/form/textfield"
+                                                    fieldDescription="An image that will be used to inform the AI."
+                                                    fieldLabel="Source Content ('Data' for the instructions)" rows="10"
+                                                    name="./sourceImage"
+                                                    granite:class="composum-ai-source-image">
+                                            </imagediv>
+                                        </items>
+                                    </image-container>
                                 </items>
                             </sourceFieldset>
                         </items>
@@ -241,6 +257,7 @@
                     <generateActionbar
                             jcr:primaryType="nt:unstructured"
                             margin="{Boolean}false"
+                            granite:class="composum-ai-actionbar"
                             sling:resourceType="granite/ui/components/coral/foundation/actionbar">
                         <primary
                                 jcr:primaryType="nt:unstructured">

aem/ui.content/src/main/content/META-INF/vault/filter.xml

@@ -3,5 +3,5 @@
     <filter root="/conf/composum-ai"/>
     <filter root="/content/composum-ai"/>
     <filter root="/content/dam/composum-ai"/>
-    <filter root="/content/experience-fragments/composum-ai" mode="merge"/>
+    <filter root="/content/experience-fragments/composum-ai"/>
 </workspaceFilter>

aem/ui.content/src/main/content/jcr_root/conf/composum-ai/settings/dialogs/contentcreation/contentselectors.json

@@ -0,0 +1,9 @@
+{
+  "widget": "The text field you were editing",
+  "component": "The component you were editing, including subcomponents",
+  "page": "Current page text",
+  "lastoutput": "Current suggestion shown in this dialog (for iterative improvement)",
+  "url": "Text content of an external URL",
+  "empty": "No additional content",
+  "-": "Manually entered source content"
+}

aem/ui.content/src/main/content/jcr_root/conf/composum-ai/settings/dialogs/contentcreation/predefinedprompts/.content.xml

@@ -70,4 +70,7 @@ Do also fix orthographical and grammar errors."></improve>
                       value="Convert the following text from Markdown to HTML:
 
 "></markdown_to_html>
+    <describeImage jcr:primaryType="nt:unstructured" sling:resourceType="nt:unstructured"
+                   text="Describe Image"
+                   value="Please describe the following image in a way that a blind person can understand it."></describeImage>
 </jcr:root>