From e4ada3cad22bc0e840ec446259b5d472df2c4db0 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 10:29:02 +0700 Subject: [PATCH 01/28] feat: bump chat-ui with inline artifact --- .../code_generator/src/app/workflow.ts | 28 +++++++++++-------- .../ui/chat/chat-message-content.tsx | 1 - packages/server/src/utils/events.ts | 18 ++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 733faf1e0..48b1201d3 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,5 +1,5 @@ -import { extractLastArtifact } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; +import { extractLastArtifact, toInlineAnnotationCode } from "../utils"; import { agentStreamEvent, @@ -262,6 +262,7 @@ ${user_msg} const response = await llm.complete({ prompt, + // stream: true, // TODO: streaming artifact }); // Extract the code from the response @@ -278,19 +279,24 @@ ${user_msg} content: `Updated the code: \n${response.text}`, }); - // To show the Canvas panel for the artifact + // Show inline artifact sendEvent( - artifactEvent.with({ - type: "artifact", - data: { - type: "code", - created_at: Date.now(), + agentStreamEvent.with({ + delta: toInlineAnnotationCode({ + type: "artifact", data: { - language: planData.requirement.language || "", - file_name: planData.requirement.file_name || "", - code, + type: "code", + created_at: Date.now(), + data: { + language: planData.requirement.language || "", + file_name: planData.requirement.file_name || "", + code, + }, }, - }, + }), + response: "", + currentAgentName: "assistant", + raw: code, }), ); diff --git a/packages/server/next/app/components/ui/chat/chat-message-content.tsx b/packages/server/next/app/components/ui/chat/chat-message-content.tsx index 3ba488ed7..caf132633 100644 --- a/packages/server/next/app/components/ui/chat/chat-message-content.tsx +++ b/packages/server/next/app/components/ui/chat/chat-message-content.tsx @@ -19,7 +19,6 @@ export function ChatMessageContent({ - diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index c20751a2b..66d9f7603 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -4,6 +4,8 @@ import type { Message } from "ai"; import { MetadataMode, type Metadata, type NodeWithScore } from "llamaindex"; import { z } from "zod"; +const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown + // Events that appended to stream as annotations export type SourceEventNode = { id: string; @@ -211,3 +213,19 @@ export function extractLastArtifact( return artifacts[artifacts.length - 1]; } + +/** + * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. + * The language key is `annotation` and the code block is wrapped in backticks. + * The prefix `0:` ensures it will be treated as inline markdown. Example: + * + * 0:\`\`\`annotation + * \{ + * "type": "artifact", + * "data": \{...\} + * \} + * \`\`\` + */ +export function toInlineAnnotationCode(item: object) { + return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; +} From 03c7c4505da82c18f0c8f6e00425e30e7e6f14a2 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 11:24:07 +0700 Subject: [PATCH 02/28] bump chat-ui 0.5.0 --- packages/server/package.json | 2 +- packages/server/project-config/package.json | 2 +- pnpm-lock.yaml | 318 ++++++++++++++++++-- 3 files changed, 301 insertions(+), 21 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index bbe896723..52c961540 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,7 +65,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.4.9", + "@llamaindex/chat-ui": "0.5.0", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/packages/server/project-config/package.json b/packages/server/project-config/package.json index f5c5bb2e6..00f32bc2f 100644 --- a/packages/server/project-config/package.json +++ b/packages/server/project-config/package.json @@ -41,7 +41,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.4.9", + "@llamaindex/chat-ui": "0.5.0", "@llamaindex/env": "~0.1.30", "@llamaindex/openai": "~0.4.0", "@llamaindex/readers": "~3.1.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa5c327c0..fc2a761fa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,8 +181,8 @@ importers: specifier: ^5.0.1 version: 5.0.1(react-hook-form@7.56.1(react@19.1.0)) '@llamaindex/chat-ui': - specifier: 0.4.9 - version: 0.4.9(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + specifier: 0.5.0 + version: 0.5.0(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1189,8 +1189,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.4.9': - resolution: {integrity: sha512-KEdydC+aJ22VK/TltxIHlMWbWLfh6I0YkyVd1D/CS3FRfLt8l9jfQ/YjY10MiEd8oc1fFfk6ek/FhVWe9Szstg==} + '@llamaindex/chat-ui@0.5.0': + resolution: {integrity: sha512-vuX4hjDizhm0fBavahF7sPu2kr2BHZ2pjs30RdeVMktV390goTut3Y3vku1nEMMA0zjne4WjRlPweXGJ/qSztQ==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -2488,6 +2488,9 @@ packages: '@types/mdast@3.0.15': resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdurl@2.0.0': resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} @@ -4568,6 +4571,9 @@ packages: mdast-util-from-markdown@1.3.1: resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} + mdast-util-from-markdown@2.0.2: + resolution: {integrity: sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==} + mdast-util-gfm-autolink-literal@1.0.3: resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==} @@ -4592,15 +4598,24 @@ packages: mdast-util-phrasing@3.0.1: resolution: {integrity: sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg==} + mdast-util-phrasing@4.1.0: + resolution: {integrity: sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==} + mdast-util-to-hast@12.3.0: resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} mdast-util-to-markdown@1.5.0: resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==} + mdast-util-to-markdown@2.1.2: + resolution: {integrity: sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==} + mdast-util-to-string@3.2.0: resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} + mdast-util-to-string@4.0.0: + resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + mdurl@2.0.0: resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} @@ -4625,6 +4640,9 @@ packages: micromark-core-commonmark@1.1.0: resolution: {integrity: sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==} + micromark-core-commonmark@2.0.3: + resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + micromark-extension-gfm-autolink-literal@1.0.5: resolution: {integrity: sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==} @@ -4652,63 +4670,123 @@ packages: micromark-factory-destination@1.1.0: resolution: {integrity: sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==} + micromark-factory-destination@2.0.1: + resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} + micromark-factory-label@1.1.0: resolution: {integrity: sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==} + micromark-factory-label@2.0.1: + resolution: {integrity: sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==} + micromark-factory-space@1.1.0: resolution: {integrity: sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==} + micromark-factory-space@2.0.1: + resolution: {integrity: sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==} + micromark-factory-title@1.1.0: resolution: {integrity: sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==} + micromark-factory-title@2.0.1: + resolution: {integrity: sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==} + micromark-factory-whitespace@1.1.0: resolution: {integrity: sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==} + micromark-factory-whitespace@2.0.1: + resolution: {integrity: sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==} + micromark-util-character@1.2.0: resolution: {integrity: sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==} + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + micromark-util-chunked@1.1.0: resolution: {integrity: sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==} + micromark-util-chunked@2.0.1: + resolution: {integrity: sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==} + micromark-util-classify-character@1.1.0: resolution: {integrity: sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==} + micromark-util-classify-character@2.0.1: + resolution: {integrity: sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==} + micromark-util-combine-extensions@1.1.0: resolution: {integrity: sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==} + micromark-util-combine-extensions@2.0.1: + resolution: {integrity: sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==} + micromark-util-decode-numeric-character-reference@1.1.0: resolution: {integrity: sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==} + micromark-util-decode-numeric-character-reference@2.0.2: + resolution: {integrity: sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==} + micromark-util-decode-string@1.1.0: resolution: {integrity: sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==} + micromark-util-decode-string@2.0.1: + resolution: {integrity: sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==} + micromark-util-encode@1.1.0: resolution: {integrity: sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==} + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + micromark-util-html-tag-name@1.2.0: resolution: {integrity: sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==} + micromark-util-html-tag-name@2.0.1: + resolution: {integrity: sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==} + micromark-util-normalize-identifier@1.1.0: resolution: {integrity: sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==} + micromark-util-normalize-identifier@2.0.1: + resolution: {integrity: sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==} + micromark-util-resolve-all@1.1.0: resolution: {integrity: sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==} + micromark-util-resolve-all@2.0.1: + resolution: {integrity: sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==} + micromark-util-sanitize-uri@1.2.0: resolution: {integrity: sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==} + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + micromark-util-subtokenize@1.1.0: resolution: {integrity: sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==} + micromark-util-subtokenize@2.1.0: + resolution: {integrity: sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==} + micromark-util-symbol@1.1.0: resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==} + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + micromark-util-types@1.1.0: resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==} + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + micromark@3.2.0: resolution: {integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==} + micromark@4.0.2: + resolution: {integrity: sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -5529,14 +5607,17 @@ packages: remark-parse@10.0.2: resolution: {integrity: sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw==} + remark-parse@11.0.0: + resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} + remark-rehype@10.1.0: resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} - remark-stringify@10.0.3: - resolution: {integrity: sha512-koyOzCMYoUHudypbj4XpnAKFbkddRMYZHwghnxd7ue5210WzGw6kOBwauJTRUMq16jsovXx8dYNvSSWP89kZ3A==} + remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} - remark@14.0.3: - resolution: {integrity: sha512-bfmJW1dmR2LvaMJuAnE88pZP9DktIFYXazkTfOIKZzi3Knk9lT0roItIA24ydOucI3bV/g/tXBA6hzqq3FV9Ew==} + remark@15.0.1: + resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} @@ -6133,6 +6214,9 @@ packages: unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + unified@11.0.5: + resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} + unist-util-find-after@5.0.0: resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} @@ -7219,7 +7303,7 @@ snapshots: p-retry: 6.2.1 zod: 3.24.3 - '@llamaindex/chat-ui@0.4.9(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + '@llamaindex/chat-ui@0.5.0(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 @@ -7253,11 +7337,13 @@ snapshots: react: 19.1.0 react-markdown: 8.0.7(@types/react@19.1.2)(react@19.1.0) rehype-katex: 7.0.1 - remark: 14.0.3 + remark: 15.0.1 remark-code-import: 1.2.0 remark-gfm: 3.0.1 remark-math: 5.1.1 + remark-parse: 11.0.0 tailwind-merge: 2.6.0 + unist-util-visit: 5.0.0 vaul: 0.9.9(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) transitivePeerDependencies: - '@babel/runtime' @@ -8609,6 +8695,10 @@ snapshots: dependencies: '@types/unist': 2.0.11 + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + '@types/mdurl@2.0.0': {} '@types/ms@2.1.0': {} @@ -11030,6 +11120,23 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-from-markdown@2.0.2: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + mdast-util-to-string: 4.0.0 + micromark: 4.0.2 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-decode-string: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-stringify-position: 4.0.0 + transitivePeerDependencies: + - supports-color + mdast-util-gfm-autolink-literal@1.0.3: dependencies: '@types/mdast': 3.0.15 @@ -11085,6 +11192,11 @@ snapshots: '@types/mdast': 3.0.15 unist-util-is: 5.2.1 + mdast-util-phrasing@4.1.0: + dependencies: + '@types/mdast': 4.0.4 + unist-util-is: 6.0.0 + mdast-util-to-hast@12.3.0: dependencies: '@types/hast': 2.3.10 @@ -11107,10 +11219,26 @@ snapshots: unist-util-visit: 4.1.2 zwitch: 2.0.4 + mdast-util-to-markdown@2.1.2: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + longest-streak: 3.1.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-string: 4.0.0 + micromark-util-classify-character: 2.0.1 + micromark-util-decode-string: 2.0.1 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + mdast-util-to-string@3.2.0: dependencies: '@types/mdast': 3.0.15 + mdast-util-to-string@4.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdurl@2.0.0: {} memoize-one@5.2.1: {} @@ -11142,6 +11270,25 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-core-commonmark@2.0.3: + dependencies: + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + micromark-factory-destination: 2.0.1 + micromark-factory-label: 2.0.1 + micromark-factory-space: 2.0.1 + micromark-factory-title: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-html-tag-name: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-extension-gfm-autolink-literal@1.0.5: dependencies: micromark-util-character: 1.2.0 @@ -11216,6 +11363,12 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-destination@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-label@1.1.0: dependencies: micromark-util-character: 1.2.0 @@ -11223,11 +11376,23 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-factory-label@2.0.1: + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-space@1.1.0: dependencies: micromark-util-character: 1.2.0 micromark-util-types: 1.1.0 + micromark-factory-space@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-types: 2.0.2 + micromark-factory-title@1.1.0: dependencies: micromark-factory-space: 1.1.0 @@ -11235,6 +11400,13 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-title@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-whitespace@1.1.0: dependencies: micromark-factory-space: 1.1.0 @@ -11242,30 +11414,61 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-whitespace@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-character@1.2.0: dependencies: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-chunked@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-chunked@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-classify-character@1.1.0: dependencies: micromark-util-character: 1.2.0 micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-classify-character@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-combine-extensions@1.1.0: dependencies: micromark-util-chunked: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-combine-extensions@2.0.1: + dependencies: + micromark-util-chunked: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-decode-numeric-character-reference@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-decode-numeric-character-reference@2.0.2: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-decode-string@1.1.0: dependencies: decode-named-character-reference: 1.1.0 @@ -11273,24 +11476,49 @@ snapshots: micromark-util-decode-numeric-character-reference: 1.1.0 micromark-util-symbol: 1.1.0 + micromark-util-decode-string@2.0.1: + dependencies: + decode-named-character-reference: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-symbol: 2.0.1 + micromark-util-encode@1.1.0: {} + micromark-util-encode@2.0.1: {} + micromark-util-html-tag-name@1.2.0: {} + micromark-util-html-tag-name@2.0.1: {} + micromark-util-normalize-identifier@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-normalize-identifier@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-resolve-all@1.1.0: dependencies: micromark-util-types: 1.1.0 + micromark-util-resolve-all@2.0.1: + dependencies: + micromark-util-types: 2.0.2 + micromark-util-sanitize-uri@1.2.0: dependencies: micromark-util-character: 1.2.0 micromark-util-encode: 1.1.0 micromark-util-symbol: 1.1.0 + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-subtokenize@1.1.0: dependencies: micromark-util-chunked: 1.1.0 @@ -11298,10 +11526,21 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-util-subtokenize@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-symbol@1.1.0: {} + micromark-util-symbol@2.0.1: {} + micromark-util-types@1.1.0: {} + micromark-util-types@2.0.2: {} + micromark@3.2.0: dependencies: '@types/debug': 4.1.12 @@ -11324,6 +11563,28 @@ snapshots: transitivePeerDependencies: - supports-color + micromark@4.0.2: + dependencies: + '@types/debug': 4.1.12 + debug: 4.4.0(supports-color@5.5.0) + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-combine-extensions: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-encode: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + transitivePeerDependencies: + - supports-color + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -12144,6 +12405,15 @@ snapshots: transitivePeerDependencies: - supports-color + remark-parse@11.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.2 + micromark-util-types: 2.0.2 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-rehype@10.1.0: dependencies: '@types/hast': 2.3.10 @@ -12151,18 +12421,18 @@ snapshots: mdast-util-to-hast: 12.3.0 unified: 10.1.2 - remark-stringify@10.0.3: + remark-stringify@11.0.0: dependencies: - '@types/mdast': 3.0.15 - mdast-util-to-markdown: 1.5.0 - unified: 10.1.2 + '@types/mdast': 4.0.4 + mdast-util-to-markdown: 2.1.2 + unified: 11.0.5 - remark@14.0.3: + remark@15.0.1: dependencies: - '@types/mdast': 3.0.15 - remark-parse: 10.0.2 - remark-stringify: 10.0.3 - unified: 10.1.2 + '@types/mdast': 4.0.4 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 transitivePeerDependencies: - supports-color @@ -12838,6 +13108,16 @@ snapshots: trough: 2.2.0 vfile: 5.3.7 + unified@11.0.5: + dependencies: + '@types/unist': 3.0.3 + bail: 2.0.2 + devlop: 1.1.0 + extend: 3.0.2 + is-plain-obj: 4.1.0 + trough: 2.2.0 + vfile: 6.0.3 + unist-util-find-after@5.0.0: dependencies: '@types/unist': 3.0.3 From b2f804c8af5a4eadf96fbc7924d84199cad7c4e0 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 11:51:52 +0700 Subject: [PATCH 03/28] update extractLastArtifact --- .../code_generator/src/app/workflow.ts | 31 ++++---- packages/server/src/utils/events.ts | 77 ++++++++----------- packages/server/src/utils/index.ts | 1 + packages/server/src/utils/inline.ts | 74 ++++++++++++++++++ 4 files changed, 123 insertions(+), 60 deletions(-) create mode 100644 packages/server/src/utils/inline.ts diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 48b1201d3..175c330c2 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,5 +1,9 @@ import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; -import { extractLastArtifact, toInlineAnnotationCode } from "../utils"; +import { + CodeArtifact, + extractLastArtifact, + toInlineAnnotation, +} from "../utils"; import { agentStreamEvent, @@ -52,26 +56,16 @@ const synthesizeAnswerEvent = workflowEvent(); const uiEvent = workflowEvent(); -const artifactEvent = workflowEvent<{ - type: "artifact"; - data: { - type: "code"; - created_at: number; - data: { - language: string; - file_name: string; - code: string; - }; - }; -}>(); - export function workflowFactory(reqBody: any) { const llm = Settings.llm; - const { withState, getContext } = createStatefulMiddleware(() => { + const { withState, getContext } = createStatefulMiddleware<{ + memory: ChatMemoryBuffer; + lastArtifact: CodeArtifact | undefined; + }>(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: extractLastArtifact(reqBody), + lastArtifact: undefined, }; }); const workflow = withState(createWorkflow()); @@ -87,6 +81,9 @@ export function workflowFactory(reqBody: any) { state.memory.set(chatHistory); state.memory.put({ role: "user", content: userInput }); + const messages = await state.memory.getMessages(); + state.lastArtifact = extractLastArtifact(messages, "code"); + return planEvent.with({ userInput: userInput, context: state.lastArtifact @@ -282,7 +279,7 @@ ${user_msg} // Show inline artifact sendEvent( agentStreamEvent.with({ - delta: toInlineAnnotationCode({ + delta: toInlineAnnotation({ type: "artifact", data: { type: "code", diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 66d9f7603..a626d7a41 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,10 +1,20 @@ import { randomUUID } from "@llamaindex/env"; import { workflowEvent } from "@llamaindex/workflow"; -import type { Message } from "ai"; -import { MetadataMode, type Metadata, type NodeWithScore } from "llamaindex"; +import { + MetadataMode, + type ChatMessage, + type Metadata, + type NodeWithScore, +} from "llamaindex"; import { z } from "zod"; +import { getInlineAnnotations } from "./inline"; -const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown +export const AnnotationSchema = z.object({ + type: z.string(), + data: z.any(), +}); + +export type Annotation = z.infer; // Events that appended to stream as annotations export type SourceEventNode = { @@ -150,49 +160,46 @@ export const artifactAnnotationSchema = z.object({ data: artifactSchema, }); -export function extractAllArtifacts(messages: Message[]): Artifact[] { - const allArtifacts: Artifact[] = []; - - for (const message of messages) { - const artifacts = - message.annotations - ?.filter( - ( - annotation, - ): annotation is z.infer => - artifactAnnotationSchema.safeParse(annotation).success, - ) - .map((annotation) => annotation.data as Artifact) ?? []; - - allArtifacts.push(...artifacts); - } +export function extractArtifactsFromMessage(message: ChatMessage): Artifact[] { + const inlineAnnotations = getInlineAnnotations(message); + const artifacts = inlineAnnotations.filter( + (annotation): annotation is z.infer => { + return artifactAnnotationSchema.safeParse(annotation).success; + }, + ); + return artifacts.map((artifact) => artifact.data); +} - return allArtifacts; +export function extractArtifactsFromAllMessages( + messages: ChatMessage[], +): Artifact[] { + return messages + .flatMap((message) => extractArtifactsFromMessage(message)) + .sort((a, b) => a.created_at - b.created_at); } export function extractLastArtifact( - requestBody: unknown, + messages: ChatMessage[], type: "code", ): CodeArtifact | undefined; export function extractLastArtifact( - requestBody: unknown, + messages: ChatMessage[], type: "document", ): DocumentArtifact | undefined; export function extractLastArtifact( - requestBody: unknown, + messages: ChatMessage[], type?: ArtifactType, ): Artifact | undefined; export function extractLastArtifact( - requestBody: unknown, + messages: ChatMessage[], type?: ArtifactType, ): CodeArtifact | DocumentArtifact | Artifact | undefined { - const { messages } = (requestBody as { messages?: Message[] }) ?? {}; - if (!messages) return undefined; + if (messages?.length === 0) return undefined; - const artifacts = extractAllArtifacts(messages); + const artifacts = extractArtifactsFromAllMessages(messages); if (!artifacts.length) return undefined; if (type) { @@ -213,19 +220,3 @@ export function extractLastArtifact( return artifacts[artifacts.length - 1]; } - -/** - * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. - * The language key is `annotation` and the code block is wrapped in backticks. - * The prefix `0:` ensures it will be treated as inline markdown. Example: - * - * 0:\`\`\`annotation - * \{ - * "type": "artifact", - * "data": \{...\} - * \} - * \`\`\` - */ -export function toInlineAnnotationCode(item: object) { - return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; -} diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 8b5184b00..3e69d4652 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1,6 +1,7 @@ export * from "./events"; export * from "./file"; export * from "./gen-ui"; +export * from "./inline"; export * from "./prompts"; export * from "./request"; export * from "./stream"; diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts new file mode 100644 index 000000000..3dbbbf38b --- /dev/null +++ b/packages/server/src/utils/inline.ts @@ -0,0 +1,74 @@ +import { type ChatMessage } from "llamaindex"; +import { type Annotation, AnnotationSchema } from "./events"; + +const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown + +export function getInlineAnnotations(message: ChatMessage): Annotation[] { + const markdownContent = getMessageMarkdownContent(message); + + const inlineAnnotations: Annotation[] = []; + + // Regex to match annotation code blocks + // Matches ```annotation followed by content until closing ``` + const annotationRegex = new RegExp( + `\`\`\`${INLINE_ANNOTATION_KEY}\\s*\\n([\\s\\S]*?)\\n\`\`\``, + "g", + ); + + let match; + while ((match = annotationRegex.exec(markdownContent)) !== null) { + const jsonContent = match[1]?.trim(); + + if (!jsonContent) { + continue; + } + + try { + // Parse the JSON content + const parsed = JSON.parse(jsonContent); + + // Validate against the annotation schema + const validated = AnnotationSchema.parse(parsed); + + // Extract the artifact data + inlineAnnotations.push(validated); + } catch (error) { + // Skip invalid annotations - they might be malformed JSON or invalid schema + console.warn("Failed to parse annotation:", error); + } + } + + return inlineAnnotations; +} + +/** + * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. + * The language key is `annotation` and the code block is wrapped in backticks. + * The prefix `0:` ensures it will be treated as inline markdown. Example: + * + * 0:\`\`\`annotation + * \{ + * "type": "artifact", + * "data": \{...\} + * \} + * \`\`\` + */ +export function toInlineAnnotation(item: object) { + return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; +} + +function getMessageMarkdownContent(message: ChatMessage): string { + let markdownContent = ""; + + if (typeof message.content === "string") { + markdownContent = message.content; + } else { + message.content.forEach((item) => { + if (item.type === "text") { + markdownContent = item.text; + } + }); + } + + return markdownContent; +} From b9f61c77e94664120d0e8253d2ef5bcea5975770 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 11:54:32 +0700 Subject: [PATCH 04/28] fix: imports --- .../use-cases/typescript/code_generator/src/app/workflow.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 175c330c2..fb6041250 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,9 +1,9 @@ -import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { CodeArtifact, extractLastArtifact, toInlineAnnotation, -} from "../utils"; +} from "@llamaindex/server"; +import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { agentStreamEvent, From 4be5c0e1cab0fff47337502454111d452a2674d7 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 11:57:20 +0700 Subject: [PATCH 05/28] fix: circle import --- packages/server/src/utils/events.ts | 7 ------- packages/server/src/utils/inline.ts | 9 ++++++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index a626d7a41..3ab2869bb 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -9,13 +9,6 @@ import { import { z } from "zod"; import { getInlineAnnotations } from "./inline"; -export const AnnotationSchema = z.object({ - type: z.string(), - data: z.any(), -}); - -export type Annotation = z.infer; - // Events that appended to stream as annotations export type SourceEventNode = { id: string; diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts index 3dbbbf38b..9c8aa1265 100644 --- a/packages/server/src/utils/inline.ts +++ b/packages/server/src/utils/inline.ts @@ -1,8 +1,15 @@ import { type ChatMessage } from "llamaindex"; -import { type Annotation, AnnotationSchema } from "./events"; +import { z } from "zod"; const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown +export const AnnotationSchema = z.object({ + type: z.string(), + data: z.any(), +}); + +export type Annotation = z.infer; + export function getInlineAnnotations(message: ChatMessage): Annotation[] { const markdownContent = getMessageMarkdownContent(message); From b493548a134eca95e44d2e791b375c7546ac82ba Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 13:44:22 +0700 Subject: [PATCH 06/28] missing export --- packages/server/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index e08c57892..b0d3f9abe 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -2,4 +2,5 @@ export * from "./server"; export * from "./types"; export * from "./utils/events"; export { generateEventComponent } from "./utils/gen-ui"; +export * from "./utils/inline"; export * from "./utils/prompts"; From e3187fac1ad09dcbc518ba8e59e85d9eb4c01ee2 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 14:20:22 +0700 Subject: [PATCH 07/28] update document gen workflow --- .../code_generator/src/app/workflow.ts | 1 - .../document_generator/src/app/workflow.ts | 57 +++++++++---------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index fb6041250..73df9c018 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -259,7 +259,6 @@ ${user_msg} const response = await llm.complete({ prompt, - // stream: true, // TODO: streaming artifact }); // Extract the code from the response diff --git a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts index 70ba93cb9..b7072abd4 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts @@ -1,4 +1,8 @@ -import { extractLastArtifact } from "@llamaindex/server"; +import { + DocumentArtifact, + extractLastArtifact, + toInlineAnnotation, +} from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -55,26 +59,16 @@ const synthesizeAnswerEvent = workflowEvent<{ const uiEvent = workflowEvent(); -const artifactEvent = workflowEvent<{ - type: "artifact"; - data: { - type: "document"; - created_at: number; - data: { - title: string; - content: string; - type: "markdown" | "html"; - }; - }; -}>(); - export function workflowFactory(reqBody: any) { const llm = Settings.llm; - const { withState, getContext } = createStatefulMiddleware(() => { + const { withState, getContext } = createStatefulMiddleware<{ + memory: ChatMemoryBuffer; + lastArtifact: DocumentArtifact | undefined; + }>(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: extractLastArtifact(reqBody), + lastArtifact: undefined, }; }); const workflow = withState(createWorkflow()); @@ -87,11 +81,11 @@ export function workflowFactory(reqBody: any) { if (!userInput) { throw new Error("Missing user input to start the workflow"); } - state.memory.set(chatHistory); - state.memory.put({ role: "user", content: userInput }); + const messages = await state.memory.getMessages(); + state.lastArtifact = extractLastArtifact(messages, "document"); return planEvent.with({ - userInput, + userInput: userInput, context: state.lastArtifact ? JSON.stringify(state.lastArtifact) : undefined, @@ -250,19 +244,24 @@ export function workflowFactory(reqBody: any) { content: `Generated document: \n${response.text}`, }); - // To show the Canvas panel for the artifact + // Show inline artifact sendEvent( - artifactEvent.with({ - type: "artifact", - data: { - type: "document", - created_at: Date.now(), + agentStreamEvent.with({ + delta: toInlineAnnotation({ + type: "artifact", data: { - title: requirement.title, - content: content, - type: docType, + type: "document", + created_at: Date.now(), + data: { + title: requirement.title, + content: content, + type: docType, + }, }, - }, + }), + response: "", + currentAgentName: "assistant", + raw: content, }), ); } From 472dbc4c2132667234d4f5767eb6198f632070c6 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 16:10:45 +0700 Subject: [PATCH 08/28] remove artifactEvent for annotations --- packages/server/src/utils/events.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 3ab2869bb..3e92c55bf 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -118,11 +118,6 @@ export type DocumentArtifact = Artifact & { type: "document"; }; -export const artifactEvent = workflowEvent<{ - type: "artifact"; - data: Artifact; -}>(); - export const codeArtifactSchema = z.object({ type: z.literal("code"), data: z.object({ From dc73a34f61cc7bd4ea7beb7dff31f1751ba623be Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 16:15:34 +0700 Subject: [PATCH 09/28] update document --- packages/server/README.md | 100 +++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index 8b91a2c57..8b37b90c5 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -223,63 +223,87 @@ new LlamaIndexServer({ ## Sending Artifacts to the UI -In addition to UI events for custom components, LlamaIndex Server supports a special `ArtifactEvent` to send structured data like generated documents or code snippets to the UI. These artifacts are displayed in a dedicated "Canvas" panel in the chat interface. +LlamaIndex Server supports sending structured data like generated documents or code snippets to the UI as inline artifacts. These artifacts are displayed in a dedicated "Canvas" panel in the chat interface and are embedded directly in the stream response. -### Artifact Event Structure +### Inline Artifact Structure -To send an artifact, your workflow needs to emit an event with `type: "artifact"`. The `data` payload of this event should include: +To send an artifact, you need to use the `toInlineAnnotation` function from `@llamaindex/server` within an `agentStreamEvent`. The artifact data should include: -- `type`: A string indicating the type of artifact (e.g., `"document"`, `"code"`). -- `created_at`: A timestamp (e.g., `Date.now()`) indicating when the artifact was created. -- `data`: An object containing the specific details of the artifact. The structure of this object depends on the artifact `type`. +- `type`: Always set to `"artifact"` for the top-level type +- `data`: An object containing: + - `type`: A string indicating the specific type of artifact (e.g., `"document"`, `"code"`) + - `created_at`: A timestamp (e.g., `Date.now()`) indicating when the artifact was created + - `data`: An object containing the specific details of the artifact, structure depends on the artifact type -### Defining and Sending an ArtifactEvent +### Sending Inline Artifacts -First, define your artifact event using `workflowEvent` from `@llamaindex/workflow`: +First, import the necessary functions: ```typescript -import { workflowEvent } from "@llamaindex/workflow"; - -// Example for a document artifact -const artifactEvent = workflowEvent<{ - type: "artifact"; // Must be "artifact" - data: { - type: "document"; // Custom type for your artifact (e.g., "document", "code") - created_at: number; - data: { - // Specific data for the document artifact type - title: string; - content: string; - type: "markdown" | "html"; // document format - }; - }; -}>(); +import { toInlineAnnotation } from "@llamaindex/server"; +import { agentStreamEvent } from "@llamaindex/workflow"; ``` -Then, within your workflow logic, use `sendEvent` (obtained from `getContext()`) to emit the event: +Then, within your workflow logic, use `sendEvent` to emit the artifact inline: ```typescript -// Assuming 'sendEvent' is available in your workflow handler -// and 'documentDetails' contains the content for the artifact. +// Example for a document artifact +sendEvent( + agentStreamEvent.with({ + delta: toInlineAnnotation({ + type: "artifact", + data: { + type: "document", // Specific artifact type + created_at: Date.now(), + data: { + title: "My Generated Document", + content: "# Hello World\nThis is a markdown document.", + type: "markdown", // document format: "markdown" | "html" + }, + }, + }), + response: "", + currentAgentName: "assistant", + raw: "", // Optional: raw content for debugging + }), +); +// Example for a code artifact sendEvent( - artifactEvent.with({ - type: "artifact", // This top-level type must be "artifact" - data: { - type: "document", // This is your specific artifact type - created_at: Date.now(), + agentStreamEvent.with({ + delta: toInlineAnnotation({ + type: "artifact", data: { - title: "My Generated Document", - content: "# Hello World -This is a markdown document.", - type: "markdown", + type: "code", // Specific artifact type + created_at: Date.now(), + data: { + language: "typescript", + file_name: "MyComponent.tsx", + code: `import React from "react"; + +export default function MyComponent() { + return
Hello World
; +}`, + }, }, - }, + }), + response: "", + currentAgentName: "assistant", + raw: "", }), ); ``` -This will send the artifact to the LlamaIndex Server UI, where it will be rendered in the [ChatCanvasPanel](/packages/server/next/app/components/ui/chat/canvas/panel.tsx) by a renderer depending on the artifact type. For type `document` this is using the [DocumentArtifactViewer](https://github.com/run-llama/chat-ui/blob/bacb75fc6edceacf742fba18632404a2483b5a81/packages/chat-ui/src/chat/canvas/artifacts/document.tsx#L17). +The `toInlineAnnotation` function wraps the artifact data in a special code block format that the UI can parse and render appropriately. This approach embeds artifacts directly in the response stream, making them part of the natural conversation flow. + +### Supported Artifact Types + +Common artifact types include: + +- **`document`**: For markdown or HTML documents with `title`, `content`, and `type` fields +- **`code`**: For code snippets with `language`, `file_name`, and `code` fields + +The artifacts will be automatically rendered in the [ChatCanvasPanel](/packages/server/next/app/components/ui/chat/canvas/panel.tsx) by the appropriate renderer based on the artifact type. ## Default Endpoints and Features From f59b46df620002efac107879417461917dc078e7 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 3 Jun 2025 16:44:51 +0700 Subject: [PATCH 10/28] bump chat-ui 0.5.1 to fix parsing $ --- packages/server/package.json | 2 +- packages/server/project-config/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 52c961540..afd056a4b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,7 +65,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.0", + "@llamaindex/chat-ui": "0.5.1", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/packages/server/project-config/package.json b/packages/server/project-config/package.json index 00f32bc2f..f7c7fe03f 100644 --- a/packages/server/project-config/package.json +++ b/packages/server/project-config/package.json @@ -41,7 +41,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.0", + "@llamaindex/chat-ui": "0.5.1", "@llamaindex/env": "~0.1.30", "@llamaindex/openai": "~0.4.0", "@llamaindex/readers": "~3.1.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc2a761fa..f60f59575 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,8 +181,8 @@ importers: specifier: ^5.0.1 version: 5.0.1(react-hook-form@7.56.1(react@19.1.0)) '@llamaindex/chat-ui': - specifier: 0.5.0 - version: 0.5.0(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + specifier: 0.5.1 + version: 0.5.1(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1189,8 +1189,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.5.0': - resolution: {integrity: sha512-vuX4hjDizhm0fBavahF7sPu2kr2BHZ2pjs30RdeVMktV390goTut3Y3vku1nEMMA0zjne4WjRlPweXGJ/qSztQ==} + '@llamaindex/chat-ui@0.5.1': + resolution: {integrity: sha512-Qwc0flO3qecgitYSgnIi7ZGlOD5WYqYnCMLVOrNGAm2mAZpdQAwog9Vp5Yl/Ig/requmRtrJkHVN48BVEme+nw==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -7303,7 +7303,7 @@ snapshots: p-retry: 6.2.1 zod: 3.24.3 - '@llamaindex/chat-ui@0.5.0(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + '@llamaindex/chat-ui@0.5.1(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 From 75409c91060e4d76034c14f801fca80625fc8d84 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 09:52:53 +0700 Subject: [PATCH 11/28] bump chat-ui 0.5.2 --- packages/server/package.json | 2 +- packages/server/project-config/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index afd056a4b..29fac104f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,7 +65,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.1", + "@llamaindex/chat-ui": "0.5.2", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/packages/server/project-config/package.json b/packages/server/project-config/package.json index f7c7fe03f..ae5083908 100644 --- a/packages/server/project-config/package.json +++ b/packages/server/project-config/package.json @@ -41,7 +41,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.1", + "@llamaindex/chat-ui": "0.5.2", "@llamaindex/env": "~0.1.30", "@llamaindex/openai": "~0.4.0", "@llamaindex/readers": "~3.1.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f60f59575..5c3a3de61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,8 +181,8 @@ importers: specifier: ^5.0.1 version: 5.0.1(react-hook-form@7.56.1(react@19.1.0)) '@llamaindex/chat-ui': - specifier: 0.5.1 - version: 0.5.1(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + specifier: 0.5.2 + version: 0.5.2(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1189,8 +1189,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.5.1': - resolution: {integrity: sha512-Qwc0flO3qecgitYSgnIi7ZGlOD5WYqYnCMLVOrNGAm2mAZpdQAwog9Vp5Yl/Ig/requmRtrJkHVN48BVEme+nw==} + '@llamaindex/chat-ui@0.5.2': + resolution: {integrity: sha512-zIOaf5cuhDtllrcDEDaGNRq2bon8STUYoex4n+zzowR6eirRRYPYoUpRBNBK7fZi4BJhBPmqTS/tkXzjw908Gw==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -7303,7 +7303,7 @@ snapshots: p-retry: 6.2.1 zod: 3.24.3 - '@llamaindex/chat-ui@0.5.1(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + '@llamaindex/chat-ui@0.5.2(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 From bdd624242f59efe888700a87a38061a02973d90a Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:03:40 +0700 Subject: [PATCH 12/28] toArtifactEvent internal --- .../code_generator/src/app/workflow.ts | 31 ++++++--------- .../document_generator/src/app/workflow.ts | 39 +++++++------------ packages/server/src/utils/events.ts | 16 +++++++- 3 files changed, 40 insertions(+), 46 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 73df9c018..539895154 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,7 +1,7 @@ import { CodeArtifact, extractLastArtifact, - toInlineAnnotation, + toArtifactEvent, } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; @@ -276,25 +276,16 @@ ${user_msg} }); // Show inline artifact - sendEvent( - agentStreamEvent.with({ - delta: toInlineAnnotation({ - type: "artifact", - data: { - type: "code", - created_at: Date.now(), - data: { - language: planData.requirement.language || "", - file_name: planData.requirement.file_name || "", - code, - }, - }, - }), - response: "", - currentAgentName: "assistant", - raw: code, - }), - ); + const artifact: CodeArtifact = { + type: "code", + created_at: Date.now(), + data: { + language: planData.requirement.language || "", + file_name: planData.requirement.file_name || "", + code, + }, + }; + sendEvent(toArtifactEvent(artifact)); return synthesizeAnswerEvent.with({}); }); diff --git a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts index b7072abd4..5ad752d32 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts @@ -1,8 +1,3 @@ -import { - DocumentArtifact, - extractLastArtifact, - toInlineAnnotation, -} from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -14,6 +9,11 @@ import { workflowEvent, } from "@llamaindex/workflow"; +import { + DocumentArtifact, + extractLastArtifact, + toArtifactEvent, +} from "@llamaindex/server"; import { z } from "zod"; export const DocumentRequirementSchema = z.object({ @@ -245,25 +245,16 @@ export function workflowFactory(reqBody: any) { }); // Show inline artifact - sendEvent( - agentStreamEvent.with({ - delta: toInlineAnnotation({ - type: "artifact", - data: { - type: "document", - created_at: Date.now(), - data: { - title: requirement.title, - content: content, - type: docType, - }, - }, - }), - response: "", - currentAgentName: "assistant", - raw: content, - }), - ); + const artifact: DocumentArtifact = { + type: "document", + created_at: Date.now(), + data: { + title: requirement.title, + content: content, + type: docType, + }, + }; + sendEvent(toArtifactEvent(artifact)); } return synthesizeAnswerEvent.with({ diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 3e92c55bf..1362fc8fc 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,5 +1,5 @@ import { randomUUID } from "@llamaindex/env"; -import { workflowEvent } from "@llamaindex/workflow"; +import { agentStreamEvent, workflowEvent } from "@llamaindex/workflow"; import { MetadataMode, type ChatMessage, @@ -7,7 +7,7 @@ import { type NodeWithScore, } from "llamaindex"; import { z } from "zod"; -import { getInlineAnnotations } from "./inline"; +import { getInlineAnnotations, toInlineAnnotation } from "./inline"; // Events that appended to stream as annotations export type SourceEventNode = { @@ -90,6 +90,18 @@ export function toAgentRunEvent(input: { }); } +export function toArtifactEvent(data: Artifact) { + return agentStreamEvent.with({ + delta: toInlineAnnotation({ + type: "artifact", + data, + }), + response: "", + currentAgentName: "assistant", + raw: data, + }); +} + export type ArtifactType = "code" | "document"; export type Artifact = { From 2165247dab8de2aa87ea8dd609d74f1b135cbbab Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:08:10 +0700 Subject: [PATCH 13/28] update doc to use toArtifactEvent --- packages/server/README.md | 69 +++++++++++++++------------------------ 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index 8b37b90c5..a6bec31a7 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -240,61 +240,46 @@ To send an artifact, you need to use the `toInlineAnnotation` function from `@ll First, import the necessary functions: ```typescript -import { toInlineAnnotation } from "@llamaindex/server"; -import { agentStreamEvent } from "@llamaindex/workflow"; +import { + toArtifactEvent, + CodeArtifact, + DocumentArtifact, +} from "@llamaindex/server"; ``` -Then, within your workflow logic, use `sendEvent` to emit the artifact inline: +Then, within your workflow logic, use `sendEvent` to emit the artifact: ```typescript // Example for a document artifact -sendEvent( - agentStreamEvent.with({ - delta: toInlineAnnotation({ - type: "artifact", - data: { - type: "document", // Specific artifact type - created_at: Date.now(), - data: { - title: "My Generated Document", - content: "# Hello World\nThis is a markdown document.", - type: "markdown", // document format: "markdown" | "html" - }, - }, - }), - response: "", - currentAgentName: "assistant", - raw: "", // Optional: raw content for debugging - }), -); +const documentArtifact: DocumentArtifact = { + type: "document", // Specific artifact type + created_at: Date.now(), + data: { + title: "My Generated Document", + content: "# Hello World\nThis is a markdown document.", + type: "markdown", // document format: "markdown" | "html" + }, +}; +sendEvent(toArtifactEvent(documentArtifact)); // Example for a code artifact -sendEvent( - agentStreamEvent.with({ - delta: toInlineAnnotation({ - type: "artifact", - data: { - type: "code", // Specific artifact type - created_at: Date.now(), - data: { - language: "typescript", - file_name: "MyComponent.tsx", - code: `import React from "react"; +const codeArtifact: CodeArtifact = { + type: "code", // Specific artifact type + created_at: Date.now(), + data: { + language: "typescript", + file_name: "MyComponent.tsx", + code: `import React from "react"; export default function MyComponent() { return
Hello World
; }`, - }, - }, - }), - response: "", - currentAgentName: "assistant", - raw: "", - }), -); + }, +}; +sendEvent(toArtifactEvent(codeArtifact)); ``` -The `toInlineAnnotation` function wraps the artifact data in a special code block format that the UI can parse and render appropriately. This approach embeds artifacts directly in the response stream, making them part of the natural conversation flow. +The `toArtifactEvent` function automatically wraps the artifact data in the proper format for the UI to parse and render appropriately. This approach embeds artifacts directly in the response stream, making them part of the natural conversation flow. ### Supported Artifact Types From 844c2974ca7078d0c72919e263883f419f1e3791 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:24:23 +0700 Subject: [PATCH 14/28] do workflow transformmation internal --- .../code_generator/src/app/workflow.ts | 42 ++++++++--------- .../document_generator/src/app/workflow.ts | 45 +++++++++---------- packages/server/src/utils/events.ts | 19 +++----- packages/server/src/utils/workflow.ts | 13 ++++++ 4 files changed, 58 insertions(+), 61 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 539895154..e47dec0a1 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,8 +1,4 @@ -import { - CodeArtifact, - extractLastArtifact, - toArtifactEvent, -} from "@llamaindex/server"; +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -59,13 +55,10 @@ const uiEvent = workflowEvent(); export function workflowFactory(reqBody: any) { const llm = Settings.llm; - const { withState, getContext } = createStatefulMiddleware<{ - memory: ChatMemoryBuffer; - lastArtifact: CodeArtifact | undefined; - }>(() => { + const { withState, getContext } = createStatefulMiddleware(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: undefined, + lastArtifact: extractLastArtifact(reqBody?.messages || []), }; }); const workflow = withState(createWorkflow()); @@ -81,9 +74,6 @@ export function workflowFactory(reqBody: any) { state.memory.set(chatHistory); state.memory.put({ role: "user", content: userInput }); - const messages = await state.memory.getMessages(); - state.lastArtifact = extractLastArtifact(messages, "code"); - return planEvent.with({ userInput: userInput, context: state.lastArtifact @@ -275,17 +265,21 @@ ${user_msg} content: `Updated the code: \n${response.text}`, }); - // Show inline artifact - const artifact: CodeArtifact = { - type: "code", - created_at: Date.now(), - data: { - language: planData.requirement.language || "", - file_name: planData.requirement.file_name || "", - code, - }, - }; - sendEvent(toArtifactEvent(artifact)); + // To show the Canvas panel for the artifact + sendEvent( + artifactEvent.with({ + type: "artifact", + data: { + type: "code", + created_at: Date.now(), + data: { + language: planData.requirement.language || "", + file_name: planData.requirement.file_name || "", + code, + }, + }, + }), + ); return synthesizeAnswerEvent.with({}); }); diff --git a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts index 5ad752d32..21fa16418 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts @@ -1,3 +1,4 @@ +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -9,11 +10,6 @@ import { workflowEvent, } from "@llamaindex/workflow"; -import { - DocumentArtifact, - extractLastArtifact, - toArtifactEvent, -} from "@llamaindex/server"; import { z } from "zod"; export const DocumentRequirementSchema = z.object({ @@ -62,13 +58,10 @@ const uiEvent = workflowEvent(); export function workflowFactory(reqBody: any) { const llm = Settings.llm; - const { withState, getContext } = createStatefulMiddleware<{ - memory: ChatMemoryBuffer; - lastArtifact: DocumentArtifact | undefined; - }>(() => { + const { withState, getContext } = createStatefulMiddleware(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: undefined, + lastArtifact: extractLastArtifact(reqBody?.messages || []), }; }); const workflow = withState(createWorkflow()); @@ -81,11 +74,11 @@ export function workflowFactory(reqBody: any) { if (!userInput) { throw new Error("Missing user input to start the workflow"); } - const messages = await state.memory.getMessages(); - state.lastArtifact = extractLastArtifact(messages, "document"); + state.memory.set(chatHistory); + state.memory.put({ role: "user", content: userInput }); return planEvent.with({ - userInput: userInput, + userInput, context: state.lastArtifact ? JSON.stringify(state.lastArtifact) : undefined, @@ -244,17 +237,21 @@ export function workflowFactory(reqBody: any) { content: `Generated document: \n${response.text}`, }); - // Show inline artifact - const artifact: DocumentArtifact = { - type: "document", - created_at: Date.now(), - data: { - title: requirement.title, - content: content, - type: docType, - }, - }; - sendEvent(toArtifactEvent(artifact)); + // To show the Canvas panel for the artifact + sendEvent( + artifactEvent.with({ + type: "artifact", + data: { + type: "document", + created_at: Date.now(), + data: { + title: requirement.title, + content: content, + type: docType, + }, + }, + }), + ); } return synthesizeAnswerEvent.with({ diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 1362fc8fc..b42fa6e12 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,5 +1,5 @@ import { randomUUID } from "@llamaindex/env"; -import { agentStreamEvent, workflowEvent } from "@llamaindex/workflow"; +import { workflowEvent } from "@llamaindex/workflow"; import { MetadataMode, type ChatMessage, @@ -7,7 +7,7 @@ import { type NodeWithScore, } from "llamaindex"; import { z } from "zod"; -import { getInlineAnnotations, toInlineAnnotation } from "./inline"; +import { getInlineAnnotations } from "./inline"; // Events that appended to stream as annotations export type SourceEventNode = { @@ -90,17 +90,10 @@ export function toAgentRunEvent(input: { }); } -export function toArtifactEvent(data: Artifact) { - return agentStreamEvent.with({ - delta: toInlineAnnotation({ - type: "artifact", - data, - }), - response: "", - currentAgentName: "assistant", - raw: data, - }); -} +export const artifactEvent = workflowEvent<{ + type: "artifact"; + data: Artifact; +}>(); export type ArtifactType = "code" | "document"; diff --git a/packages/server/src/utils/workflow.ts b/packages/server/src/utils/workflow.ts index 566c8f330..b63e7e7e6 100644 --- a/packages/server/src/utils/workflow.ts +++ b/packages/server/src/utils/workflow.ts @@ -1,4 +1,5 @@ import { + agentStreamEvent, agentToolCallEvent, agentToolCallResultEvent, run, @@ -15,12 +16,14 @@ import { type NodeWithScore, } from "llamaindex"; import { + artifactEvent, sourceEvent, toAgentRunEvent, toSourceEvent, type SourceEventNode, } from "./events"; import { downloadFile } from "./file"; +import { toInlineAnnotation } from "./inline"; export async function runWorkflow( workflow: Workflow, @@ -74,6 +77,16 @@ function processWorkflowStream( transformedEvent = toSourceEvent(sourceNodes); } } + // Handle artifact events, transform to agentStreamEvent + else if (artifactEvent.include(event)) { + const artifactAnnotation = event.data; + transformedEvent = agentStreamEvent.with({ + delta: toInlineAnnotation(artifactAnnotation), + response: "", + currentAgentName: "assistant", + raw: artifactAnnotation, + }); + } // Post-process for llama-cloud files if (sourceEvent.include(transformedEvent)) { const sourceNodesForDownload = transformedEvent.data.data.nodes; // These are SourceEventNode[] From 41898dcf109423a995a8e4728be5bcf047f6fdc7 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:31:35 +0700 Subject: [PATCH 15/28] revert doc --- packages/server/README.md | 95 ++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 52 deletions(-) diff --git a/packages/server/README.md b/packages/server/README.md index a6bec31a7..8b91a2c57 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -223,72 +223,63 @@ new LlamaIndexServer({ ## Sending Artifacts to the UI -LlamaIndex Server supports sending structured data like generated documents or code snippets to the UI as inline artifacts. These artifacts are displayed in a dedicated "Canvas" panel in the chat interface and are embedded directly in the stream response. +In addition to UI events for custom components, LlamaIndex Server supports a special `ArtifactEvent` to send structured data like generated documents or code snippets to the UI. These artifacts are displayed in a dedicated "Canvas" panel in the chat interface. -### Inline Artifact Structure +### Artifact Event Structure -To send an artifact, you need to use the `toInlineAnnotation` function from `@llamaindex/server` within an `agentStreamEvent`. The artifact data should include: +To send an artifact, your workflow needs to emit an event with `type: "artifact"`. The `data` payload of this event should include: -- `type`: Always set to `"artifact"` for the top-level type -- `data`: An object containing: - - `type`: A string indicating the specific type of artifact (e.g., `"document"`, `"code"`) - - `created_at`: A timestamp (e.g., `Date.now()`) indicating when the artifact was created - - `data`: An object containing the specific details of the artifact, structure depends on the artifact type +- `type`: A string indicating the type of artifact (e.g., `"document"`, `"code"`). +- `created_at`: A timestamp (e.g., `Date.now()`) indicating when the artifact was created. +- `data`: An object containing the specific details of the artifact. The structure of this object depends on the artifact `type`. -### Sending Inline Artifacts +### Defining and Sending an ArtifactEvent -First, import the necessary functions: +First, define your artifact event using `workflowEvent` from `@llamaindex/workflow`: ```typescript -import { - toArtifactEvent, - CodeArtifact, - DocumentArtifact, -} from "@llamaindex/server"; -``` +import { workflowEvent } from "@llamaindex/workflow"; -Then, within your workflow logic, use `sendEvent` to emit the artifact: - -```typescript // Example for a document artifact -const documentArtifact: DocumentArtifact = { - type: "document", // Specific artifact type - created_at: Date.now(), - data: { - title: "My Generated Document", - content: "# Hello World\nThis is a markdown document.", - type: "markdown", // document format: "markdown" | "html" - }, -}; -sendEvent(toArtifactEvent(documentArtifact)); - -// Example for a code artifact -const codeArtifact: CodeArtifact = { - type: "code", // Specific artifact type - created_at: Date.now(), +const artifactEvent = workflowEvent<{ + type: "artifact"; // Must be "artifact" data: { - language: "typescript", - file_name: "MyComponent.tsx", - code: `import React from "react"; - -export default function MyComponent() { - return
Hello World
; -}`, - }, -}; -sendEvent(toArtifactEvent(codeArtifact)); + type: "document"; // Custom type for your artifact (e.g., "document", "code") + created_at: number; + data: { + // Specific data for the document artifact type + title: string; + content: string; + type: "markdown" | "html"; // document format + }; + }; +}>(); ``` -The `toArtifactEvent` function automatically wraps the artifact data in the proper format for the UI to parse and render appropriately. This approach embeds artifacts directly in the response stream, making them part of the natural conversation flow. - -### Supported Artifact Types +Then, within your workflow logic, use `sendEvent` (obtained from `getContext()`) to emit the event: -Common artifact types include: - -- **`document`**: For markdown or HTML documents with `title`, `content`, and `type` fields -- **`code`**: For code snippets with `language`, `file_name`, and `code` fields +```typescript +// Assuming 'sendEvent' is available in your workflow handler +// and 'documentDetails' contains the content for the artifact. + +sendEvent( + artifactEvent.with({ + type: "artifact", // This top-level type must be "artifact" + data: { + type: "document", // This is your specific artifact type + created_at: Date.now(), + data: { + title: "My Generated Document", + content: "# Hello World +This is a markdown document.", + type: "markdown", + }, + }, + }), +); +``` -The artifacts will be automatically rendered in the [ChatCanvasPanel](/packages/server/next/app/components/ui/chat/canvas/panel.tsx) by the appropriate renderer based on the artifact type. +This will send the artifact to the LlamaIndex Server UI, where it will be rendered in the [ChatCanvasPanel](/packages/server/next/app/components/ui/chat/canvas/panel.tsx) by a renderer depending on the artifact type. For type `document` this is using the [DocumentArtifactViewer](https://github.com/run-llama/chat-ui/blob/bacb75fc6edceacf742fba18632404a2483b5a81/packages/chat-ui/src/chat/canvas/artifacts/document.tsx#L17). ## Default Endpoints and Features From 1f0f01a15d9362d5c2b4e6ffc1fe20b89d04dc86 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:37:02 +0700 Subject: [PATCH 16/28] keep contract --- .../typescript/code_generator/src/app/workflow.ts | 2 +- .../typescript/document_generator/src/app/workflow.ts | 2 +- packages/server/src/utils/events.ts | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index e47dec0a1..95f960fda 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -58,7 +58,7 @@ export function workflowFactory(reqBody: any) { const { withState, getContext } = createStatefulMiddleware(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: extractLastArtifact(reqBody?.messages || []), + lastArtifact: extractLastArtifact(reqBody), }; }); const workflow = withState(createWorkflow()); diff --git a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts index 21fa16418..ca2316578 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts @@ -61,7 +61,7 @@ export function workflowFactory(reqBody: any) { const { withState, getContext } = createStatefulMiddleware(() => { return { memory: new ChatMemoryBuffer({ llm }), - lastArtifact: extractLastArtifact(reqBody?.messages || []), + lastArtifact: extractLastArtifact(reqBody), }; }); const workflow = withState(createWorkflow()); diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index b42fa6e12..2699658c1 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -172,25 +172,26 @@ export function extractArtifactsFromAllMessages( } export function extractLastArtifact( - messages: ChatMessage[], + requestBody: unknown, type: "code", ): CodeArtifact | undefined; export function extractLastArtifact( - messages: ChatMessage[], + requestBody: unknown, type: "document", ): DocumentArtifact | undefined; export function extractLastArtifact( - messages: ChatMessage[], + requestBody: unknown, type?: ArtifactType, ): Artifact | undefined; export function extractLastArtifact( - messages: ChatMessage[], + requestBody: unknown, type?: ArtifactType, ): CodeArtifact | DocumentArtifact | Artifact | undefined { - if (messages?.length === 0) return undefined; + const { messages } = (requestBody as { messages?: ChatMessage[] }) ?? {}; + if (!messages) return undefined; const artifacts = extractArtifactsFromAllMessages(messages); if (!artifacts.length) return undefined; From 1c598071dc6145f009545fed1ca697b38e617755 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 10:42:39 +0700 Subject: [PATCH 17/28] fix format --- packages/server/src/utils/events.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 2699658c1..86b31d52d 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -90,11 +90,6 @@ export function toAgentRunEvent(input: { }); } -export const artifactEvent = workflowEvent<{ - type: "artifact"; - data: Artifact; -}>(); - export type ArtifactType = "code" | "document"; export type Artifact = { @@ -123,6 +118,11 @@ export type DocumentArtifact = Artifact & { type: "document"; }; +export const artifactEvent = workflowEvent<{ + type: "artifact"; + data: Artifact; +}>(); + export const codeArtifactSchema = z.object({ type: z.literal("code"), data: z.object({ From 37d52368de3d52234b2a4b922986ca9f7140e96b Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 11:30:54 +0700 Subject: [PATCH 18/28] update get_last_artifact to extract inline annotations in Python --- .../llama_index/server/models/artifacts.py | 10 +-- .../llama_index/server/utils/inline.py | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 python/llama-index-server/llama_index/server/utils/inline.py diff --git a/python/llama-index-server/llama_index/server/models/artifacts.py b/python/llama-index-server/llama_index/server/models/artifacts.py index 431b12e8b..882beeba1 100644 --- a/python/llama-index-server/llama_index/server/models/artifacts.py +++ b/python/llama-index-server/llama_index/server/models/artifacts.py @@ -3,8 +3,9 @@ from typing import Literal, Optional, Union from llama_index.core.workflow.events import Event -from llama_index.server.models.chat import ChatAPIMessage -from pydantic import BaseModel +from .chat import ChatAPIMessage +from ..utils.inline import get_inline_annotations +from pydantic import BaseModel, ValidationError logger = logging.getLogger(__name__) @@ -33,10 +34,9 @@ class Artifact(BaseModel): @classmethod def from_message(cls, message: ChatAPIMessage) -> Optional["Artifact"]: - if not message.annotations or not isinstance(message.annotations, list): - return None + inline_annotations = get_inline_annotations(message) - for annotation in message.annotations: + for annotation in inline_annotations: if isinstance(annotation, dict) and annotation.get("type") == "artifact": try: artifact = cls.model_validate(annotation.get("data")) diff --git a/python/llama-index-server/llama_index/server/utils/inline.py b/python/llama-index-server/llama_index/server/utils/inline.py new file mode 100644 index 000000000..856c576e6 --- /dev/null +++ b/python/llama-index-server/llama_index/server/utils/inline.py @@ -0,0 +1,66 @@ +import json +import re +from typing import Any, List + +from pydantic import ValidationError + +from llama_index.server.models.chat import ChatAPIMessage + +INLINE_ANNOTATION_KEY = ( + "annotation" # the language key to detect inline annotation code in markdown +) + + +def get_inline_annotations(message: ChatAPIMessage) -> List[Any]: + """Extract inline annotations from a chat message.""" + markdown_content = message.content + + inline_annotations: List[Any] = [] + + # Regex to match annotation code blocks + # Matches ```annotation followed by content until closing ``` + annotation_regex = re.compile( + rf"```{re.escape(INLINE_ANNOTATION_KEY)}\s*\n([\s\S]*?)\n```", re.MULTILINE + ) + + for match in annotation_regex.finditer(markdown_content): + json_content = match.group(1).strip() if match.group(1) else None + + if not json_content: + continue + + try: + # Parse the JSON content + parsed = json.loads(json_content) + + # Check for required fields in the parsed annotation + if ( + not isinstance(parsed, dict) + or "type" not in parsed + or "data" not in parsed + ): + continue + + # Extract the annotation data + inline_annotations.append(parsed) + except (json.JSONDecodeError, ValidationError) as error: + # Skip invalid annotations - they might be malformed JSON or invalid schema + print(f"Failed to parse annotation: {error}") + + return inline_annotations + + +def to_inline_annotation(item: dict) -> str: + """ + To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. + The language key is `annotation` and the code block is wrapped in backticks. + The prefix `0:` ensures it will be treated as inline markdown. Example: + + 0:```annotation + { + "type": "artifact", + "data": {...} + } + ``` + """ + return f"\n```{INLINE_ANNOTATION_KEY}\n{json.dumps(item)}\n```\n" From d8668bfaaf26392f6d8edd3d63cd2244d2c012f4 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 11:37:23 +0700 Subject: [PATCH 19/28] fix imports --- .../llama-index-server/llama_index/server/models/artifacts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/llama-index-server/llama_index/server/models/artifacts.py b/python/llama-index-server/llama_index/server/models/artifacts.py index 882beeba1..1fd686eaa 100644 --- a/python/llama-index-server/llama_index/server/models/artifacts.py +++ b/python/llama-index-server/llama_index/server/models/artifacts.py @@ -3,9 +3,9 @@ from typing import Literal, Optional, Union from llama_index.core.workflow.events import Event -from .chat import ChatAPIMessage +from llama_index.server.models.chat import ChatAPIMessage +from pydantic import BaseModel from ..utils.inline import get_inline_annotations -from pydantic import BaseModel, ValidationError logger = logging.getLogger(__name__) From 3ac65caa2eabdd0ff5bd60d51fdb66a3736b12f8 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 12:15:07 +0700 Subject: [PATCH 20/28] Transforms ArtifactEvent to AgentStream with inline annotation format --- .../server/api/callbacks/__init__.py | 2 + .../api/callbacks/artifact_transform.py | 37 +++++++++++++++++++ .../llama_index/server/api/routers/chat.py | 2 + 3 files changed, 41 insertions(+) create mode 100644 python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py diff --git a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py index 196e61b29..58a505433 100644 --- a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py +++ b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py @@ -1,4 +1,5 @@ from llama_index.server.api.callbacks.agent_call_tool import AgentCallTool +from llama_index.server.api.callbacks.artifact_transform import ArtifactTransform from llama_index.server.api.callbacks.base import EventCallback from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall @@ -12,4 +13,5 @@ "SuggestNextQuestions", "LlamaCloudFileDownload", "AgentCallTool", + "ArtifactTransform", ] diff --git a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py new file mode 100644 index 000000000..88730a22f --- /dev/null +++ b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py @@ -0,0 +1,37 @@ +import logging +from typing import Any + +from llama_index.core.agent.workflow.workflow_events import AgentStream +from llama_index.server.api.callbacks.base import EventCallback +from llama_index.server.models.artifacts import ArtifactEvent +from llama_index.server.utils.inline import to_inline_annotation + +logger = logging.getLogger("uvicorn") + + +class ArtifactTransform(EventCallback): + """ + Transforms ArtifactEvent to AgentStream with inline annotation format. + """ + + async def run(self, event: Any) -> Any: + if isinstance(event, ArtifactEvent): + # Create the artifact annotation + artifact_annotation = { + "type": "artifact", + "data": event.data.model_dump(), + } + + # Transform to AgentStream with inline annotation format + return AgentStream( + delta=to_inline_annotation(artifact_annotation), + response="", + current_agent_name="assistant", + tool_calls=[], + raw=artifact_annotation, + ) + return event + + @classmethod + def from_default(cls, *args: Any, **kwargs: Any) -> "ArtifactTransform": + return cls() diff --git a/python/llama-index-server/llama_index/server/api/routers/chat.py b/python/llama-index-server/llama_index/server/api/routers/chat.py index 18970860a..5d1786fbe 100644 --- a/python/llama-index-server/llama_index/server/api/routers/chat.py +++ b/python/llama-index-server/llama_index/server/api/routers/chat.py @@ -17,6 +17,7 @@ ) from llama_index.server.api.callbacks import ( AgentCallTool, + ArtifactTransform, EventCallback, LlamaCloudFileDownload, SourceNodesFromToolCall, @@ -72,6 +73,7 @@ async def chat( callbacks: list[EventCallback] = [ AgentCallTool(), + ArtifactTransform(), SourceNodesFromToolCall(), LlamaCloudFileDownload(background_tasks), ] From 980f212d54187878d81084b5b43a280b121a7e72 Mon Sep 17 00:00:00 2001 From: Thuc Pham <51660321+thucpn@users.noreply.github.com> Date: Wed, 4 Jun 2025 12:21:57 +0700 Subject: [PATCH 21/28] Create thick-turtles-deny.md --- .changeset/thick-turtles-deny.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/thick-turtles-deny.md diff --git a/.changeset/thick-turtles-deny.md b/.changeset/thick-turtles-deny.md new file mode 100644 index 000000000..3c761ad2c --- /dev/null +++ b/.changeset/thick-turtles-deny.md @@ -0,0 +1,7 @@ +--- +"create-llama": patch +"@llamaindex/server": patch +"@create-llama/llama-index-server": patch +--- + +feat: bump chat-ui with inline artifact From cded3ae41437ee0b858788d1dd2f97baf481e6fc Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 13:23:02 +0700 Subject: [PATCH 22/28] donot use relative imports --- packages/server/src/utils/inline.ts | 2 +- .../llama-index-server/llama_index/server/models/artifacts.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts index 9c8aa1265..b7758dde0 100644 --- a/packages/server/src/utils/inline.ts +++ b/packages/server/src/utils/inline.ts @@ -72,7 +72,7 @@ function getMessageMarkdownContent(message: ChatMessage): string { } else { message.content.forEach((item) => { if (item.type === "text") { - markdownContent = item.text; + markdownContent += item.text; } }); } diff --git a/python/llama-index-server/llama_index/server/models/artifacts.py b/python/llama-index-server/llama_index/server/models/artifacts.py index 1fd686eaa..ecc146928 100644 --- a/python/llama-index-server/llama_index/server/models/artifacts.py +++ b/python/llama-index-server/llama_index/server/models/artifacts.py @@ -5,7 +5,7 @@ from llama_index.core.workflow.events import Event from llama_index.server.models.chat import ChatAPIMessage from pydantic import BaseModel -from ..utils.inline import get_inline_annotations +from llama_index.server.utils.inline import get_inline_annotations logger = logging.getLogger(__name__) From 31cc131eb24fc9021e9fe0cfd70e31d7403e28f1 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 14:09:13 +0700 Subject: [PATCH 23/28] toInlineAnnotationEvent --- packages/server/src/utils/events.ts | 17 +++++++++++++++-- packages/server/src/utils/inline.ts | 2 +- packages/server/src/utils/workflow.ts | 11 ++--------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 86b31d52d..771132a12 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,5 +1,9 @@ import { randomUUID } from "@llamaindex/env"; -import { workflowEvent } from "@llamaindex/workflow"; +import { + agentStreamEvent, + workflowEvent, + type WorkflowEventData, +} from "@llamaindex/workflow"; import { MetadataMode, type ChatMessage, @@ -7,7 +11,7 @@ import { type NodeWithScore, } from "llamaindex"; import { z } from "zod"; -import { getInlineAnnotations } from "./inline"; +import { getInlineAnnotations, toInlineAnnotation } from "./inline"; // Events that appended to stream as annotations export type SourceEventNode = { @@ -90,6 +94,15 @@ export function toAgentRunEvent(input: { }); } +export function toInlineAnnotationEvent(event: WorkflowEventData) { + return agentStreamEvent.with({ + delta: toInlineAnnotation(event.data), + response: "", + currentAgentName: "assistant", + raw: event.data, + }); +} + export type ArtifactType = "code" | "document"; export type Artifact = { diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts index b7758dde0..dd6840b7a 100644 --- a/packages/server/src/utils/inline.ts +++ b/packages/server/src/utils/inline.ts @@ -60,7 +60,7 @@ export function getInlineAnnotations(message: ChatMessage): Annotation[] { * \} * \`\`\` */ -export function toInlineAnnotation(item: object) { +export function toInlineAnnotation(item: unknown) { return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; } diff --git a/packages/server/src/utils/workflow.ts b/packages/server/src/utils/workflow.ts index b63e7e7e6..69139a61d 100644 --- a/packages/server/src/utils/workflow.ts +++ b/packages/server/src/utils/workflow.ts @@ -1,5 +1,4 @@ import { - agentStreamEvent, agentToolCallEvent, agentToolCallResultEvent, run, @@ -19,11 +18,11 @@ import { artifactEvent, sourceEvent, toAgentRunEvent, + toInlineAnnotationEvent, toSourceEvent, type SourceEventNode, } from "./events"; import { downloadFile } from "./file"; -import { toInlineAnnotation } from "./inline"; export async function runWorkflow( workflow: Workflow, @@ -79,13 +78,7 @@ function processWorkflowStream( } // Handle artifact events, transform to agentStreamEvent else if (artifactEvent.include(event)) { - const artifactAnnotation = event.data; - transformedEvent = agentStreamEvent.with({ - delta: toInlineAnnotation(artifactAnnotation), - response: "", - currentAgentName: "assistant", - raw: artifactAnnotation, - }); + transformedEvent = toInlineAnnotationEvent(event); } // Post-process for llama-cloud files if (sourceEvent.include(transformedEvent)) { From 27de9418ff9f98333e9f3447cfed14be0dc4a358 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 14:20:56 +0700 Subject: [PATCH 24/28] to_inline_annotation_event in python --- .../server/api/callbacks/artifact_transform.py | 18 ++---------------- .../llama_index/server/utils/inline.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py index 88730a22f..071c9d0ed 100644 --- a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py +++ b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py @@ -1,10 +1,9 @@ import logging from typing import Any -from llama_index.core.agent.workflow.workflow_events import AgentStream from llama_index.server.api.callbacks.base import EventCallback from llama_index.server.models.artifacts import ArtifactEvent -from llama_index.server.utils.inline import to_inline_annotation +from llama_index.server.utils.inline import to_inline_annotation_event logger = logging.getLogger("uvicorn") @@ -16,20 +15,7 @@ class ArtifactTransform(EventCallback): async def run(self, event: Any) -> Any: if isinstance(event, ArtifactEvent): - # Create the artifact annotation - artifact_annotation = { - "type": "artifact", - "data": event.data.model_dump(), - } - - # Transform to AgentStream with inline annotation format - return AgentStream( - delta=to_inline_annotation(artifact_annotation), - response="", - current_agent_name="assistant", - tool_calls=[], - raw=artifact_annotation, - ) + return to_inline_annotation_event(event) return event @classmethod diff --git a/python/llama-index-server/llama_index/server/utils/inline.py b/python/llama-index-server/llama_index/server/utils/inline.py index 856c576e6..195531761 100644 --- a/python/llama-index-server/llama_index/server/utils/inline.py +++ b/python/llama-index-server/llama_index/server/utils/inline.py @@ -4,7 +4,9 @@ from pydantic import ValidationError +from llama_index.core.workflow.events import Event from llama_index.server.models.chat import ChatAPIMessage +from llama_index.core.agent.workflow.workflow_events import AgentStream INLINE_ANNOTATION_KEY = ( "annotation" # the language key to detect inline annotation code in markdown @@ -64,3 +66,17 @@ def to_inline_annotation(item: dict) -> str: ``` """ return f"\n```{INLINE_ANNOTATION_KEY}\n{json.dumps(item)}\n```\n" + + +def to_inline_annotation_event(event: Event) -> AgentStream: + """ + Convert an event to an AgentStream with inline annotation format. + """ + event_dict = event.model_dump() + return AgentStream( + delta=to_inline_annotation(event_dict), + response="", + current_agent_name="assistant", + tool_calls=[], + raw=event_dict, + ) From eff763191e11c23d84721a3a25fc36984712fb8f Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 14:24:17 +0700 Subject: [PATCH 25/28] refactor: move toInlineAnnotationEvent to inline.ts --- packages/server/src/utils/events.ts | 17 ++--------------- packages/server/src/utils/inline.ts | 10 ++++++++++ packages/server/src/utils/workflow.ts | 2 +- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index 771132a12..86b31d52d 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,9 +1,5 @@ import { randomUUID } from "@llamaindex/env"; -import { - agentStreamEvent, - workflowEvent, - type WorkflowEventData, -} from "@llamaindex/workflow"; +import { workflowEvent } from "@llamaindex/workflow"; import { MetadataMode, type ChatMessage, @@ -11,7 +7,7 @@ import { type NodeWithScore, } from "llamaindex"; import { z } from "zod"; -import { getInlineAnnotations, toInlineAnnotation } from "./inline"; +import { getInlineAnnotations } from "./inline"; // Events that appended to stream as annotations export type SourceEventNode = { @@ -94,15 +90,6 @@ export function toAgentRunEvent(input: { }); } -export function toInlineAnnotationEvent(event: WorkflowEventData) { - return agentStreamEvent.with({ - delta: toInlineAnnotation(event.data), - response: "", - currentAgentName: "assistant", - raw: event.data, - }); -} - export type ArtifactType = "code" | "document"; export type Artifact = { diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts index dd6840b7a..ffa4bec69 100644 --- a/packages/server/src/utils/inline.ts +++ b/packages/server/src/utils/inline.ts @@ -1,3 +1,4 @@ +import { agentStreamEvent, type WorkflowEventData } from "@llamaindex/workflow"; import { type ChatMessage } from "llamaindex"; import { z } from "zod"; @@ -64,6 +65,15 @@ export function toInlineAnnotation(item: unknown) { return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; } +export function toInlineAnnotationEvent(event: WorkflowEventData) { + return agentStreamEvent.with({ + delta: toInlineAnnotation(event.data), + response: "", + currentAgentName: "assistant", + raw: event.data, + }); +} + function getMessageMarkdownContent(message: ChatMessage): string { let markdownContent = ""; diff --git a/packages/server/src/utils/workflow.ts b/packages/server/src/utils/workflow.ts index 69139a61d..4771efe15 100644 --- a/packages/server/src/utils/workflow.ts +++ b/packages/server/src/utils/workflow.ts @@ -18,11 +18,11 @@ import { artifactEvent, sourceEvent, toAgentRunEvent, - toInlineAnnotationEvent, toSourceEvent, type SourceEventNode, } from "./events"; import { downloadFile } from "./file"; +import { toInlineAnnotationEvent } from "./inline"; export async function runWorkflow( workflow: Workflow, From e20f198707137c2e4de34ffcebc379129f758ea7 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 4 Jun 2025 15:41:38 +0700 Subject: [PATCH 26/28] update comment --- packages/server/src/utils/inline.ts | 3 +-- python/llama-index-server/llama_index/server/utils/inline.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts index ffa4bec69..db4d596b2 100644 --- a/packages/server/src/utils/inline.ts +++ b/packages/server/src/utils/inline.ts @@ -52,9 +52,8 @@ export function getInlineAnnotations(message: ChatMessage): Annotation[] { /** * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. * The language key is `annotation` and the code block is wrapped in backticks. - * The prefix `0:` ensures it will be treated as inline markdown. Example: * - * 0:\`\`\`annotation + * \`\`\`annotation * \{ * "type": "artifact", * "data": \{...\} diff --git a/python/llama-index-server/llama_index/server/utils/inline.py b/python/llama-index-server/llama_index/server/utils/inline.py index 195531761..daa97ce45 100644 --- a/python/llama-index-server/llama_index/server/utils/inline.py +++ b/python/llama-index-server/llama_index/server/utils/inline.py @@ -56,9 +56,8 @@ def to_inline_annotation(item: dict) -> str: """ To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. The language key is `annotation` and the code block is wrapped in backticks. - The prefix `0:` ensures it will be treated as inline markdown. Example: - 0:```annotation + ```annotation { "type": "artifact", "data": {...} From 4dca89ddb4eb8f83fcd426785447fd2c72282e4b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Wed, 4 Jun 2025 16:32:22 +0700 Subject: [PATCH 27/28] rename ArtifactTransform to InlineAnnotationTransformer --- .../llama_index/server/api/callbacks/__init__.py | 6 ++++-- .../llama_index/server/api/callbacks/artifact_transform.py | 7 ++++--- .../llama_index/server/api/routers/chat.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py index 58a505433..bd4275887 100644 --- a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py +++ b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py @@ -1,5 +1,7 @@ from llama_index.server.api.callbacks.agent_call_tool import AgentCallTool -from llama_index.server.api.callbacks.artifact_transform import ArtifactTransform +from llama_index.server.api.callbacks.artifact_transform import ( + InlineAnnotationTransformer, +) from llama_index.server.api.callbacks.base import EventCallback from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall @@ -13,5 +15,5 @@ "SuggestNextQuestions", "LlamaCloudFileDownload", "AgentCallTool", - "ArtifactTransform", + "InlineAnnotationTransformer", ] diff --git a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py index 071c9d0ed..ccd0197ba 100644 --- a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py +++ b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py @@ -8,16 +8,17 @@ logger = logging.getLogger("uvicorn") -class ArtifactTransform(EventCallback): +class InlineAnnotationTransformer(EventCallback): """ - Transforms ArtifactEvent to AgentStream with inline annotation format. + Transforms an event to AgentStream with inline annotation format. """ async def run(self, event: Any) -> Any: + # handle for ArtifactEvent specifically as it's only supported by inline annotation if isinstance(event, ArtifactEvent): return to_inline_annotation_event(event) return event @classmethod - def from_default(cls, *args: Any, **kwargs: Any) -> "ArtifactTransform": + def from_default(cls, *args: Any, **kwargs: Any) -> "InlineAnnotationTransformer": return cls() diff --git a/python/llama-index-server/llama_index/server/api/routers/chat.py b/python/llama-index-server/llama_index/server/api/routers/chat.py index 5d1786fbe..8ccf207bd 100644 --- a/python/llama-index-server/llama_index/server/api/routers/chat.py +++ b/python/llama-index-server/llama_index/server/api/routers/chat.py @@ -17,8 +17,8 @@ ) from llama_index.server.api.callbacks import ( AgentCallTool, - ArtifactTransform, EventCallback, + InlineAnnotationTransformer, LlamaCloudFileDownload, SourceNodesFromToolCall, SuggestNextQuestions, @@ -73,7 +73,7 @@ async def chat( callbacks: list[EventCallback] = [ AgentCallTool(), - ArtifactTransform(), + InlineAnnotationTransformer(), SourceNodesFromToolCall(), LlamaCloudFileDownload(background_tasks), ] From 5c1de3c3ee94bd68803a1909b7e5f7d60aca4593 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 5 Jun 2025 09:50:31 +0700 Subject: [PATCH 28/28] add codegen example --- packages/server/examples/code-gen/README.md | 22 ++ .../examples/code-gen/components/ui_event.jsx | 132 +++++++ packages/server/examples/code-gen/index.ts | 20 ++ .../examples/code-gen/src/app/workflow.ts | 337 ++++++++++++++++++ 4 files changed, 511 insertions(+) create mode 100644 packages/server/examples/code-gen/README.md create mode 100644 packages/server/examples/code-gen/components/ui_event.jsx create mode 100644 packages/server/examples/code-gen/index.ts create mode 100644 packages/server/examples/code-gen/src/app/workflow.ts diff --git a/packages/server/examples/code-gen/README.md b/packages/server/examples/code-gen/README.md new file mode 100644 index 000000000..8ac84b5d3 --- /dev/null +++ b/packages/server/examples/code-gen/README.md @@ -0,0 +1,22 @@ +This example demonstrates how to use the code generation workflow. + +```ts +new LlamaIndexServer({ + workflow: workflowFactory, + uiConfig: { + starterQuestions: [ + "Generate a calculator app", + "Create a simple todo list app", + ], + componentsDir: "components", + }, + port: 3000, +}).start(); +``` + +Export OpenAI API key and start the server in dev mode. + +```bash +export OPENAI_API_KEY= +npx nodemon --exec tsx index.ts +``` diff --git a/packages/server/examples/code-gen/components/ui_event.jsx b/packages/server/examples/code-gen/components/ui_event.jsx new file mode 100644 index 000000000..9b2db6683 --- /dev/null +++ b/packages/server/examples/code-gen/components/ui_event.jsx @@ -0,0 +1,132 @@ +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { Skeleton } from "@/components/ui/skeleton"; +import { cn } from "@/lib/utils"; +import { Markdown } from "@llamaindex/chat-ui/widgets"; +import { ListChecks, Loader2, Wand2 } from "lucide-react"; +import { useEffect, useState } from "react"; + +const STAGE_META = { + plan: { + icon: ListChecks, + badgeText: "Step 1/2: Planning", + gradient: "from-blue-100 via-blue-50 to-white", + progress: 33, + iconBg: "bg-blue-100 text-blue-600", + badge: "bg-blue-100 text-blue-700", + }, + generate: { + icon: Wand2, + badgeText: "Step 2/2: Generating", + gradient: "from-violet-100 via-violet-50 to-white", + progress: 66, + iconBg: "bg-violet-100 text-violet-600", + badge: "bg-violet-100 text-violet-700", + }, +}; + +function ArtifactWorkflowCard({ event }) { + const [visible, setVisible] = useState(event?.state !== "completed"); + const [fade, setFade] = useState(false); + + useEffect(() => { + if (event?.state === "completed") { + setVisible(false); + } else { + setVisible(true); + setFade(false); + } + }, [event?.state]); + + if (!event || !visible) return null; + + const { state, requirement } = event; + const meta = STAGE_META[state]; + + if (!meta) return null; + + return ( +
+ + +
+ +
+ + + {meta.badgeText} + + + + + {state === "plan" && ( +
+ +
+ Analyzing your request... +
+ +
+ )} + {state === "generate" && ( +
+
+ + + Working on the requirement: + +
+
+ {requirement ? ( + + ) : ( + + No requirements available yet. + + )} +
+
+ )} + +
+ +
+ +
+ ); +} + +export default function Component({ events }) { + const aggregateEvents = () => { + if (!events || events.length === 0) return null; + return events[events.length - 1]; + }; + + const event = aggregateEvents(); + + return ; +} diff --git a/packages/server/examples/code-gen/index.ts b/packages/server/examples/code-gen/index.ts new file mode 100644 index 000000000..6cbb9227e --- /dev/null +++ b/packages/server/examples/code-gen/index.ts @@ -0,0 +1,20 @@ +import { OpenAI } from "@llamaindex/openai"; +import { LlamaIndexServer } from "@llamaindex/server"; +import { Settings } from "llamaindex"; +import { workflowFactory } from "./src/app/workflow"; + +Settings.llm = new OpenAI({ + model: "gpt-4o-mini", +}); + +new LlamaIndexServer({ + workflow: workflowFactory, + uiConfig: { + starterQuestions: [ + "Generate a calculator app", + "Create a simple todo list app", + ], + componentsDir: "components", + }, + port: 3000, +}).start(); diff --git a/packages/server/examples/code-gen/src/app/workflow.ts b/packages/server/examples/code-gen/src/app/workflow.ts new file mode 100644 index 000000000..e16af7bfc --- /dev/null +++ b/packages/server/examples/code-gen/src/app/workflow.ts @@ -0,0 +1,337 @@ +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; +import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; + +import { + agentStreamEvent, + createStatefulMiddleware, + createWorkflow, + startAgentEvent, + stopAgentEvent, + workflowEvent, +} from "@llamaindex/workflow"; + +import { z } from "zod"; + +export const RequirementSchema = z.object({ + next_step: z.enum(["answering", "coding"]), + language: z.string().nullable().optional(), + file_name: z.string().nullable().optional(), + requirement: z.string(), +}); + +export type Requirement = z.infer; + +export const UIEventSchema = z.object({ + type: z.literal("ui_event"), + data: z.object({ + state: z + .enum(["plan", "generate", "completed"]) + .describe( + "The current state of the workflow: 'plan', 'generate', or 'completed'.", + ), + requirement: z + .string() + .optional() + .describe( + "An optional requirement creating or updating a code, if applicable.", + ), + }), +}); + +export type UIEvent = z.infer; +const planEvent = workflowEvent<{ + userInput: MessageContent; + context?: string | undefined; +}>(); + +const generateArtifactEvent = workflowEvent<{ + requirement: Requirement; +}>(); + +const synthesizeAnswerEvent = workflowEvent(); + +const uiEvent = workflowEvent(); + +export function workflowFactory(reqBody: unknown) { + const llm = Settings.llm; + + const { withState, getContext } = createStatefulMiddleware(() => { + return { + memory: new ChatMemoryBuffer({ llm }), + lastArtifact: extractLastArtifact(reqBody), + }; + }); + const workflow = withState(createWorkflow()); + + workflow.handle([startAgentEvent], async ({ data }) => { + const { userInput, chatHistory = [] } = data; + // Prepare chat history + const { state } = getContext(); + // Put user input to the memory + if (!userInput) { + throw new Error("Missing user input to start the workflow"); + } + state.memory.set(chatHistory); + state.memory.put({ role: "user", content: userInput }); + + return planEvent.with({ + userInput: userInput, + context: state.lastArtifact + ? JSON.stringify(state.lastArtifact) + : undefined, + }); + }); + + workflow.handle([planEvent], async ({ data: planData }) => { + const { sendEvent } = getContext(); + const { state } = getContext(); + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "plan", + }, + }), + ); + const user_msg = planData.userInput; + const context = planData.context + ? `## The context is: \n${planData.context}\n` + : ""; + const prompt = ` +You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation. +You are helping user with their code artifact. To update the code, you need to plan a coding step. + +Follow these instructions: +1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be. +2. The next step must be one of the following two options: + - "coding": To make the changes to the current code. + - "answering": If you don't need to update the current code or need clarification from the user. +Important: Avoid telling the user to update the code themselves, you are the one who will update the code (by planning a coding step). +3. If the next step is "coding", you may specify the language ("typescript" or "python") and file_name if known, otherwise set them to null. +4. The requirement must be provided clearly what is the user request and what need to be done for the next step in details + as precise and specific as possible, don't be stingy with in the requirement. +5. If the next step is "answering", set language and file_name to null, and the requirement should describe what to answer or explain to the user. +6. Be concise; only return the requirements for the next step. +7. The requirements must be in the following format: + \`\`\`json + { + "next_step": "answering" | "coding", + "language": "typescript" | "python" | null, + "file_name": string | null, + "requirement": string + } + \`\`\` + +## Example 1: +User request: Create a calculator app. +You should return: +\`\`\`json +{ + "next_step": "coding", + "language": "typescript", + "file_name": "calculator.tsx", + "requirement": "Generate code for a calculator app that has a simple UI with a display and button layout. The display should show the current input and the result. The buttons should include basic operators, numbers, clear, and equals. The calculation should work correctly." +} +\`\`\` + +## Example 2: +User request: Explain how the game loop works. +Context: You have already generated the code for a snake game. +You should return: +\`\`\`json +{ + "next_step": "answering", + "language": null, + "file_name": null, + "requirement": "The user is asking about the game loop. Explain how the game loop works." +} +\`\`\` + +${context} + +Now, plan the user's next step for this request: +${user_msg} +`; + + const response = await llm.complete({ + prompt, + }); + // parse the response to Requirement + // 1. use regex to find the json block + const jsonBlock = response.text.match(/```json\s*([\s\S]*?)\s*```/); + if (!jsonBlock) { + throw new Error("No JSON block found in the response."); + } + const requirement = RequirementSchema.parse(JSON.parse(jsonBlock[1])); + state.memory.put({ + role: "assistant", + content: `The plan for next step: \n${response.text}`, + }); + + if (requirement.next_step === "coding") { + return generateArtifactEvent.with({ + requirement, + }); + } else { + return synthesizeAnswerEvent.with({}); + } + }); + + workflow.handle([generateArtifactEvent], async ({ data: planData }) => { + const { sendEvent } = getContext(); + const { state } = getContext(); + + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "generate", + requirement: planData.requirement.requirement, + }, + }), + ); + + const previousArtifact = state.lastArtifact + ? JSON.stringify(state.lastArtifact) + : "There is no previous artifact"; + const requirementText = planData.requirement.requirement; + + const prompt = ` + You are a skilled developer who can help user with coding. + You are given a task to generate or update a code for a given requirement. + + ## Follow these instructions: + **1. Carefully read the user's requirements.** + If any details are ambiguous or missing, make reasonable assumptions and clearly reflect those in your output. + If the previous code is provided: + + Carefully analyze the code with the request to make the right changes. + + Avoid making a lot of changes from the previous code if the request is not to write the code from scratch again. + **2. For code requests:** + - If the user does not specify a framework or language, default to a React component using the Next.js framework. + - For Next.js, use Shadcn UI components, Typescript, @types/node, @types/react, @types/react-dom, PostCSS, and TailwindCSS. + The import pattern should be: + \`\`\`typescript + import { ComponentName } from "@/components/ui/component-name" + import { Markdown } from "@llamaindex/chat-ui" + import { cn } from "@/lib/utils" + \`\`\` + - Ensure the code is idiomatic, production-ready, and includes necessary imports. + - Only generate code relevant to the user's request—do not add extra boilerplate. + **3. Don't be verbose on response** + - No other text or comments only return the code which wrapped by \`\`\`language\`\`\` block. + - If the user's request is to update the code, only return the updated code. + **4. Only the following languages are allowed: "typescript", "python".** + **5. If there is no code to update, return the reason without any code block.** + + ## Example: + \`\`\`typescript + import React from "react"; + import { Button } from "@/components/ui/button"; + import { cn } from "@/lib/utils"; + + export default function MyComponent() { + return ( +
+ +
+ ); + } + \`\`\` + + The previous code is: + {previousArtifact} + + Now, i have to generate the code for the following requirement: + {requirement} + ` + .replace("{previousArtifact}", previousArtifact) + .replace("{requirement}", requirementText); + + const response = await llm.complete({ + prompt, + }); + + // Extract the code from the response + const codeMatch = response.text.match(/```(\w+)([\s\S]*)```/); + if (!codeMatch) { + return synthesizeAnswerEvent.with({}); + } + + const code = codeMatch[2].trim(); + + // Put the generated code to the memory + state.memory.put({ + role: "assistant", + content: `Updated the code: \n${response.text}`, + }); + + // To show the Canvas panel for the artifact + sendEvent( + artifactEvent.with({ + type: "artifact", + data: { + type: "code", + created_at: Date.now(), + data: { + language: planData.requirement.language || "", + file_name: planData.requirement.file_name || "", + code, + }, + }, + }), + ); + + return synthesizeAnswerEvent.with({}); + }); + + workflow.handle([synthesizeAnswerEvent], async () => { + const { sendEvent } = getContext(); + const { state } = getContext(); + + const chatHistory = await state.memory.getMessages(); + const messages = [ + ...chatHistory, + { + role: "system" as const, + content: ` + You are a helpful assistant who is responsible for explaining the work to the user. + Based on the conversation history, provide an answer to the user's question. + The user has access to the code so avoid mentioning the whole code again in your response. + `, + }, + ]; + + const responseStream = await llm.chat({ + messages, + stream: true, + }); + + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "completed", + }, + }), + ); + + let response = ""; + for await (const chunk of responseStream) { + response += chunk.delta; + sendEvent( + agentStreamEvent.with({ + delta: chunk.delta, + response: "", + currentAgentName: "assistant", + raw: chunk, + }), + ); + } + + return stopAgentEvent.with({ + result: response, + }); + }); + + return workflow; +}