Skip to content

Commit

Permalink
feat: paste long context as plaintext files (#1549)
Browse files Browse the repository at this point in the history 
* feat: paste long context as a file

* fix: anim sources

* fix: animations

* feat: actually inject plaintext files in the prompt

* filter out files that are plain text

* feat: use custom MIME type for clipboard content

* feat: add better UI affordance for pasting

* fix: cleanup animations
  • Loading branch information
@nsarrazin
nsarrazin authored Oct 30, 2024
1 parent b1355da commit 632ef40
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 27 deletions.
4 changes: 2 additions & 2 deletions src/lib/components/chat/ChatMessage.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@
{#if message.files?.length}
<div class="flex h-fit flex-wrap gap-x-5 gap-y-2">
{#each message.files as file}
<UploadedFile {file} canClose={false} isPreview={false} />
<UploadedFile {file} canClose={false} />
{/each}
</div>
{/if}
Expand Down Expand Up @@ -410,7 +410,7 @@
{#if message.files?.length}
<div class="flex w-fit gap-4 px-5">
{#each message.files as file}
<UploadedFile {file} canClose={false} isPreview={false} />
<UploadedFile {file} canClose={false} />
{/each}
</div>
{/if}
Expand Down
48 changes: 46 additions & 2 deletions src/lib/components/chat/ChatWindow.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
import type { ToolFront } from "$lib/types/Tool";
import ModelSwitch from "./ModelSwitch.svelte";
import { fly } from "svelte/transition";
import { cubicInOut } from "svelte/easing";
export let messages: Message[] = [];
export let loading = false;
export let pending = false;
Expand All @@ -55,6 +58,7 @@
let message: string;
let timeout: ReturnType<typeof setTimeout>;
let isSharedRecently = false;
$: pastedLongContent = false;
$: $page.params.id && (isSharedRecently = false);
const dispatch = createEventDispatcher<{
Expand Down Expand Up @@ -86,6 +90,21 @@
};
const onPaste = (e: ClipboardEvent) => {
const textContent = e.clipboardData?.getData("text");
if (textContent && textContent.length > 256) {
e.preventDefault();
pastedLongContent = true;
setTimeout(() => {
pastedLongContent = false;
}, 1000);
const pastedFile = new File([textContent], "Pasted Content", {
type: "application/vnd.chatui.clipboard",
});
files = [...files, pastedFile];
}
if (!e.clipboardData) {
return;
}
Expand Down Expand Up @@ -344,7 +363,10 @@
class="dark:via-gray-80 pointer-events-none absolute inset-x-0 bottom-0 z-0 mx-auto flex w-full max-w-3xl flex-col items-center justify-center bg-gradient-to-t from-white via-white/80 to-white/0 px-3.5 py-4 dark:border-gray-800 dark:from-gray-900 dark:to-gray-900/0 max-md:border-t max-md:bg-white max-md:dark:bg-gray-900 sm:px-5 md:py-8 xl:max-w-4xl [&>*]:pointer-events-auto"
>
{#if sources?.length && !loading}
<div class="flex flex-row flex-wrap justify-center gap-2.5 max-md:pb-3">
<div
in:fly|local={sources.length === 1 ? { y: -20, easing: cubicInOut } : undefined}
class="flex flex-row flex-wrap justify-center gap-2.5 rounded-xl max-md:pb-3"
>
{#each sources as source, index}
{#await source then src}
<UploadedFile
Expand Down Expand Up @@ -409,7 +431,10 @@
{#if onDrag && isFileUploadEnabled}
<FileDropzone bind:files bind:onDrag mimeTypes={activeMimeTypes} />
{:else}
<div class="flex w-full flex-1 border-none bg-transparent">
<div
class="flex w-full flex-1 rounded-xl border-none bg-transparent"
class:paste-glow={pastedLongContent}
>
{#if lastIsError}
<ChatInput value="Sorry, something went wrong. Please try again." disabled={true} />
{:else}
Expand Down Expand Up @@ -508,3 +533,22 @@
</div>
</div>
</div>

<style lang="postcss">
.paste-glow {
animation: glow 1s cubic-bezier(0.4, 0, 0.2, 1) forwards;
will-change: box-shadow;
}
@keyframes glow {
0% {
box-shadow: 0 0 0 0 rgba(59, 130, 246, 0.8);
}
50% {
box-shadow: 0 0 20px 4px rgba(59, 130, 246, 0.6);
}
100% {
box-shadow: 0 0 0 0 rgba(59, 130, 246, 0);
}
}
</style>
115 changes: 93 additions & 22 deletions src/lib/components/chat/UploadedFile.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
import CarbonClose from "~icons/carbon/close";
import CarbonDocumentBlank from "~icons/carbon/document-blank";
import CarbonDownload from "~icons/carbon/download";
import CarbonDocument from "~icons/carbon/document";
import Modal from "../Modal.svelte";
import AudioPlayer from "../players/AudioPlayer.svelte";
import EosIconsLoading from "~icons/eos-icons/loading";
export let file: MessageFile;
export let canClose = true;
export let isPreview = false;
$: showModal = false;
$: urlNotTrailing = $page.url.pathname.replace(/\/$/, "");
Expand All @@ -38,33 +38,74 @@
const isVideo = (mime: string) =>
mime.startsWith("video/") || mime === "mp4" || mime === "x-mpeg";
$: isClickable = isImage(file.mime) && !isPreview;
const isPlainText = (mime: string) =>
mime === "text/plain" ||
mime === "text/csv" ||
mime === "text/markdown" ||
mime === "application/json" ||
mime === "application/xml" ||
mime === "application/vnd.chatui.clipboard";
$: isClickable = isImage(file.mime) || isPlainText(file.mime);
</script>

{#if showModal && isClickable}
<!-- show the image file full screen, click outside to exit -->
<Modal width="sm:max-w-[500px]" on:close={() => (showModal = false)}>
{#if file.type === "hash"}
<img
src={urlNotTrailing + "/output/" + file.value}
alt="input from user"
class="aspect-auto"
/>
{:else}
<!-- handle the case where this is a base64 encoded image -->
<img
src={`data:${file.mime};base64,${file.value}`}
alt="input from user"
class="aspect-auto"
/>
<Modal width="sm:max-w-[800px]" on:close={() => (showModal = false)}>
{#if isImage(file.mime)}
{#if file.type === "hash"}
<img
src={urlNotTrailing + "/output/" + file.value}
alt="input from user"
class="aspect-auto"
/>
{:else}
<!-- handle the case where this is a base64 encoded image -->
<img
src={`data:${file.mime};base64,${file.value}`}
alt="input from user"
class="aspect-auto"
/>
{/if}
{:else if isPlainText(file.mime)}
<div class="relative flex h-full w-full flex-col gap-4 p-4">
<h3 class="-mb-2 pt-2 text-xl font-bold">{file.name}</h3>
<button
class="absolute right-4 top-4 text-xl text-gray-500 hover:text-gray-800"
on:click={() => (showModal = false)}
>
<CarbonClose class="text-xl" />
</button>
{#if file.type === "hash"}
{#await fetch(urlNotTrailing + "/output/" + file.value).then((res) => res.text())}
<div class="flex h-full w-full items-center justify-center">
<EosIconsLoading class="text-xl" />
</div>
{:then result}
<pre
class="w-full whitespace-pre-wrap break-words pt-0 text-sm"
class:font-sans={file.mime === "text/plain" ||
file.mime === "application/vnd.chatui.clipboard"}
class:font-mono={file.mime !== "text/plain" &&
file.mime !== "application/vnd.chatui.clipboard"}>{result}</pre>
{/await}
{:else}
<pre
class="w-full whitespace-pre-wrap break-words pt-0 text-sm"
class:font-sans={file.mime === "text/plain" ||
file.mime === "application/vnd.chatui.clipboard"}
class:font-mono={file.mime !== "text/plain" &&
file.mime !== "application/vnd.chatui.clipboard"}>{atob(file.value)}</pre>
{/if}
</div>
{/if}
</Modal>
{/if}

<button on:click={() => (showModal = true)} disabled={!isClickable}>
<button on:click={() => (showModal = true)} disabled={!isClickable} class:clickable={isClickable}>
<div class="group relative flex items-center rounded-xl shadow-sm">
{#if isImage(file.mime)}
<div class=" overflow-hidden rounded-xl" class:size-24={isPreview} class:size-48={!isPreview}>
<div class="size-48 overflow-hidden rounded-xl">
<img
src={file.type === "base64"
? `data:${file.mime};base64,${file.value}`
Expand Down Expand Up @@ -92,9 +133,31 @@
controls
/>
</div>
{:else if isPlainText(file.mime)}
<div
class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
class:hoverable={isClickable}
>
<div
class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
>
<CarbonDocument class="text-base text-gray-700 dark:text-gray-300" />
</div>
<dl class="flex flex-col items-start truncate leading-tight">
<dd class="text-sm">
{truncateMiddle(file.name, 28)}
</dd>
{#if file.mime === "application/vnd.chatui.clipboard"}
<dt class="text-xs text-gray-400">Clipboard source</dt>
{:else}
<dt class="text-xs text-gray-400">{file.mime}</dt>
{/if}
</dl>
</div>
{:else if file.mime === "octet-stream"}
<div
class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
class:hoverable={isClickable}
>
<div
class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
Expand All @@ -120,13 +183,14 @@
{:else}
<div
class="flex h-14 w-72 items-center gap-2 overflow-hidden rounded-xl border border-gray-200 bg-white p-2 dark:border-gray-800 dark:bg-gray-900"
class:hoverable={isClickable}
>
<div
class="grid size-10 flex-none place-items-center rounded-lg bg-gray-100 dark:bg-gray-800"
>
<CarbonDocumentBlank class="text-base text-gray-700 dark:text-gray-300" />
</div>
<dl class="flex flex-col truncate leading-tight">
<dl class="flex flex-col items-start truncate leading-tight">
<dd class="text-sm">
{truncateMiddle(file.name, 28)}
</dd>
Expand All @@ -137,11 +201,18 @@
<!-- add a button on top that removes the image -->
{#if canClose}
<button
class="invisible absolute -right-2 -top-2 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
on:click={() => dispatch("close")}
class="absolute -right-2 -top-2 z-10 grid size-6 place-items-center rounded-full border bg-black group-hover:visible dark:border-gray-700"
class:invisible={navigator.maxTouchPoints === 0}
on:click|stopPropagation|preventDefault={() => dispatch("close")}
>
<CarbonClose class=" text-xs text-white" />
</button>
{/if}
</div>
</button>

<style lang="postcss">
.hoverable {
@apply hover:bg-gray-500/10;
}
</style>
21 changes: 20 additions & 1 deletion src/lib/server/endpoints/preprocessMessages.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ export async function preprocessMessages(
): Promise<EndpointMessage[]> {
return Promise.resolve(messages)
.then((msgs) => addWebSearchContext(msgs, webSearch))
.then((msgs) => downloadFiles(msgs, convId));
.then((msgs) => downloadFiles(msgs, convId))
.then((msgs) => injectClipboardFiles(msgs));
}

function addWebSearchContext(messages: Message[], webSearch: Message["webSearch"]) {
Expand Down Expand Up @@ -54,3 +55,21 @@ async function downloadFiles(messages: Message[], convId: ObjectId): Promise<End
)
);
}

async function injectClipboardFiles(messages: EndpointMessage[]) {
return Promise.all(
messages.map((message) => {
const plaintextFiles = message.files
?.filter((file) => file.mime === "application/vnd.chatui.clipboard")
.map((file) => Buffer.from(file.value, "base64").toString("utf-8"));

if (!plaintextFiles || plaintextFiles.length === 0) return message;

return {
...message,
content: `${plaintextFiles.join("\n\n")}\n\n${message.content}`,
files: message.files?.filter((file) => file.mime !== "application/vnd.chatui.clipboard"),
};
})
);
}

0 comments on commit 632ef40

Please sign in to comment.