diff --git a/.env.sample b/.env.sample index 7579b324..6ec5c103 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,3 @@ -OPENAI_API_KEY= BACKEND_ALLOW_ORIGINS= BACKEND_HOST=localhost diff --git a/docetl/console.py b/docetl/console.py index da389f44..da1d6aa8 100644 --- a/docetl/console.py +++ b/docetl/console.py @@ -89,7 +89,14 @@ def get_console(): highlight=False, ) else: - return Console() + class NoOpConsole(Console): + def post_optimizer_status(self, *args, **kwargs): + pass + + def post_optimizer_rationale(self, *args, **kwargs): + pass + + return NoOpConsole() DOCETL_CONSOLE = get_console() diff --git a/docetl/optimizers/map_optimizer/optimizer.py b/docetl/optimizers/map_optimizer/optimizer.py index d27872e9..a4eaabdc 100644 --- a/docetl/optimizers/map_optimizer/optimizer.py +++ b/docetl/optimizers/map_optimizer/optimizer.py @@ -223,8 +223,10 @@ def optimize( ) self.console.post_optimizer_rationale( assessment.get("needs_improvement", True), - "\n".join(assessment.get("reasons", [])), - validator_prompt + "\n".join(assessment.get("reasons", [])) + + "\n\n" + + "\n".join(assessment.get("improvements", [])), + validator_prompt, ) # Check if improvement is needed based on the assessment diff --git a/docetl/optimizers/map_optimizer/prompt_generators.py b/docetl/optimizers/map_optimizer/prompt_generators.py index 91e18834..0b3aeae4 100644 --- a/docetl/optimizers/map_optimizer/prompt_generators.py +++ b/docetl/optimizers/map_optimizer/prompt_generators.py @@ -387,17 +387,17 @@ def _get_combine_prompt( combine_prompt = result["combine_prompt"] # Confirm with the user that this prompt is good & ask them to edit - if self.runner.status: - self.runner.status.stop() + # if self.runner.status: + # self.runner.status.stop() - combine_prompt = Prompt.ask( - f"Here is the prompt generated for the reduce operation:\n```\n{combine_prompt}\n```\n\nPress enter to confirm, or type in the prompt you would like to use instead.", - default=combine_prompt, - console=self.console, - ) + # combine_prompt = Prompt.ask( + # f"Here is the prompt generated for the reduce operation:\n```\n{combine_prompt}\n```\n\nPress enter to confirm, or type in the prompt you would like to use instead.", + # default=combine_prompt, + # console=self.console, + # ) - if self.runner.status: - self.runner.status.start() + # if self.runner.status: + # self.runner.status.start() # Determine if the combine operation is associative system_prompt_associative = ( diff --git a/website/src/app/playground/page.tsx b/website/src/app/playground/page.tsx index a7c11e56..232484bd 100644 --- a/website/src/app/playground/page.tsx +++ b/website/src/app/playground/page.tsx @@ -1,7 +1,7 @@ "use client"; import dynamic from "next/dynamic"; -import React, { useEffect, useState, useRef } from "react"; +import React, { useEffect, useState } from "react"; import { Scroll, Info, Save } from "lucide-react"; import { Button } from "@/components/ui/button"; import { @@ -433,12 +433,18 @@ const CodeEditorPipelineApp: React.FC = () => { {showChat && setShowChat(false)} />} {/* Main content */} - - {/* File Explorer and Bookmarks */} + {showFileExplorer && ( - - - + + + { @@ -461,7 +467,11 @@ const CodeEditorPipelineApp: React.FC = () => { withHandle className="h-2 bg-gray-200 hover:bg-gray-300 transition-colors duration-200" /> - + @@ -474,10 +484,13 @@ const CodeEditorPipelineApp: React.FC = () => { /> )} - {/* Pipeline GUI and Output */} - - - + + + {showOutput && ( @@ -487,24 +500,31 @@ const CodeEditorPipelineApp: React.FC = () => { /> )} {showOutput && ( - + )} - {/* Dataset View */} - {showDatasetView && ( - - )} {showDatasetView && currentFile && ( - - - + <> + + + + + )} @@ -518,7 +538,7 @@ const WrappedCodeEditorPipelineApp: React.FC = () => { useEffect(() => { setIsLocalhost( window.location.hostname === "localhost" || - window.location.hostname === "127.0.0.1" + window.location.hostname === "127.0.0.1" ); }, []); diff --git a/website/src/components/AnsiRenderer.tsx b/website/src/components/AnsiRenderer.tsx index 4e261333..56611caf 100644 --- a/website/src/components/AnsiRenderer.tsx +++ b/website/src/components/AnsiRenderer.tsx @@ -50,24 +50,17 @@ const AnsiRenderer: React.FC = ({ return (
-
+
       
-
+
{ const { terminalOutput, setTerminalOutput, optimizerProgress } = @@ -24,9 +25,9 @@ export const ConsoleContent: React.FC = () => { const { readyState } = useWebSocket(); return ( -
+
{optimizerProgress && ( -
+
{/* Animated gradient border */}
{
)} -
+
- -
); }; @@ -253,14 +244,14 @@ export const Output: React.FC = () => { }, [outputs, operation?.output?.schema]); const TableContent = () => ( -
+
{isLoadingOutputs ? (
Loading outputs...
) : outputs.length > 0 ? ( - + { /> ) : ( -

- No outputs available. -

+
+

No outputs available.

+
)}
); @@ -294,16 +285,18 @@ export const Output: React.FC = () => { if (!visualizationColumn || !operation) { return ( -

- No visualization data available. -

+
+

+ No visualization data available. +

+
); } if (operation.type === "reduce") { const reduceKeys = operation.otherKwargs?.reduce_key || []; return ( -
+
{outputs .sort( (a, b) => @@ -354,7 +347,7 @@ export const Output: React.FC = () => { }, [outputs, visualizationColumn.name]); return ( -
+
{Object.entries(groupedData) .sort(([, groupA], [, groupB]) => groupB.length - groupA.length) .map(([key, group]: [string, any[]]) => ( @@ -394,9 +387,11 @@ export const Output: React.FC = () => { ); } else { return ( -

- Visualization not supported for this operation type. -

+
+

+ Visualization not supported for this operation type. +

+
); } }; @@ -428,67 +423,119 @@ export const Output: React.FC = () => { inputCount > 0 ? (outputCount / inputCount).toFixed(2) : "N/A"; return ( -
-
-

- Output{" "} - {opName && - {opName}} -

-
- - {inputCount} inputs | {outputCount} outputs | Selectivity:{" "} - {selectivityFactor}x - - - {/* - - - - -
-

Output

- +
+
+
+

+ Output{" "} + {opName && - {opName}} +

+
+
+
+ {inputCount} + in +
+ +
+ {outputCount} + out +
+ + + +
+ 1 && + "text-emerald-600", + selectivityFactor !== "N/A" && + Number(selectivityFactor) < 1 && + "text-rose-600", + selectivityFactor === "N/A" && "text-gray-900" + )} + > + {selectivityFactor}× + +
+
+ +

Output to input ratio

+ {selectivityFactor !== "N/A" && ( +

+ {Number(selectivityFactor) > 1 + ? "Operation increases data volume" + : Number(selectivityFactor) < 1 + ? "Operation reduces data volume" + : "Operation maintains data volume"} +

+ )} +
+
+
- -
*/} + + + + + + +

Download as CSV

+
+
+
+
- - - - Console - {readyState === WebSocket.OPEN && ( - - - - )} - - Table - - Visualize Input Distribution - - - - - - - - - -
+ + +
+ + + Console + {readyState === WebSocket.OPEN && ( + + + + )} + + Table + + Visualize Input Distribution + + +
+ +
+ + + + + + + -
- + +
); diff --git a/website/src/components/PipelineGui.tsx b/website/src/components/PipelineGui.tsx index def89e4c..80d23593 100644 --- a/website/src/components/PipelineGui.tsx +++ b/website/src/components/PipelineGui.tsx @@ -87,7 +87,7 @@ const PipelineGUI: React.FC = () => { } = usePipelineContext(); const [isSettingsOpen, setIsSettingsOpen] = useState(false); const [tempPipelineName, setTempPipelineName] = useState(pipelineName); - const [tempOptimizerModel, setTempOptimizerModel] = useState(defaultModel); + const [tempOptimizerModel, setTempOptimizerModel] = useState(optimizerModel); const [tempSampleSize, setTempSampleSize] = useState( sampleSize?.toString() || "" ); @@ -518,7 +518,7 @@ const PipelineGUI: React.FC = () => { }; return ( -
+
diff --git a/website/src/components/ResizableDataTable.tsx b/website/src/components/ResizableDataTable.tsx index 0adf7a87..3846208c 100644 --- a/website/src/components/ResizableDataTable.tsx +++ b/website/src/components/ResizableDataTable.tsx @@ -1,4 +1,10 @@ -import React, { useState, useEffect, useCallback } from "react"; +import React, { + useState, + useEffect, + useCallback, + useRef, + useMemo, +} from "react"; import { flexRender, getCoreRowModel, @@ -28,8 +34,10 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { TABLE_SETTINGS_KEY } from "@/app/localStorageKeys"; +import ReactMarkdown from "react-markdown"; +import debounce from "lodash/debounce"; -export type DataType = Record; +export type DataType = Record; export type ColumnType = ColumnDef & { initialWidth?: number; }; @@ -57,13 +65,7 @@ interface ResizableRow extends Row { setSize: (size: number) => void; } -const RowResizer = ({ - row, - saveSettings, -}: { - row: ResizableRow; - saveSettings: () => void; -}) => { +const RowResizer = ({ row }: { row: ResizableRow }) => { return ( @@ -104,6 +106,39 @@ interface ResizableDataTableProps { startingRowHeight?: number; } +interface MarkdownCellProps { + content: string; +} + +const MarkdownCell = React.memo(({ content }: MarkdownCellProps) => { + return ( + ( +
+ {children} +
+ ), + h2: ({ children }) => ( +
+ {children} +
+ ), + h3: ({ children }) => ( +
+ {children} +
+ ), + h4: ({ children }) => ( +
{children}
+ ), + }} + > + {content} +
+ ); +}); + function ResizableDataTable({ data, columns, @@ -169,6 +204,12 @@ function ResizableDataTable({ return 0; }); + const [isResizing, setIsResizing] = useState(false); + const debouncedSetIsResizing = useMemo( + () => debounce((value: boolean) => setIsResizing(value), 150), + [] + ); + const table = useReactTable({ data, // Replace columns with sortedColumns here @@ -196,6 +237,8 @@ function ResizableDataTable({ pageSize: 5, }, }, + onColumnSizingStart: () => setIsResizing(true), + onColumnSizingEnd: () => debouncedSetIsResizing(false), }); return ( @@ -203,7 +246,7 @@ function ResizableDataTable({
- @@ -224,7 +267,7 @@ function ResizableDataTable({
{data.length > 0 && ( -
+
@@ -338,7 +389,6 @@ function ResizableDataTable({ }); }, }} - saveSettings={saveSettings} /> ))}