feat: ScriptQueryEditor

frontend/src2/query/Query.vue

@@ -4,6 +4,7 @@ import { WorkbookQuery } from '../types/workbook.types'
 import NativeQueryEditor from './components/NativeQueryEditor.vue'
 import QueryBuilder from './components/QueryBuilder.vue'
 import useQuery from './query'
+import ScriptQueryEditor from './components/ScriptQueryEditor.vue'
 
 const props = defineProps<{ query: WorkbookQuery }>()
 const query = useQuery(props.query)
@@ -19,4 +20,5 @@ const is_builder_query = computed(
 <template>
 	<QueryBuilder v-if="is_builder_query" />
 	<NativeQueryEditor v-else-if="query.doc.is_native_query" />
+	<ScriptQueryEditor v-else-if="query.doc.is_script_query" />
 </template>

frontend/src2/query/components/NativeQueryEditor.vue

@@ -17,7 +17,10 @@ const operation = query.getSQLOperation()
 const data_source = ref(operation ? operation.data_source : '')
 const sql = ref(operation ? operation.raw_sql : '')
 function execute() {
-	query.setSQLQuery(sql.value, data_source.value)
+	query.setSQL({
+		raw_sql: sql.value,
+		data_source: data_source.value,
+	})
 }
 
 const columns = computed(() => query.result.columns)

frontend/src2/query/components/ScriptQueryEditor.vue

@@ -0,0 +1,113 @@
+<script setup lang="ts">
+import { useTimeAgo } from '@vueuse/core'
+import { LoadingIndicator } from 'frappe-ui'
+import { Play, RefreshCw, Wand2 } from 'lucide-vue-next'
+import { computed, inject, ref } from 'vue'
+import Code from '../../components/Code.vue'
+import DataTable from '../../components/DataTable.vue'
+import { Query } from '../query'
+import ContentEditable from '../../components/ContentEditable.vue'
+
+const query = inject<Query>('query')!
+
+const operation = query.getCodeOperation()
+const code = ref(operation ? operation.code : '')
+function execute() {
+	query.setCode({ code: code.value })
+}
+
+const columns = computed(() => query.result.columns)
+const rows = computed(() => query.result.formattedRows)
+const previewRowCount = computed(() => query.result.rows.length.toLocaleString())
+const totalRowCount = computed(() =>
+	query.result.totalRowCount ? query.result.totalRowCount.toLocaleString() : ''
+)
+
+const exampleCode = `# This is an example script that fetches data from a URL and logs the data to the script log
+
+def fetch_data_from_url():
+		# URL of the CSV file
+		csv_url = "https://example.com/data.csv"
+
+		try:
+				# Read data from the CSV file into a Pandas DataFrame
+				df = pandas.read_csv(csv_url)
+
+				# use the log function to log messages to the script log
+				log(df)
+
+				# return the DataFrame
+				return df
+
+		except Exception as e:
+				log("An error occurred:", str(e))
+				return None
+
+# Call the function to execute the script and
+# then convert the data into a Pandas DataFrame or a List of lists with first row as column names
+results = fetch_data_from_url()`
+</script>
+
+<template>
+	<div class="flex flex-1 flex-col gap-4 overflow-hidden p-4">
+		<div class="relative flex h-[55%] w-full flex-col rounded border">
+			<div class="flex flex-shrink-0 items-center gap-1 border-b p-1">
+				<ContentEditable
+					class="flex h-7 cursor-text items-center justify-center rounded bg-white px-2 text-base text-gray-800 focus-visible:ring-1 focus-visible:ring-gray-600"
+					v-model="query.doc.title"
+					placeholder="Untitled Dashboard"
+				></ContentEditable>
+			</div>
+			<div class="flex-1 overflow-hidden">
+				<Code v-model="code" language="python" :placeholder="exampleCode" />
+			</div>
+			<div class="flex flex-shrink-0 gap-1 border-t p-1">
+				<Button @click="execute" label="Run">
+					<template #prefix>
+						<Play class="h-3.5 w-3.5 text-gray-700" stroke-width="1.5" />
+					</template>
+				</Button>
+			</div>
+		</div>
+		<div
+			v-show="query.result.executedSQL"
+			class="tnum flex flex-shrink-0 items-center gap-2 text-sm text-gray-600"
+		>
+			<div class="h-2 w-2 rounded-full bg-green-500"></div>
+			<div>
+				<span v-if="query.result.timeTaken == -1"> Fetched from cache </span>
+				<span v-else> Fetched in {{ query.result.timeTaken }}s </span>
+				<span> {{ useTimeAgo(query.result.lastExecutedAt).value }} </span>
+			</div>
+		</div>
+		<div class="relative flex w-full flex-1 flex-col overflow-hidden rounded border">
+			<div
+				v-if="query.executing"
+				class="absolute top-10 z-10 flex w-full items-center justify-center rounded bg-gray-50/30 backdrop-blur-sm"
+			>
+				<LoadingIndicator class="h-8 w-8 text-gray-700" />
+			</div>
+
+			<DataTable :columns="columns" :rows="rows" :on-export="query.downloadResults">
+				<template #footer-left>
+					<div class="tnum flex items-center gap-2 text-sm text-gray-600">
+						<span> Showing {{ previewRowCount }} of </span>
+						<span v-if="!totalRowCount" class="inline-block">
+							<Tooltip text="Load Count">
+								<RefreshCw
+									v-if="!query.fetchingCount"
+									class="h-3.5 w-3.5 cursor-pointer transition-all hover:text-gray-800"
+									stroke-width="1.5"
+									@click="query.fetchResultCount"
+								/>
+								<LoadingIndicator v-else class="h-3.5 w-3.5 text-gray-600" />
+							</Tooltip>
+						</span>
+						<span v-else> {{ totalRowCount }} </span>
+						rows
+					</div>
+				</template>
+			</DataTable>
+		</div>
+	</div>
+</template>

frontend/src2/query/helpers.ts

@@ -11,6 +11,7 @@ import {
 	GitBranch,
 	Indent,
 	Repeat,
+	ScrollText,
 	TextCursorInput,
 	XSquareIcon,
 } from 'lucide-vue-next'
@@ -21,6 +22,8 @@ import dayjs from '../helpers/dayjs'
 import {
 	Cast,
 	CastArgs,
+	Code,
+	CodeArgs,
 	Column,
 	CustomOperation,
 	CustomOperationArgs,
@@ -328,12 +331,23 @@ export const query_operation_types = {
 	sql: {
 		label: 'SQL',
 		type: 'sql',
-		icon: Braces,
+		icon: ScrollText,
 		color: 'gray',
 		class: 'text-gray-600 bg-gray-100',
 		init: (args: SQLArgs): SQL => ({ type: 'sql', ...args }),
 		getDescription: (op: SQL) => {
-			return `${op.raw_sql}`
+			return "SQL"
+		},
+	},
+	code: {
+		label: 'Code',
+		type: 'code',
+		icon: Braces,
+		color: 'gray',
+		class: 'text-gray-600 bg-gray-100',
+		init: (args: CodeArgs): Code => ({ type: 'code', ...args }),
+		getDescription: (op: Code) => {
+			return "Code"
 		},
 	},
 }
@@ -354,3 +368,4 @@ export const order_by = query_operation_types.order_by.init
 export const limit = query_operation_types.limit.init
 export const custom_operation = query_operation_types.custom_operation.init
 export const sql = query_operation_types.sql.init
+export const code = query_operation_types.code.init

frontend/src2/query/query.ts

@@ -6,6 +6,7 @@ import { confirmDialog } from '../helpers/confirm_dialog'
 import { FIELDTYPES } from '../helpers/constants'
 import { createToast } from '../helpers/toasts'
 import {
+	CodeArgs,
 	ColumnDataType,
 	CustomOperationArgs,
 	Dimension,
@@ -23,6 +24,7 @@ import {
 	Rename,
 	SelectArgs,
 	SourceArgs,
+	SQLArgs,
 	SummarizeArgs,
 	UnionArgs,
 } from '../types/query.types'
@@ -46,6 +48,7 @@ import {
 	sql,
 	summarize,
 	union,
+	code
 } from './helpers'
 
 const queries = new Map<string, Query>()
@@ -107,7 +110,10 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 		downloadResults,
 
 		getSQLOperation,
-		setSQLQuery,
+		setSQL,
+
+		getCodeOperation,
+		setCode,
 
 		dimensions: computed(() => ({} as Dimension[])),
 		measures: computed(() => ({} as Measure[])),
@@ -705,12 +711,26 @@ export function makeQuery(workbookQuery: WorkbookQuery) {
 		return query.doc.operations.find((op) => op.type === 'sql')
 	}
 
-	function setSQLQuery(raw_sql: string, data_source: string) {
+	function setSQL(args: SQLArgs) {
+		query.doc.operations = []
+		if (args.raw_sql.trim().length) {
+			query.doc.operations.push(sql(args))
+			query.activeOperationIdx = 0
+		} else {
+			query.activeOperationIdx = -1
+		}
+	}
+
+	function getCodeOperation() {
+		return query.doc.operations.find((op) => op.type === 'code')
+	}
+
+	function setCode(args: CodeArgs) {
 		query.doc.operations = []
-		const op = sql({ raw_sql, data_source })
-		if (raw_sql.trim().length) {
-			query.doc.operations.push(op)
+		if (args.code.trim().length) {
+			query.doc.operations.push(code(args))
 			query.activeOperationIdx = 0
+			query.execute()
 		} else {
 			query.activeOperationIdx = -1
 		}

frontend/src2/types/query.types.ts

@@ -143,6 +143,9 @@ export type CustomOperation = { type: 'custom_operation' } & CustomOperationArgs
 export type SQLArgs = { raw_sql: string, data_source: string }
 export type SQL = { type: 'sql' } & SQLArgs
 
+export type CodeArgs = { code: string }
+export type Code = { type: 'code' } & CodeArgs
+
 export type Operation =
 	| Source
 	| Filter
@@ -160,6 +163,7 @@ export type Operation =
 	| PivotWider
 	| CustomOperation
 	| SQL
+	| Code
 
 export type QueryResultRow = Record<string, any>
 export type QueryResultColumn = {

frontend/src2/workbook/WorkbookQueryEmptyState.vue

@@ -34,7 +34,7 @@ const emit = defineEmits({
 					<p class="text-sm text-gray-500">Create queries with raw SQL</p>
 				</div>
 			</div>
-			<!-- <div
+			<div
 				class="flex w-full cursor-pointer items-center gap-4 rounded border border-transparent bg-white p-2 shadow-sm transition-all hover:border-gray-300"
 				@click="emit('select', 'script-editor')"
 			>
@@ -45,7 +45,7 @@ const emit = defineEmits({
 					<p class="font-medium text-gray-700">Script Editor</p>
 					<p class="text-sm text-gray-500">Create queries with a python script</p>
 				</div>
-			</div> -->
+			</div>
 		</div>
 	</div>
 </template>

frontend/src2/workbook/WorkbookSidebar.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { LayoutPanelTop, ScrollText, Table2 } from 'lucide-vue-next'
+import { Braces, LayoutPanelTop, ScrollText, Table2 } from 'lucide-vue-next'
 import { inject } from 'vue'
 import ChartIcon from '../charts/components/ChartIcon.vue'
 import WorkbookSidebarListSection from './WorkbookSidebarListSection.vue'
@@ -31,6 +31,11 @@ const workbook = inject(workbookKey) as Workbook
 					class="h-4 w-4 text-gray-700"
 					stroke-width="1.5"
 				/>
+				<Braces
+					v-else-if="item.is_script_query"
+					class="h-4 w-4 text-gray-700"
+					stroke-width="1.5"
+				/>
 				<Table2 v-else class="h-4 w-4 text-gray-700" stroke-width="1.5" />
 			</template>
 		</WorkbookSidebarListSection>

insights/insights/doctype/insights_data_source_v3/ibis_utils.py

@@ -74,6 +74,8 @@ def perform_operation(self, operation):
             return self.apply_custom_operation(operation)
         elif operation.type == "sql":
             return self.apply_sql(operation)
+        elif operation.type == "code":
+            return self.apply_code(operation)
         return self.query
 
     def get_table_or_query(self, table_args):
@@ -434,6 +436,30 @@ def apply_sql(self, sql_args):
         db = ds._get_ibis_backend()
         return db.sql(raw_sql)
 
+    def apply_code(self, code_args):
+        code = code_args.code
+
+        pandas = frappe._dict()
+        pandas.DataFrame = pd.DataFrame
+        pandas.read_csv = pd.read_csv
+        pandas.json_normalize = pd.json_normalize
+        # prevent users from writing to disk
+        pandas.DataFrame.to_csv = lambda *args, **kwargs: None
+        pandas.DataFrame.to_json = lambda *args, **kwargs: None
+
+        results = []
+        _, _locals = safe_exec(
+            code,
+            _globals={"pandas": pandas},
+            _locals={"results": results},
+            restrict_commit_rollback=True,
+        )
+        results = _locals["results"]
+        if results is None or len(results) == 0:
+            results = [{"error": "No results"}]
+
+        return ibis.memtable(results, name=make_digest(code))
+
     def translate_measure(self, measure):
         if measure.column_name == "count" and measure.aggregation == "count":
             first_column = self.query.columns[0]
@@ -631,7 +657,7 @@ def exec_with_return(
     _globals = _globals or {}
     _locals = _locals or {}
     if last_expression:
-        safe_exec(ast.unparse(a), _globals, _locals)
+        safe_exec(ast.unparse(a), _globals, _locals, restrict_commit_rollback=True)
         return safe_eval(last_expression, _globals, _locals)
     else:
         return safe_eval(code, _globals, _locals)