feat: connect remote duckdb file as data source

frontend/src2/components/Form.vue

@@ -10,6 +10,7 @@ const props = defineProps<{
 		placeholder?: string
 		required?: boolean
 		defaultValue?: any
+		description?: string
 	}[]
 	actions?: {
 		label: string
@@ -42,10 +43,12 @@ defineExpose({
 		<div class="flex flex-col gap-4">
 			<div class="relative" v-for="field in fields" :key="field.name">
 				<FormControl
+					autocomplete="off"
 					:type="field.type"
 					:label="field.label"
 					:options="field.options"
 					:placeholder="field.placeholder"
+					:description="field.description"
 					v-model="form[field.name]"
 				/>
 				<span

frontend/src2/data_source/ConnectDuckDBDialog.vue

@@ -0,0 +1,106 @@
+<script setup lang="ts">
+import { computed, ref } from 'vue'
+import Form from '../components/Form.vue'
+import useDataSourceStore from './data_source'
+import { DuckDBDataSource } from './data_source.types'
+
+const show = defineModel({
+	default: false,
+})
+
+const database = ref<DuckDBDataSource>({
+	database_type: 'DuckDB',
+	title: '',
+	database_name: '',
+})
+
+const form = ref()
+const fields = [
+	{
+		name: 'title',
+		label: 'Title',
+		type: 'text',
+		placeholder: 'My Database',
+		required: true,
+	},
+	{
+		label: 'File URL',
+		name: 'database_name',
+		type: 'text',
+		placeholder: 'https://example.com/file.duckdb',
+		required: true,
+		description: 'Enter the URL of the DuckDB file',
+	},
+]
+
+const isValidFileURL = computed(() => {
+	const url = new URL(database.value.database_name)
+	return url.protocol === 'https:' || url.protocol === 'http:'
+})
+
+const sources = useDataSourceStore()
+
+const connected = ref<boolean | null>(null)
+const connectButton = computed(() => {
+	const _button = {
+		label: 'Connect',
+		disabled:
+			form.value?.hasRequiredFields === false || !isValidFileURL.value || sources.testing,
+		loading: sources.testing,
+		variant: 'subtle',
+		theme: 'gray',
+		onClick() {
+			sources.testConnection(database.value).then((result: boolean) => {
+				connected.value = Boolean(result)
+			})
+		},
+	}
+
+	if (sources.testing) {
+		_button.label = 'Connecting...'
+	} else if (connected.value) {
+		_button.label = 'Connected'
+		_button.variant = 'outline'
+		_button.theme = 'green'
+	} else if (connected.value === false) {
+		_button.label = 'Failed, Retry?'
+		_button.variant = 'outline'
+		_button.theme = 'red'
+	}
+
+	return _button
+})
+
+const submitButton = computed(() => {
+	return {
+		label: 'Add Data Source',
+		disabled:
+			form.value?.hasRequiredFields === false ||
+			!isValidFileURL.value ||
+			!connected.value ||
+			sources.creating,
+		loading: sources.creating,
+		variant: connected.value ? 'solid' : 'subtle',
+		onClick() {
+			sources.createDataSource(database.value).then(() => {
+				show.value = false
+			})
+		},
+	}
+})
+</script>
+
+<template>
+	<Dialog v-model="show" :options="{ title: 'Connect to DuckDB' }">
+		<template #body-content>
+			<Form
+				ref="form"
+				class="flex-1"
+				v-model="database"
+				:fields="fields"
+				:actions="[connectButton, submitButton]"
+			>
+			</Form>
+		</template>
+	</Dialog>
+</template>

frontend/src2/data_source/DataSourceList.vue

@@ -11,6 +11,7 @@ import ConnectPostgreSQLDialog from './ConnectPostgreSQLDialog.vue'
 import useDataSourceStore, { getDatabaseLogo } from './data_source'
 import { DataSourceListItem } from './data_source.types'
 import UploadCSVFileDialog from './UploadCSVFileDialog.vue'
+import ConnectDuckDBDialog from './ConnectDuckDBDialog.vue'
 
 const dataSourceStore = useDataSourceStore()
 dataSourceStore.getSources()
@@ -28,6 +29,7 @@ const filteredDataSources = computed(() => {
 const showNewSourceDialog = ref(false)
 const showNewMariaDBDialog = ref(false)
 const showNewPostgreSQLDialog = ref(false)
+const showNewDuckDBDialog = ref(false)
 const showCSVFileUploadDialog = ref(false)
 
 const sourceTypes = [
@@ -49,6 +51,15 @@ const sourceTypes = [
 			showNewPostgreSQLDialog.value = true
 		},
 	},
+	{
+		label: 'DuckDB',
+		icon: getDatabaseLogo('DuckDB'),
+		description: 'Connect to DuckDB database',
+		onClick: () => {
+			showNewSourceDialog.value = false
+			showNewDuckDBDialog.value = true
+		},
+	},
 	{
 		label: 'Upload CSV',
 		icon: <CSVIcon class="h-8 w-8" />,
@@ -149,5 +160,6 @@ document.title = 'Data Sources | Insights'
 
 	<ConnectMariaDBDialog v-model="showNewMariaDBDialog" />
 	<ConnectPostgreSQLDialog v-model="showNewPostgreSQLDialog" />
+	<ConnectDuckDBDialog v-model="showNewDuckDBDialog" />
 	<UploadCSVFileDialog v-model="showCSVFileUploadDialog" />
 </template>

frontend/src2/data_source/data_source.ts

@@ -39,6 +39,9 @@ const creating = ref(false)
 function createDataSource(data_source: DataSource) {
 	creating.value = true
 	return call('insights.api.data_sources.create_data_source', { data_source })
+		.then(() => {
+			fetchSources()
+		})
 		.catch((error: Error) => {
 			showErrorToast(error)
 		})

frontend/src2/data_source/data_source.types.ts

@@ -40,4 +40,9 @@ export type PostgreSQLDataSource = BaseDataSource & {
 	use_ssl: boolean
 }
 
-export type DataSource = MariaDBDataSource | PostgreSQLDataSource
+export type DuckDBDataSource = BaseDataSource & {
+	database_type: 'DuckDB'
+	database_name: string
+}
+
+export type DataSource = MariaDBDataSource | PostgreSQLDataSource | DuckDBDataSource

insights/insights/doctype/insights_data_source_v3/connectors/duckdb.py

@@ -8,6 +8,16 @@
 
 
 def get_duckdb_connection(data_source, read_only=True):
-    path = os.path.realpath(get_files_path(is_private=1))
-    path = os.path.join(path, f"{data_source.database_name}.duckdb")
-    return ibis.duckdb.connect(path, read_only=read_only)
+    name = data_source.name or data_source.title
+    db_name = data_source.database_name
+
+    if db_name.startswith("http"):
+        db = ibis.duckdb.connect()
+        db.load_extension("httpfs")
+        db.attach(db_name, name, read_only=True)
+        db.raw_sql(f"USE {name}")
+        return db
+    else:
+        path = os.path.realpath(get_files_path(is_private=1))
+        path = os.path.join(path, f"{db_name}.duckdb")
+        return ibis.duckdb.connect(path, read_only=read_only)

insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py

@@ -212,6 +212,8 @@ def get_quoted_db_name(self):
         database_name = self.database_name
         if self.is_site_db:
             database_name = frappe.conf.db_name
+        if self.database_type == "DuckDB":
+            return None
         if not database_name:
             return None
         quote_start = self._get_ibis_backend().dialect.QUOTE_START