From 19efc83455c8f4b03dd77f0d58bd5b4ff6b88514 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 4 Sep 2024 16:06:15 -0700 Subject: [PATCH 1/3] Set File Types to Sync from Obsidian via Khoj Plugin Settings Page Useful to limit file types to sync with Khoj. Avoids hitting indexed data limits, especially for users on the Khoj cloud free tier Closes #893 --- src/interface/obsidian/src/settings.ts | 48 ++++++++++++++++++++++++++ src/interface/obsidian/src/utils.ts | 23 +++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 85e51187c..25b7116c2 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -10,6 +10,11 @@ export interface UserInfo { email?: string; } +interface SyncFileTypes { + markdown: boolean; + images: boolean; + pdf: boolean; +} export interface KhojSetting { resultsCount: number; khojUrl: string; @@ -17,6 +22,7 @@ export interface KhojSetting { connectedToBackend: boolean; autoConfigure: boolean; lastSync: Map; + syncFileType: SyncFileTypes; userInfo: UserInfo | null; } @@ -27,6 +33,11 @@ export const DEFAULT_SETTINGS: KhojSetting = { connectedToBackend: false, autoConfigure: true, lastSync: new Map(), + syncFileType: { + markdown: true, + images: true, + pdf: true, + }, userInfo: null, } @@ -96,6 +107,43 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.settings.resultsCount = value; await this.plugin.saveSettings(); })); + + // Add new "Sync" heading + containerEl.createEl('h3', {text: 'Sync'}); + + // Add setting to sync markdown notes + new Setting(containerEl) + .setName('Sync Notes') + .setDesc('Index Markdown files in your vault with Khoj.') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.syncFileType.markdown) + .onChange(async (value) => { + this.plugin.settings.syncFileType.markdown = value; + await this.plugin.saveSettings(); + })); + + // Add setting to sync images + new Setting(containerEl) + .setName('Sync Images') + .setDesc('Index images in your vault with Khoj.') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.syncFileType.images) + .onChange(async (value) => { + this.plugin.settings.syncFileType.images = value; + await this.plugin.saveSettings(); + })); + + // Add setting to sync PDFs + new Setting(containerEl) + .setName('Sync PDFs') + .setDesc('Index PDF files in your vault with Khoj.') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.syncFileType.pdf) + .onChange(async (value) => { + this.plugin.settings.syncFileType.pdf = value; + await this.plugin.saveSettings(); + })); + new Setting(containerEl) .setName('Auto Sync') .setDesc('Automatically index your vault with Khoj.') diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 0f4d10ed2..6e45cc0a5 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -48,14 +48,29 @@ function filenameToMimeType (filename: TFile): string { } } -export const supportedImageFilesTypes = ['png', 'jpg', 'jpeg']; -export const supportedBinaryFileTypes = ['pdf'].concat(supportedImageFilesTypes); -export const supportedFileTypes = ['md', 'markdown'].concat(supportedBinaryFileTypes); +export const fileTypeToExtension = { + 'pdf': ['pdf'], + 'image': ['png', 'jpg', 'jpeg'], + 'markdown': ['md', 'markdown'], +}; +export const supportedImageFilesTypes = fileTypeToExtension.image; +export const supportedBinaryFileTypes = fileTypeToExtension.pdf.concat(supportedImageFilesTypes); +export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedBinaryFileTypes); export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) - const files = vault.getFiles().filter(file => supportedFileTypes.includes(file.extension)); + const files = vault.getFiles() + // Filter supported file types for syncing + .filter(file => supportedFileTypes.includes(file.extension)) + // Filter user configured file types for syncing + .filter(file => { + if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown; + if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf; + if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images; + return false; + }); + let countOfFilesToIndex = 0; let countOfFilesToDelete = 0; lastSync = lastSync.size > 0 ? lastSync : new Map(); From cb425a073dda99c2304015f45f1196a0140a8dc6 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 4 Sep 2024 17:28:31 -0700 Subject: [PATCH 2/3] Use rich text error to better guide when exceed data sync limits in Obsidian When user exceeds data sync limits. Show error notice with - Link to web app settings page to upgrade subscription - Link to Khoj plugin settings in Obsidian to configure file types to sync from vault to Khoj --- src/interface/obsidian/src/utils.ts | 39 ++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 6e45cc0a5..501391627 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -120,7 +120,37 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las if (!response.ok) { if (response.status === 429) { - error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`; + let response_text = await response.text(); + if (response_text.includes("Too much data")) { + const errorFragment = document.createDocumentFragment(); + errorFragment.appendChild(document.createTextNode("❗️Exceeded data sync limits. To resolve this either:")); + const bulletList = document.createElement('ul'); + + const limitFilesItem = document.createElement('li'); + const settingsPrefixText = document.createTextNode("Limit files to sync from "); + const settingsLink = document.createElement('a'); + settingsLink.textContent = "Khoj settings"; + settingsLink.href = "#"; + settingsLink.addEventListener('click', (e) => { + e.preventDefault(); + openKhojPluginSettings(); + }); + limitFilesItem.appendChild(settingsPrefixText); + limitFilesItem.appendChild(settingsLink); + bulletList.appendChild(limitFilesItem); + + const upgradeItem = document.createElement('li'); + const upgradeLink = document.createElement('a'); + upgradeLink.href = `${setting.khojUrl}/settings#subscription`; + upgradeLink.textContent = 'Upgrade your subscription'; + upgradeLink.target = '_blank'; + upgradeItem.appendChild(upgradeLink); + bulletList.appendChild(upgradeItem); + errorFragment.appendChild(bulletList); + error_message = errorFragment; + } else { + error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`; + } break; } else if (response.status === 404) { error_message = `❗️Could not connect to Khoj server. Ensure you can connect to it.`; @@ -155,6 +185,13 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las return lastSync; } +export async function openKhojPluginSettings(): Promise + { + const setting = this.app.setting; + await setting.open(); + setting.openTabById('khoj'); +} + export async function createNote(name: string, newLeaf = false): Promise { try { let pathPrefix: string From bc26cf8b2f2f818ad54147a02a1b5bf666e243dc Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Wed, 4 Sep 2024 17:16:52 -0700 Subject: [PATCH 3/3] Only show updated index notice on success in Obsidian plugin Previously it'd show indexing success notice on error and success --- src/interface/obsidian/src/settings.ts | 3 +-- src/interface/obsidian/src/utils.ts | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 25b7116c2..6f70d609c 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -189,9 +189,8 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.registerInterval(progress_indicator); this.plugin.settings.lastSync = await updateContentIndex( - this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true + this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true, true ); - new Notice('✅ Updated Khoj index.'); // Reset button once index is updated window.clearInterval(progress_indicator); diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 501391627..2395c5292 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -57,7 +57,7 @@ export const supportedImageFilesTypes = fileTypeToExtension.image; export const supportedBinaryFileTypes = fileTypeToExtension.pdf.concat(supportedImageFilesTypes); export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedBinaryFileTypes); -export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false): Promise> { +export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false, userTriggered: boolean = false): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) const files = vault.getFiles() @@ -179,6 +179,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las if (error_message) { new Notice(error_message); } else { + if (userTriggered) new Notice('✅ Updated Khoj index.'); console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`); }