Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set File Types to Sync from Obsidian via Khoj Plugin Settings Page #904

Merged
merged 3 commits into from
Sep 6, 2024
Merged

Set File Types to Sync from Obsidian via Khoj Plugin Settings Page #904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19efc83
Set File Types to Sync from Obsidian via Khoj Plugin Settings Page
debanjum Sep 4, 2024
cb425a0
Use rich text error to better guide when exceed data sync limits in O…
debanjum Sep 5, 2024
bc26cf8
Only show updated index notice on success in Obsidian plugin
debanjum Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 49 additions & 2 deletions src/interface/obsidian/src/settings.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,19 @@ export interface UserInfo {
email?: string;
}

interface SyncFileTypes {
markdown: boolean;
images: boolean;
pdf: boolean;
}
export interface KhojSetting {
resultsCount: number;
khojUrl: string;
khojApiKey: string;
connectedToBackend: boolean;
autoConfigure: boolean;
lastSync: Map<TFile, number>;
syncFileType: SyncFileTypes;
userInfo: UserInfo | null;
}

Expand All @@ -27,6 +33,11 @@ export const DEFAULT_SETTINGS: KhojSetting = {
connectedToBackend: false,
autoConfigure: true,
lastSync: new Map(),
syncFileType: {
markdown: true,
images: true,
pdf: true,
},
userInfo: null,
}

Expand Down Expand Up @@ -96,6 +107,43 @@ export class KhojSettingTab extends PluginSettingTab {
this.plugin.settings.resultsCount = value;
await this.plugin.saveSettings();
}));

// Add new "Sync" heading
containerEl.createEl('h3', {text: 'Sync'});

// Add setting to sync markdown notes
new Setting(containerEl)
.setName('Sync Notes')
.setDesc('Index Markdown files in your vault with Khoj.')
.addToggle(toggle => toggle
.setValue(this.plugin.settings.syncFileType.markdown)
.onChange(async (value) => {
this.plugin.settings.syncFileType.markdown = value;
await this.plugin.saveSettings();
}));

// Add setting to sync images
new Setting(containerEl)
.setName('Sync Images')
.setDesc('Index images in your vault with Khoj.')
.addToggle(toggle => toggle
.setValue(this.plugin.settings.syncFileType.images)
.onChange(async (value) => {
this.plugin.settings.syncFileType.images = value;
await this.plugin.saveSettings();
}));

// Add setting to sync PDFs
new Setting(containerEl)
.setName('Sync PDFs')
.setDesc('Index PDF files in your vault with Khoj.')
.addToggle(toggle => toggle
.setValue(this.plugin.settings.syncFileType.pdf)
.onChange(async (value) => {
this.plugin.settings.syncFileType.pdf = value;
await this.plugin.saveSettings();
}));

new Setting(containerEl)
.setName('Auto Sync')
.setDesc('Automatically index your vault with Khoj.')
Expand Down Expand Up @@ -141,9 +189,8 @@ export class KhojSettingTab extends PluginSettingTab {
this.plugin.registerInterval(progress_indicator);

this.plugin.settings.lastSync = await updateContentIndex(
this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true
this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true, true
);
new Notice('✅ Updated Khoj index.');
debanjum marked this conversation as resolved.
Show resolved Hide resolved

// Reset button once index is updated
window.clearInterval(progress_indicator);
Expand Down
67 changes: 60 additions & 7 deletions src/interface/obsidian/src/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,29 @@ function filenameToMimeType (filename: TFile): string {
}
}

export const supportedImageFilesTypes = ['png', 'jpg', 'jpeg'];
export const supportedBinaryFileTypes = ['pdf'].concat(supportedImageFilesTypes);
export const supportedFileTypes = ['md', 'markdown'].concat(supportedBinaryFileTypes);

export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
export const fileTypeToExtension = {
'pdf': ['pdf'],
'image': ['png', 'jpg', 'jpeg'],
'markdown': ['md', 'markdown'],
};
export const supportedImageFilesTypes = fileTypeToExtension.image;
export const supportedBinaryFileTypes = fileTypeToExtension.pdf.concat(supportedImageFilesTypes);
export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedBinaryFileTypes);
debanjum marked this conversation as resolved.
Show resolved Hide resolved

export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false, userTriggered: boolean = false): Promise<Map<TFile, number>> {
// Get all markdown, pdf files in the vault
console.log(`Khoj: Updating Khoj content index...`)
const files = vault.getFiles().filter(file => supportedFileTypes.includes(file.extension));
const files = vault.getFiles()
// Filter supported file types for syncing
.filter(file => supportedFileTypes.includes(file.extension))
// Filter user configured file types for syncing
.filter(file => {
if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown;
if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf;
if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images;
return false;
});

let countOfFilesToIndex = 0;
let countOfFilesToDelete = 0;
lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();
Expand Down Expand Up @@ -105,7 +120,37 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las

if (!response.ok) {
if (response.status === 429) {
error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`;
let response_text = await response.text();
if (response_text.includes("Too much data")) {
const errorFragment = document.createDocumentFragment();
errorFragment.appendChild(document.createTextNode("❗️Exceeded data sync limits. To resolve this either:"));
const bulletList = document.createElement('ul');

const limitFilesItem = document.createElement('li');
const settingsPrefixText = document.createTextNode("Limit files to sync from ");
const settingsLink = document.createElement('a');
settingsLink.textContent = "Khoj settings";
settingsLink.href = "#";
settingsLink.addEventListener('click', (e) => {
e.preventDefault();
openKhojPluginSettings();
});
limitFilesItem.appendChild(settingsPrefixText);
limitFilesItem.appendChild(settingsLink);
bulletList.appendChild(limitFilesItem);

const upgradeItem = document.createElement('li');
const upgradeLink = document.createElement('a');
upgradeLink.href = `${setting.khojUrl}/settings#subscription`;
upgradeLink.textContent = 'Upgrade your subscription';
upgradeLink.target = '_blank';
upgradeItem.appendChild(upgradeLink);
bulletList.appendChild(upgradeItem);
errorFragment.appendChild(bulletList);
error_message = errorFragment;
} else {
error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`;
}
break;
} else if (response.status === 404) {
error_message = `❗️Could not connect to Khoj server. Ensure you can connect to it.`;
Expand Down Expand Up @@ -134,12 +179,20 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
if (error_message) {
new Notice(error_message);
} else {
if (userTriggered) new Notice('✅ Updated Khoj index.');
debanjum marked this conversation as resolved.
Show resolved Hide resolved
console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`);
}

return lastSync;
}

export async function openKhojPluginSettings(): Promise<void>
{
const setting = this.app.setting;
await setting.open();
setting.openTabById('khoj');
}

export async function createNote(name: string, newLeaf = false): Promise<void> {
try {
let pathPrefix: string
Expand Down
Loading