Skip to content

Commit

Permalink
Move to capturePage over display media for screenshot handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tyriar
Tyriar committed Oct 14, 2024
1 parent 5272848 commit 8b7a797
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 59 deletions.
51 changes: 1 addition & 50 deletions src/vs/code/electron-main/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { app, BrowserWindow, desktopCapturer, protocol, session, Session, systemPreferences, screen, WebFrameMain } from 'electron';
import { app, BrowserWindow, protocol, session, Session, systemPreferences, WebFrameMain } from 'electron';
import { addUNCHostToAllowlist, disableUNCAccessRestrictions } from '../../base/node/unc.js';
import { validatedIpcMain } from '../../base/parts/ipc/electron-main/ipcMain.js';
import { hostname, release } from 'os';
Expand Down Expand Up @@ -196,55 +196,6 @@ export class CodeApplication extends Disposable {
}
return false;
});
session.defaultSession.setDisplayMediaRequestHandler(async (request, callback) => {

// Get the currently focused window
const focusedWindow = BrowserWindow.getFocusedWindow();

if (!focusedWindow) {
return;
}

// Get the bounds (position and size) of the focused window
const windowBounds = focusedWindow.getBounds();

// Get all the screen sources
const screens = await desktopCapturer.getSources({ types: ['screen'] });

// Get the display that contains the focused window
const displays = screen.getAllDisplays();

// Find the screen that contains the focused window
for (const display of displays) {
const displayBounds = display.bounds;

// Check if the window is within the display's bounds. The center of the window is
// used since maximizing actually causes the window to go beyond the screen. There
// is also the case where a window could be spread across multiple screens.
const windowCenter = {
x: windowBounds.x + windowBounds.width / 2,
y: windowBounds.y + windowBounds.height / 2,
};
if (
windowCenter.x >= displayBounds.x &&
windowCenter.x <= displayBounds.x + displayBounds.width &&
windowCenter.y >= displayBounds.y &&
windowCenter.y <= displayBounds.y + displayBounds.height
) {
// Match the display to the screen source
for (const source of screens) {
if (source.display_id === display.id.toString()) {
// Found the screen containing the focused window
callback({ video: source, audio: 'loopback' });
return;
}
}
}
}

// Fallback: if no matching screen is found, return the first screen
callback({ video: screens[0], audio: 'loopback' });
});

//#endregion

Expand Down
3 changes: 3 additions & 0 deletions src/vs/platform/native/common/native.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@ export interface ICommonNativeHostService {

hasWSLFeatureInstalled(): Promise<boolean>;

// Screenshots
getScreenshot(windowId: number | undefined): Promise<ArrayBufferLike | undefined>;

// Process
getProcessId(): Promise<number | undefined>;
killProcess(pid: number, code: string): Promise<void>;
Expand Down
11 changes: 11 additions & 0 deletions src/vs/platform/native/electron-main/nativeHostMainService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,17 @@ export class NativeHostMainService extends Disposable implements INativeHostMain
//#endregion


//#region Screenshots

async getScreenshot(windowId: number | undefined): Promise<ArrayBufferLike | undefined> {
const window = this.windowById(undefined, windowId);
const captured = await window?.win?.webContents.capturePage();
return captured?.toJPEG(95);
}

//#endregion


//#region Process

async getProcessId(windowId: number | undefined): Promise<number | undefined> {
Expand Down
32 changes: 23 additions & 9 deletions src/vs/workbench/contrib/chat/browser/actions/chatContextActions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { IConfigurationService } from '../../../../../platform/configuration/com
import { ContextKeyExpr, IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js';
import { KeybindingWeight } from '../../../../../platform/keybinding/common/keybindingsRegistry.js';
import { ILabelService } from '../../../../../platform/label/common/label.js';
import { INativeHostService } from '../../../../../platform/native/common/native.js';
import { AnythingQuickAccessProviderRunOptions } from '../../../../../platform/quickinput/common/quickAccess.js';
import { IQuickInputService, IQuickPickItem, IQuickPickItemWithResource, IQuickPickSeparator, QuickPickItem } from '../../../../../platform/quickinput/common/quickInput.js';
import { ActiveEditorContext } from '../../../../common/contextkeys.js';
Expand Down Expand Up @@ -271,7 +272,7 @@ export class AttachContextAction extends Action2 {
`:${item.range.startLineNumber}`);
}

private async _attachContext(widget: IChatWidget, commandService: ICommandService, clipboardService: IClipboardService, editorService: IEditorService, labelService: ILabelService, viewsService: IViewsService, chatEditingService: IChatEditingService | undefined, ...picks: IChatContextQuickPickItem[]) {
private async _attachContext(widget: IChatWidget, commandService: ICommandService, clipboardService: IClipboardService, editorService: IEditorService, labelService: ILabelService, viewsService: IViewsService, chatEditingService: IChatEditingService | undefined, nativeHostService: INativeHostService, ...picks: IChatContextQuickPickItem[]) {
const toAttach: IChatRequestVariableEntry[] = [];
for (const pick of picks) {
if (isISymbolQuickPickItem(pick) && pick.symbol) {
Expand Down Expand Up @@ -349,11 +350,23 @@ export class AttachContextAction extends Action2 {
}
}
} else if (isScreenshotQuickPickItem(pick)) {
const variable = await getScreenshotAsVariable();
if (!variable) {
return;
// TODO: Ensure this works properly when falling back
const blob = await nativeHostService.getScreenshot(undefined);
if (blob) {
toAttach.push({
id: 'screenshot-focused-window',
name: localize('screenshot', 'Screenshot'),
value: new Uint8Array(blob),
isImage: true,
isDynamic: true
});
} else {
const variable = await getScreenshotAsVariable();
if (!variable) {
return;
}
toAttach.push(variable);
}
toAttach.push(variable);
} else {
// Anything else is an attachment
const attachmentPick = pick as IAttachmentQuickPickItem;
Expand Down Expand Up @@ -422,6 +435,7 @@ export class AttachContextAction extends Action2 {
const labelService = accessor.get(ILabelService);
const contextKeyService = accessor.get(IContextKeyService);
const viewsService = accessor.get(IViewsService);
const nativeHostService = accessor.get(INativeHostService);

const context: { widget?: IChatWidget; showFilesOnly?: boolean; placeholder?: string } | undefined = args[0];
const widget = context?.widget ?? widgetService.lastFocusedWidget;
Expand Down Expand Up @@ -565,19 +579,19 @@ export class AttachContextAction extends Action2 {
const second = extractTextFromIconLabel(b.label).toUpperCase();

return compare(first, second);
}), clipboardService, editorService, labelService, viewsService, chatEditingService, '', context?.placeholder);
}), clipboardService, editorService, labelService, viewsService, chatEditingService, nativeHostService, '', context?.placeholder);
}

private _show(quickInputService: IQuickInputService, commandService: ICommandService, widget: IChatWidget, quickChatService: IQuickChatService, quickPickItems: (IChatContextQuickPickItem | QuickPickItem)[] | undefined, clipboardService: IClipboardService, editorService: IEditorService, labelService: ILabelService, viewsService: IViewsService, chatEditingService: IChatEditingService | undefined, query: string = '', placeholder?: string) {
private _show(quickInputService: IQuickInputService, commandService: ICommandService, widget: IChatWidget, quickChatService: IQuickChatService, quickPickItems: (IChatContextQuickPickItem | QuickPickItem)[] | undefined, clipboardService: IClipboardService, editorService: IEditorService, labelService: ILabelService, viewsService: IViewsService, chatEditingService: IChatEditingService | undefined, nativeHostService: INativeHostService, query: string = '', placeholder?: string) {
const providerOptions: AnythingQuickAccessProviderRunOptions = {
handleAccept: (item: IChatContextQuickPickItem) => {
if ('prefix' in item) {
this._show(quickInputService, commandService, widget, quickChatService, quickPickItems, clipboardService, editorService, labelService, viewsService, chatEditingService, item.prefix, placeholder);
this._show(quickInputService, commandService, widget, quickChatService, quickPickItems, clipboardService, editorService, labelService, viewsService, chatEditingService, nativeHostService, item.prefix, placeholder);
} else {
if (!clipboardService) {
return;
}
this._attachContext(widget, commandService, clipboardService, editorService, labelService, viewsService, chatEditingService, item);
this._attachContext(widget, commandService, clipboardService, editorService, labelService, viewsService, chatEditingService, nativeHostService, item);
if (isQuickChat(widget)) {
quickChatService.open();
}
Expand Down

0 comments on commit 8b7a797

Please sign in to comment.