Skip to content

Commit

Permalink
Settings search - split remote settings search into multiple requests…
Browse files Browse the repository at this point in the history 
… to stay under the filter size limit
  • Loading branch information
@roblourens
roblourens committed Jan 29, 2018
1 parent 4807dae commit 757ff83
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 73 deletions.
6 changes: 4 additions & 2 deletions src/vs/workbench/parts/preferences/browser/preferencesEditor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,14 +316,16 @@ export class PreferencesEditor extends BaseEditor {
let data = {
filter,
durations,
counts
counts,
requestCount: metadata && metadata['nlpResult'] && metadata['nlpResult'].requestCount
};

/* __GDPR__
"defaultSettings.filter" : {
"filter": { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
"durations" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
"counts" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
"counts" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
"requestCount" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
}
*/
this.telemetryService.publicLog('defaultSettings.filter', data);
Expand Down
1 change: 1 addition & 0 deletions src/vs/workbench/parts/preferences/common/preferences.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ export interface IFilterMetadata {
timestamp: number;
duration: number;
scoredResults: IScoredResults;
requestCount?: number;

/** The name of the server that actually served the request */
context: string;
Expand Down
174 changes: 103 additions & 71 deletions src/vs/workbench/parts/preferences/electron-browser/preferencesSearch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,16 @@ interface IRemoteSearchProviderOptions {
newExtensionsOnly: boolean;
}

interface IBingRequestDetails {
url: string;
body?: string;
hasMoreFilters?: boolean;
}

class RemoteSearchProvider implements ISearchProvider {
// Must keep extension filter size under 8kb. 42 extension filters + core buildnum filter puts us there.
private static MAX_EXTENSION_FILTERS = 42;
// Must keep extension filter size under 8kb. 42 filters puts us there.
private static MAX_REQUEST_FILTERS = 42;
private static MAX_REQUESTS = 10;

private _remoteSearchP: TPromise<IFilterMetadata>;

Expand All @@ -144,7 +151,7 @@ class RemoteSearchProvider implements ISearchProvider {
@ILogService private logService: ILogService
) {
this._remoteSearchP = this.options.filter ?
this.getSettingsFromBing(this.options.filter) :
this.getSettingsForFilter(this.options.filter) :
TPromise.wrap(null);
}

Expand Down Expand Up @@ -185,76 +192,100 @@ class RemoteSearchProvider implements ISearchProvider {
});
}

private getSettingsFromBing(filter: string): TPromise<IFilterMetadata> {
private async getSettingsForFilter(filter: string): TPromise<IFilterMetadata> {
const allRequestDetails: IBingRequestDetails[] = [];

// Only send MAX_REQUESTS requests in total just to keep it sane
for (let i = 0; i < RemoteSearchProvider.MAX_REQUESTS; i++) {
const details = await this.prepareRequest(filter, i);
allRequestDetails.push(details);
if (!details.hasMoreFilters) {
break;
}
}

return TPromise.join(allRequestDetails.map(details => this.getSettingsFromBing(details))).then(allResponses => {
// Merge all IFilterMetadata
const metadata = allResponses[0];
metadata.requestCount = 1;

for (let response of allResponses.slice(1)) {
metadata.requestCount++;
metadata.scoredResults = { ...metadata.scoredResults, ...response.scoredResults };
}

return metadata;
});
}

private getSettingsFromBing(details: IBingRequestDetails): TPromise<IFilterMetadata> {
this.logService.debug(`Searching settings via ${details.url}`);
if (details.body) {
this.logService.debug(`Body: ${details.body}`);
}

const requestType = details.body ? 'post' : 'get';
const start = Date.now();
return this.prepareRequest(filter).then(details => {
this.logService.debug(`Searching settings via ${details.url}`);
if (details.body) {
this.logService.debug(`Body: ${details.body}`);
return this.requestService.request({
type: requestType,
url: details.url,
data: details.body,
headers: {
'User-Agent': 'request',
'Content-Type': 'application/json; charset=utf-8',
'api-key': this.options.endpoint.key
},
timeout: 5000
}).then(context => {
if (context.res.statusCode >= 300) {
throw new Error(`${details} returned status code: ${context.res.statusCode}`);
}

const requestType = details.body ? 'post' : 'get';
return this.requestService.request({
type: requestType,
url: details.url,
data: details.body,
headers: {
'User-Agent': 'request',
'Content-Type': 'application/json; charset=utf-8',
'api-key': this.options.endpoint.key
},
timeout: 5000
}).then(context => {
if (context.res.statusCode >= 300) {
throw new Error(`${details} returned status code: ${context.res.statusCode}`);
}
return asJson(context);
}).then((result: any) => {
const timestamp = Date.now();
const duration = timestamp - start;
const remoteSettings: IRemoteSetting[] = (result.value || [])
.map(r => {
const key = JSON.parse(r.setting || r.Setting);
const packageId = r['packageid'];
const id = getSettingKey(key, packageId);

const value = r['value'];
const defaultValue = value ? JSON.parse(value) : value;

const packageName = r['packagename'];
let extensionName: string;
let extensionPublisher: string;
if (packageName && packageName.indexOf('##') >= 0) {
[extensionPublisher, extensionName] = packageName.split('##');
}

return asJson(context);
}).then((result: any) => {
const timestamp = Date.now();
const duration = timestamp - start;
const remoteSettings: IRemoteSetting[] = (result.value || [])
.map(r => {
const key = JSON.parse(r.setting || r.Setting);
const packageId = r['packageid'];
const id = getSettingKey(key, packageId);

const value = r['value'];
const defaultValue = value ? JSON.parse(value) : value;

const packageName = r['packagename'];
let extensionName: string;
let extensionPublisher: string;
if (packageName && packageName.indexOf('##') >= 0) {
[extensionPublisher, extensionName] = packageName.split('##');
}

return <IRemoteSetting>{
key,
id,
defaultValue,
score: r['@search.score'],
description: JSON.parse(r['details']),
packageId,
extensionName,
extensionPublisher
};
});

const scoredResults = Object.create(null);
remoteSettings.forEach(s => {
scoredResults[s.id] = s;
return <IRemoteSetting>{
key,
id,
defaultValue,
score: r['@search.score'],
description: JSON.parse(r['details']),
packageId,
extensionName,
extensionPublisher
};
});

return <IFilterMetadata>{
requestUrl: details.url,
requestBody: details.body,
duration,
timestamp,
scoredResults,
context: result['@odata.context']
};
const scoredResults = Object.create(null);
remoteSettings.forEach(s => {
scoredResults[s.id] = s;
});

return <IFilterMetadata>{
requestUrl: details.url,
requestBody: details.body,
duration,
timestamp,
scoredResults,
context: result['@odata.context']
};
});
}

Expand All @@ -272,7 +303,7 @@ class RemoteSearchProvider implements ISearchProvider {
};
}

private async prepareRequest(query: string): TPromise<{ url: string, body?: string }> {
private async prepareRequest(query: string, filterPage = 0): TPromise<IBingRequestDetails> {
query = escapeSpecialChars(query);
const boost = 10;
const userQuery = `(${query})^${boost}`;
Expand All @@ -283,18 +314,18 @@ class RemoteSearchProvider implements ISearchProvider {
const encodedQuery = encodeURIComponent(userQuery + ' || ' + query);
let url = `${this.options.endpoint.urlBase}?`;

const buildNumber = this.environmentService.settingsSearchBuildId;
if (this.options.endpoint.key) {
url += `${API_VERSION}&${QUERY_TYPE}`;
}

const filters = this.options.newExtensionsOnly ?
[`diminish eq 'latest'`] :
await this.getVersionFilters(buildNumber);
await this.getVersionFilters(this.environmentService.settingsSearchBuildId);

const filterStr = filters
.slice(0, RemoteSearchProvider.MAX_EXTENSION_FILTERS)
.slice(filterPage * RemoteSearchProvider.MAX_REQUEST_FILTERS, (filterPage + 1) * RemoteSearchProvider.MAX_REQUEST_FILTERS)
.join(' or ');
const hasMoreFilters = filters.length > (filterPage + 1) * RemoteSearchProvider.MAX_REQUEST_FILTERS;

const body = JSON.stringify({
query: encodedQuery,
Expand All @@ -303,7 +334,8 @@ class RemoteSearchProvider implements ISearchProvider {

return {
url,
body
body,
hasMoreFilters
};
}

Expand Down

0 comments on commit 757ff83

Please sign in to comment.