Skip to content

Commit

Permalink
[json] when downloading JSON Schema files, do Conditional GETs. Fixes #…
Browse files Browse the repository at this point in the history
  • Loading branch information
aeschli committed Nov 19, 2021
1 parent d3aabc5 commit f13f583
Show file tree
Hide file tree
Showing 5 changed files with 229 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@

import { ExtensionContext, Uri } from 'vscode';
import { LanguageClientOptions } from 'vscode-languageclient';
import { startClient, LanguageClientConstructor } from '../jsonClient';
import { startClient, LanguageClientConstructor, SchemaRequestService } from '../jsonClient';
import { LanguageClient } from 'vscode-languageclient/browser';
import { RequestService } from '../requests';

declare const Worker: {
new(stringUrl: string): any;
Expand All @@ -24,15 +23,16 @@ export function activate(context: ExtensionContext) {
return new LanguageClient(id, name, clientOptions, worker);
};

const http: RequestService = {
const schemaRequests: SchemaRequestService = {
getContent(uri: string) {
return fetch(uri, { mode: 'cors' })
.then(function (response: any) {
return response.text();
});
}
};
startClient(context, newLanguageClient, { http });

startClient(context, newLanguageClient, { schemaRequests });

} catch (e) {
console.log(e);
Expand Down
14 changes: 10 additions & 4 deletions extensions/json-language-features/client/src/jsonClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
} from 'vscode-languageclient';

import { hash } from './utils/hash';
import { RequestService, joinPath } from './requests';
import { joinPath } from './requests';
import { createLanguageStatusItem } from './languageStatus';

namespace VSCodeContentRequest {
Expand Down Expand Up @@ -96,10 +96,16 @@ export interface TelemetryReporter {
export type LanguageClientConstructor = (name: string, description: string, clientOptions: LanguageClientOptions) => CommonLanguageClient;

export interface Runtime {
http: RequestService;
schemaRequests: SchemaRequestService;
telemetry?: TelemetryReporter
}

export interface SchemaRequestService {
getContent(uri: string): Promise<string>;
}

export const languageServerDescription = localize('jsonserver.name', 'JSON Language Server');

export function startClient(context: ExtensionContext, newLanguageClient: LanguageClientConstructor, runtime: Runtime) {

const toDispose = context.subscriptions;
Expand Down Expand Up @@ -198,7 +204,7 @@ export function startClient(context: ExtensionContext, newLanguageClient: Langua
};

// Create the language client and start the client.
const client = newLanguageClient('json', localize('jsonserver.name', 'JSON Language Server'), clientOptions);
const client = newLanguageClient('json', languageServerDescription, clientOptions);
client.registerProposedFeatures();

const disposable = client.start();
Expand Down Expand Up @@ -228,7 +234,7 @@ export function startClient(context: ExtensionContext, newLanguageClient: Langua
*/
runtime.telemetry.sendTelemetryEvent('json.schema', { schemaURL: uriPath });
}
return runtime.http.getContent(uriPath).catch(e => {
return runtime.schemaRequests.getContent(uriPath).catch(e => {
return Promise.reject(new ResponseError(4, e.toString()));
});
} else {
Expand Down
126 changes: 107 additions & 19 deletions extensions/json-language-features/client/src/node/jsonClientMain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,26 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { ExtensionContext } from 'vscode';
import { startClient, LanguageClientConstructor } from '../jsonClient';
import { ExtensionContext, OutputChannel, window, workspace } from 'vscode';
import { startClient, LanguageClientConstructor, SchemaRequestService, languageServerDescription } from '../jsonClient';
import { ServerOptions, TransportKind, LanguageClientOptions, LanguageClient } from 'vscode-languageclient/node';

import * as fs from 'fs';
import { xhr, XHRResponse, getErrorStatusDescription } from 'request-light';
import { promises as fs } from 'fs';
import * as path from 'path';
import { xhr, XHRResponse, getErrorStatusDescription, Headers } from 'request-light';

import TelemetryReporter from 'vscode-extension-telemetry';
import { RequestService } from '../requests';
import { JSONSchemaCache } from './schemaCache';

let telemetry: TelemetryReporter | undefined;

// this method is called when vs code is activated
export function activate(context: ExtensionContext) {

const clientPackageJSON = getPackageInfo(context);
export async function activate(context: ExtensionContext) {
const clientPackageJSON = await getPackageInfo(context);
telemetry = new TelemetryReporter(clientPackageJSON.name, clientPackageJSON.version, clientPackageJSON.aiKey);

const outputChannel = window.createOutputChannel(languageServerDescription);

const serverMain = `./server/${clientPackageJSON.main.indexOf('/dist/') !== -1 ? 'dist' : 'out'}/node/jsonServerMain`;
const serverModule = context.asAbsolutePath(serverMain);

Expand All @@ -35,10 +37,15 @@ export function activate(context: ExtensionContext) {
};

const newLanguageClient: LanguageClientConstructor = (id: string, name: string, clientOptions: LanguageClientOptions) => {
clientOptions.outputChannel = outputChannel;
return new LanguageClient(id, name, serverOptions, clientOptions);
};
const log = getLog(outputChannel);
context.subscriptions.push(log);

startClient(context, newLanguageClient, { http: getHTTPRequestService(), telemetry });
const schemaRequests = await getSchemaRequestService(context, log);

startClient(context, newLanguageClient, { schemaRequests, telemetry });
}

export function deactivate(): Promise<any> {
Expand All @@ -52,32 +59,113 @@ interface IPackageInfo {
main: string;
}

function getPackageInfo(context: ExtensionContext): IPackageInfo {
async function getPackageInfo(context: ExtensionContext): Promise<IPackageInfo> {
const location = context.asAbsolutePath('./package.json');
try {
return JSON.parse(fs.readFileSync(location).toString());
return JSON.parse((await fs.readFile(location)).toString());
} catch (e) {
console.log(`Problems reading ${location}: ${e}`);
return { name: '', version: '', aiKey: '', main: '' };
}
}

function getHTTPRequestService(): RequestService {
interface Log {
trace(message: string): void;
dispose(): void;
}

const traceSetting = 'json.trace.server';
function getLog(outputChannel: OutputChannel): Log {
let trace = workspace.getConfiguration().get(traceSetting) === 'verbose';
const configListener = workspace.onDidChangeConfiguration(e => {
if (e.affectsConfiguration(traceSetting)) {
trace = workspace.getConfiguration().get(traceSetting) === 'verbose';
}
});
return {
getContent(uri: string, _encoding?: string): Promise<string> {
const headers = { 'Accept-Encoding': 'gzip, deflate' };
return xhr({ url: uri, followRedirects: 5, headers }).then(response => {
return response.responseText;
}, (error: XHRResponse) => {
trace(message: string) {
if (trace) {
outputChannel.appendLine(message);
}
},
dispose: () => configListener.dispose()
};
}

const retryTimeoutInDays = 2; // 2 days
const retryTimeoutInMs = retryTimeoutInDays * 24 * 60 * 60 * 1000;

async function getSchemaRequestService(context: ExtensionContext, log: Log): Promise<SchemaRequestService> {
let cache: JSONSchemaCache | undefined = undefined;
const globalStorage = context.globalStorageUri;
if (globalStorage.scheme === 'file') {
const schemaCacheLocation = path.join(globalStorage.fsPath, 'json-schema-cache');
await fs.mkdir(schemaCacheLocation, { recursive: true });

cache = new JSONSchemaCache(schemaCacheLocation, context.globalState);
log.trace(`[json schema cache] initial state: ${JSON.stringify(cache.getCacheInfo(), null, ' ')}`);
}

const isXHRResponse = (error: any): error is XHRResponse => typeof error?.status === 'number';

const request = async (uri: string, etag?: string): Promise<string> => {
const headers: Headers = { 'Accept-Encoding': 'gzip, deflate' };
if (etag) {
headers['If-None-Match'] = etag;
}
try {
log.trace(`[json schema cache] Requesting schema ${uri} etag ${etag}...`);

const response = await xhr({ url: uri, followRedirects: 5, headers });
if (cache) {
const etag = response.headers['etag'];
if (typeof etag === 'string') {
log.trace(`[json schema cache] Storing schema ${uri} etag ${etag} in cache`);
await cache.putSchema(uri, etag, response.responseText);
} else {
log.trace(`[json schema cache] Response: schema ${uri} no etag`);
}
}
return response.responseText;
} catch (error: unknown) {
if (isXHRResponse(error)) {
if (error.status === 304 && etag && cache) {

log.trace(`[json schema cache] Response: schema ${uri} unchanged etag ${etag}`);

const content = await cache.getSchema(uri, etag);
if (content) {
log.trace(`[json schema cache] Get schema ${uri} etag ${etag} from cache`);
return content;
}
return request(uri);
}

let status = getErrorStatusDescription(error.status);
if (status && error.responseText) {
status = `${status}\n${error.responseText.substring(0, 200)}`;
}
if (!status) {
status = error.toString();
}
return Promise.reject(status);
});
log.trace(`[json schema cache] Respond schema ${uri} error ${status}`);

throw status;
}
throw error;
}
};

return {
getContent: async (uri: string) => {
if (cache && /^https?:\/\/json\.schemastore\.org\//.test(uri)) {
const content = await cache.getSchemaIfAccessedSince(uri, retryTimeoutInMs);
if (content) {
log.trace(`[json schema cache] Schema ${uri} from cache without request (last accessed less than ${retryTimeoutInDays} days ago)`);
return content;
}
}
return request(uri, cache?.getETag(uri));
}
};
}
108 changes: 108 additions & 0 deletions extensions/json-language-features/client/src/node/schemaCache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { promises as fs } from 'fs';
import * as path from 'path';
import { createHash } from 'crypto';
import { Memento } from 'vscode';

interface CacheEntry {
etag: string;
fileName: string;
accessTime: number;
}

interface CacheInfo {
[schemaUri: string]: CacheEntry;
}

const MEMENTO_KEY = 'json-schema-cache';

export class JSONSchemaCache {
private readonly cacheInfo: CacheInfo;

constructor(private readonly schemaCacheLocation: string, private readonly globalState: Memento) {
this.cacheInfo = globalState.get<CacheInfo>(MEMENTO_KEY, {});
}

getETag(schemaUri: string): string | undefined {
return this.cacheInfo[schemaUri]?.etag;
}

async putSchema(schemaUri: string, etag: string, schemaContent: string): Promise<void> {
try {
const fileName = getCacheFileName(schemaUri);
await fs.writeFile(path.join(this.schemaCacheLocation, fileName), schemaContent);
const entry: CacheEntry = { etag, fileName, accessTime: new Date().getTime() };
this.cacheInfo[schemaUri] = entry;
} catch (e) {
delete this.cacheInfo[schemaUri];
} finally {
await this.updateMemento();
}
}

async getSchemaIfAccessedSince(schemaUri: string, expirationDuration: number): Promise<string | undefined> {
const cacheEntry = this.cacheInfo[schemaUri];
if (cacheEntry && cacheEntry.accessTime + expirationDuration >= new Date().getTime()) {
return this.loadSchemaFile(schemaUri, cacheEntry);
}
return undefined;
}

async getSchema(schemaUri: string, etag: string): Promise<string | undefined> {
const cacheEntry = this.cacheInfo[schemaUri];
if (cacheEntry) {
if (cacheEntry.etag === etag) {
return this.loadSchemaFile(schemaUri, cacheEntry);
} else {
this.deleteSchemaFile(schemaUri, cacheEntry);
}
}
return undefined;
}

private async loadSchemaFile(schemaUri: string, cacheEntry: CacheEntry): Promise<string | undefined> {
const cacheLocation = path.join(this.schemaCacheLocation, cacheEntry.fileName);
try {
const content = (await fs.readFile(cacheLocation)).toString();
cacheEntry.accessTime = new Date().getTime();
return content;
} catch (e) {
delete this.cacheInfo[schemaUri];
return undefined;
} finally {
await this.updateMemento();
}
}

private async deleteSchemaFile(schemaUri: string, cacheEntry: CacheEntry): Promise<void> {
const cacheLocation = path.join(this.schemaCacheLocation, cacheEntry.fileName);
delete this.cacheInfo[schemaUri];
await this.updateMemento();
try {
await fs.rm(cacheLocation);
} catch (e) {
// ignore
}
}


// for debugging
public getCacheInfo() {
return this.cacheInfo;
}

private async updateMemento() {
try {
await this.globalState.update(MEMENTO_KEY, this.cacheInfo);
} catch (e) {
// ignore
}
}
}
function getCacheFileName(uri: string): string {
return `${createHash('MD5').update(uri).digest('hex')}.schema.json`;
}
4 changes: 0 additions & 4 deletions extensions/json-language-features/client/src/requests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@

import { Uri } from 'vscode';

export interface RequestService {
getContent(uri: string, encoding?: string): Promise<string>;
}

export function getScheme(uri: string) {
return uri.substr(0, uri.indexOf(':'));
}
Expand Down

0 comments on commit f13f583

Please sign in to comment.