Skip to content

Commit

Permalink
Support loading additional TS lib files
Browse files Browse the repository at this point in the history
Fixes #3726

This PR provides support for referencing other lib files that are not
used by default in Deno via the compiler APIs.
  • Loading branch information
kitsonk committed Feb 18, 2020
1 parent 3d5bed3 commit 2b111e6
Show file tree
Hide file tree
Showing 22 changed files with 284 additions and 67 deletions.
3 changes: 1 addition & 2 deletions cli/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,12 @@ fn op_fetch_asset(
) -> impl Fn(&[u8], Option<ZeroCopyBuf>) -> CoreOp {
move |control: &[u8], zero_copy_buf: Option<ZeroCopyBuf>| -> CoreOp {
assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op.
let custom_assets = custom_assets.clone();
let name = std::str::from_utf8(control).unwrap();

let asset_code = if let Some(source_code) = deno_typescript::get_asset(name)
{
source_code.to_string()
} else if let Some(asset_path) = custom_assets.get(name) {
} else if let Some(asset_path) = custom_assets.clone().get(name) {
let source_code_vec =
std::fs::read(&asset_path).expect("Asset not found");
let source_code = std::str::from_utf8(&source_code_vec).unwrap();
Expand Down
27 changes: 9 additions & 18 deletions cli/file_fetcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ impl SourceFileFetcher {
maybe_referrer: Option<ModuleSpecifier>,
) -> Pin<Box<SourceFileFuture>> {
let module_url = specifier.as_url().to_owned();
debug!("fetch_source_file. specifier {} ", &module_url);
debug!("fetch_source_file_async specifier: {} ", &module_url);

// Check if this file was already fetched and can be retrieved from in-process cache.
if let Some(source_file) = self.source_file_cache.get(specifier.to_string())
Expand Down Expand Up @@ -368,18 +368,13 @@ impl SourceFileFetcher {
}
Ok(c) => c,
};
let media_type = map_content_type(
&filepath,
source_code_headers.mime_type.as_ref().map(String::as_str),
);
let media_type =
map_content_type(&filepath, source_code_headers.mime_type.as_deref());
let types_url = match media_type {
msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
&module_url,
&source_code,
source_code_headers
.x_typescript_types
.as_ref()
.map(String::as_str),
source_code_headers.x_typescript_types.as_deref(),
),
_ => None,
};
Expand Down Expand Up @@ -515,17 +510,13 @@ impl SourceFileFetcher {
.location
.join(dir.deps_cache.get_cache_filename(&module_url));

let media_type = map_content_type(
&filepath,
maybe_content_type.as_ref().map(String::as_str),
);
let media_type =
map_content_type(&filepath, maybe_content_type.as_deref());

let types_url = match media_type {
msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
&module_url,
&source,
x_typescript_types.as_ref().map(String::as_str),
),
msg::MediaType::JavaScript | msg::MediaType::JSX => {
get_types_url(&module_url, &source, x_typescript_types.as_deref())
}
_ => None,
};

Expand Down
4 changes: 4 additions & 0 deletions cli/js/compiler_api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ export interface CompilerOptions {
* Does not apply to `"esnext"` target. */
useDefineForClassFields?: boolean;

/** List of library files to be included in the compilation. If omitted,
* then the Deno main runtime libs are used. */
lib?: string[];

/** The locale to use to show error messages. */
locale?: string;

Expand Down
16 changes: 16 additions & 0 deletions cli/js/compiler_api_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,22 @@ test(async function compilerApiCompileOptions() {
assert(actual["/foo.js"].startsWith("define("));
});

test(async function compilerApiCompileLib() {
const [diagnostics, actual] = await compile(
"/foo.ts",
{
"/foo.ts": `console.log(document.getElementById("foo"));
console.log(Deno.args);`
},
{
lib: ["dom", "es2018", "deno.ns"]
}
);
assert(diagnostics == null);
assert(actual);
assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
});

test(async function transpileOnlyApi() {
const actual = await transpileOnly({
"foo.ts": `export enum Foo { Foo, Bar, Baz };\n`
Expand Down
20 changes: 7 additions & 13 deletions cli/js/compiler_bootstrap.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.

import { ASSETS, CompilerHostTarget, Host } from "./compiler_host.ts";
import { CompilerHostTarget, Host } from "./compiler_host.ts";
import { ASSETS } from "./compiler_sourcefile.ts";
import { getAsset } from "./compiler_util.ts";

// NOTE: target doesn't really matter here,
Expand All @@ -14,18 +15,11 @@ const options = host.getCompilationSettings();

// This is a hacky way of adding our libs to the libs available in TypeScript()
// as these are internal APIs of TypeScript which maintain valid libs
/* eslint-disable @typescript-eslint/no-explicit-any */
(ts as any).libs.push(
"deno_ns",
"deno_window",
"deno_worker",
"deno_shared_globals"
);
(ts as any).libMap.set("deno_ns", "lib.deno.ns.d.ts");
(ts as any).libMap.set("deno_window", "lib.deno.window.d.ts");
(ts as any).libMap.set("deno_worker", "lib.deno.worker.d.ts");
(ts as any).libMap.set("deno_shared_globals", "lib.deno.shared_globals.d.ts");
/* eslint-enable @typescript-eslint/no-explicit-any */
ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals");
ts.libMap.set("deno.ns", "lib.deno.ns.d.ts");
ts.libMap.set("deno.window", "lib.deno.window.d.ts");
ts.libMap.set("deno.worker", "lib.deno.worker.d.ts");
ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts");

// this pre-populates the cache at snapshot time of our library files, so they
// are available in the future when needed.
Expand Down
21 changes: 14 additions & 7 deletions cli/js/compiler_host.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.

import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
import { ASSETS, MediaType, SourceFile } from "./compiler_sourcefile.ts";
import { OUT_DIR, WriteFileCallback, getAsset } from "./compiler_util.ts";
import { cwd } from "./dir.ts";
import { assert, notImplemented } from "./util.ts";
Expand All @@ -18,8 +18,14 @@ export enum CompilerHostTarget {
}

export interface CompilerHostOptions {
/** Flag determines if the host should assume a single bundle output. */
bundle?: boolean;

/** Determines what the default library that should be used when type checking
* TS code. */
target: CompilerHostTarget;

/** A function to be used when the program emit occurs to write out files. */
writeFile: WriteFileCallback;
}

Expand All @@ -28,8 +34,6 @@ export interface ConfigureResponse {
diagnostics?: ts.Diagnostic[];
}

export const ASSETS = "$asset$";

/** Options that need to be used when generating a bundle (either trusted or
* runtime). */
export const defaultBundlerOptions: ts.CompilerOptions = {
Expand Down Expand Up @@ -96,7 +100,6 @@ const ignoredCompilerOptions: readonly string[] = [
"inlineSources",
"init",
"isolatedModules",
"lib",
"listEmittedFiles",
"listFiles",
"mapRoot",
Expand Down Expand Up @@ -141,7 +144,10 @@ export class Host implements ts.CompilerHost {
private _writeFile: WriteFileCallback;

private _getAsset(filename: string): SourceFile {
const url = filename.split("/").pop()!;
const lastSegment = filename.split("/").pop()!;
const url = ts.libMap.has(lastSegment)
? ts.libMap.get(lastSegment)!
: lastSegment;
const sourceFile = SourceFile.get(url);
if (sourceFile) {
return sourceFile;
Expand All @@ -150,7 +156,7 @@ export class Host implements ts.CompilerHost {
const sourceCode = getAsset(name);
return new SourceFile({
url,
filename,
filename: `${ASSETS}/${name}`,
mediaType: MediaType.TypeScript,
sourceCode
});
Expand Down Expand Up @@ -230,6 +236,7 @@ export class Host implements ts.CompilerHost {
}

getDefaultLibFileName(_options: ts.CompilerOptions): string {
util.log("compiler::host.getDefaultLibFileName()");
switch (this._target) {
case CompilerHostTarget.Main:
case CompilerHostTarget.Runtime:
Expand Down Expand Up @@ -259,7 +266,7 @@ export class Host implements ts.CompilerHost {
if (!sourceFile.tsSourceFile) {
assert(sourceFile.sourceCode != null);
sourceFile.tsSourceFile = ts.createSourceFile(
fileName,
fileName.startsWith(ASSETS) ? sourceFile.filename : fileName,
sourceFile.sourceCode,
languageVersion
);
Expand Down
14 changes: 12 additions & 2 deletions cli/js/compiler_sourcefile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ export interface SourceFileJson {
sourceCode: string;
}

export const ASSETS = "$asset$";

/** Returns the TypeScript Extension enum for a given media type. */
function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
switch (mediaType) {
Expand Down Expand Up @@ -109,7 +111,7 @@ export class SourceFile {
this.processed = true;
const files = (this.importedFiles = [] as Array<[string, string]>);

function process(references: ts.FileReference[]): void {
function process(references: Array<{ fileName: string }>): void {
for (const { fileName } of references) {
files.push([fileName, fileName]);
}
Expand All @@ -133,7 +135,15 @@ export class SourceFile {
process(importedFiles);
}
process(referencedFiles);
process(libReferenceDirectives);
// built in libs comes across as `"dom"` for example, and should be filtered
// out during pre-processing as they are either already cached or they will
// be lazily fetched by the compiler host. Ones that contain full files are
// not filtered out and will be fetched as normal.
process(
libReferenceDirectives.filter(
({ fileName }) => !ts.libMap.has(fileName.toLowerCase())
)
);
process(typeReferenceDirectives);
return files;
}
Expand Down
22 changes: 13 additions & 9 deletions cli/js/compiler_util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,24 +92,28 @@ function cache(
}

let OP_FETCH_ASSET: number;
const encoder = new TextEncoder();
const decoder = new TextDecoder();

/**
* This op is called only during snapshotting.
*
* We really don't want to depend on JSON dispatch
* during snapshotting, so this op exchanges strings with Rust
* as raw byte arrays.
*/
/** Retrieve an asset from Rust. */
export function getAsset(name: string): string {
// this path should only be called for assets that are lazily loaded at
// runtime
if (dispatch.OP_FETCH_ASSET) {
util.log("compiler_util::getAsset", name);
return sendSync(dispatch.OP_FETCH_ASSET, { name }).sourceCode;
}

// this path should only be taken during snapshotting
if (!OP_FETCH_ASSET) {
const ops = core.ops();
const opFetchAsset = ops["fetch_asset"];
assert(opFetchAsset, "OP_FETCH_ASSET is not registered");
OP_FETCH_ASSET = opFetchAsset;
}

const encoder = new TextEncoder();
const decoder = new TextDecoder();
// We really don't want to depend on JSON dispatch during snapshotting, so
// this op exchanges strings with Rust as raw byte arrays.
const sourceCodeBytes = core.dispatch(OP_FETCH_ASSET, encoder.encode(name));
return decoder.decode(sourceCodeBytes!);
}
Expand Down
5 changes: 1 addition & 4 deletions cli/js/dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export let OP_APPLY_SOURCE_MAP: number;
export let OP_FORMAT_ERROR: number;
export let OP_CACHE: number;
export let OP_RESOLVE_MODULES: number;
export let OP_FETCH_ASSET: number;
export let OP_FETCH_SOURCE_FILES: number;
export let OP_OPEN: number;
export let OP_CLOSE: number;
Expand Down Expand Up @@ -76,10 +77,6 @@ export let OP_SIGNAL_BIND: number;
export let OP_SIGNAL_UNBIND: number;
export let OP_SIGNAL_POLL: number;

/** **WARNING:** This is only available during the snapshotting process and is
* unavailable at runtime. */
export let OP_FETCH_ASSET: number;

const PLUGIN_ASYNC_HANDLER_MAP: Map<number, AsyncHandler> = new Map();

export function setPluginAsyncHandler(
Expand Down
6 changes: 5 additions & 1 deletion cli/js/lib.deno.ns.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ declare namespace Deno {
/** UNSTABLE: might move to Deno.symbols */
export const EOF: unique symbol;

/** UNSTABLE: might move to Deno.symbols */
/** UNSTABLE: might move to Deno.symbols */
export type EOF = typeof EOF;

/** UNSTABLE: maybe remove "SEEK_" prefix. Maybe capitalization wrong. */
Expand Down Expand Up @@ -1917,6 +1917,10 @@ declare namespace Deno {
* Does not apply to `"esnext"` target. */
useDefineForClassFields?: boolean;

/** List of library files to be included in the compilation. If omitted,
* then the Deno main runtime libs are used. */
lib?: string[];

/** The locale to use to show error messages. */
locale?: string;

Expand Down
5 changes: 4 additions & 1 deletion cli/js/lib.deno.shared_globals.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */

/// <reference no-default-lib="true" />
/// <reference lib="deno_ns" />
// TODO: we need to remove this, but Fetch::Response::Body implements Reader
// which requires Deno.EOF, and we shouldn't be leaking that, but https_proxy
// at the least requires the Reader interface on Body, which it shouldn't
/// <reference lib="deno.ns" />
/// <reference lib="esnext" />

// https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope
Expand Down
4 changes: 2 additions & 2 deletions cli/js/lib.deno.window.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */

/// <reference no-default-lib="true" />
/// <reference lib="deno_ns" />
/// <reference lib="deno_shared_globals" />
/// <reference lib="deno.ns" />
/// <reference lib="deno.shared_globals" />
/// <reference lib="esnext" />

declare interface Window extends WindowOrWorkerGlobalScope {
Expand Down
2 changes: 1 addition & 1 deletion cli/js/lib.deno.worker.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
/* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */

/// <reference no-default-lib="true" />
/// <reference lib="deno_shared_globals" />
/// <reference lib="deno.shared_globals" />
/// <reference lib="esnext" />

declare interface DedicatedWorkerGlobalScope extends WindowOrWorkerGlobalScope {
Expand Down
7 changes: 7 additions & 0 deletions cli/js/ts_global.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,11 @@ declare global {
namespace ts {
export = ts_;
}

namespace ts {
// this are marked @internal in TypeScript, but we need to access them,
// there is a risk these could change in future versions of TypeScript
export const libs: string[];
export const libMap: Map<string, string>;
}
}
Loading

0 comments on commit 2b111e6

Please sign in to comment.