Skip to content

Support alternative file extensions #1231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 26, 2020
Merged

Support alternative file extensions #1231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f38056
Support alternative file extensions
dcodeIO Apr 17, 2020
0351e51
accept extensions without leading dot
dcodeIO Apr 22, 2020
9c5b271
Merge branch 'master' into extension
dcodeIO Apr 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cli/asc.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ interface CompilerOptions {
printrtti?: boolean;
/** Disables terminal colors. */
noColors?: boolean;
/** Specifies an alternative file extension. */
extension?: string;
}

/** Compiler API options. */
Expand Down
114 changes: 70 additions & 44 deletions cli/asc.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,21 @@ const EOL = process.platform === "win32" ? "\r\n" : "\n";
const SEP = process.platform === "win32" ? "\\" : "/";
const binaryen = global.binaryen || (global.binaryen = require("binaryen"));

// Sets up an extension with its definition counterpart and relevant regexes.
function setupExtension(extension) {
if (!extension.startsWith(".")) extension = "." + extension;
return {
ext: extension,
ext_d: ".d" + extension,
re: new RegExp("\\" + extension + "$"),
re_d: new RegExp("\\.d\\" + extension + "$"),
re_except_d: new RegExp("^(?!.*\\.d\\" + extension + "$).*\\" + extension + "$"),
re_index: new RegExp("(?:^|[\\\\\\/])index\\" + extension + "$")
};
}

const defaultExtension = setupExtension(".ts");

// Proxy Binaryen's ready event
Object.defineProperty(exports, "ready", {
get: function() { return binaryen.ready; }
Expand Down Expand Up @@ -109,23 +124,23 @@ exports.defaultShrinkLevel = 1;
exports.libraryFiles = exports.isBundle ? BUNDLE_LIBRARY : (() => { // set up if not a bundle
const libDir = path.join(__dirname, "..", "std", "assembly");
const bundled = {};
find.files(libDir, find.TS_EXCEPT_DTS)
.forEach(file => bundled[file.replace(/\.ts$/, "")] = fs.readFileSync(path.join(libDir, file), "utf8" ));
find.files(libDir, defaultExtension.re_except_d)
.forEach(file => bundled[file.replace(defaultExtension.re, "")] = fs.readFileSync(path.join(libDir, file), "utf8" ));
return bundled;
})();

/** Bundled definition files. */
exports.definitionFiles = exports.isBundle ? BUNDLE_DEFINITIONS : (() => { // set up if not a bundle
const stdDir = path.join(__dirname, "..", "std");
return {
"assembly": fs.readFileSync(path.join(stdDir, "assembly", "index.d.ts"), "utf8"),
"portable": fs.readFileSync(path.join(stdDir, "portable", "index.d.ts"), "utf8")
"assembly": fs.readFileSync(path.join(stdDir, "assembly", "index" + defaultExtension.ext_d), "utf8"),
"portable": fs.readFileSync(path.join(stdDir, "portable", "index" + defaultExtension.ext_d), "utf8")
};
})();

/** Convenience function that parses and compiles source strings directly. */
exports.compileString = (sources, options) => {
if (typeof sources === "string") sources = { "input.ts": sources };
if (typeof sources === "string") sources = { ["input" + defaultExtension.ext]: sources };
const output = Object.create({
stdout: createMemoryStream(),
stderr: createMemoryStream()
Expand Down Expand Up @@ -169,6 +184,7 @@ exports.main = function main(argv, options, callback) {
const writeFile = options.writeFile || writeFileNode;
const listFiles = options.listFiles || listFilesNode;
const stats = options.stats || createStats();
let extension = defaultExtension;

// Output must be specified if not present in the environment
if (!stdout) throw Error("'options.stdout' must be specified");
Expand Down Expand Up @@ -213,6 +229,15 @@ exports.main = function main(argv, options, callback) {
return callback(null);
}

// Use another extension if requested
if (typeof args.extension === "string") {
if (/^\.?[0-9a-zA-Z]{1,14}$/.test(args.extension)) {
extension = setupExtension(args.extension);
} else {
return callback(Error("Invalid extension: " + args.extension));
}
}

// Print the help message if requested or no source files are provided
if (args.help || !argv.length) {
var out = args.help ? stdout : stderr;
Expand All @@ -222,9 +247,9 @@ exports.main = function main(argv, options, callback) {
" " + color.cyan("asc") + " [entryFile ...] [options]",
"",
color.white("EXAMPLES"),
" " + color.cyan("asc") + " hello.ts",
" " + color.cyan("asc") + " hello.ts -b hello.wasm -t hello.wat",
" " + color.cyan("asc") + " hello1.ts hello2.ts -b -O > hello.wasm",
" " + color.cyan("asc") + " hello" + extension.ext,
" " + color.cyan("asc") + " hello" + extension.ext + " -b hello.wasm -t hello.wat",
" " + color.cyan("asc") + " hello1" + extension.ext + " hello2" + extension.ext + " -b -O > hello.wasm",
"",
color.white("OPTIONS"),
].concat(
Expand Down Expand Up @@ -319,7 +344,7 @@ exports.main = function main(argv, options, callback) {
let transformArgs = args.transform;
for (let i = 0, k = transformArgs.length; i < k; ++i) {
let filename = transformArgs[i].trim();
if (!tsNodeRegistered && filename.endsWith('.ts')) {
if (!tsNodeRegistered && filename.endsWith(".ts")) { // ts-node requires .ts specifically
require("ts-node").register({ transpileOnly: true, skipProject: true, compilerOptions: { target: "ES2016" } });
tsNodeRegistered = true;
}
Expand Down Expand Up @@ -363,7 +388,7 @@ exports.main = function main(argv, options, callback) {
if (libPath.indexOf("/") >= 0) return; // in sub-directory: imported on demand
stats.parseCount++;
stats.parseTime += measure(() => {
assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + ".ts", false);
assemblyscript.parse(program, exports.libraryFiles[libPath], exports.libraryPrefix + libPath + extension.ext, false);
});
});
const customLibDirs = [];
Expand All @@ -374,7 +399,7 @@ exports.main = function main(argv, options, callback) {
for (let i = 0, k = customLibDirs.length; i < k; ++i) { // custom
let libDir = customLibDirs[i];
let libFiles;
if (libDir.endsWith(".ts")) {
if (libDir.endsWith(extension.ext)) {
libFiles = [ path.basename(libDir) ];
libDir = path.dirname(libDir);
} else {
Expand All @@ -385,7 +410,7 @@ exports.main = function main(argv, options, callback) {
let libText = readFile(libPath, libDir);
if (libText === null) return callback(Error("Library file '" + libPath + "' not found."));
stats.parseCount++;
exports.libraryFiles[libPath.replace(/\.ts$/, "")] = libText;
exports.libraryFiles[libPath.replace(extension.re, "")] = libText;
stats.parseTime += measure(() => {
assemblyscript.parse(program, libText, exports.libraryPrefix + libPath, false);
});
Expand All @@ -406,13 +431,13 @@ exports.main = function main(argv, options, callback) {
const libraryPrefix = exports.libraryPrefix;
const libraryFiles = exports.libraryFiles;

// Try file.ts, file/index.ts, file.d.ts
// Try file.ext, file/index.ext, file.d.ext
if (!internalPath.startsWith(libraryPrefix)) {
if ((sourceText = readFile(sourcePath = internalPath + ".ts", baseDir)) == null) {
if ((sourceText = readFile(sourcePath = internalPath + "/index.ts", baseDir)) == null) {
// portable d.ts: uses the .js file next to it in JS or becomes an import in Wasm
sourcePath = internalPath + ".ts";
sourceText = readFile(internalPath + ".d.ts", baseDir);
if ((sourceText = readFile(sourcePath = internalPath + extension.ext, baseDir)) == null) {
if ((sourceText = readFile(sourcePath = internalPath + "/index" + extension.ext, baseDir)) == null) {
// portable d.ext: uses the .js file next to it in JS or becomes an import in Wasm
sourcePath = internalPath + extension.ext;
sourceText = readFile(internalPath + extension.ext_d, baseDir);
}
}

Expand All @@ -422,18 +447,18 @@ exports.main = function main(argv, options, callback) {
const indexName = plainName + "/index";
if (libraryFiles.hasOwnProperty(plainName)) {
sourceText = libraryFiles[plainName];
sourcePath = libraryPrefix + plainName + ".ts";
sourcePath = libraryPrefix + plainName + extension.ext;
} else if (libraryFiles.hasOwnProperty(indexName)) {
sourceText = libraryFiles[indexName];
sourcePath = libraryPrefix + indexName + ".ts";
sourcePath = libraryPrefix + indexName + extension.ext;
} else { // custom lib dirs
for (const libDir of customLibDirs) {
if ((sourceText = readFile(plainName + ".ts", libDir)) != null) {
sourcePath = libraryPrefix + plainName + ".ts";
if ((sourceText = readFile(plainName + extension.ext, libDir)) != null) {
sourcePath = libraryPrefix + plainName + extension.ext;
break;
} else {
if ((sourceText = readFile(indexName + ".ts", libDir)) != null) {
sourcePath = libraryPrefix + indexName + ".ts";
if ((sourceText = readFile(indexName + extension.ext, libDir)) != null) {
sourcePath = libraryPrefix + indexName + extension.ext;
break;
}
}
Expand Down Expand Up @@ -463,25 +488,25 @@ exports.main = function main(argv, options, callback) {
try {
let json = JSON.parse(jsonText);
if (typeof json.ascMain === "string") {
mainPath = json.ascMain.replace(/[\/\\]index\.ts$/, "");
mainPath = json.ascMain.replace(extension.re_index, "");
packageMains.set(packageName, mainPath);
}
} catch (e) { }
}
}
const mainDir = path.join(currentPath, packageName, mainPath);
const plainName = filePath;
if ((sourceText = readFile(path.join(mainDir, plainName + ".ts"), baseDir)) != null) {
sourcePath = libraryPrefix + packageName + "/" + plainName + ".ts";
packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName));
if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, plainName + ".ts") + EOL);
if ((sourceText = readFile(path.join(mainDir, plainName + extension.ext), baseDir)) != null) {
sourcePath = libraryPrefix + packageName + "/" + plainName + extension.ext;
packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName));
if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, plainName + extension.ext) + EOL);
break;
} else if (!isPackageRoot) {
const indexName = filePath + "/index";
if ((sourceText = readFile(path.join(mainDir, indexName + ".ts"), baseDir)) !== null) {
sourcePath = libraryPrefix + packageName + "/" + indexName + ".ts";
packageBases.set(sourcePath.replace(/\.ts$/, ""), path.join(currentPath, packageName));
if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, indexName + ".ts") + EOL);
if ((sourceText = readFile(path.join(mainDir, indexName + extension.ext), baseDir)) !== null) {
sourcePath = libraryPrefix + packageName + "/" + indexName + extension.ext;
packageBases.set(sourcePath.replace(extension.re, ""), path.join(currentPath, packageName));
if (args.traceResolution) stderr.write(" -> " + path.join(mainDir, indexName + extension.ext) + EOL);
break;
}
}
Expand Down Expand Up @@ -519,33 +544,34 @@ exports.main = function main(argv, options, callback) {
let runtimeText = exports.libraryFiles[runtimePath];
if (runtimeText == null) {
runtimePath = runtimeName;
runtimeText = readFile(runtimePath + ".ts", baseDir);
runtimeText = readFile(runtimePath + extension.ext, baseDir);
if (runtimeText == null) return callback(Error("Runtime '" + runtimeName + "' not found."));
} else {
runtimePath = "~lib/" + runtimePath;
}
stats.parseCount++;
stats.parseTime += measure(() => {
assemblyscript.parse(program, runtimeText, runtimePath, true);
assemblyscript.parse(program, runtimeText, runtimePath + extension.ext, true);
});
}

// Include entry files
for (let i = 0, k = argv.length; i < k; ++i) {
const filename = argv[i];

let sourcePath = String(filename).replace(/\\/g, "/").replace(/(\.ts|\/)$/, "");
let sourcePath = String(filename).replace(/\\/g, "/").replace(extension.re, "").replace(/[\\\/]$/, "");

// Setting the path to relative path
sourcePath = path.isAbsolute(sourcePath) ? path.relative(baseDir, sourcePath) : sourcePath;

// Try entryPath.ts, then entryPath/index.ts
let sourceText = readFile(sourcePath + ".ts", baseDir);
// Try entryPath.ext, then entryPath/index.ext
let sourceText = readFile(sourcePath + extension.ext, baseDir);
if (sourceText == null) {
sourceText = readFile(sourcePath + "/index.ts", baseDir);
if (sourceText == null) return callback(Error("Entry file '" + sourcePath + ".ts' not found."));
sourcePath += "/index.ts";
sourceText = readFile(sourcePath + "/index" + extension.ext, baseDir);
if (sourceText == null) return callback(Error("Entry file '" + sourcePath + extension.ext + "' not found."));
sourcePath += "/index" + extension.ext;
} else {
sourcePath += ".ts";
sourcePath += extension.ext;
}

stats.parseCount++;
Expand Down Expand Up @@ -741,7 +767,7 @@ exports.main = function main(argv, options, callback) {
map.sourceRoot = "./" + basename;
let contents = [];
map.sources.forEach((name, index) => {
let text = assemblyscript.getSource(program, name.replace(/\.ts$/, ""));
let text = assemblyscript.getSource(program, name.replace(extension.re, ""));
if (text == null) return callback(Error("Source of file '" + name + "' not found."));
contents[index] = text;
});
Expand Down Expand Up @@ -882,7 +908,7 @@ exports.main = function main(argv, options, callback) {
var files;
try {
stats.readTime += measure(() => {
files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => /^(?!.*\.d\.ts$).*\.ts$/.test(file));
files = fs.readdirSync(path.join(baseDir, dirname)).filter(file => extension.re_except_d.test(file))
});
return files;
} catch (e) {
Expand Down
4 changes: 4 additions & 0 deletions cli/asc.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,10 @@
"type": "s",
"default": "."
},
"extension": {
"description": "Specifies an alternative file extension to use.",
"type": "s"
},
"noUnsafe": {
"description": [
"Disallows the use of unsafe features in user code.",
Expand Down
2 changes: 0 additions & 2 deletions cli/util/find.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,3 @@
*/

export function files(dirname: string, filter?: ((name: string) => bool) | RegExp): string[];
export const TS: RegExp;
export const TS_EXCEPT_DTS: RegExp;
3 changes: 0 additions & 3 deletions cli/util/find.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,3 @@ function findFiles(dirname, filter) {
}

exports.files = findFiles;

exports.TS = /\.ts$/;
exports.TS_EXCEPT_DTS = /(?:(?!\.d).{2}|^.{0,1})\.ts$/;
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@
"check": "npm run check:config && npm run check:compiler && tsc --noEmit --target ESNEXT --module commonjs --experimentalDecorators tests/require/index",
"check:config": "tsc --noEmit -p src --diagnostics --listFiles",
"check:compiler": "tslint -c tslint.json --project src --formatters-dir lib/lint/formatters --format as",
"test": "npm run test:parser && npm run test:compiler && npm run test:packages",
"test": "npm run test:parser && npm run test:compiler && npm run test:packages && npm run test:extension",
"test:parser": "node tests/parser",
"test:compiler": "node tests/compiler",
"test:packages": "cd tests/packages && npm run test",
"test:extension": "cd tests/extension && npm run test",
"make": "npm run clean && npm test && npm run build && npm test",
"all": "npm run check && npm run make",
"docs": "typedoc --tsconfig tsconfig-docs.json --mode modules --name \"AssemblyScript Compiler API\" --out ./docs/api --ignoreCompilerErrors --excludeNotExported --excludePrivate --excludeExternals --exclude **/std/** --includeDeclarations --readme src/README.md",
Expand Down
23 changes: 18 additions & 5 deletions src/ast.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ import {
import {
normalizePath,
resolvePath,
CharCode
CharCode,
isTrivialAlphanum
} from "./util";

/** Indicates the kind of a node. */
Expand Down Expand Up @@ -716,7 +717,7 @@ export abstract class Node {
} else { // absolute
if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
}
node.internalPath = mangleInternalPath(normalizedPath);
node.internalPath = normalizedPath;
} else {
node.internalPath = null;
}
Expand Down Expand Up @@ -804,7 +805,7 @@ export abstract class Node {
} else { // absolute in library
if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
}
node.internalPath = mangleInternalPath(normalizedPath);
node.internalPath = normalizedPath;
return node;
}

Expand All @@ -825,7 +826,7 @@ export abstract class Node {
} else {
if (!normalizedPath.startsWith(LIBRARY_PREFIX)) normalizedPath = LIBRARY_PREFIX + normalizedPath;
}
node.internalPath = mangleInternalPath(normalizedPath);
node.internalPath = normalizedPath;
return node;
}

Expand Down Expand Up @@ -2088,7 +2089,19 @@ export function findDecorator(kind: DecoratorKind, decorators: DecoratorNode[] |

/** Mangles an external to an internal path. */
export function mangleInternalPath(path: string): string {
if (path.endsWith(".ts")) path = path.substring(0, path.length - 3);
var pos = path.lastIndexOf(".");
var len = path.length;
if (pos >= 0 && len - pos >= 2) { // at least one char plus dot
let cur = pos;
while (++cur < len) {
if (!isTrivialAlphanum(path.charCodeAt(cur))) {
assert(false); // not a valid external path
return path;
}
}
return path.substring(0, pos);
}
assert(false); // not an external path
return path;
}

Expand Down
2 changes: 1 addition & 1 deletion src/parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ export class Parser extends DiagnosticEmitter {
/** Whether this is an entry file. */
isEntry: bool
): void {
// the frontend gives us paths with .ts endings
// the frontend gives us paths with file extensions
var normalizedPath = normalizePath(path);
var internalPath = mangleInternalPath(normalizedPath);
// check if already processed
Expand Down
2 changes: 1 addition & 1 deletion src/program.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ export class Program extends DiagnosticEmitter {
) {
super(diagnostics);
this.options = options;
var nativeSource = new Source(LIBRARY_SUBST, "[native code]", SourceKind.LIBRARY_ENTRY);
var nativeSource = new Source(LIBRARY_SUBST + ".wasm", "[native code]", SourceKind.LIBRARY_ENTRY);
this.nativeSource = nativeSource;
var nativeFile = new File(this, nativeSource);
this.nativeFile = nativeFile;
Expand Down
Loading