-
Notifications
You must be signed in to change notification settings - Fork 8
/
extractFromJS.js
157 lines (137 loc) · 5.43 KB
/
extractFromJS.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
// Author: Michael Pradel
// Adapted from https://github.com/michaelpradel/DeepBugs
(function() {
const usage = `
Two usage modes:
1) extractFromJS.js <what> --parallel N <fileList.txt> <dir>
Analyze all files in <dir> that are listed in <fileList.txt>, using N parallel instances.
If <fileList.txt> is "all", analyze all files in <dir>.
2) extractFromJS.js <what> --files <list of files>:
Analyze the list of files.
The <what> argument must be one of:
tokens
`;
const process = require("process");
const fs = require("fs");
const walkSync = require("walk-sync");
const {spawn} = require('child_process');
const util = require("./jsExtractionUtil");
const out_folder_name ="../"
const filesPerParallelInstance = 200;
const fileToIDFileName = "fileIDs.json";
function spawnSingleInstance(worklist, what) {
console.log("Left in worklist: " + worklist.length + ". Spawning an instance.");
const jsFiles = worklist.pop();
if (jsFiles) {
const argsToPass = [process.argv[1], what, "--files"].concat(jsFiles);
const cmd = spawn("node", argsToPass);
cmd.on("close", (code) => {
console.log("Instance has finished with exit code " + code);
if (worklist.length > 0) {
spawnSingleInstance(worklist, what);
}
});
cmd.stdout.on('data', (data) => {
console.log(`${data}`);
});
cmd.stderr.on('data', (data) => {
console.log(`${data}`);
});
}
}
function spawnInstances(nbInstances, jsFiles, what) {
const worklist = [];
for (let i = 0; i < jsFiles.length; i += filesPerParallelInstance) {
const chunkOfJSFiles = jsFiles.slice(i, i + filesPerParallelInstance);
worklist.push(chunkOfJSFiles);
}
for (let instance = 0; instance < nbInstances; instance++) {
spawnSingleInstance(worklist, what);
}
}
function getOrCreateFileToID(files) {
let fileToID;
try {
fileToID = JSON.parse(fs.readFileSync(fileToIDFileName, {encoding:"utf8"}));
} catch (_) {
fileToID = {};
}
let maxID = 1;
for (let file in fileToID) {
if (fileToID[file] > maxID) maxID = fileToID[file];
}
let haveAdded = false;
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (!fileToID.hasOwnProperty(file)) {
maxID += 1;
fileToID[file] = maxID;
haveAdded = true;
}
}
if (haveAdded) fs.writeFileSync(out_folder_name+fileToIDFileName, JSON.stringify(fileToID, 0, 2));
return fileToID;
}
function visitFile(jsFile, extractor, allData, fileID) {
//console.log(jsFile,extractor, allData, fileID)
const code = fs.readFileSync(jsFile);
//console.log(code)
const tokens = util.getTokens(code);
const ast = util.getAST(code);
if (tokens && ast) {
const locationMap = util.computeLocationMap(tokens);
extractor.visitCode(ast, locationMap, jsFile, allData, fileID);
} else {
console.log("Ignoring file with parse errors: " + jsFile);
}
}
// read command line arguments
const args = process.argv.slice(2);
const what = args[0];
if (["tokens"].indexOf(what) === -1) {
console.log(usage);
process.exit(1);
}
if (args[1] === "--parallel") {
if (args.length !== 5) {
console.log(usage);
process.exit(1);
}
const nbInstances = args[2];
const fileListFile = args[3];
const dir = args[4];
// filter to use only files in file list
const relativeJsFiles = walkSync(dir, {globs:["**/*.js"], directories:false});
let jsFiles;
if (fileListFile === "all") {
jsFiles = relativeJsFiles.map(f => dir + (dir.endsWith("/") ? "" : "/") + f);
} else {
console.log(fileListFile)
const filesToConsider = new Set(fs.readFileSync(fileListFile, {encoding:"utf8"}).split(/\r?\n/));
console.log(relativeJsFiles)
console.log(filesToConsider)
console.log(dir)
jsFiles = relativeJsFiles.map(f => dir + (dir.endsWith("/") ? "" : "/") + f).filter(p => filesToConsider.has(p));
}
console.log("Total number of files: " + jsFiles.length);
getOrCreateFileToID(jsFiles);
spawnInstances(nbInstances, jsFiles, what);
} else if (args[1] === "--files") {
let extractor;
if (what === "tokens") extractor = require("./extractorOfTokens");
const allData = [];
const jsFiles = args.slice(2);
let fileToID = getOrCreateFileToID(jsFiles);
for (let i = 0; i < jsFiles.length; i++) {
const jsFile = jsFiles[i];
const fileID = fileToID[jsFile]; // assumption: at most 6 digits long
extractor.visitFile(jsFile, allData);
}
const fileName = what + "_" + Date.now() + ".json";
console.log("Writing " + allData.length + " items to file " + fileName);
fs.writeFileSync(fileName, JSON.stringify(allData, null, 2));
} else {
console.log(usage);
process.exit(1);
}
})();