Eliminate duplicate css files using hashing.

Parsing css files is by far the most time consuming part of the aggregator. Over 90% of the wall time is used parsing css into ASTs. In a test project with around 250 css files, with many duplicates, parsing takes around 14 seconds on s 2,5GHz Core i5 M processer, while all the other tasks combined take less than two seconds. Therefore we want to do as little parsing as possible.

The changes in this commit compares the xxHash of each file with a set of hashes of already processed files. Duplicates are then eliminated before they are handed over to the css parser. This results in a factor 8 speedup of the aggregation function on the test project, which now completes in around two seconds.

Author: andersea

package.json

@@ -46,6 +46,7 @@
     "tslint": "latest"
   },
   "dependencies": {
-    "css": "^2.2.1"
+    "css": "^2.2.1",
+    "xxhashjs": "^0.2.1"
   }
 }

src/cssAggregator.ts

@@ -1,6 +1,7 @@
 import { workspace } from 'vscode'
 import { parse, Rule } from 'css';
 import { readFile } from 'fs'
+let XXH = require('xxhashjs').h32;
 
 function flatten<T>(nestedArray: T[][]): T[] {
     if (nestedArray.length === 0) {
@@ -28,8 +29,11 @@ function findClassName(selector: string): string {
 
 export default function () {
 
+    let startTime = process.hrtime();
+    let cssHashSet = new Set();
+
     return workspace.findFiles('**/*.css', '').then(uris => {
-        
+
         let cssTextPromises = uris.map(uri =>
             <PromiseLike<string>>new Promise<string>((resolve, reject) =>
                 readFile(uri.fsPath, workspace.getConfiguration('files').get('encoding'), (err, data) => {
@@ -42,40 +46,57 @@ export default function () {
             )
         );
 
-    let cssASTPromises = cssTextPromises.map(
-        cssTextPromise => cssTextPromise.then(cssText => parse(cssText), console.log)
-    );
+        let uniqueCssTextsPromise = Promise.all(cssTextPromises).then(
+            cssTexts => cssTexts.filter(cssText => {
+                let hash = XXH(cssText, 0x1337).toNumber();
+                if (cssHashSet.has(hash)) {
+                    return false;
+                } else {
+                    cssHashSet.add(hash);
+                    return true;
+                }
+            })
+        );
+
+        let cssASTsPromise = uniqueCssTextsPromise.then(
+            uniqueCssTexts => uniqueCssTexts.map(cssText => parse(cssText)),
+            console.log
+        );
 
-    let rulesPromises = cssASTPromises.map(
-        cssASTPromise => cssASTPromise.then(cssAST =>
-            <Rule[]>(cssAST.stylesheet.rules.filter(node => (<Rule>node).type === 'rule')),
-            console.log)
-    );
+        let rulesPromise = cssASTsPromise.then(cssASTs => {
+            if (cssASTs.length > 0) {
+                return flatten(cssASTs.map(
+                    cssAST => <Rule[]>(cssAST.stylesheet.rules.filter(node => (<Rule>node).type === 'rule'))
+                ))
+            } else {
+                return [];
+            }
+        }, console.log);
 
-    let selectorsPromises = rulesPromises.map(
-        rulesPromise => rulesPromise.then(rules => {
+        let selectorsPromise = rulesPromise.then(rules => {
             if (rules.length > 0) {
                 return flatten(rules.map(rule => rule.selectors));
             } else {
                 return [];
             }
-        }, console.log)
-    );
+        }, console.log);
 
-    let cssClassesPromises = selectorsPromises.map(
-        selectorsPromise => selectorsPromise.then(selectors =>
+        let cssClassesPromise = selectorsPromise.then(selectors =>
             selectors.map(selector => findClassName(selector)).filter(value => value !== ""),
-            console.log)
-    );
+            console.log
+        );
+
+        return cssClassesPromise.then(cssClasses => {
+            let uniqueCssClasses = Array.from(new Set(cssClasses));
+
+            let elapsedTime = process.hrtime(startTime);
+
+            console.log(`Elapsed time: ${elapsedTime[0]} s ${Math.trunc(elapsedTime[1] / 1e6)} ms`);
+            console.log(`Files processed: ${cssHashSet.size}`);
+            console.log(`cssClasses discovered: ${uniqueCssClasses.length}`);
+            
 
-    return Promise.all(cssClassesPromises).then(cssClassesInAllFiles => {
-        let allcssClasses = Array.from(new Set(
-            cssClassesInAllFiles
-                .filter(cssClasses => cssClasses.length > 0)
-                .reduce((p, c) => p.concat(c))
-        ));
-       
-        return allcssClasses;
+            return uniqueCssClasses;
+        }, console.log);
     }, console.log);
-}, console.log);
 }