feat: electron-compile support

Close #807

README.md

@@ -17,6 +17,7 @@ A complete solution to package and build a ready for distribution Electron app f
 * Pack in a distributable format [already packaged app](#pack-only-in-a-distributable-format).
 * Separate [build steps](https://github.com/electron-userland/electron-builder/issues/1102#issuecomment-271845854).
 * Build and publish in parallel, using hard links on CI server to reduce IO and disk space usage.
+* [electron-compile](https://github.com/electron/electron-compile) support (compile for release-time on the fly on build).
 
 | Question | Answer |
 |--------|-------|

package.json

@@ -31,7 +31,7 @@
     "ajv": "^5.0.4-beta.0",
     "ajv-keywords": "^2.0.1-beta.2",
     "archiver": "^1.3.0",
-    "aws-sdk": "^2.28.0",
+    "aws-sdk": "^2.29.0",
     "bluebird-lst": "^1.0.2",
     "chalk": "^1.1.3",
     "chromium-pickle-js": "^0.2.0",
@@ -87,8 +87,8 @@
     "jest-junit": "^1.4.0",
     "jsdoc-to-markdown": "^3.0.0",
     "path-sort": "^0.1.0",
-    "source-map-support": "^0.4.13",
-    "ts-babel": "^2.0.0",
+    "source-map-support": "^0.4.14",
+    "ts-babel": "^3.0.0",
     "tslint": "^4.5.1",
     "typescript": "^2.2.1",
     "typescript-json-schema": "0.10.0",

packages/electron-builder-core/src/core.ts

@@ -164,8 +164,6 @@ export interface AsarOptions {
   smartUnpack?: boolean
 
   ordering?: string | null
-
-  extraMetadata?: any | null
 }
 
 export interface BeforeBuildContext {

packages/electron-builder-util/package.json

@@ -19,7 +19,7 @@
     "debug": "2.6.3",
     "node-emoji": "^1.5.1",
     "electron-builder-http": "~0.0.0-semantic-release",
-    "source-map-support": "^0.4.13",
+    "source-map-support": "^0.4.14",
     "7zip-bin": "^2.0.4",
     "ini": "^1.3.4",
     "tunnel-agent": "^0.6.0"

packages/electron-builder/package.json

@@ -63,6 +63,7 @@
     "isbinaryfile": "^3.0.2",
     "js-yaml": "^3.8.2",
     "minimatch": "^3.0.3",
+    "mime": "^1.3.4",
     "node-forge": "^0.7.0",
     "normalize-package-data": "^2.3.6",
     "parse-color": "^1.0.0",

packages/electron-builder/src/appInfo.ts

@@ -18,7 +18,7 @@ export class AppInfo {
     return this.info.config
   }
 
-  constructor(public metadata: Metadata, private info: BuildInfo, buildVersion?: string | null) {
+  constructor(public readonly metadata: Metadata, private readonly info: BuildInfo, buildVersion?: string | null) {
     this.version = metadata.version!
 
     this.buildNumber = this.config.buildVersion || process.env.TRAVIS_BUILD_NUMBER || process.env.APPVEYOR_BUILD_NUMBER || process.env.CIRCLE_BUILD_NUM || process.env.BUILD_NUMBER

packages/electron-builder/src/asar.ts

@@ -77,24 +77,6 @@ export class AsarFilesystem {
     this.offset.add(UINT64(node.size))
   }
 
-  // listFiles() {
-  //   const files: Array<string> = []
-  //   const fillFilesFromHeader = (p: string, json: Node) => {
-  //     if (json.files == null) {
-  //       return
-  //     }
-  //
-  //     for (const f in json.files) {
-  //       const fullPath = path.join(p, f)
-  //       files.push(fullPath)
-  //       fillFilesFromHeader(fullPath, json.files[f]!)
-  //     }
-  //   }
-  //
-  //   fillFilesFromHeader("/", this.header)
-  //   return files
-  // }
-
   getNode(p: string) {
     const node = this.searchNodeFromDirectory(path.dirname(p))
     return node.files![path.basename(p)]

packages/electron-builder/src/asarUtil.ts

@@ -1,23 +1,18 @@
 import BluebirdPromise from "bluebird-lst"
 import { AsarOptions } from "electron-builder-core"
 import { debug } from "electron-builder-util"
-import { deepAssign } from "electron-builder-util/out/deepAssign"
 import { CONCURRENCY, FileCopier, Filter, MAX_FILE_REQUESTS, statOrNull, walk } from "electron-builder-util/out/fs"
 import { log } from "electron-builder-util/out/log"
-import { createReadStream, createWriteStream, ensureDir, readFile, readJson, readlink, stat, Stats, writeFile } from "fs-extra-p"
+import { createReadStream, createWriteStream, ensureDir, readFile, readlink, stat, Stats, writeFile } from "fs-extra-p"
 import * as path from "path"
 import { AsarFilesystem, Node, readAsar } from "./asar"
+import { FileTransformer } from "./fileTransformer"
 
 const isBinaryFile: any = BluebirdPromise.promisify(require("isbinaryfile"))
 const pickle = require ("chromium-pickle-js")
 
 const NODE_MODULES_PATTERN = `${path.sep}node_modules${path.sep}`
 
-export async function createAsarArchive(src: string, resourcesPath: string, options: AsarOptions, filter: Filter, unpackPattern: Filter | null): Promise<any> {
-  // sort files to minimize file change (i.e. asar file is not changed dramatically on small change)
-  await new AsarPackager(src, resourcesPath, options, unpackPattern).pack(filter)
-}
-
 function addValue(map: Map<string, Array<string>>, key: string, value: string) {
   let list = map.get(key)
   if (list == null) {
@@ -47,17 +42,16 @@ function writeUnpackedFiles(filesToUnpack: Array<UnpackedFileTask>, fileCopier:
   })
 }
 
-class AsarPackager {
-  private readonly toPack: Array<string> = []
+export class AsarPackager {
   private readonly fs = new AsarFilesystem(this.src)
-  private readonly changedFiles = new Map<string, string>()
   private readonly outFile: string
 
   constructor(private readonly src: string, destination: string, private readonly options: AsarOptions, private readonly unpackPattern: Filter | null) {
     this.outFile = path.join(destination, "app.asar")
   }
 
-  async pack(filter: Filter) {
+  // sort files to minimize file change (i.e. asar file is not changed dramatically on small change)
+  async pack(filter: Filter, transformer: ((path: string) => any) | null) {
     const metadata = new Map<string, Stats>()
     const files = await walk(this.src, filter, (file, fileStat) => {
       metadata.set(file, fileStat)
@@ -83,12 +77,11 @@ class AsarPackager {
       }
       return null
     })
-    await this.createPackageFromFiles(this.options.ordering == null ? files : await this.order(files), metadata)
-    await this.writeAsarFile()
+
+    await this.createPackageFromFiles(this.options.ordering == null ? files : await this.order(files), metadata, transformer)
   }
 
-  async detectUnpackedDirs(files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, unpackedDest: string, fileIndexToModulePackageData: Map<number, BluebirdPromise<string>>) {
-    const packageJsonStringLength = "package.json".length
+  async detectUnpackedDirs(files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, unpackedDest: string) {
     const dirToCreate = new Map<string, Array<string>>()
 
     /* tslint:disable:rule1 prefer-const */
@@ -109,11 +102,6 @@ class AsarPackager {
       }
 
       const nodeModuleDir = file.substring(0, nextSlashIndex)
-
-      if (file.length === (nodeModuleDir.length + 1 + packageJsonStringLength) && file.endsWith("package.json")) {
-        fileIndexToModulePackageData.set(i, <BluebirdPromise<string>>readJson(file).then(it => cleanupPackageJson(it)))
-      }
-
       if (autoUnpackDirs.has(nodeModuleDir)) {
         const fileParent = path.dirname(file)
         if (fileParent !== nodeModuleDir && !autoUnpackDirs.has(fileParent)) {
@@ -149,11 +137,7 @@ class AsarPackager {
       }
       autoUnpackDirs.add(nodeModuleDir)
     }
-
-    if (fileIndexToModulePackageData.size > 0) {
-      await BluebirdPromise.all(<any>fileIndexToModulePackageData.values())
-    }
-
+
     if (dirToCreate.size > 0) {
       // child directories should be not created asynchronously - parent directories should be created first
       await BluebirdPromise.map(dirToCreate.keys(), async (it) => {
@@ -164,21 +148,20 @@ class AsarPackager {
     }
   }
 
-  async createPackageFromFiles(files: Array<string>, metadata: Map<string, Stats>) {
+  private async createPackageFromFiles(files: Array<string>, metadata: Map<string, Stats>, transformer: FileTransformer | null) {
     // search auto unpacked dir
     const unpackedDirs = new Set<string>()
     const unpackedDest = `${this.outFile}.unpacked`
-    const fileIndexToModulePackageData = new Map<number, BluebirdPromise<string>>()
     await ensureDir(path.dirname(this.outFile))
 
     if (this.options.smartUnpack !== false) {
-      await this.detectUnpackedDirs(files, metadata, unpackedDirs, unpackedDest, fileIndexToModulePackageData)
+      await this.detectUnpackedDirs(files, metadata, unpackedDirs, unpackedDest)
     }
 
     const dirToCreateForUnpackedFiles = new Set<string>(unpackedDirs)
-
+
+    const transformedFiles = transformer == null ? new Array(files.length) : await BluebirdPromise.map(files, it => it.includes("/node_modules/") || it.includes("/bower_components/") || !metadata.get(it)!.isFile() ? null : transformer(it), CONCURRENCY)
     const filesToUnpack: Array<UnpackedFileTask> = []
-    const mainPackageJson = path.join(this.src, "package.json")
     const fileCopier = new FileCopier()
     /* tslint:disable:rule1 prefer-const */
     for (let i = 0, n = files.length; i < n; i++) {
@@ -187,34 +170,15 @@ class AsarPackager {
       if (stat.isFile()) {
         const fileParent = path.dirname(file)
         const dirNode = this.fs.getOrCreateNode(fileParent)
-        const packageDataPromise = fileIndexToModulePackageData.get(i)
-        let newData: string | null = null
-        if (packageDataPromise == null) {
-          if (file === mainPackageJson) {
-            const mainPackageData = await readJson(file)
-            if (this.options.extraMetadata != null) {
-              deepAssign(mainPackageData, this.options.extraMetadata)
-            }
-
-            // https://github.com/electron-userland/electron-builder/issues/1212
-            const serializedDataIfChanged = cleanupPackageJson(mainPackageData)
-            if (serializedDataIfChanged != null) {
-              newData = serializedDataIfChanged
-            }
-            else if (this.options.extraMetadata != null) {
-              newData = JSON.stringify(mainPackageData, null, 2)
-            }
-          }
-        }
-        else {
-          newData = packageDataPromise.value()
-        }
-
-        const fileSize = newData == null ? stat.size : Buffer.byteLength(newData)
+
+        const newData = transformedFiles == null ? null : transformedFiles[i]
         const node = this.fs.getOrCreateNode(file)
-        node.size = fileSize
+        node.size = newData == null ? stat.size : Buffer.byteLength(newData)
         if (dirNode.unpacked || (this.unpackPattern != null && this.unpackPattern(file, stat))) {
           node.unpacked = true
+          if (newData != null) {
+            transformedFiles[i] = null
+          }
 
           if (!dirNode.unpacked && !dirToCreateForUnpackedFiles.has(fileParent)) {
             dirToCreateForUnpackedFiles.add(fileParent)
@@ -229,11 +193,11 @@ class AsarPackager {
           }
         }
         else {
-          if (newData != null) {
-            this.changedFiles.set(file, newData)
+          if (newData == null) {
+            transformedFiles[i] = true
           }
+
           this.fs.insertFileNode(node, stat, file)
-          this.toPack.push(file)
         }
       }
       else if (stat.isDirectory()) {
@@ -263,11 +227,13 @@ class AsarPackager {
     if (filesToUnpack.length > 0) {
       await writeUnpackedFiles(filesToUnpack, fileCopier)
     }
+
+    await this.writeAsarFile(files, transformedFiles)
   }
 
-  private writeAsarFile(): Promise<any> {
+  private writeAsarFile(files: Array<string>, transformedFiles: Array<string | Buffer | null | true>): Promise<any> {
     const headerPickle = pickle.createEmpty()
-    headerPickle.writeString(JSON.stringify(this.fs.header))
+    headerPickle.writeString(JSON.stringify(this.fs.header, (name, value) => name === "data" ? undefined : value))
     const headerBuf = headerPickle.toBuffer()
 
     const sizePickle = pickle.createEmpty()
@@ -280,30 +246,31 @@ class AsarPackager {
       writeStream.on("close", resolve)
       writeStream.write(sizeBuf)
 
-      let w: (list: Array<any>, index: number) => void
-      w = (list: Array<any>, index: number) => {
-        if (list.length === index) {
+      const w = (index: number) => {
+        let data
+        while (index < files.length && (data = transformedFiles[index++]) == null) {
+        }
+
+        if (index >= files.length) {
           writeStream.end()
           return
         }
 
-        const file = list[index]
-
-        const data = this.changedFiles.get(file)
-        if (data != null) {
-          writeStream.write(data, () => w(list, index + 1))
+        const file = files[index - 1]
+        if (data !== true && data != null) {
+          writeStream.write(data, () => w(index))
           return
         }
 
         const readStream = createReadStream(file)
         readStream.on("error", reject)
-        readStream.once("end", () => w(list, index + 1))
+        readStream.once("end", () => w(index))
         readStream.pipe(writeStream, {
           end: false
         })
       }
 
-      writeStream.write(headerBuf, () => w(this.toPack, 0))
+      writeStream.write(headerBuf, () => w(0))
     })
   }
 
@@ -348,27 +315,6 @@ class AsarPackager {
   }
 }
 
-function cleanupPackageJson(data: any): any {
-  try {
-    let changed = false
-    for (const prop of Object.getOwnPropertyNames(data)) {
-      if (prop[0] === "_" || prop === "dist" || prop === "gitHead" || prop === "keywords" || prop === "build" || prop === "devDependencies" || prop === "scripts") {
-        delete data[prop]
-        changed = true
-      }
-    }
-
-    if (changed) {
-      return JSON.stringify(data, null, 2)
-    }
-  }
-  catch (e) {
-    debug(e)
-  }
-
-  return null
-}
-
 export async function checkFileInArchive(asarFile: string, relativeFile: string, messagePrefix: string) {
   function error(text: string) {
     return new Error(`${messagePrefix} "${relativeFile}" in the "${asarFile}" ${text}`)