feat: linked dirs outside of projects (e.g. linked modules)

Closes electron-userland#675

packages/electron-builder-util/src/fs.ts

@@ -56,27 +56,51 @@ export async function walk(initialDirPath: string, filter?: Filter | null, consu
     childNames.sort()
 
     const dirs: Array<string> = []
-    await BluebirdPromise.map(childNames, name => {
+    // our handler is async, but we should add sorted files, so, we add file to result not in the mapper, but after map
+    const sortedFilePaths = await BluebirdPromise.map(childNames, name => {
       const filePath = dirPath + path.sep + name
       return lstat(filePath)
         .then(stat => {
           if (filter != null && !filter(filePath, stat)) {
-            return
+            return null
           }
 
-          if (stat.isDirectory()) {
-            dirs.push(filePath)
+          const consumerResult = consumer == null ? null : consumer(filePath, stat, dirPath)
+          if (consumerResult == null || !("then" in consumerResult)) {
+            if (stat.isDirectory()) {
+              dirs.push(name)
+              return null
+            }
+            else {
+              return filePath
+            }
           }
           else {
-            result.push(filePath)
+            return (<Promise<any>>consumerResult)
+              .then(it => {
+                // asarUtil can return modified stat (symlink handling)
+                if (it != null && "isDirectory" in it && (<Stats>it).isDirectory()) {
+                  dirs.push(name)
+                  return null
+                }
+                else {
+                  result.push(filePath)
+                  return filePath
+                }
+              })
           }
-
-          return consumer == null ? null : consumer(filePath, stat, dirPath)
         })
     }, CONCURRENCY)
 
-    for (let i = dirs.length - 1; i > -1; i--) {
-      queue.push(dirs[i])
+    for (const child of sortedFilePaths) {
+      if (child != null) {
+        result.push(child)
+      }
+    }
+
+    dirs.sort()
+    for (const child of dirs) {
+      queue.push(dirPath + path.sep + child)
     }
   }
 

packages/electron-builder/src/asarUtil.ts

@@ -1,6 +1,6 @@
 import { AsarFileInfo, listPackage, statFile, AsarOptions } from "asar-electron-builder"
 import { debug } from "electron-builder-util"
-import { readFile, Stats, createWriteStream, ensureDir, createReadStream, readJson, writeFile, realpath } from "fs-extra-p"
+import { readFile, Stats, createWriteStream, ensureDir, createReadStream, readJson, writeFile, readlink, stat } from "fs-extra-p"
 import BluebirdPromise from "bluebird-lst-c"
 import * as path from "path"
 import { log } from "electron-builder-util/out/log"
@@ -12,7 +12,7 @@ const pickle = require ("chromium-pickle-js")
 const Filesystem = require("asar-electron-builder/lib/filesystem")
 const UINT64 = require("cuint").UINT64
 
-const NODE_MODULES_PATTERN = path.sep + "node_modules" + path.sep
+const NODE_MODULES_PATTERN = `${path.sep}node_modules${path.sep}`
 
 export async function createAsarArchive(src: string, resourcesPath: string, options: AsarOptions, filter: Filter, unpackPattern: Filter | null): Promise<any> {
   // sort files to minimize file change (i.e. asar file is not changed dramatically on small change)
@@ -54,26 +54,40 @@ class AsarPackager {
   private readonly changedFiles = new Map<string, string>()
   private readonly outFile: string
 
-  private srcRealPath: Promise<string>
-
   constructor(private readonly src: string, destination: string, private readonly options: AsarOptions, private readonly unpackPattern: Filter | null) {
     this.outFile = path.join(destination, "app.asar")
   }
 
   async pack(filter: Filter) {
     const metadata = new Map<string, Stats>()
-    const files = await walk(this.src, filter, (it, stat) => metadata.set(it, stat))
+    const files = await walk(this.src, filter, (file, fileStat) => {
+      metadata.set(file, fileStat)
+      if (fileStat.isSymbolicLink()) {
+        return readlink(file)
+          .then((linkTarget): any => {
+            // http://unix.stackexchange.com/questions/105637/is-symlinks-target-relative-to-the-destinations-parent-directory-and-if-so-wh
+            const resolved = path.resolve(path.dirname(file), linkTarget)
+            const link = path.relative(this.src, linkTarget)
+            if (link.startsWith("..")) {
+              // outside of project, linked module (https://github.com/electron-userland/electron-builder/issues/675)
+              return stat(resolved)
+                .then(targetFileStat => {
+                  metadata.set(file, targetFileStat)
+                  return targetFileStat
+                })
+            }
+            else {
+              (<any>fileStat).relativeLink = link
+            }
+            return null
+          })
+      }
+      return null
+    })
     await this.createPackageFromFiles(this.options.ordering == null ? files : await this.order(files), metadata)
     await this.writeAsarFile()
   }
 
-  getSrcRealPath(): Promise<string> {
-    if (this.srcRealPath == null) {
-      this.srcRealPath = realpath(this.src)
-    }
-    return this.srcRealPath
-  }
-
   async detectUnpackedDirs(files: Array<string>, metadata: Map<string, Stats>, autoUnpackDirs: Set<string>, unpackedDest: string, fileIndexToModulePackageData: Map<number, BluebirdPromise<string>>) {
     const packageJsonStringLength = "package.json".length
     const dirToCreate = new Map<string, Array<string>>()
@@ -241,7 +255,7 @@ class AsarPackager {
         this.fs.insertDirectory(file, unpacked)
       }
       else if (stat.isSymbolicLink()) {
-        await this.addLink(file)
+        this.fs.searchNodeFromPath(file).link = (<any>stat).relativeLink
       }
     }
 
@@ -250,17 +264,6 @@ class AsarPackager {
     }
   }
 
-  private async addLink(file: string) {
-    const realFile = await realpath(file)
-    const link = path.relative(await this.getSrcRealPath(), realFile)
-    if (link.startsWith("..")) {
-      throw new Error(`${realFile}: file links out of the package`)
-    }
-    else {
-      this.fs.searchNodeFromPath(file).link = link
-    }
-  }
-
   private writeAsarFile(): Promise<any> {
     const headerPickle = pickle.createEmpty()
     headerPickle.writeString(JSON.stringify(this.fs.header))

packages/electron-builder/src/macPackager.ts

@@ -81,7 +81,7 @@ export default class MacPackager extends PlatformPackager<MacOptions> {
         .then(async() => {
           const publishConfigs = await this.publishConfigs
           if (publishConfigs != null) {
-            await writeFile(path.join(appOutDir, "resources", "app-update.yml"), safeDump(publishConfigs[0]))
+            await writeFile(path.join(this.getOSXResourcesDir(appOutDir), "app-update.yml"), safeDump(publishConfigs[0]))
           }
         })
         .then(() => this.sign(appOutDir, null))

packages/electron-builder/src/platformPackager.ts

@@ -101,6 +101,8 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
 
   readonly appInfo: AppInfo
 
+  private _publishConfigs: Promise<Array<PublishConfiguration> | null>
+
   constructor(readonly info: BuildInfo) {
     this.config = info.config
     this.platformSpecificBuildOptions = this.normalizePlatformSpecificBuildOptions((<any>this.config)[this.platform.buildConfigurationKey])
@@ -410,7 +412,7 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     return this.platform === Platform.MAC ? this.getOSXResourcesDir(appOutDir) : path.join(appOutDir, "resources")
   }
 
-  private getOSXResourcesDir(appOutDir: string): string {
+  protected getOSXResourcesDir(appOutDir: string): string {
     return path.join(appOutDir, `${this.appInfo.productFilename}.app`, "Contents", "Resources")
   }
 
@@ -540,8 +542,6 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     return getRepositoryInfo(this.appInfo.metadata, this.info.devMetadata)
   }
 
-  private _publishConfigs: Promise<Array<PublishConfiguration> | null>
-
   get publishConfigs(): Promise<Array<PublishConfiguration> | null> {
     if (this._publishConfigs == null) {
       this._publishConfigs = this.computePublishConfigs()

test/out/__snapshots__/globTest.js.snap

@@ -3,3 +3,10 @@ Object {
   "link": "index.js",
 }
 `;
+
+exports[`test outside link 1`] = `
+Object {
+  "offset": "5290",
+  "size": 4,
+}
+`;

test/src/globTest.ts

@@ -1,5 +1,5 @@
 import { outputFile, symlink, writeFile, mkdirs } from "fs-extra-p"
-import { assertPack, modifyPackageJson, app, PackedContext } from "./helpers/packTester"
+import { assertPack, modifyPackageJson, app, PackedContext, getTempFile } from "./helpers/packTester"
 import BluebirdPromise from "bluebird-lst-c"
 import * as path from "path"
 import { assertThat } from "./helpers/fileAssert"
@@ -61,6 +61,19 @@ test.ifNotWindows("link", app({
   },
 }))
 
+test.ifNotWindows("outside link", app({
+  targets: Platform.LINUX.createTarget(DIR_TARGET),
+}, {
+  projectDirCreated: async (projectDir) => {
+    const tempDir = getTempFile()
+    await outputFile(path.join(tempDir, "foo"), "data")
+    await symlink(tempDir, path.join(projectDir, "o-dir"))
+  },
+  packed: async context => {
+    expect(statFile(path.join(context.getResources(Platform.LINUX), "app.asar"), "o-dir/foo", false)).toMatchSnapshot()
+  },
+}))
+
 // https://github.com/electron-userland/electron-builder/issues/611
 test.ifDevOrLinuxCi("failed peer dep", () => {
   return assertPack("test-app-one", {

test/src/helpers/packTester.ts

@@ -71,6 +71,10 @@ export function appTwo(packagerOptions: PackagerOptions, checkOptions: AssertPac
   return () => assertPack("test-app", packagerOptions, checkOptions)
 }
 
+export function getTempFile() {
+  return path.join(testDir, `${(tmpDirCounter++).toString(16)}`)
+}
+
 export async function assertPack(fixtureName: string, packagerOptions: PackagerOptions, checkOptions: AssertPackOptions = {}): Promise<void> {
   if (checkOptions.signed) {
     packagerOptions = signed(packagerOptions)
@@ -85,7 +89,7 @@ export async function assertPack(fixtureName: string, packagerOptions: PackagerO
   let dirToDelete: string | null = null
   if (useTempDir) {
     // non-macOS test uses the same dir as macOS test, but we cannot share node_modules (because tests executed in parallel)
-    const dir = customTmpDir == null ? path.join(testDir, `${(tmpDirCounter++).toString(16)}`) : path.resolve(customTmpDir)
+    const dir = customTmpDir == null ? getTempFile() : path.resolve(customTmpDir)
     if (customTmpDir == null) {
       dirToDelete = dir
     }