Arm support 2625

buildSrc/DesktopBuilder.js

@@ -23,22 +23,23 @@ const projectRoot = path.resolve(path.join(buildSrc, ".."))
  * @param dirname directory this was called from
  * @param version application version that gets built
  * @param platform: {"linux"|"win32"|"darwin"} - Canonical platform name of the desktop target to be built
+ * @param architecture: {"arm64"|"x64"|"universal"} the instruction set used in the built desktop binary
  * @param updateUrl where the client should pull its updates from, if any
  * @param nameSuffix suffix used to distinguish test-, prod- or snapshot builds on the same machine
  * @param notarize for the MacOs notarization feature
  * @param outDir where copy the finished artifacts
  * @param unpacked output desktop client without packing it into an installer
  * @returns {Promise<void>}
  */
-export async function buildDesktop({ dirname, version, platform, updateUrl, nameSuffix, notarize, outDir, unpacked, disableMinify }) {
+export async function buildDesktop({ dirname, version, platform, architecture, updateUrl, nameSuffix, notarize, outDir, unpacked, disableMinify }) {
 	// The idea is that we
 	// - build desktop code into build/desktop
 	// - package the whole dist directory into the app
 	// - move installers out of the dist into build/desktop-whatever
 	// - cleanup dist directory
 	// It's messy
 
-	console.log(`Building ${platform} desktop client for v${version}`)
+	console.log(`Building ${architecture} ${platform} desktop client for v${version}`)
 	updateUrl = updateUrl?.toString()
 	const updateSubDir = `desktop${nameSuffix}`
 	const distDir = path.join(dirname, "build")
@@ -61,6 +62,7 @@ export async function buildDesktop({ dirname, version, platform, updateUrl, name
 		unpacked,
 		sign: (process.env.DEBUG_SIGN && updateUrl !== "") || !!process.env.JENKINS_HOME,
 		linux: platform === "linux",
+		architecture,
 	})
 	console.log("updateUrl is", updateUrl)
 	await fs.promises.writeFile("./build/package.json", JSON.stringify(content), "utf-8")
@@ -76,7 +78,7 @@ export async function buildDesktop({ dirname, version, platform, updateUrl, name
 	}
 
 	console.log("Bundling desktop client")
-	await rollupDesktop(dirname, path.join(distDir, "desktop"), version, platform, disableMinify)
+	await rollupDesktop(dirname, path.join(distDir, "desktop"), version, platform, architecture, disableMinify)
 
 	console.log("Starting installer build...")
 	if (process.platform.startsWith("darwin")) {
@@ -112,7 +114,7 @@ export async function buildDesktop({ dirname, version, platform, updateUrl, name
 	])
 }
 
-async function rollupDesktop(dirname, outDir, version, platform, disableMinify) {
+async function rollupDesktop(dirname, outDir, version, platform, architecture, disableMinify) {
 	platform = getCanonicalPlatformName(platform)
 	const mainBundle = await rollup({
 		input: [path.join(dirname, "src/desktop/DesktopMain.ts"), path.join(dirname, "src/desktop/sqlworker.ts")],
@@ -125,12 +127,14 @@ async function rollupDesktop(dirname, outDir, version, platform, disableMinify)
 				rootDir: projectRoot,
 				dstPath: "./build/desktop/",
 				platform,
+				architecture,
 				nodeModule: "better-sqlite3",
 			}),
 			copyNativeModulePlugin({
 				rootDir: projectRoot,
 				dstPath: "./build/desktop/",
 				platform,
+				architecture,
 				nodeModule: "keytar",
 			}),
 			typescript({

buildSrc/DevBuild.js

@@ -97,13 +97,15 @@ async function buildDesktopPart({ version }) {
 					environment: "electron",
 					dstPath: "./build/desktop/better_sqlite3.node",
 					platform: process.platform,
+					architecture: process.arch,
 					nativeBindingPath: "./better_sqlite3.node",
 				}),
 				keytarNativePlugin({
 					environment: "electron",
 					dstPath: "./build/desktop/keytar.node",
 					nativeBindingPath: "./keytar.node",
 					platform: process.platform,
+					architecture: process.arch,
 				}),
 				preludeEnvPlugin(env.create({ staticUrl: null, version, mode: "Desktop", dist: false, domainConfigs })),
 				externalTranslationsPlugin(),
@@ -122,6 +124,7 @@ async function buildDesktopPart({ version }) {
 			iconPath: path.join(desktopIconsPath, "logo-solo-red.png"),
 			sign: false,
 			linux: process.platform === "linux",
+			architecture: "x64",
 		})
 		const content = JSON.stringify(packageJSON, null, 2)
 

buildSrc/buildUtils.js

@@ -159,6 +159,24 @@ export function getCanonicalPlatformName(platformName) {
 	}
 }
 
+/**
+ * Checks whether the combination of OS & architecture is supported by the build system
+ * @param platformName {"darwin"|"win32"|"linux"}
+ * @param architecture {"arm"|"arm64"|"ia32"|"mips"|"mipsel"|"ppc"|"ppc64"|"riscv64"|"s390"|"s390x"|"x64"|"universal"}
+ * @returns {boolean}
+ */
+export function checkArchitectureIsSupported(platformName, architecture) {
+	switch (architecture) {
+		case "x64":
+			return true
+		case "arm64":
+		case "universal":
+			return platformName === "darwin"
+		default:
+			return false
+	}
+}
+
 export async function runStep(name, cmd) {
 	const before = Date.now()
 	console.log("Build >", name)

buildSrc/electron-package-json-template.js

@@ -8,7 +8,7 @@ import { getElectronVersion } from "./buildUtils.js"
  * 2. copied to app-desktop/build from dist.js (DesktopBuilder)
  */
 
-export default async function generateTemplate({ nameSuffix, version, updateUrl, iconPath, sign, notarize, unpacked, linux }) {
+export default async function generateTemplate({ nameSuffix, version, updateUrl, iconPath, sign, notarize, unpacked, linux, architecture }) {
 	const appName = "tutanota-desktop" + nameSuffix
 	const appId = "de.tutao.tutanota" + nameSuffix
 	if (process.env.JENKINS_HOME && process.env.DEBUG_SIGN) throw new Error("Tried to DEBUG_SIGN in CI!")
@@ -111,7 +111,7 @@ export default async function generateTemplate({ nameSuffix, version, updateUrl,
 				target: [
 					{
 						target: unpacked ? "dir" : "nsis",
-						arch: "x64",
+						arch: architecture,
 					},
 				],
 			},
@@ -135,15 +135,15 @@ export default async function generateTemplate({ nameSuffix, version, updateUrl,
 					LSUIElement: 1, //hide dock icon on startup
 				},
 				target: unpacked
-					? [{ target: "dir", arch: "x64" }]
+					? [{ target: "dir", arch: architecture }]
 					: [
 							{
 								target: "zip",
-								arch: "x64",
+								arch: architecture,
 							},
 							{
 								target: "dmg",
-								arch: "x64",
+								arch: architecture,
 							},
 					  ],
 			},
@@ -157,7 +157,7 @@ export default async function generateTemplate({ nameSuffix, version, updateUrl,
 				target: [
 					{
 						target: unpacked ? "dir" : "AppImage",
-						arch: "x64",
+						arch: architecture,
 					},
 				],
 			},

buildSrc/esbuildUtils.js

@@ -8,7 +8,7 @@ import { aliasPath as esbuildPluginAliasPath } from "esbuild-plugin-alias-path"
  * Little plugin that obtains compiled keytar, copies it to dstPath and sets the path to nativeBindingPath.
  * We do not use default file loader from esbuild, it is much simpler and reliable to do it manually.
  */
-export function keytarNativePlugin({ environment, dstPath, nativeBindingPath, platform }) {
+export function keytarNativePlugin({ environment, dstPath, nativeBindingPath, platform, architecture }) {
 	return {
 		name: "keytar-native-plugin",
 		setup(build) {
@@ -19,6 +19,7 @@ export function keytarNativePlugin({ environment, dstPath, nativeBindingPath, pl
 					rootDir: process.cwd(),
 					log: console.log.bind(console),
 					platform,
+					architecture,
 					copyTarget: "keytar",
 				})
 				await fs.promises.mkdir(path.dirname(dstPath), { recursive: true })
@@ -41,7 +42,7 @@ export function keytarNativePlugin({ environment, dstPath, nativeBindingPath, pl
  * anyway.
  * It will also replace `buildOptions.sqliteNativePath` with the nativeBindingPath
  */
-export function sqliteNativePlugin({ environment, dstPath, nativeBindingPath, platform }) {
+export function sqliteNativePlugin({ environment, dstPath, nativeBindingPath, platform, architecture }) {
 	return {
 		name: "sqlite-native-plugin",
 		setup(build) {
@@ -58,6 +59,7 @@ export function sqliteNativePlugin({ environment, dstPath, nativeBindingPath, pl
 					rootDir: process.cwd(),
 					log: console.log.bind(console),
 					platform,
+					architecture,
 					copyTarget: "better_sqlite3",
 				})
 				await fs.promises.mkdir(path.dirname(dstPath), { recursive: true })

buildSrc/mac-entitlements.plist

@@ -4,6 +4,8 @@
   <dict>
     <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
     <true/>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
     <key>com.apple.security.cs.disable-library-validation</key>
     <true/>
   </dict>

buildSrc/nativeLibraryProvider.js

@@ -40,7 +40,8 @@ function validateOpts(opts) {
 
 async function cli(nodeModule, { environment, rootDir, forceRebuild, useExisting, copyTarget }) {
 	const platform = getCanonicalPlatformName(process.platform)
-	const path = await getCachedLibPath({ rootDir, nodeModule, environment, platform }, console.log.bind(console))
+	const architecture = process.arch
+	const path = await getCachedLibPath({ rootDir, nodeModule, environment, platform, architecture }, console.log.bind(console))
 
 	if (forceRebuild) {
 		await fs.promises.rm(path, { force: true })
@@ -53,6 +54,7 @@ async function cli(nodeModule, { environment, rootDir, forceRebuild, useExisting
 		log: console.log.bind(console),
 		useExisting,
 		platform,
+		architecture,
 		copyTarget,
 	})
 }
@@ -64,6 +66,7 @@ async function cli(nodeModule, { environment, rootDir, forceRebuild, useExisting
  * to avoid rebuilding between different invocations (e.g. running desktop and running tests).
  * @param environment {"electron"|"node"}
  * @param platform {"win32"|"linux"|"darwin"} platform to compile for in case of cross compilation
+ * @param architecture: {"arm64"|"x64"|"universal"} the instruction set used in the built desktop binary
  * @param rootDir {string} path to the root of the project
  * @param nodeModule {string} name of the npm module to rebuild
  * @param log {(...string) => void}
@@ -72,8 +75,8 @@ async function cli(nodeModule, { environment, rootDir, forceRebuild, useExisting
  * @param prebuildTarget: {{ runtime: string, version: number} | undefined} Target parameters to use when getting a prebuild
  * @returns {Promise<string>} path to cached native module
  */
-export async function getNativeLibModulePath({ environment, platform, rootDir, nodeModule, log, noBuild, copyTarget }) {
-	const libPath = await getCachedLibPath({ rootDir, nodeModule, environment, platform }, log)
+export async function getNativeLibModulePath({ environment, platform, architecture, rootDir, nodeModule, log, noBuild, copyTarget }) {
+	const libPath = await getCachedLibPath({ rootDir, nodeModule, environment, platform, architecture }, log)
 
 	if (await fileExists(libPath)) {
 		log(`Using cached ${nodeModule} at`, libPath)
@@ -104,6 +107,8 @@ export async function getNativeLibModulePath({ environment, platform, rootDir, n
 				rootDir,
 				log,
 				nodeModule,
+				copyTarget,
+				architecture,
 			})
 		}
 
@@ -120,26 +125,68 @@ export async function getNativeLibModulePath({ environment, platform, rootDir, n
  * @param nodeModule {string} the node module being built. Must be installed, and must be a native module project with a `binding.gyp` at the root
  * @param environment {"node"|"electron"} Used to determine which node version to use
  * @param rootDir {string} the root dir of the project
+ * @param architecture the architecture to build for: "x64" | "arm64" | "universal"
  * @param log {(string) => void} a logger
  * @returns {Promise<void>}
  */
-export async function buildNativeModule({ nodeModule, environment, rootDir, log }) {
-	await callProgram({
-		command: "npm exec",
-		args: [
-			"--",
-			"node-gyp",
-			"rebuild",
-			"--release",
-			"--build-from-source",
-			`--arch=${process.arch}`,
-			...(environment === "electron"
-				? ["--runtime=electron", "--dist-url=https://www.electronjs.org/headers", `--target=${await getElectronVersion(log)}`]
-				: []),
-		],
-		cwd: await getModuleDir(rootDir, nodeModule),
-		log,
-	})
+export async function buildNativeModule({ nodeModule, copyTarget, environment, rootDir, log, architecture }) {
+	const moduleDir = await getModuleDir(rootDir, nodeModule)
+	const electronVersion = await getElectronVersion(log)
+	const doBuild = (arch) =>
+		callProgram({
+			command: "npm exec",
+			args: [
+				"--",
+				"node-gyp",
+				"rebuild",
+				"--release",
+				"--build-from-source",
+				`--arch=${arch}`,
+				...(environment === "electron" ? ["--runtime=electron", "--dist-url=https://www.electronjs.org/headers", `--target=${electronVersion}`] : []),
+			],
+			cwd: moduleDir,
+			log,
+		})
+
+	if (architecture === "universal") {
+		if (copyTarget === "keytar") {
+			// this is a hack to get us out of a pickle with incompatible macos SDKs
+			// we reuse the keytar of tutanota-desktop 3.118.13 for the
+			// mac build because any newer build crashes the process when loaded.
+			// we will replace keytar very soon.
+			const artifactPath = path.join(moduleDir, "build/Release", `${copyTarget}.node`)
+			const armArtifactPath = path.join(moduleDir, `${copyTarget}-arm64.node`)
+			const intelArtifactPath = path.join(moduleDir, `${copyTarget}-3.118.13.node`)
+			await doBuild("arm64")
+			await fs.promises.copyFile(artifactPath, armArtifactPath)
+			await callProgram({
+				command: "lipo",
+				args: ["-create", "-output", artifactPath, intelArtifactPath, armArtifactPath],
+				cwd: path.join(rootDir, "native-cache"),
+				log,
+			})
+		} else {
+			const artifactPath = path.join(moduleDir, "build/Release", `${copyTarget}.node`)
+			const armArtifactPath = path.join(moduleDir, `${copyTarget}-arm64.node`)
+			const intelArtifactPath = path.join(moduleDir, `${copyTarget}-x64.node`)
+			// this is a hack to get us out of a pickle with incompatible macos SDKs
+			// we reuse the keytar of tutanota-desktop 3.118.13 for the
+			// mac build because any newer build crashes the process when loaded.
+			// we will replace keytar very soon.
+			await doBuild("x64")
+			await fs.promises.copyFile(artifactPath, intelArtifactPath)
+			await doBuild("arm64")
+			await fs.promises.copyFile(artifactPath, armArtifactPath)
+			await callProgram({
+				command: "lipo",
+				args: ["-create", "-output", artifactPath, intelArtifactPath, armArtifactPath],
+				cwd: path.join(rootDir, "native-cache"),
+				log,
+			})
+		}
+	} else {
+		await doBuild(architecture)
+	}
 }
 
 /**
@@ -241,15 +288,19 @@ function callProgram({ command, args, cwd, log }) {
  * @param nodeModule
  * @param environment
  * @param platform
+ * @param architecture: {"arm64"|"x64"|"universal"} the instruction set used in the built desktop binary
  * @returns {Promise<string>}
  */
-async function getCachedLibPath({ rootDir, nodeModule, environment, platform }, log) {
+async function getCachedLibPath({ rootDir, nodeModule, environment, platform, architecture }, log) {
 	const dir = path.join(rootDir, "native-cache", environment)
 	const libraryVersion = await getInstalledModuleVersion(nodeModule, log)
 	await fs.promises.mkdir(dir, { recursive: true })
 
 	if (environment === "electron") {
-		return path.resolve(dir, `${nodeModule}-${libraryVersion}-electron-${await getInstalledModuleVersion("electron", log)}-${platform}.node`)
+		return path.resolve(
+			dir,
+			`${nodeModule}-${libraryVersion}-electron-${await getInstalledModuleVersion("electron", log)}-${platform}-${architecture}.node`,
+		)
 	} else {
 		return path.resolve(dir, `${nodeModule}-${libraryVersion}-${platform}.node`)
 	}

buildSrc/nativeLibraryRollupPlugin.js

@@ -3,7 +3,7 @@ import path from "node:path"
 import { getNativeLibModulePath } from "./nativeLibraryProvider.js"
 
 /** copy either a fresh build or a cached version of a native module for the platform client being built into the build directory.*/
-export function copyNativeModulePlugin({ rootDir, dstPath, platform, nodeModule }, log = console.log.bind(console)) {
+export function copyNativeModulePlugin({ rootDir, dstPath, platform, architecture, nodeModule }, log = console.log.bind(console)) {
 	return {
 		name: "copy-native-module-plugin",
 		async buildStart() {
@@ -13,7 +13,8 @@ export function copyNativeModulePlugin({ rootDir, dstPath, platform, nodeModule
 				rootDir,
 				log,
 				platform,
-				// for some reason, the binary produced by better-sqlite3 is called better_sqlite3.node
+				architecture,
+				// because its name is used as a C identifier, the binary produced by better-sqlite3 is called better_sqlite3.node
 				copyTarget: nodeModule.replace("-", "_"),
 			})
 			const normalDst = path.join(path.normalize(dstPath), `${nodeModule}.node`)