From cbee5e6fd55a8c6f408c3c71c2e1ec034cf01a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 17 Jan 2021 17:10:21 +0100 Subject: [PATCH 1/8] explicitly pass path to current binary --- cli/main.rs | 2 ++ cli/tools/standalone.rs | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/main.rs b/cli/main.rs index 3502414a9685c4..04335bc8dc167c 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -337,9 +337,11 @@ async fn compile_command( colors::green("Compile"), module_specifier.to_string() ); + let original_binary_path = std::env::current_exe()?; tools::standalone::create_standalone_binary( bundle_str, run_flags, + &original_binary_path, output.clone(), ) .await?; diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index 141b3e820555ac..9c79f8100db051 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -11,6 +11,7 @@ use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; +use std::path::Path; use std::path::PathBuf; use crate::standalone::Metadata; @@ -21,6 +22,7 @@ use crate::standalone::MAGIC_TRAILER; pub async fn create_standalone_binary( source_code: String, flags: Flags, + original_binary_path: &Path, output: PathBuf, ) -> Result<(), AnyError> { let mut source_code = source_code.as_bytes().to_vec(); @@ -39,7 +41,6 @@ pub async fn create_standalone_binary( ca_data, }; let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec(); - let original_binary_path = std::env::current_exe()?; let mut original_bin = tokio::fs::read(original_binary_path).await?; let bundle_pos = original_bin.len(); From 8b5446081f5fcbd642d7b6a5a36dc9777407a555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 17 Jan 2021 17:21:44 +0100 Subject: [PATCH 2/8] tepm --- cli/flags.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/cli/flags.rs b/cli/flags.rs index 9fe25df50da650..052e4cbc552326 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -29,6 +29,8 @@ pub enum DenoSubcommand { source_file: String, output: Option, args: Vec, + target: Option, + lite: bool, }, Completions { buf: Box<[u8]>, @@ -447,11 +449,13 @@ fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let args = script.split_off(1); let source_file = script[0].to_string(); let output = matches.value_of("output").map(PathBuf::from); + let lite = matches.is_present("lite"); flags.subcommand = DenoSubcommand::Compile { source_file, output, args, + lite, }; } @@ -893,11 +897,24 @@ fn compile_subcommand<'a, 'b>() -> App<'a, 'b> { .help("Output file (defaults to $PWD/)") .takes_value(true) ) + .arg( + Arg::with_name("target") + .long("target") + .help("Target OS architecture") + .takes_value(true) + .possible_values(&["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "x86_64-apple-darwin"]) + ) + .arg( + Arg::with_name("lite") + .long("lite") + .help("Use lite runtime") + ) .about("Compile the script into a self contained executable") .long_about( "Compiles the given script into a self contained executable. deno compile --unstable https://deno.land/std/http/file_server.ts deno compile --unstable --output /usr/local/bin/color_util https://deno.land/std/examples/colors.ts + deno compile --unstable --lite --target x86_64-unknown-linux-gnu https://deno.land/std/http/file_server.ts Any flags passed which affect runtime behavior, such as '--unstable', '--allow-*', '--v8-flags', etc. are encoded into the output executable and used @@ -909,9 +926,7 @@ The executable name is inferred by default: - If the file stem is something generic like 'main', 'mod', 'index' or 'cli', and the path has no parent, take the file name of the parent path. Otherwise settle with the generic name. - - If the resulting name has an '@...' suffix, strip it. - -Cross compiling binaries for different platforms is not currently possible.", + - If the resulting name has an '@...' suffix, strip it.", ) } From 461e2fee22a7b674f659b33fd152b8707934a14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 17 Jan 2021 18:46:13 +0100 Subject: [PATCH 3/8] download prebuilt binaries --- cli/flags.rs | 2 + cli/main.rs | 24 +++++++---- cli/tools/standalone.rs | 93 ++++++++++++++++++++++++++++++++++++++--- cli/tools/upgrade.rs | 2 +- 4 files changed, 108 insertions(+), 13 deletions(-) diff --git a/cli/flags.rs b/cli/flags.rs index 052e4cbc552326..7b538de201368e 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -450,12 +450,14 @@ fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let source_file = script[0].to_string(); let output = matches.value_of("output").map(PathBuf::from); let lite = matches.is_present("lite"); + let target = matches.value_of("target").map(String::from); flags.subcommand = DenoSubcommand::Compile { source_file, output, args, lite, + target, }; } diff --git a/cli/main.rs b/cli/main.rs index 04335bc8dc167c..ba70cebe9d6b7b 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -299,6 +299,8 @@ async fn compile_command( source_file: String, output: Option, args: Vec, + target: Option, + lite: bool, ) -> Result<(), AnyError> { if !flags.unstable { exit_unstable("compile"); @@ -311,6 +313,7 @@ async fn compile_command( let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?; let program_state = ProgramState::new(flags.clone())?; + let deno_dir = &program_state.dir; let output = output.or_else(|| { infer_name_from_url(module_specifier.as_url()).map(PathBuf::from) @@ -337,17 +340,21 @@ async fn compile_command( colors::green("Compile"), module_specifier.to_string() ); - let original_binary_path = std::env::current_exe()?; - tools::standalone::create_standalone_binary( + + // Select base binary based on `target` and `lite` arguments + let original_binary = + tools::standalone::get_base_binary(deno_dir, target, lite).await?; + + let final_bin = tools::standalone::create_standalone_binary( + original_binary, bundle_str, run_flags, - &original_binary_path, - output.clone(), - ) - .await?; + )?; info!("{} {}", colors::green("Emit"), output.display()); + tools::standalone::write_standalone_binary(output.clone(), final_bin).await?; + Ok(()) } @@ -1164,7 +1171,10 @@ fn get_subcommand( source_file, output, args, - } => compile_command(flags, source_file, output, args).boxed_local(), + lite, + target, + } => compile_command(flags, source_file, output, args, target, lite) + .boxed_local(), DenoSubcommand::Fmt { check, files, diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index 9c79f8100db051..ca51d32c7952b6 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -1,10 +1,13 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. +use crate::deno_dir::DenoDir; use crate::flags::DenoSubcommand; use crate::flags::Flags; use deno_core::error::bail; use deno_core::error::AnyError; use deno_core::serde_json; +use deno_runtime::deno_fetch::reqwest::Client; +use std::env; use std::fs::read; use std::fs::File; use std::io::Read; @@ -17,14 +20,86 @@ use std::path::PathBuf; use crate::standalone::Metadata; use crate::standalone::MAGIC_TRAILER; +pub async fn get_base_binary( + deno_dir: &DenoDir, + target: Option, + lite: bool, +) -> Result, AnyError> { + if target.is_none() && !lite { + let path = std::env::current_exe()?; + return Ok(tokio::fs::read(path).await?); + } + + let target = target.unwrap_or_else(|| env!("TARGET").to_string()); + let binary_name = if lite { + format!("denort-{}.zip", target) + } else { + format!("deno-{}.zip", target) + }; + + let binary_path_suffix = if crate::version::is_canary() { + format!("canary/{}/{}", crate::version::GIT_COMMIT_HASH, binary_name) + } else { + format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) + }; + + let download_directory = deno_dir.root.join("dl"); + let binary_path = download_directory.join(&binary_path_suffix); + + if !binary_path.exists() { + download_base_binary(&download_directory, &binary_path_suffix).await?; + } + + let archive_data = tokio::fs::read(binary_path).await?; + let base_binary_path = crate::tools::upgrade::unpack(archive_data)?; + let base_binary = tokio::fs::read(base_binary_path).await?; + Ok(base_binary) +} + +async fn download_base_binary( + output_directory: &Path, + binary_path_suffix: &str, +) -> Result<(), AnyError> { + let download_url = format!("https://dl.deno.land/{}", binary_path_suffix); + + let client_builder = Client::builder(); + + // TODO: + // // If we have been provided a CA Certificate, add it into the HTTP client + // if let Some(ca_file) = ca_file { + // let buf = std::fs::read(ca_file)?; + // let cert = reqwest::Certificate::from_pem(&buf)?; + // client_builder = client_builder.add_root_certificate(cert); + // } + + let client = client_builder.build()?; + + println!("Checking {}", &download_url); + + let res = client.get(&download_url).send().await?; + + let binary_content = if res.status().is_success() { + println!("Download has been found"); + res.bytes().await?.to_vec() + } else { + println!("Download could not be found, aborting"); + std::process::exit(1) + }; + + std::fs::create_dir_all(&output_directory)?; + let output_path = output_directory.join(binary_path_suffix); + std::fs::create_dir_all(&output_path.parent().unwrap())?; + tokio::fs::write(output_path, binary_content).await?; + Ok(()) +} + /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. -pub async fn create_standalone_binary( +pub fn create_standalone_binary( + mut original_bin: Vec, source_code: String, flags: Flags, - original_binary_path: &Path, - output: PathBuf, -) -> Result<(), AnyError> { +) -> Result, AnyError> { let mut source_code = source_code.as_bytes().to_vec(); let ca_data = match &flags.ca_file { Some(ca_file) => Some(read(ca_file)?), @@ -41,7 +116,6 @@ pub async fn create_standalone_binary( ca_data, }; let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec(); - let mut original_bin = tokio::fs::read(original_binary_path).await?; let bundle_pos = original_bin.len(); let metadata_pos = bundle_pos + source_code.len(); @@ -56,6 +130,15 @@ pub async fn create_standalone_binary( final_bin.append(&mut metadata); final_bin.append(&mut trailer); + Ok(final_bin) +} + +/// This function writes out a final binary to specified path. If output path +/// is not already standalone binary it will return error instead. +pub async fn write_standalone_binary( + output: PathBuf, + final_bin: Vec, +) -> Result<(), AnyError> { let output = if cfg!(windows) && output.extension().unwrap_or_default() != "exe" { PathBuf::from(output.display().to_string() + ".exe") diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index 58c283ecd4b5ff..dcbbb717a34185 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -176,7 +176,7 @@ async fn download_package( } } -fn unpack(archive_data: Vec) -> Result { +pub fn unpack(archive_data: Vec) -> Result { // We use into_path so that the tempdir is not automatically deleted. This is // useful for debugging upgrade, but also so this function can return a path // to the newly uncompressed file without fear of the tempdir being deleted. From 7e19067e65c9b44ed2f697dd74100ac2a2780a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 17 Jan 2021 18:51:13 +0100 Subject: [PATCH 4/8] fix test --- cli/flags.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/flags.rs b/cli/flags.rs index 7b538de201368e..9eb694c76ee04d 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -3335,6 +3335,7 @@ mod tests { let r = flags_from_vec(svec![ "deno", "compile", + "--lite", "https://deno.land/std/examples/colors.ts" ]); assert_eq!( @@ -3344,6 +3345,8 @@ mod tests { source_file: "https://deno.land/std/examples/colors.ts".to_string(), output: None, args: vec![], + target: None, + lite: true, }, ..Flags::default() } @@ -3361,6 +3364,8 @@ mod tests { source_file: "https://deno.land/std/examples/colors.ts".to_string(), output: Some(PathBuf::from("colors")), args: svec!["foo", "bar"], + target: None, + lite: false, }, unstable: true, import_map_path: Some("import_map.json".to_string()), From 4d5a6e53f08e0127a4cab1b9e89789f92f62759d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 18 Jan 2021 14:10:39 +0100 Subject: [PATCH 5/8] reset CI From 3337805fd734eecde0cf007dae5feda46e720fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 18 Jan 2021 18:14:15 +0100 Subject: [PATCH 6/8] update docs --- cli/flags.rs | 8 +++++++- cli/tools/standalone.rs | 9 --------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cli/flags.rs b/cli/flags.rs index 9eb694c76ee04d..af11c4221218f1 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -928,7 +928,13 @@ The executable name is inferred by default: - If the file stem is something generic like 'main', 'mod', 'index' or 'cli', and the path has no parent, take the file name of the parent path. Otherwise settle with the generic name. - - If the resulting name has an '@...' suffix, strip it.", + - If the resulting name has an '@...' suffix, strip it. + +This commands supports cross-compiling to different target architectures using `--target` flag. +On the first invocation with deno will download proper binary and cache it in $DENO_DIR. + +It is possible to use \"lite\" binaries when compiling by passing `--lite` flag; these are stripped down versions +of the deno binary that do not contain built-in tooling (eg. formatter, linter). This feature is experimental.", ) } diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index ca51d32c7952b6..c94e596de09d9e 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -63,15 +63,6 @@ async fn download_base_binary( let download_url = format!("https://dl.deno.land/{}", binary_path_suffix); let client_builder = Client::builder(); - - // TODO: - // // If we have been provided a CA Certificate, add it into the HTTP client - // if let Some(ca_file) = ca_file { - // let buf = std::fs::read(ca_file)?; - // let cert = reqwest::Certificate::from_pem(&buf)?; - // client_builder = client_builder.add_root_certificate(cert); - // } - let client = client_builder.build()?; println!("Checking {}", &download_url); From e3e13e59eb1ef979162f9d966a979fec3da30774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 18 Jan 2021 20:13:35 +0100 Subject: [PATCH 7/8] fix --lite unarchive --- cli/tools/standalone.rs | 9 +++------ cli/tools/upgrade.rs | 12 +++++++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index c94e596de09d9e..8cc574e5054a43 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -31,11 +31,8 @@ pub async fn get_base_binary( } let target = target.unwrap_or_else(|| env!("TARGET").to_string()); - let binary_name = if lite { - format!("denort-{}.zip", target) - } else { - format!("deno-{}.zip", target) - }; + let exe_name = if lite { "denort" } else { "deno" }; + let binary_name = format!("{}-{}.zip", exe_name, target); let binary_path_suffix = if crate::version::is_canary() { format!("canary/{}/{}", crate::version::GIT_COMMIT_HASH, binary_name) @@ -51,7 +48,7 @@ pub async fn get_base_binary( } let archive_data = tokio::fs::read(binary_path).await?; - let base_binary_path = crate::tools::upgrade::unpack(archive_data)?; + let base_binary_path = crate::tools::upgrade::unpack(archive_data, exe_name)?; let base_binary = tokio::fs::read(base_binary_path).await?; Ok(base_binary) } diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs index dcbbb717a34185..ab49c06e9d0087 100644 --- a/cli/tools/upgrade.rs +++ b/cli/tools/upgrade.rs @@ -111,7 +111,7 @@ pub async fn upgrade_command( println!("Deno is upgrading to version {}", &install_version); let old_exe_path = std::env::current_exe()?; - let new_exe_path = unpack(archive_data)?; + let new_exe_path = unpack(archive_data, "deno")?; let permissions = fs::metadata(&old_exe_path)?.permissions(); fs::set_permissions(&new_exe_path, permissions)?; check_exe(&new_exe_path)?; @@ -176,13 +176,17 @@ async fn download_package( } } -pub fn unpack(archive_data: Vec) -> Result { +pub fn unpack( + archive_data: Vec, + exe_name: &str, +) -> Result { // We use into_path so that the tempdir is not automatically deleted. This is // useful for debugging upgrade, but also so this function can return a path // to the newly uncompressed file without fear of the tempdir being deleted. let temp_dir = TempDir::new()?.into_path(); let exe_ext = if cfg!(windows) { "exe" } else { "" }; - let exe_path = temp_dir.join("deno").with_extension(exe_ext); + let archive_path = temp_dir.join(exe_name).with_extension(".zip"); + let exe_path = temp_dir.join(exe_name).with_extension(exe_ext); assert!(!exe_path.exists()); let archive_ext = Path::new(&*ARCHIVE_NAME) @@ -191,7 +195,6 @@ pub fn unpack(archive_data: Vec) -> Result { .unwrap(); let unpack_status = match archive_ext { "zip" if cfg!(windows) => { - let archive_path = temp_dir.join("deno.zip"); fs::write(&archive_path, &archive_data)?; Command::new("powershell.exe") .arg("-NoLogo") @@ -217,7 +220,6 @@ pub fn unpack(archive_data: Vec) -> Result { .wait()? } "zip" => { - let archive_path = temp_dir.join("deno.zip"); fs::write(&archive_path, &archive_data)?; Command::new("unzip") .current_dir(&temp_dir) From dedfc79a52b4ab89a77cf1bd1be39c008f2ebdcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 18 Jan 2021 22:03:05 +0100 Subject: [PATCH 8/8] update flags --- cli/flags.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/flags.rs b/cli/flags.rs index af11c4221218f1..754ccec8439976 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -914,9 +914,9 @@ fn compile_subcommand<'a, 'b>() -> App<'a, 'b> { .about("Compile the script into a self contained executable") .long_about( "Compiles the given script into a self contained executable. - deno compile --unstable https://deno.land/std/http/file_server.ts + deno compile --unstable -A https://deno.land/std/http/file_server.ts deno compile --unstable --output /usr/local/bin/color_util https://deno.land/std/examples/colors.ts - deno compile --unstable --lite --target x86_64-unknown-linux-gnu https://deno.land/std/http/file_server.ts + deno compile --unstable --lite --target x86_64-unknown-linux-gnu -A https://deno.land/std/http/file_server.ts Any flags passed which affect runtime behavior, such as '--unstable', '--allow-*', '--v8-flags', etc. are encoded into the output executable and used @@ -934,7 +934,8 @@ This commands supports cross-compiling to different target architectures using ` On the first invocation with deno will download proper binary and cache it in $DENO_DIR. It is possible to use \"lite\" binaries when compiling by passing `--lite` flag; these are stripped down versions -of the deno binary that do not contain built-in tooling (eg. formatter, linter). This feature is experimental.", +of the deno binary that do not contain built-in tooling (eg. formatter, linter). This feature is experimental. +", ) }