From dd6b8e1b8ad777753d311a2f9a287626d78de7d9 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 6 Nov 2024 08:34:42 +0100 Subject: [PATCH 1/3] improve url source --- docs/reference/recipe_file.md | 11 +++ examples/rich/recipe.yaml | 10 ++- src/recipe/parser/source.rs | 2 + src/source/url_source.rs | 146 ++++++++++++++++++++++------------ 4 files changed, 115 insertions(+), 54 deletions(-) diff --git a/docs/reference/recipe_file.md b/docs/reference/recipe_file.md index 26ddbb4ca..b2abf8e36 100644 --- a/docs/reference/recipe_file.md +++ b/docs/reference/recipe_file.md @@ -186,6 +186,17 @@ If an extracted archive contains only 1 folder at its top level, its contents will be moved 1 level up, so that the extracted package contents sit in the root of the work folder. +##### Specifying a file name + +For URL and local paths you can specify a file name. If the source is an archive and a file name is set, automatic extraction is disabled. + +```yaml +source: + url: https://pypi.python.org/packages/source/b/bsdiff4/bsdiff4-1.1.4.tar.gz + # will put the file in the work directory as `bsdiff4-1.1.4.tar.gz` + file_name: bsdiff4-1.1.4.tar.gz +``` + #### Source from `git` ```yaml diff --git a/examples/rich/recipe.yaml b/examples/rich/recipe.yaml index 8262d6d1a..b0be667dc 100644 --- a/examples/rich/recipe.yaml +++ b/examples/rich/recipe.yaml @@ -8,10 +8,12 @@ package: version: ${{ version }} source: - - url: - - https://example.com/rich-${{ version }}.tar.gz # this will give a 404! - - https://pypi.io/packages/source/r/rich/rich-${{ version }}.tar.gz - sha256: d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 + url: file:///Users/wolfv/Downloads/rich-13.9.4.tar.gz + sha256: 439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098 + # - url: + # - https://example.com/rich-${{ version }}.tar.gz # this will give a 404! + # - https://pypi.io/packages/source/r/rich/rich-${{ version }}.tar.gz + # sha256: d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 build: # Thanks to `noarch: python` this package works on all platforms diff --git a/src/recipe/parser/source.rs b/src/recipe/parser/source.rs index 7f799d6ec..70c894e49 100644 --- a/src/recipe/parser/source.rs +++ b/src/recipe/parser/source.rs @@ -401,9 +401,11 @@ pub struct UrlSource { /// Optionally a file name to rename the downloaded file (does not apply to archives) #[serde(skip_serializing_if = "Option::is_none")] file_name: Option, + /// Patches to apply to the source code #[serde(default, skip_serializing_if = "Vec::is_empty")] patches: Vec, + /// Optionally a folder name under the `work` directory to place the source code #[serde(skip_serializing_if = "Option::is_none")] target_directory: Option, diff --git a/src/source/url_source.rs b/src/source/url_source.rs index b53f2211c..23f6b23d2 100644 --- a/src/source/url_source.rs +++ b/src/source/url_source.rs @@ -3,10 +3,12 @@ use std::{ ffi::OsStr, fs, + io::{Read as _, Write as _}, path::{Path, PathBuf}, }; use crate::{ + console_utils::LoggingOutputHandler, recipe::parser::UrlSource, source::extract::{extract_tar, extract_zip}, tool_configuration, @@ -15,48 +17,48 @@ use tokio::io::AsyncWriteExt; use super::{checksum::Checksum, extract::is_tarball, SourceError}; -fn split_filename(filename: &str) -> (String, String) { - let stem = Path::new(filename) +/// Splits a path into stem and extension, handling special cases like .tar.gz +fn split_path(path: &Path) -> std::io::Result<(String, String)> { + let stem = path .file_stem() - .and_then(|os_str| os_str.to_str()) - .unwrap_or("") - .to_string(); + .and_then(|s| s.to_str()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid path stem"))?; - let stem_without_tar = stem.trim_end_matches(".tar"); + let (stem_no_tar, is_tar) = if let Some(s) = stem.strip_suffix(".tar") { + (s, true) + } else { + (stem, false) + }; - let extension = Path::new(filename) - .extension() - .and_then(|os_str| os_str.to_str()) - .unwrap_or("") - .to_string(); + let extension = path.extension().and_then(|s| s.to_str()).unwrap_or(""); - let full_extension = if stem != stem_without_tar { + let full_extension = if is_tar { format!(".tar.{}", extension) } else if !extension.is_empty() { format!(".{}", extension) } else { - "".to_string() + String::new() }; - // remove any dots from the stem - let stem_without_tar = stem_without_tar.replace('.', "_"); - - (stem_without_tar.to_string(), full_extension) + Ok((stem_no_tar.replace('.', "_"), full_extension)) } +/// Generates a cache name from URL and checksum fn cache_name_from_url( url: &url::Url, checksum: &Checksum, with_extension: bool, ) -> Option { let filename = url.path_segments()?.filter(|x| !x.is_empty()).last()?; - let (stem, extension) = split_filename(filename); - let checksum = checksum.to_hex(); - if with_extension { - Some(format!("{}_{}{}", stem, &checksum[0..8], extension)) + + let (stem, extension) = split_path(Path::new(filename)).ok()?; + let checksum_hex = checksum.to_hex(); + + Some(if with_extension { + format!("{}_{}{}", stem, &checksum_hex[..8], extension) } else { - Some(format!("{}_{}", stem, &checksum[0..8])) - } + format!("{}_{}", stem, &checksum_hex[..8]) + }) } async fn fetch_remote( @@ -85,6 +87,7 @@ async fn fetch_remote( .with_prefix("Downloading") .with_style(tool_configuration.fancy_log_handler.default_bytes_style()), ); + progress_bar.set_message( url.path_segments() .and_then(|segs| segs.last()) @@ -143,6 +146,37 @@ fn extract_to_cache( Ok(path.to_path_buf()) } +fn copy_with_progress( + source: &Path, + dest: &Path, + progress_handler: &LoggingOutputHandler, +) -> std::io::Result { + let file_size = source.metadata()?.len(); + let progress_bar = progress_handler.add_progress_bar( + indicatif::ProgressBar::new(file_size) + .with_prefix("Copying") + .with_style(progress_handler.default_bytes_style()), + ); + + let mut reader = std::io::BufReader::new(fs::File::open(source)?); + let mut writer = std::io::BufWriter::new(fs::File::create(dest)?); + let mut buffer = vec![0; 8192]; + let mut copied = 0u64; + + while let Ok(n) = reader.read(&mut buffer) { + if n == 0 { + break; + } + writer.write_all(&buffer[..n])?; + copied += n as u64; + progress_bar.set_position(copied); + } + + progress_bar.finish(); + writer.flush()?; + Ok(copied) +} + pub(crate) async fn url_src( source: &UrlSource, cache_dir: &Path, @@ -155,6 +189,12 @@ pub(crate) async fn url_src( let mut last_error = None; for url in source.urls() { + let cache_name = PathBuf::from(cache_name_from_url(url, &checksum, true).ok_or( + SourceError::UnknownErrorStr("Failed to build cache name from url"), + )?); + + let cache_name = cache_dir.join(cache_name); + if url.scheme() == "file" { let local_path = url.to_file_path().map_err(|_| { SourceError::Io(std::io::Error::new( @@ -171,37 +211,43 @@ pub(crate) async fn url_src( return Err(SourceError::ValidationFailed); } + // copy file to cache + copy_with_progress( + &local_path, + &cache_name, + &tool_configuration.fancy_log_handler, + )?; + tracing::info!("Using local source file."); - return Ok(local_path); + } else { + let metadata = fs::metadata(&cache_name); + if metadata.is_ok() && metadata?.is_file() && checksum.validate(&cache_name) { + tracing::info!("Found valid source cache file."); + } else { + match fetch_remote(url, &cache_name, tool_configuration).await { + Ok(_) => { + tracing::info!("Downloaded file from {}", url); + + if !checksum.validate(&cache_name) { + tracing::error!("Checksum validation failed!"); + fs::remove_file(&cache_name)?; + return Err(SourceError::ValidationFailed); + } + } + Err(e) => { + last_error = Some(e); + continue; + } + } + } } - let cache_name = PathBuf::from(cache_name_from_url(url, &checksum, true).ok_or( - SourceError::UnknownErrorStr("Failed to build cache name from url"), - )?); - let cache_name = cache_dir.join(cache_name); - - let metadata = fs::metadata(&cache_name); - if metadata.is_ok() && metadata?.is_file() && checksum.validate(&cache_name) { - tracing::info!("Found valid source cache file."); + // If the source has a file name, we skip the extraction step + if source.file_name().is_some() { + return Ok(cache_name); + } else { return extract_to_cache(&cache_name, tool_configuration); } - - match fetch_remote(url, &cache_name, tool_configuration).await { - Ok(_) => { - tracing::info!("Downloaded file from {}", url); - - if !checksum.validate(&cache_name) { - tracing::error!("Checksum validation failed!"); - fs::remove_file(&cache_name)?; - return Err(SourceError::ValidationFailed); - } - - return extract_to_cache(&cache_name, tool_configuration); - } - Err(e) => { - last_error = Some(e); - } - } } if let Some(last_error) = last_error { @@ -232,7 +278,7 @@ mod tests { ]; for (filename, expected) in test_cases { - let (name, ending) = split_filename(filename); + let (name, ending) = split_path(Path::new(filename)).unwrap(); assert_eq!( (name.as_str(), ending.as_str()), expected, From ff4dfd39af441ccdf7362dc1e8cadab7a355a2cd Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 6 Nov 2024 08:34:59 +0100 Subject: [PATCH 2/3] revert example --- examples/rich/recipe.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/examples/rich/recipe.yaml b/examples/rich/recipe.yaml index b0be667dc..8262d6d1a 100644 --- a/examples/rich/recipe.yaml +++ b/examples/rich/recipe.yaml @@ -8,12 +8,10 @@ package: version: ${{ version }} source: - url: file:///Users/wolfv/Downloads/rich-13.9.4.tar.gz - sha256: 439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098 - # - url: - # - https://example.com/rich-${{ version }}.tar.gz # this will give a 404! - # - https://pypi.io/packages/source/r/rich/rich-${{ version }}.tar.gz - # sha256: d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 + - url: + - https://example.com/rich-${{ version }}.tar.gz # this will give a 404! + - https://pypi.io/packages/source/r/rich/rich-${{ version }}.tar.gz + sha256: d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 build: # Thanks to `noarch: python` this package works on all platforms From d58e9fa8fb8dc28e9a1c42b444563a37795e6097 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 6 Nov 2024 09:18:42 +0100 Subject: [PATCH 3/3] fix cli docs --- docs/reference/cli.md | 7 ++++++ docs/tutorials/javascript.md | 41 ++++++++++++++++++++++++++++++++++++ docs/tutorials/python.md | 3 +++ 3 files changed, 51 insertions(+) create mode 100644 docs/tutorials/javascript.md diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7f72caa28..e12448eb4 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -42,6 +42,13 @@ This document contains the help content for the `rattler-build` command-line pro Use plain logging output +- `--wrap-log-lines ` + + Wrap log lines at the terminal width. This is automatically disabled on CI (by detecting the `CI` environment variable) + + - Possible values: `true`, `false` + + - `--color ` Enable or disable colored output from rattler-build. Also honors the `CLICOLOR` and `CLICOLOR_FORCE` environment variable diff --git a/docs/tutorials/javascript.md b/docs/tutorials/javascript.md new file mode 100644 index 000000000..bccdfe2ad --- /dev/null +++ b/docs/tutorials/javascript.md @@ -0,0 +1,41 @@ +# Packaging a Javascript (NPM/NodeJS) package + +This tutorial will guide you though making a NodeJS package with `rattler-build`. + +## Building a NodeJS Package + +In this example, we will build a package for the NodeJS package `bibtex-tidy`. +We use `nodejs` in build and run requirements, and install the package using `npm`. +NPM comes as part of the NodeJS installation, so we do not need to install it separately. + +```yaml title="recipe.yaml" +context: + version: "1.14.0" + +package: + name: bibtex-tidy + version: ${{ version }} + +source: + url: https://registry.npmjs.org/bibtex-tidy/-/bibtex-tidy-${{ version }}.tgz + sha256: 0a2c1bb73911a7cee36a30ce1fc86feffe39b2d39acd4c94d02aac6f84a00285 + # we do not extract the source code and install the tarball directly as that works better + file_name: bibtex-tidy-${{ version }}.tgz + +build: + number: 0 + script: + # we use NPM to globally install the bibtex-tidy package + - npm install -g bibtex-tidy-${{ version }}.tgz --prefix ${{ PREFIX }} + +requirements: + build: + - nodejs + run: + - nodejs + +tests: + - script: + - bibtex-tidy --version +``` + diff --git a/docs/tutorials/python.md b/docs/tutorials/python.md index 2c2707f04..03b5d593e 100644 --- a/docs/tutorials/python.md +++ b/docs/tutorials/python.md @@ -4,6 +4,7 @@ Writing a Python package is fairly straightforward, especially for "Python-only" In the second example we will build a package for `numpy` which contains compiled code. ## A Python-only package + The following recipe uses the `noarch: python` setting to build a `noarch` package that can be installed on any platform without modification. This is very handy for packages that are pure Python and do not contain any compiled extensions. @@ -63,6 +64,7 @@ about: installed correctly and can be imported. ### Running the recipe + To build this recipe, simply run: ```bash @@ -180,6 +182,7 @@ for /f %%f in ('dir /b /S .\dist') do ( ``` ### Running the recipe + Running this recipe with the variant config file will build a total of 2 `numpy` packages: ```bash