fix: url source extraction for file:/// urls

docs/reference/cli.md

@@ -42,6 +42,13 @@ This document contains the help content for the `rattler-build` command-line pro
 			Use plain logging output
 
 
+- `--wrap-log-lines <WRAP_LOG_LINES>`
+
+	Wrap log lines at the terminal width. This is automatically disabled on CI (by detecting the `CI` environment variable)
+
+	- Possible values: `true`, `false`
+
+
 - `--color <COLOR>`
 
 	Enable or disable colored output from rattler-build. Also honors the `CLICOLOR` and `CLICOLOR_FORCE` environment variable

docs/reference/recipe_file.md

@@ -186,6 +186,17 @@ If an extracted archive contains only 1 folder at its top level, its contents
 will be moved 1 level up, so that the extracted package contents sit in the root
 of the work folder.
 
+##### Specifying a file name
+
+For URL and local paths you can specify a file name. If the source is an archive and a file name is set, automatic extraction is disabled.
+
+```yaml
+source:
+  url: https://pypi.python.org/packages/source/b/bsdiff4/bsdiff4-1.1.4.tar.gz
+  # will put the file in the work directory as `bsdiff4-1.1.4.tar.gz`
+  file_name: bsdiff4-1.1.4.tar.gz
+```
+
 #### Source from `git`
 
 ```yaml

docs/tutorials/javascript.md

@@ -0,0 +1,41 @@
+# Packaging a Javascript (NPM/NodeJS) package
+
+This tutorial will guide you though making a NodeJS package with `rattler-build`.
+
+## Building a NodeJS Package
+
+In this example, we will build a package for the NodeJS package `bibtex-tidy`.
+We use `nodejs` in build and run requirements, and install the package using `npm`.
+NPM comes as part of the NodeJS installation, so we do not need to install it separately.
+
+```yaml title="recipe.yaml"
+context:
+  version: "1.14.0"
+
+package:
+  name: bibtex-tidy
+  version: ${{ version }}
+
+source:
+  url: https://registry.npmjs.org/bibtex-tidy/-/bibtex-tidy-${{ version }}.tgz
+  sha256: 0a2c1bb73911a7cee36a30ce1fc86feffe39b2d39acd4c94d02aac6f84a00285
+  # we do not extract the source code and install the tarball directly as that works better
+  file_name: bibtex-tidy-${{ version }}.tgz
+
+build:
+  number: 0
+  script:
+    # we use NPM to globally install the bibtex-tidy package
+    - npm install -g bibtex-tidy-${{ version }}.tgz --prefix ${{ PREFIX }}
+
+requirements:
+  build:
+    - nodejs
+  run:
+    - nodejs
+
+tests:
+  - script:
+    - bibtex-tidy --version
+```
+

docs/tutorials/python.md

@@ -4,6 +4,7 @@ Writing a Python package is fairly straightforward, especially for "Python-only"
 In the second example we will build a package for `numpy` which contains compiled code.
 
 ## A Python-only package
+
 The following recipe uses the `noarch: python` setting to build a `noarch` package that can be installed on any platform without modification.
 This is very handy for packages that are pure Python and do not contain any compiled extensions.
 
@@ -63,6 +64,7 @@ about:
    installed correctly and can be imported.
 
 ### Running the recipe
+
 To build this recipe, simply run:
 
 ```bash
@@ -180,6 +182,7 @@ for /f %%f in ('dir /b /S .\dist') do (
 ```
 
 ### Running the recipe
+
 Running this recipe with the variant config file will build a total of 2 `numpy` packages:
 
 ```bash

src/recipe/parser/source.rs

@@ -401,9 +401,11 @@ pub struct UrlSource {
     /// Optionally a file name to rename the downloaded file (does not apply to archives)
     #[serde(skip_serializing_if = "Option::is_none")]
     file_name: Option<String>,
+
     /// Patches to apply to the source code
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     patches: Vec<PathBuf>,
+
     /// Optionally a folder name under the `work` directory to place the source code
     #[serde(skip_serializing_if = "Option::is_none")]
     target_directory: Option<PathBuf>,

src/source/url_source.rs

@@ -3,10 +3,12 @@
 use std::{
     ffi::OsStr,
     fs,
+    io::{Read as _, Write as _},
     path::{Path, PathBuf},
 };
 
 use crate::{
+    console_utils::LoggingOutputHandler,
     recipe::parser::UrlSource,
     source::extract::{extract_tar, extract_zip},
     tool_configuration,
@@ -15,48 +17,48 @@ use tokio::io::AsyncWriteExt;
 
 use super::{checksum::Checksum, extract::is_tarball, SourceError};
 
-fn split_filename(filename: &str) -> (String, String) {
-    let stem = Path::new(filename)
+/// Splits a path into stem and extension, handling special cases like .tar.gz
+fn split_path(path: &Path) -> std::io::Result<(String, String)> {
+    let stem = path
         .file_stem()
-        .and_then(|os_str| os_str.to_str())
-        .unwrap_or("")
-        .to_string();
+        .and_then(|s| s.to_str())
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid path stem"))?;
 
-    let stem_without_tar = stem.trim_end_matches(".tar");
+    let (stem_no_tar, is_tar) = if let Some(s) = stem.strip_suffix(".tar") {
+        (s, true)
+    } else {
+        (stem, false)
+    };
 
-    let extension = Path::new(filename)
-        .extension()
-        .and_then(|os_str| os_str.to_str())
-        .unwrap_or("")
-        .to_string();
+    let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("");
 
-    let full_extension = if stem != stem_without_tar {
+    let full_extension = if is_tar {
         format!(".tar.{}", extension)
     } else if !extension.is_empty() {
         format!(".{}", extension)
     } else {
-        "".to_string()
+        String::new()
     };
 
-    // remove any dots from the stem
-    let stem_without_tar = stem_without_tar.replace('.', "_");
-
-    (stem_without_tar.to_string(), full_extension)
+    Ok((stem_no_tar.replace('.', "_"), full_extension))
 }
 
+/// Generates a cache name from URL and checksum
 fn cache_name_from_url(
     url: &url::Url,
     checksum: &Checksum,
     with_extension: bool,
 ) -> Option<String> {
     let filename = url.path_segments()?.filter(|x| !x.is_empty()).last()?;
-    let (stem, extension) = split_filename(filename);
-    let checksum = checksum.to_hex();
-    if with_extension {
-        Some(format!("{}_{}{}", stem, &checksum[0..8], extension))
+
+    let (stem, extension) = split_path(Path::new(filename)).ok()?;
+    let checksum_hex = checksum.to_hex();
+
+    Some(if with_extension {
+        format!("{}_{}{}", stem, &checksum_hex[..8], extension)
     } else {
-        Some(format!("{}_{}", stem, &checksum[0..8]))
-    }
+        format!("{}_{}", stem, &checksum_hex[..8])
+    })
 }
 
 async fn fetch_remote(
@@ -85,6 +87,7 @@ async fn fetch_remote(
             .with_prefix("Downloading")
             .with_style(tool_configuration.fancy_log_handler.default_bytes_style()),
     );
+
     progress_bar.set_message(
         url.path_segments()
             .and_then(|segs| segs.last())
@@ -143,6 +146,37 @@ fn extract_to_cache(
     Ok(path.to_path_buf())
 }
 
+fn copy_with_progress(
+    source: &Path,
+    dest: &Path,
+    progress_handler: &LoggingOutputHandler,
+) -> std::io::Result<u64> {
+    let file_size = source.metadata()?.len();
+    let progress_bar = progress_handler.add_progress_bar(
+        indicatif::ProgressBar::new(file_size)
+            .with_prefix("Copying")
+            .with_style(progress_handler.default_bytes_style()),
+    );
+
+    let mut reader = std::io::BufReader::new(fs::File::open(source)?);
+    let mut writer = std::io::BufWriter::new(fs::File::create(dest)?);
+    let mut buffer = vec![0; 8192];
+    let mut copied = 0u64;
+
+    while let Ok(n) = reader.read(&mut buffer) {
+        if n == 0 {
+            break;
+        }
+        writer.write_all(&buffer[..n])?;
+        copied += n as u64;
+        progress_bar.set_position(copied);
+    }
+
+    progress_bar.finish();
+    writer.flush()?;
+    Ok(copied)
+}
+
 pub(crate) async fn url_src(
     source: &UrlSource,
     cache_dir: &Path,
@@ -155,6 +189,12 @@ pub(crate) async fn url_src(
 
     let mut last_error = None;
     for url in source.urls() {
+        let cache_name = PathBuf::from(cache_name_from_url(url, &checksum, true).ok_or(
+            SourceError::UnknownErrorStr("Failed to build cache name from url"),
+        )?);
+
+        let cache_name = cache_dir.join(cache_name);
+
         if url.scheme() == "file" {
             let local_path = url.to_file_path().map_err(|_| {
                 SourceError::Io(std::io::Error::new(
@@ -171,37 +211,43 @@ pub(crate) async fn url_src(
                 return Err(SourceError::ValidationFailed);
             }
 
+            // copy file to cache
+            copy_with_progress(
+                &local_path,
+                &cache_name,
+                &tool_configuration.fancy_log_handler,
+            )?;
+
             tracing::info!("Using local source file.");
-            return Ok(local_path);
+        } else {
+            let metadata = fs::metadata(&cache_name);
+            if metadata.is_ok() && metadata?.is_file() && checksum.validate(&cache_name) {
+                tracing::info!("Found valid source cache file.");
+            } else {
+                match fetch_remote(url, &cache_name, tool_configuration).await {
+                    Ok(_) => {
+                        tracing::info!("Downloaded file from {}", url);
+
+                        if !checksum.validate(&cache_name) {
+                            tracing::error!("Checksum validation failed!");
+                            fs::remove_file(&cache_name)?;
+                            return Err(SourceError::ValidationFailed);
+                        }
+                    }
+                    Err(e) => {
+                        last_error = Some(e);
+                        continue;
+                    }
+                }
+            }
         }
 
-        let cache_name = PathBuf::from(cache_name_from_url(url, &checksum, true).ok_or(
-            SourceError::UnknownErrorStr("Failed to build cache name from url"),
-        )?);
-        let cache_name = cache_dir.join(cache_name);
-
-        let metadata = fs::metadata(&cache_name);
-        if metadata.is_ok() && metadata?.is_file() && checksum.validate(&cache_name) {
-            tracing::info!("Found valid source cache file.");
+        // If the source has a file name, we skip the extraction step
+        if source.file_name().is_some() {
+            return Ok(cache_name);
+        } else {
             return extract_to_cache(&cache_name, tool_configuration);
         }
-
-        match fetch_remote(url, &cache_name, tool_configuration).await {
-            Ok(_) => {
-                tracing::info!("Downloaded file from {}", url);
-
-                if !checksum.validate(&cache_name) {
-                    tracing::error!("Checksum validation failed!");
-                    fs::remove_file(&cache_name)?;
-                    return Err(SourceError::ValidationFailed);
-                }
-
-                return extract_to_cache(&cache_name, tool_configuration);
-            }
-            Err(e) => {
-                last_error = Some(e);
-            }
-        }
     }
 
     if let Some(last_error) = last_error {
@@ -232,7 +278,7 @@ mod tests {
         ];
 
         for (filename, expected) in test_cases {
-            let (name, ending) = split_filename(filename);
+            let (name, ending) = split_path(Path::new(filename)).unwrap();
             assert_eq!(
                 (name.as_str(), ending.as_str()),
                 expected,