Add support for remaining pip-supported file extensions (#7387)

closes #7365 Summary This pull request adds support for additional file extension aliases in the SourceDistExtension and ExtensionError enums. The newly supported file extensions include .tbz, .tgz, .txz, .tar.lz, .tar.lzma. These changes align the extensions supported by the SourceDistExtension with those used in Python packaging tools, enhancing compatibility with a broader range of source distribution formats. Test Plan should be added or updated to verify that the new extensions are correctly recognized as valid source distributions and that errors are correctly raised when unsupported extensions are provided.
astral-sh · Sep 14, 2024 · 3d62154 · 3d62154
1 parent 3060fd2
commit 3d62154
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 19 deletions.
diff --git a/crates/distribution-filename/src/extension.rs b/crates/distribution-filename/src/extension.rs
@@ -33,6 +33,8 @@ pub enum SourceDistExtension {
     TarBz2,
     TarXz,
     TarZst,
+    TarLzma,
+    Tar,
 }
 
 impl DistExtension {
@@ -69,10 +71,15 @@ impl SourceDistExtension {
 
         match extension {
             "zip" => Ok(Self::Zip),
-            "gz" if is_tar(path.as_ref()) => Ok(Self::TarGz),
+            "tar" => Ok(Self::Tar),
             "tgz" => Ok(Self::TarGz),
+            "tbz" => Ok(Self::TarBz2),
+            "txz" => Ok(Self::TarXz),
+            "tlz" => Ok(Self::TarLzma),
+            "gz" if is_tar(path.as_ref()) => Ok(Self::TarGz),
             "bz2" if is_tar(path.as_ref()) => Ok(Self::TarBz2),
             "xz" if is_tar(path.as_ref()) => Ok(Self::TarXz),
+            "lz" | "lzma" if is_tar(path.as_ref()) => Ok(Self::TarLzma),
             "zst" if is_tar(path.as_ref()) => Ok(Self::TarZst),
             _ => Err(ExtensionError::SourceDist),
         }
@@ -87,14 +94,16 @@ impl Display for SourceDistExtension {
             Self::TarBz2 => f.write_str("tar.bz2"),
             Self::TarXz => f.write_str("tar.xz"),
             Self::TarZst => f.write_str("tar.zst"),
+            Self::TarLzma => f.write_str("tar.lzma"),
+            Self::Tar => f.write_str("tar"),
         }
     }
 }
 
 #[derive(Error, Debug)]
 pub enum ExtensionError {
-    #[error("`.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`")]
+    #[error("`.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`")]
     Dist,
-    #[error("`.zip`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`")]
+    #[error("`.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`")]
     SourceDist,
 }
diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs
@@ -1496,7 +1496,7 @@ mod test {
         }, {
             insta::assert_snapshot!(errors, @r###"
             Couldn't parse requirement in `<REQUIREMENTS_TXT>` at position 3
-            Expected direct URL (`https://localhost:8080/`) to end in a supported file extension: `.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`
+            Expected direct URL (`https://localhost:8080/`) to end in a supported file extension: `.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`
             https://localhost:8080/
             ^^^^^^^^^^^^^^^^^^^^^^^
             "###);

diff --git a/crates/uv-extract/src/stream.rs b/crates/uv-extract/src/stream.rs
@@ -10,7 +10,7 @@ use tracing::warn;
 
 const DEFAULT_BUF_SIZE: usize = 128 * 1024;
 
-/// Unzip a `.zip` archive into the target directory, without requiring `Seek`.
+/// Unpack a `.zip` archive into the target directory, without requiring `Seek`.
 ///
 /// This is useful for unzipping files as they're being downloaded. If the archive
 /// is already fully on disk, consider using `unzip_archive`, which can use multiple
@@ -154,7 +154,7 @@ async fn untar_in<'a>(
     Ok(())
 }
 
-/// Unzip a `.tar.gz` archive into the target directory, without requiring `Seek`.
+/// Unpack a `.tar.gz` archive into the target directory, without requiring `Seek`.
 ///
 /// This is useful for unpacking files as they're being downloaded.
 pub async fn untar_gz<R: tokio::io::AsyncRead + Unpin>(
@@ -172,7 +172,7 @@ pub async fn untar_gz<R: tokio::io::AsyncRead + Unpin>(
     Ok(untar_in(archive, target.as_ref()).await?)
 }
 
-/// Unzip a `.tar.bz2` archive into the target directory, without requiring `Seek`.
+/// Unpack a `.tar.bz2` archive into the target directory, without requiring `Seek`.
 ///
 /// This is useful for unpacking files as they're being downloaded.
 pub async fn untar_bz2<R: tokio::io::AsyncRead + Unpin>(
@@ -190,7 +190,7 @@ pub async fn untar_bz2<R: tokio::io::AsyncRead + Unpin>(
     Ok(untar_in(archive, target.as_ref()).await?)
 }
 
-/// Unzip a `.tar.zst` archive into the target directory, without requiring `Seek`.
+/// Unpack a `.tar.zst` archive into the target directory, without requiring `Seek`.
 ///
 /// This is useful for unpacking files as they're being downloaded.
 pub async fn untar_zst<R: tokio::io::AsyncRead + Unpin>(
@@ -208,7 +208,7 @@ pub async fn untar_zst<R: tokio::io::AsyncRead + Unpin>(
     Ok(untar_in(archive, target.as_ref()).await?)
 }
 
-/// Unzip a `.tar.xz` archive into the target directory, without requiring `Seek`.
+/// Unpack a `.tar.xz` archive into the target directory, without requiring `Seek`.
 ///
 /// This is useful for unpacking files as they're being downloaded.
 pub async fn untar_xz<R: tokio::io::AsyncRead + Unpin>(
@@ -227,7 +227,24 @@ pub async fn untar_xz<R: tokio::io::AsyncRead + Unpin>(
     Ok(())
 }
 
-/// Unzip a `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.zst`, or `.tar.xz` archive into the target directory,
+/// Unpack a `.tar` archive into the target directory, without requiring `Seek`.
+///
+/// This is useful for unpacking files as they're being downloaded.
+pub async fn untar<R: tokio::io::AsyncRead + Unpin>(
+    reader: R,
+    target: impl AsRef<Path>,
+) -> Result<(), Error> {
+    let mut reader = tokio::io::BufReader::with_capacity(DEFAULT_BUF_SIZE, reader);
+
+    let archive =
+        tokio_tar::ArchiveBuilder::new(&mut reader as &mut (dyn tokio::io::AsyncRead + Unpin))
+            .set_preserve_mtime(false)
+            .build();
+    untar_in(archive, target.as_ref()).await?;
+    Ok(())
+}
+
+/// Unpack a `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.zst`, or `.tar.xz` archive into the target directory,
 /// without requiring `Seek`.
 pub async fn archive<R: tokio::io::AsyncRead + Unpin>(
     reader: R,
@@ -238,13 +255,16 @@ pub async fn archive<R: tokio::io::AsyncRead + Unpin>(
         SourceDistExtension::Zip => {
             unzip(reader, target).await?;
         }
+        SourceDistExtension::Tar => {
+            untar(reader, target).await?;
+        }
         SourceDistExtension::TarGz => {
             untar_gz(reader, target).await?;
         }
         SourceDistExtension::TarBz2 => {
             untar_bz2(reader, target).await?;
         }
-        SourceDistExtension::TarXz => {
+        SourceDistExtension::TarXz | SourceDistExtension::TarLzma => {
             untar_xz(reader, target).await?;
         }
         SourceDistExtension::TarZst => {

diff --git a/crates/uv/tests/build.rs b/crates/uv/tests/build.rs
@@ -816,7 +816,7 @@ fn wheel_from_sdist() -> Result<()> {
 
     ----- stderr -----
     Building wheel from source distribution...
-    error: `dist/project-0.1.0-py3-none-any.whl` is not a valid build source. Expected to receive a source directory, or a source distribution ending in one of: `.zip`, `.tar.gz`, `.tgz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`.
+    error: `dist/project-0.1.0-py3-none-any.whl` is not a valid build source. Expected to receive a source directory, or a source distribution ending in one of: `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`.
     "###);
 
     Ok(())

diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs
@@ -11575,7 +11575,7 @@ fn invalid_tool_uv_sources() -> Result<()> {
 
     ----- stderr -----
     error: Failed to parse metadata from built wheel
-      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.tar.baz`) to end in a supported file extension: `.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`
+      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.tar.baz`) to end in a supported file extension: `.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`
     urllib3 @ https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.tar.baz
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     "###
@@ -11603,7 +11603,7 @@ fn invalid_tool_uv_sources() -> Result<()> {
 
     ----- stderr -----
     error: Failed to parse entry for: `urllib3`
-      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.tar.baz`) to end in a supported file extension: `.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`
+      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.tar.baz`) to end in a supported file extension: `.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`
     "###
     );
 

diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs
@@ -6712,7 +6712,7 @@ fn invalid_extension() {
 
     ----- stderr -----
     error: Failed to parse: `ruff @ https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6.tar.baz`
-      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6.tar.baz`) to end in a supported file extension: `.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`
+      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6.tar.baz`) to end in a supported file extension: `.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`
     ruff @ https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6.tar.baz
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     "###);
@@ -6732,7 +6732,7 @@ fn no_extension() {
 
     ----- stderr -----
     error: Failed to parse: `ruff @ https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6`
-      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6`) to end in a supported file extension: `.whl`, `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.xz`, or `.tar.zst`
+      Caused by: Expected direct URL (`https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6`) to end in a supported file extension: `.whl`, `.tar.gz`, `.zip`, `.tar.bz2`, `.tar.lz`, `.tar.lzma`, `.tar.xz`, `.tar.zst`, `.tar`, `.tbz`, `.tgz`, `.tlz`, or `.txz`
     ruff @ https://files.pythonhosted.org/packages/f7/69/96766da2cdb5605e6a31ef2734aff0be17901cefb385b885c2ab88896d76/ruff-0.5.6
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     "###);

diff --git a/docs/concepts/resolution.md b/docs/concepts/resolution.md
@@ -303,11 +303,13 @@ distributions as gzip tarball (`.tar.gz`) archives. Prior to this specification,
 formats, which need to be supported for backward compatibility, were also allowed. uv supports
 reading and extracting archives in the following formats:
 
-- bzip2 tarball (`.tar.bz2`)
 - gzip tarball (`.tar.gz`, `.tgz`)
-- xz tarball (`.tar.xz`)
-- zip (`.zip`)
+- bzip2 tarball (`.tar.bz2`, `.tbz`)
+- xz tarball (`.tar.xz`, `.txz`)
 - zstd tarball (`.tar.zst`)
+- lzip tarball (`.tar.lz`)
+- lzma tarball (`.tar.lzma`)
+- zip (`.zip`)
 
 ## Learn more