Add .unpack = false and --no-unpack to build.zig.zon and zig fetch

closes ziglang#17895

Enhances zig with the ability to fetch a dependency of any file type.

doc/build.zig.zon.md

@@ -78,6 +78,13 @@ Boolean.
 When this is set to `true`, a package is declared to be lazily fetched. This
 makes the dependency only get fetched if it is actually used.
 
+#### `unpack`
+
+Boolean.
+
+When this is set to `false`, the package is not unpacked but kept as a single
+file.
+
 ### `paths`
 
 List. Required.

lib/init/build.zig.zon

@@ -50,6 +50,10 @@
         //    // fetched. This makes the dependency only get fetched if it is
         //    // actually used.
         //    .lazy = false,
+        //
+        //    // When this is set to `false`, the package is not unpacked but kept as a single
+        //    // file.
+        //    .unpack = false,
         //},
     },
 

src/Package/Fetch.zig

@@ -33,6 +33,7 @@ location_tok: std.zig.Ast.TokenIndex,
 hash_tok: std.zig.Ast.TokenIndex,
 name_tok: std.zig.Ast.TokenIndex,
 lazy_status: LazyStatus,
+unpack: bool,
 parent_package_root: Cache.Path,
 parent_manifest_ast: ?*const std.zig.Ast,
 prog_node: std.Progress.Node,
@@ -345,12 +346,12 @@ pub fn run(f: *Fetch) RunError!void {
         .path_or_url => |path_or_url| {
             if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| {
                 var resource: Resource = .{ .dir = dir };
-                return f.runResource(path_or_url, &resource, null);
+                return f.runResource(path_or_url, &resource, null, f.unpack);
             } else |dir_err| {
                 const file_err = if (dir_err == error.NotDir) e: {
                     if (fs.cwd().openFile(path_or_url, .{})) |file| {
                         var resource: Resource = .{ .file = file };
-                        return f.runResource(path_or_url, &resource, null);
+                        return f.runResource(path_or_url, &resource, null, f.unpack);
                     } else |err| break :e err;
                 } else dir_err;
 
@@ -362,7 +363,7 @@ pub fn run(f: *Fetch) RunError!void {
                 };
                 var server_header_buffer: [header_buffer_size]u8 = undefined;
                 var resource = try f.initResource(uri, &server_header_buffer);
-                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);
+                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, f.unpack);
             }
         },
     };
@@ -424,7 +425,7 @@ pub fn run(f: *Fetch) RunError!void {
     );
     var server_header_buffer: [header_buffer_size]u8 = undefined;
     var resource = try f.initResource(uri, &server_header_buffer);
-    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);
+    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, f.unpack);
 }
 
 pub fn deinit(f: *Fetch) void {
@@ -438,6 +439,7 @@ fn runResource(
     uri_path: []const u8,
     resource: *Resource,
     remote_hash: ?Manifest.MultiHashHexDigest,
+    unpack: bool,
 ) RunError!void {
     defer resource.deinit();
     const arena = f.arena.allocator();
@@ -468,7 +470,7 @@ fn runResource(
         defer tmp_directory.handle.close();
 
         // Fetch and unpack a resource into a temporary directory.
-        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
+        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory, unpack);
 
         var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
 
@@ -712,6 +714,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                 .hash_tok = dep.hash_tok,
                 .name_tok = dep.name_tok,
                 .lazy_status = if (dep.lazy) .available else .eager,
+                .unpack = dep.unpack,
                 .parent_package_root = f.package_root,
                 .parent_manifest_ast = &f.manifest_ast,
                 .prog_node = f.prog_node,
@@ -1055,8 +1058,35 @@ fn unpackResource(
     resource: *Resource,
     uri_path: []const u8,
     tmp_directory: Cache.Directory,
+    unpack: bool,
 ) RunError!UnpackResult {
     const eb = &f.error_bundle;
+
+    if (!unpack) {
+        const basename = std.fs.path.basename(uri_path);
+        var out_file = tmp_directory.handle.createFile(
+            basename,
+            .{},
+        ) catch |err| return f.fail(f.location_tok, try eb.printString(
+            "failed to create temporary file: {s}",
+            .{@errorName(err)},
+        ));
+        defer out_file.close();
+        var buf: [std.mem.page_size]u8 = undefined;
+        while (true) {
+            const len = resource.reader().readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "read stream failed: {s}",
+                .{@errorName(err)},
+            ));
+            if (len == 0) break;
+            out_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "write temporary file failed: {s}",
+                .{@errorName(err)},
+            ));
+        }
+        return .{};
+    }
+
     const file_type = switch (resource.*) {
         .file => FileType.fromPath(uri_path) orelse
             return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),

src/Package/Manifest.zig

@@ -25,6 +25,7 @@ pub const Dependency = struct {
     node: Ast.Node.Index,
     name_tok: Ast.TokenIndex,
     lazy: bool,
+    unpack: bool,
 
     pub const Location = union(enum) {
         url: []const u8,
@@ -302,6 +303,7 @@ const Parse = struct {
             .node = node,
             .name_tok = 0,
             .lazy = false,
+            .unpack = true,
         };
         var has_location = false;
 
@@ -350,6 +352,12 @@ const Parse = struct {
                     error.ParseFailure => continue,
                     else => |e| return e,
                 };
+            } else if (mem.eql(u8, field_name, "unpack")) {
+                dep.unpack = parseBool(p, field_init) catch |err| switch (err) {
+                    error.ParseFailure => continue,
+                    else => |e| return e,
+                };
+                if (dep.unpack) return fail(p, main_tokens[field_init], "unpack cannot be set to true, omit it instead", .{});
             } else {
                 // Ignore unknown fields so that we can add fields in future zig
                 // versions without breaking older zig versions.

src/main.zig

@@ -5073,6 +5073,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .hash_tok = 0,
                     .name_tok = 0,
                     .lazy_status = .eager,
+                    .unpack = true,
                     .parent_package_root = build_mod.root,
                     .parent_manifest_ast = null,
                     .prog_node = fetch_prog_node,
@@ -6814,6 +6815,7 @@ const usage_fetch =
     \\  --save=[name]                 Add the fetched package to build.zig.zon as name
     \\  --save-exact                  Add the fetched package to build.zig.zon, storing the URL verbatim
     \\  --save-exact=[name]           Add the fetched package to build.zig.zon as name, storing the URL verbatim
+    \\  --no-unpack                   Don't unpack the package
     \\
 ;
 
@@ -6835,6 +6837,7 @@ fn cmdFetch(
         yes: ?[]const u8,
         exact: ?[]const u8,
     } = .no;
+    var unpack = true;
 
     {
         var i: usize = 0;
@@ -6859,6 +6862,8 @@ fn cmdFetch(
                     save = .{ .exact = null };
                 } else if (mem.startsWith(u8, arg, "--save-exact=")) {
                     save = .{ .exact = arg["--save-exact=".len..] };
+                } else if (mem.eql(u8, arg, "--no-unpack")) {
+                    unpack = false;
                 } else {
                     fatal("unrecognized parameter: '{s}'", .{arg});
                 }
@@ -6913,6 +6918,7 @@ fn cmdFetch(
         .hash_tok = 0,
         .name_tok = 0,
         .lazy_status = .eager,
+        .unpack = unpack,
         .parent_package_root = undefined,
         .parent_manifest_ast = null,
         .prog_node = root_prog_node,

test/standalone/build.zig.zon

@@ -69,6 +69,9 @@
         .@"extern" = .{
             .path = "extern",
         },
+        .fetch = .{
+            .path = "fetch",
+        },
         .dep_diamond = .{
             .path = "dep_diamond",
         },

test/standalone/fetch/build.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    const test_step = b.step("test", "Test it");
+    b.default_step = test_step;
+
+    {
+        const codegen_exe = b.addExecutable(.{
+            .name = "codegen",
+            .target = b.host,
+            .root_source_file = b.path("codegen.zig"),
+        });
+        const run_codegen = b.addRunArtifact(codegen_exe);
+        const example_dir = run_codegen.addOutputDirectoryArg("example");
+
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "install",
+            "--build-file",
+        });
+        run.addFileArg(example_dir.path(b, "build.zig"));
+        run.addArg("--prefix");
+        const install_dir = run.addOutputDirectoryArg("install");
+        const check_file = b.addCheckFile(install_dir.path(b, "example_dep_file.txt"), .{
+            .expected_exact = "This is an example file.\n",
+        });
+        test_step.dependOn(&check_file.step);
+    }
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "--build-file",
+        });
+        run.addFileArg(b.path("unpacktrue/build.zig"));
+        run.addCheck(.{ .expect_stderr_match = "error: unpack cannot be set to true, omit it instead" });
+        test_step.dependOn(&run.step);
+    }
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "fetch",
+            "--no-unpack",
+        });
+        run.addFileArg(b.path("example/example_dep_file.txt"));
+        run.expectStdOutEqual("12200f68aca70ebc76057200af436aab5720ec53a780713c5dc614825db42a39dbfb\n");
+        test_step.dependOn(&run.step);
+    }
+}

test/standalone/fetch/codegen.zig

@@ -0,0 +1,36 @@
+const std = @import("std");
+
+pub fn main() !void {
+    var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const arena = arena_instance.allocator();
+    const args = try std.process.argsAlloc(arena);
+    std.debug.assert(args.len == 2);
+    const out_dir = args[1];
+
+    if (!std.fs.path.isAbsolute(out_dir)) {
+        std.log.err("directory '{s}' must be absolute", .{out_dir});
+        std.process.exit(0xff);
+    }
+
+    var dir = try std.fs.openDirAbsolute(out_dir, .{});
+    defer dir.close();
+
+    try writeFile(dir, "build.zig", @embedFile("example/build.zig"));
+    try writeFile(dir, "example_dep_file.txt", @embedFile("example/example_dep_file.txt"));
+
+    {
+        const template = @embedFile("example/build.zig.zon.template");
+        const package_path_absolute = try arena.dupe(u8, out_dir);
+        for (package_path_absolute) |*c| {
+            c.* = if (c.* == '\\') '/' else c.*;
+        }
+        const content = try std.mem.replaceOwned(u8, arena, template, "<PACKAGE_PATH_ABSOLUTE>", package_path_absolute);
+        try writeFile(dir, "build.zig.zon", content);
+    }
+}
+
+fn writeFile(dir: std.fs.Dir, name: []const u8, content: []const u8) !void {
+    const file = try dir.createFile(name, .{});
+    defer file.close();
+    try file.writer().writeAll(content);
+}

test/standalone/fetch/example/build.zig

@@ -0,0 +1,9 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) void {
+    const dep = b.dependency("somedependency", .{});
+    b.getInstallStep().dependOn(&b.addInstallFile(
+        dep.path("example_dep_file.txt"),
+        "example_dep_file.txt",
+    ).step);
+}

test/standalone/fetch/example/build.zig.zon.template

@@ -0,0 +1,16 @@
+.{
+    .name = "fetch",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://<PACKAGE_PATH_ABSOLUTE>/example_dep_file.txt",
+            .hash = "12200f68aca70ebc76057200af436aab5720ec53a780713c5dc614825db42a39dbfb",
+            .unpack = false,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "example_dep_file.txt",
+    },
+}

test/standalone/fetch/example/example_dep_file.txt

@@ -0,0 +1 @@
+This is an example file.

test/standalone/fetch/unpacktrue/build.zig

@@ -0,0 +1,3 @@
+const std = @import("std");
+
+pub fn build(_: *std.Build) void {}

test/standalone/fetch/unpacktrue/build.zig.zon

@@ -0,0 +1,15 @@
+.{
+    .name = "unpacktrue",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://localhost/bar",
+            .hash = "1220f3b02ca452c26a96b48d2912b7fc907bef8d0b85c2e8f7e4a5c8bd95cdbfbae6",
+            .unpack = true,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+    },
+}