Add .unpack = false and --no-unpack to build.zig.zon and zig fetch

closes ziglang#17895 Enhances zig with the ability to fetch a dependency of any file type.
marler8997 · Nov 21, 2024 · a7abaff · a7abaff
1 parent a5d4ad1
commit a7abaff
Show file tree

Hide file tree

Showing 10 changed files with 130 additions and 7 deletions.
diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig
@@ -33,6 +33,7 @@ location_tok: std.zig.Ast.TokenIndex,
 hash_tok: std.zig.Ast.TokenIndex,
 name_tok: std.zig.Ast.TokenIndex,
 lazy_status: LazyStatus,
+unpack: bool,
 parent_package_root: Cache.Path,
 parent_manifest_ast: ?*const std.zig.Ast,
 prog_node: std.Progress.Node,
@@ -345,12 +346,12 @@ pub fn run(f: *Fetch) RunError!void {
         .path_or_url => |path_or_url| {
             if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| {
                 var resource: Resource = .{ .dir = dir };
-                return f.runResource(path_or_url, &resource, null);
+                return f.runResource(path_or_url, &resource, null, f.unpack);
             } else |dir_err| {
                 const file_err = if (dir_err == error.NotDir) e: {
                     if (fs.cwd().openFile(path_or_url, .{})) |file| {
                         var resource: Resource = .{ .file = file };
-                        return f.runResource(path_or_url, &resource, null);
+                        return f.runResource(path_or_url, &resource, null, f.unpack);
                     } else |err| break :e err;
                 } else dir_err;
 
@@ -362,7 +363,7 @@ pub fn run(f: *Fetch) RunError!void {
                 };
                 var server_header_buffer: [header_buffer_size]u8 = undefined;
                 var resource = try f.initResource(uri, &server_header_buffer);
-                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);
+                return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, f.unpack);
             }
         },
     };
@@ -424,7 +425,7 @@ pub fn run(f: *Fetch) RunError!void {
     );
     var server_header_buffer: [header_buffer_size]u8 = undefined;
     var resource = try f.initResource(uri, &server_header_buffer);
-    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);
+    return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, f.unpack);
 }
 
 pub fn deinit(f: *Fetch) void {
@@ -438,6 +439,7 @@ fn runResource(
     uri_path: []const u8,
     resource: *Resource,
     remote_hash: ?Manifest.MultiHashHexDigest,
+    unpack: bool,
 ) RunError!void {
     defer resource.deinit();
     const arena = f.arena.allocator();
@@ -468,7 +470,7 @@ fn runResource(
         defer tmp_directory.handle.close();
 
         // Fetch and unpack a resource into a temporary directory.
-        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
+        var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory, unpack);
 
         var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
 
@@ -712,6 +714,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                 .hash_tok = dep.hash_tok,
                 .name_tok = dep.name_tok,
                 .lazy_status = if (dep.lazy) .available else .eager,
+                .unpack = dep.unpack,
                 .parent_package_root = f.package_root,
                 .parent_manifest_ast = &f.manifest_ast,
                 .prog_node = f.prog_node,
@@ -915,9 +918,10 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
 
     if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
         const path = try uri.path.toRawMaybeAlloc(arena);
-        return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| {
+        const path_relative = std.mem.trimLeft(u8, path, "/");
+        return .{ .file = f.parent_package_root.openFile(path_relative, .{}) catch |err| {
             return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
-                f.parent_package_root, path, @errorName(err),
+                f.parent_package_root, path_relative, @errorName(err),
             }));
         } };
     }
@@ -1055,8 +1059,35 @@ fn unpackResource(
     resource: *Resource,
     uri_path: []const u8,
     tmp_directory: Cache.Directory,
+    unpack: bool,
 ) RunError!UnpackResult {
     const eb = &f.error_bundle;
+
+    if (!unpack) {
+        const basename = std.fs.path.basename(uri_path)
+        var out_file = tmp_directory.handle.createFile(
+            basename,
+            .{},
+        ) catch |err| return f.fail(f.location_tok, try eb.printString(
+            "failed to create tmp file: {s}",
+            .{@errorName(err)},
+        ));
+        defer out_file.close();
+        var buf: [std.mem.page_size]u8 = undefined;
+        while (true) {
+            const len = resource.reader().readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "read stream failed: {s}",
+                .{@errorName(err)},
+            ));
+            if (len == 0) break;
+            out_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString(
+                "write temporary file failed: {s}",
+                .{@errorName(err)},
+            ));
+        }
+        return .{};
+    }
+
     const file_type = switch (resource.*) {
         .file => FileType.fromPath(uri_path) orelse
             return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),

diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig
@@ -25,6 +25,7 @@ pub const Dependency = struct {
     node: Ast.Node.Index,
     name_tok: Ast.TokenIndex,
     lazy: bool,
+    unpack: bool,
 
     pub const Location = union(enum) {
         url: []const u8,
@@ -302,6 +303,7 @@ const Parse = struct {
             .node = node,
             .name_tok = 0,
             .lazy = false,
+            .unpack = true,
         };
         var has_location = false;
 
@@ -350,6 +352,12 @@ const Parse = struct {
                     error.ParseFailure => continue,
                     else => |e| return e,
                 };
+            } else if (mem.eql(u8, field_name, "unpack")) {
+                dep.unpack = parseBool(p, field_init) catch |err| switch (err) {
+                    error.ParseFailure => continue,
+                    else => |e| return e,
+                };
+                if (dep.unpack) return fail(p, main_tokens[field_init], "unpack cannot be set to true, omit it instead", .{});
             } else {
                 // Ignore unknown fields so that we can add fields in future zig
                 // versions without breaking older zig versions.

diff --git a/src/main.zig b/src/main.zig
@@ -5073,6 +5073,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .hash_tok = 0,
                     .name_tok = 0,
                     .lazy_status = .eager,
+                    .unpack = true,
                     .parent_package_root = build_mod.root,
                     .parent_manifest_ast = null,
                     .prog_node = fetch_prog_node,
@@ -6814,6 +6815,7 @@ const usage_fetch =
     \\  --save=[name]                 Add the fetched package to build.zig.zon as name
     \\  --save-exact                  Add the fetched package to build.zig.zon, storing the URL verbatim
     \\  --save-exact=[name]           Add the fetched package to build.zig.zon as name, storing the URL verbatim
+    \\  --no-unpack                   Don't unpack the package
     \\
 ;
 
@@ -6835,6 +6837,7 @@ fn cmdFetch(
         yes: ?[]const u8,
         exact: ?[]const u8,
     } = .no;
+    var unpack = true;
 
     {
         var i: usize = 0;
@@ -6859,6 +6862,8 @@ fn cmdFetch(
                     save = .{ .exact = null };
                 } else if (mem.startsWith(u8, arg, "--save-exact=")) {
                     save = .{ .exact = arg["--save-exact=".len..] };
+                } else if (mem.eql(u8, arg, "--no-unpack")) {
+                    unpack = false;
                 } else {
                     fatal("unrecognized parameter: '{s}'", .{arg});
                 }
@@ -6913,6 +6918,7 @@ fn cmdFetch(
         .hash_tok = 0,
         .name_tok = 0,
         .lazy_status = .eager,
+        .unpack = unpack,
         .parent_package_root = undefined,
         .parent_manifest_ast = null,
         .prog_node = root_prog_node,

diff --git a/test/standalone/build.zig.zon b/test/standalone/build.zig.zon
@@ -69,6 +69,9 @@
         .@"extern" = .{
             .path = "extern",
         },
+        .fetch = .{
+            .path = "fetch",
+        },
         .dep_diamond = .{
             .path = "dep_diamond",
         },

diff --git a/test/standalone/fetch/build.zig b/test/standalone/fetch/build.zig
@@ -0,0 +1,37 @@
+const std = @import("std");
+
+pub fn build(b: *std.Build) !void {
+    b.default_step = b.step("test", "Test it");
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "--build-file",
+        });
+        run.addFileArg(b.path("example/build.zig"));
+        b.default_step.dependOn(&run.step);
+    }
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "build",
+            "--build-file",
+        });
+        run.addFileArg(b.path("unpacktrue/build.zig"));
+        run.addCheck(.{ .expect_stderr_match = "error: unpack cannot be set to true, omit it instead" });
+        b.default_step.dependOn(&run.step);
+    }
+
+    {
+        const run = b.addSystemCommand(&.{
+            b.graph.zig_exe,
+            "fetch",
+            "--no-unpack",
+        });
+        run.addFileArg(b.path("example/example_dep_file"));
+        run.expectStdOutEqual("122008dbadc3bbdeaf08a92bcd337c2dc994efc3c8c1e9a7f59a23901df660e89d8f\n");
+        b.default_step.dependOn(&run.step);
+    }
+}
diff --git a/test/standalone/fetch/example/build.zig b/test/standalone/fetch/example/build.zig
@@ -0,0 +1,3 @@
+const std = @import("std");
+
+pub fn build(_: *std.Build) !void {}
diff --git a/test/standalone/fetch/example/build.zig.zon b/test/standalone/fetch/example/build.zig.zon
@@ -0,0 +1,16 @@
+.{
+    .name = "fetch",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://localhost/example_dep_file",
+            .hash = "122008dbadc3bbdeaf08a92bcd337c2dc994efc3c8c1e9a7f59a23901df660e89d8f",
+            .unpack = false,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "example_dep_file",
+    },
+}
diff --git a/test/standalone/fetch/example/example_dep_file b/test/standalone/fetch/example/example_dep_file
@@ -0,0 +1 @@
+This is an example file.
diff --git a/test/standalone/fetch/unpacktrue/build.zig b/test/standalone/fetch/unpacktrue/build.zig
@@ -0,0 +1,3 @@
+const std = @import("std");
+
+pub fn build(_: *std.Build) !void {}
diff --git a/test/standalone/fetch/unpacktrue/build.zig.zon b/test/standalone/fetch/unpacktrue/build.zig.zon
@@ -0,0 +1,15 @@
+.{
+    .name = "unpacktrue",
+    .version = "0.0.0",
+    .dependencies = .{
+        .somedependency = .{
+            .url = "file://localhost/bar",
+            .hash = "1220f3b02ca452c26a96b48d2912b7fc907bef8d0b85c2e8f7e4a5c8bd95cdbfbae6",
+            .unpack = true,
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+    },
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		const std = @import("std");

		pub fn build(_: *std.Build) !void {}