From 6e953cf85afa45a66750471c19558801c872323b Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Wed, 20 Nov 2024 21:56:55 -0700 Subject: [PATCH] Add `.unpack = false` and --no-unpack to build.zig.zon and zig fetch closes #17895 Enhances zig with the ability to fetch a dependency of any file type. --- doc/build.zig.zon.md | 7 +++ lib/init/build.zig.zon | 4 ++ src/Package/Fetch.zig | 40 ++++++++++++-- src/Package/Manifest.zig | 8 +++ src/main.zig | 6 +++ test/standalone/build.zig.zon | 3 ++ test/standalone/fetch/build.zig | 52 +++++++++++++++++++ test/standalone/fetch/codegen.zig | 36 +++++++++++++ test/standalone/fetch/example/build.zig | 9 ++++ .../fetch/example/build.zig.zon.template | 16 ++++++ .../fetch/example/example_dep_file.txt | 1 + test/standalone/fetch/unpacktrue/build.zig | 3 ++ .../standalone/fetch/unpacktrue/build.zig.zon | 15 ++++++ 13 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 test/standalone/fetch/build.zig create mode 100644 test/standalone/fetch/codegen.zig create mode 100644 test/standalone/fetch/example/build.zig create mode 100644 test/standalone/fetch/example/build.zig.zon.template create mode 100644 test/standalone/fetch/example/example_dep_file.txt create mode 100644 test/standalone/fetch/unpacktrue/build.zig create mode 100644 test/standalone/fetch/unpacktrue/build.zig.zon diff --git a/doc/build.zig.zon.md b/doc/build.zig.zon.md index d0740a923760..ffa6dd200eb0 100644 --- a/doc/build.zig.zon.md +++ b/doc/build.zig.zon.md @@ -78,6 +78,13 @@ Boolean. When this is set to `true`, a package is declared to be lazily fetched. This makes the dependency only get fetched if it is actually used. +#### `unpack` + +Boolean. + +When this is set to `false`, the package is not unpacked but kept as a single +file. + ### `paths` List. Required. diff --git a/lib/init/build.zig.zon b/lib/init/build.zig.zon index c06abec206bd..a38625c70568 100644 --- a/lib/init/build.zig.zon +++ b/lib/init/build.zig.zon @@ -50,6 +50,10 @@ // // fetched. This makes the dependency only get fetched if it is // // actually used. // .lazy = false, + // + // // When this is set to `false`, the package is not unpacked but kept as a single + // // file. + // .unpack = false, //}, }, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 95aa41a44434..036eae7d4755 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -33,6 +33,7 @@ location_tok: std.zig.Ast.TokenIndex, hash_tok: std.zig.Ast.TokenIndex, name_tok: std.zig.Ast.TokenIndex, lazy_status: LazyStatus, +unpack: bool, parent_package_root: Cache.Path, parent_manifest_ast: ?*const std.zig.Ast, prog_node: std.Progress.Node, @@ -345,12 +346,12 @@ pub fn run(f: *Fetch) RunError!void { .path_or_url => |path_or_url| { if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, f.unpack); } else |dir_err| { const file_err = if (dir_err == error.NotDir) e: { if (fs.cwd().openFile(path_or_url, .{})) |file| { var resource: Resource = .{ .file = file }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, f.unpack); } else |err| break :e err; } else dir_err; @@ -362,7 +363,7 @@ pub fn run(f: *Fetch) RunError!void { }; var server_header_buffer: [header_buffer_size]u8 = undefined; var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, f.unpack); } }, }; @@ -424,7 +425,7 @@ pub fn run(f: *Fetch) RunError!void { ); var server_header_buffer: [header_buffer_size]u8 = undefined; var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, f.unpack); } pub fn deinit(f: *Fetch) void { @@ -438,6 +439,7 @@ fn runResource( uri_path: []const u8, resource: *Resource, remote_hash: ?Manifest.MultiHashHexDigest, + unpack: bool, ) RunError!void { defer resource.deinit(); const arena = f.arena.allocator(); @@ -468,7 +470,7 @@ fn runResource( defer tmp_directory.handle.close(); // Fetch and unpack a resource into a temporary directory. - var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); + var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory, unpack); var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; @@ -712,6 +714,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .hash_tok = dep.hash_tok, .name_tok = dep.name_tok, .lazy_status = if (dep.lazy) .available else .eager, + .unpack = dep.unpack, .parent_package_root = f.package_root, .parent_manifest_ast = &f.manifest_ast, .prog_node = f.prog_node, @@ -1055,8 +1058,35 @@ fn unpackResource( resource: *Resource, uri_path: []const u8, tmp_directory: Cache.Directory, + unpack: bool, ) RunError!UnpackResult { const eb = &f.error_bundle; + + if (!unpack) { + const basename = std.fs.path.basename(uri_path); + var out_file = tmp_directory.handle.createFile( + basename, + .{}, + ) catch |err| return f.fail(f.location_tok, try eb.printString( + "failed to create temporary file: {s}", + .{@errorName(err)}, + )); + defer out_file.close(); + var buf: [std.mem.page_size]u8 = undefined; + while (true) { + const len = resource.reader().readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( + "read stream failed: {s}", + .{@errorName(err)}, + )); + if (len == 0) break; + out_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( + "write temporary file failed: {s}", + .{@errorName(err)}, + )); + } + return .{}; + } + const file_type = switch (resource.*) { .file => FileType.fromPath(uri_path) orelse return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index 4eed6cc386e6..f45c4c9dfb83 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -25,6 +25,7 @@ pub const Dependency = struct { node: Ast.Node.Index, name_tok: Ast.TokenIndex, lazy: bool, + unpack: bool, pub const Location = union(enum) { url: []const u8, @@ -302,6 +303,7 @@ const Parse = struct { .node = node, .name_tok = 0, .lazy = false, + .unpack = true, }; var has_location = false; @@ -350,6 +352,12 @@ const Parse = struct { error.ParseFailure => continue, else => |e| return e, }; + } else if (mem.eql(u8, field_name, "unpack")) { + dep.unpack = parseBool(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + if (dep.unpack) return fail(p, main_tokens[field_init], "unpack cannot be set to true, omit it instead", .{}); } else { // Ignore unknown fields so that we can add fields in future zig // versions without breaking older zig versions. diff --git a/src/main.zig b/src/main.zig index 291820cb7523..2c77ae892839 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5073,6 +5073,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .hash_tok = 0, .name_tok = 0, .lazy_status = .eager, + .unpack = true, .parent_package_root = build_mod.root, .parent_manifest_ast = null, .prog_node = fetch_prog_node, @@ -6814,6 +6815,7 @@ const usage_fetch = \\ --save=[name] Add the fetched package to build.zig.zon as name \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ --no-unpack Don't unpack the package \\ ; @@ -6835,6 +6837,7 @@ fn cmdFetch( yes: ?[]const u8, exact: ?[]const u8, } = .no; + var unpack = true; { var i: usize = 0; @@ -6859,6 +6862,8 @@ fn cmdFetch( save = .{ .exact = null }; } else if (mem.startsWith(u8, arg, "--save-exact=")) { save = .{ .exact = arg["--save-exact=".len..] }; + } else if (mem.eql(u8, arg, "--no-unpack")) { + unpack = false; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } @@ -6913,6 +6918,7 @@ fn cmdFetch( .hash_tok = 0, .name_tok = 0, .lazy_status = .eager, + .unpack = unpack, .parent_package_root = undefined, .parent_manifest_ast = null, .prog_node = root_prog_node, diff --git a/test/standalone/build.zig.zon b/test/standalone/build.zig.zon index 30ec07823b75..e3c0be375752 100644 --- a/test/standalone/build.zig.zon +++ b/test/standalone/build.zig.zon @@ -69,6 +69,9 @@ .@"extern" = .{ .path = "extern", }, + .fetch = .{ + .path = "fetch", + }, .dep_diamond = .{ .path = "dep_diamond", }, diff --git a/test/standalone/fetch/build.zig b/test/standalone/fetch/build.zig new file mode 100644 index 000000000000..eb31933b65a7 --- /dev/null +++ b/test/standalone/fetch/build.zig @@ -0,0 +1,52 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) !void { + const test_step = b.step("test", "Test it"); + b.default_step = test_step; + + { + const codegen_exe = b.addExecutable(.{ + .name = "codegen", + .target = b.host, + .root_source_file = b.path("codegen.zig"), + }); + const run_codegen = b.addRunArtifact(codegen_exe); + const example_dir = run_codegen.addOutputDirectoryArg("example"); + + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "build", + "install", + "--build-file", + }); + run.addFileArg(example_dir.path(b, "build.zig")); + run.addArg("--prefix"); + const install_dir = run.addOutputDirectoryArg("install"); + const check_file = b.addCheckFile(install_dir.path(b, "example_dep_file.txt"), .{ + .expected_exact = "This is an example file.\n", + }); + test_step.dependOn(&check_file.step); + } + + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "build", + "--build-file", + }); + run.addFileArg(b.path("unpacktrue/build.zig")); + run.addCheck(.{ .expect_stderr_match = "error: unpack cannot be set to true, omit it instead" }); + test_step.dependOn(&run.step); + } + + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "fetch", + "--no-unpack", + }); + run.addFileArg(b.path("example/example_dep_file.txt")); + run.expectStdOutEqual("12200f68aca70ebc76057200af436aab5720ec53a780713c5dc614825db42a39dbfb\n"); + test_step.dependOn(&run.step); + } +} diff --git a/test/standalone/fetch/codegen.zig b/test/standalone/fetch/codegen.zig new file mode 100644 index 000000000000..150f70f33125 --- /dev/null +++ b/test/standalone/fetch/codegen.zig @@ -0,0 +1,36 @@ +const std = @import("std"); + +pub fn main() !void { + var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); + const arena = arena_instance.allocator(); + const args = try std.process.argsAlloc(arena); + std.debug.assert(args.len == 2); + const out_dir = args[1]; + + if (!std.fs.path.isAbsolute(out_dir)) { + std.log.err("directory '{s}' must be absolute", .{out_dir}); + std.process.exit(0xff); + } + + var dir = try std.fs.openDirAbsolute(out_dir, .{}); + defer dir.close(); + + try writeFile(dir, "build.zig", @embedFile("example/build.zig")); + try writeFile(dir, "example_dep_file.txt", @embedFile("example/example_dep_file.txt")); + + { + const template = @embedFile("example/build.zig.zon.template"); + const package_path_absolute = try arena.dupe(u8, out_dir); + for (package_path_absolute) |*c| { + c.* = if (c.* == '\\') '/' else c.*; + } + const content = try std.mem.replaceOwned(u8, arena, template, "", package_path_absolute); + try writeFile(dir, "build.zig.zon", content); + } +} + +fn writeFile(dir: std.fs.Dir, name: []const u8, content: []const u8) !void { + const file = try dir.createFile(name, .{}); + defer file.close(); + try file.writer().writeAll(content); +} diff --git a/test/standalone/fetch/example/build.zig b/test/standalone/fetch/example/build.zig new file mode 100644 index 000000000000..7142939a8bfb --- /dev/null +++ b/test/standalone/fetch/example/build.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const dep = b.dependency("somedependency", .{}); + b.getInstallStep().dependOn(&b.addInstallFile( + dep.path("example_dep_file.txt"), + "example_dep_file.txt", + ).step); +} diff --git a/test/standalone/fetch/example/build.zig.zon.template b/test/standalone/fetch/example/build.zig.zon.template new file mode 100644 index 000000000000..e6d87ab7b56c --- /dev/null +++ b/test/standalone/fetch/example/build.zig.zon.template @@ -0,0 +1,16 @@ +.{ + .name = "fetch", + .version = "0.0.0", + .dependencies = .{ + .somedependency = .{ + .url = "file:///example_dep_file.txt", + .hash = "12200f68aca70ebc76057200af436aab5720ec53a780713c5dc614825db42a39dbfb", + .unpack = false, + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "example_dep_file.txt", + }, +} diff --git a/test/standalone/fetch/example/example_dep_file.txt b/test/standalone/fetch/example/example_dep_file.txt new file mode 100644 index 000000000000..e738d768073c --- /dev/null +++ b/test/standalone/fetch/example/example_dep_file.txt @@ -0,0 +1 @@ +This is an example file. diff --git a/test/standalone/fetch/unpacktrue/build.zig b/test/standalone/fetch/unpacktrue/build.zig new file mode 100644 index 000000000000..f45a6021ee7d --- /dev/null +++ b/test/standalone/fetch/unpacktrue/build.zig @@ -0,0 +1,3 @@ +const std = @import("std"); + +pub fn build(_: *std.Build) void {} diff --git a/test/standalone/fetch/unpacktrue/build.zig.zon b/test/standalone/fetch/unpacktrue/build.zig.zon new file mode 100644 index 000000000000..284ea4455ebc --- /dev/null +++ b/test/standalone/fetch/unpacktrue/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = "unpacktrue", + .version = "0.0.0", + .dependencies = .{ + .somedependency = .{ + .url = "file://localhost/bar", + .hash = "1220f3b02ca452c26a96b48d2912b7fc907bef8d0b85c2e8f7e4a5c8bd95cdbfbae6", + .unpack = true, + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + }, +}