Skip to content

Commit

Permalink
Support fetching dependencies over git+http(s)
Browse files Browse the repository at this point in the history 
Closes ziglang#14298

This commit adds support for fetching dependencies over git+http(s)
using a minimal implementation of the Git protocols and formats relevant
to fetching repository data. It currently supports only version 2 of Git's
wire protocol (documented in
[protocol-v2](https://git-scm.com/docs/protocol-v2)), which was
first introduced in Git 2.19 (2018) and made the default in 2.26 (2020).

The wire protocol behaves similarly when used over other transports,
such as SSH and the "Git protocol" (git:// URLs), so it should be
reasonably straightforward to support fetching dependencies from such
URLs if the necessary transports are implemented (e.g. ziglang#14295).
  • Loading branch information
@ianprime0509
ianprime0509 committed Sep 25, 2023
1 parent df5f051 commit e2b64ae
Show file tree
Hide file tree
Showing 4 changed files with 1,570 additions and 58 deletions.
249 changes: 191 additions & 58 deletions src/Package.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const Module = @import("Module.zig");
const Cache = std.Build.Cache;
const build_options = @import("build_options");
const Manifest = @import("Manifest.zig");
const git = @import("git.zig");

pub const Table = std.StringHashMapUnmanaged(*Package);

Expand Down Expand Up @@ -647,65 +648,12 @@ fn fetchAndUnpack(
};
defer tmp_directory.closeAndFree(gpa);

var h = std.http.Headers{ .allocator = gpa };
defer h.deinit();

var req = try http_client.request(.GET, uri, h, .{});
defer req.deinit();

try req.start();
try req.wait();

if (req.response.status != .ok) {
return report.fail(dep.url_tok, "Expected response status '200 OK' got '{} {s}'", .{
@intFromEnum(req.response.status),
req.response.status.phrase() orelse "",
});
}

const content_type = req.response.headers.getFirstValue("Content-Type") orelse
return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});

var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{
.child_reader = req.reader(),
.prog_node = &pkg_prog_node,
.unit = if (req.response.content_length) |content_length| unit: {
const kib = content_length / 1024;
const mib = kib / 1024;
if (mib > 0) {
pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
pkg_prog_node.setUnit("MiB");
break :unit .mib;
} else {
pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
pkg_prog_node.setUnit("KiB");
break :unit .kib;
}
} else .any,
};
pkg_prog_node.context.refresh();

if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
{
// I observed the gzip stream to read 1 byte at a time, so I am using a
// buffered reader on the front of it.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
// I have not checked what buffer sizes the xz decompression implementation uses
// by default, so the same logic applies for buffering the reader as for gzip.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz);
} else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
if (isTarAttachment(content_disposition)) {
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
if (mem.eql(u8, uri.scheme, "http") or mem.eql(u8, uri.scheme, "https")) {
try fetchHttp(http_client, tmp_directory, dep, uri, report, &pkg_prog_node);
} else if (mem.eql(u8, uri.scheme, "git+http") or mem.eql(u8, uri.scheme, "git+https")) {
try fetchGit(http_client, tmp_directory, dep, uri, report, &pkg_prog_node);
} else {
return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});
return report.fail(dep.url_tok, "unsupported URL scheme: {s}", .{uri.scheme});
}

// Download completed - stop showing downloaded amount as progress
Expand Down Expand Up @@ -776,6 +724,191 @@ fn fetchAndUnpack(
};
}

fn fetchHttp(
http_client: *std.http.Client,
tmp_directory: Compilation.Directory,
dep: Manifest.Dependency,
uri: std.Uri,
report: Report,
pkg_prog_node: *std.Progress.Node,
) !void {
const gpa = http_client.allocator;

var h = std.http.Headers{ .allocator = gpa };
defer h.deinit();

var req = try http_client.request(.GET, uri, h, .{});
defer req.deinit();

try req.start();
try req.wait();

if (req.response.status != .ok) {
return report.fail(dep.url_tok, "Expected response status '200 OK' got '{} {s}'", .{
@intFromEnum(req.response.status),
req.response.status.phrase() orelse "",
});
}

const content_type = req.response.headers.getFirstValue("Content-Type") orelse
return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});

var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{
.child_reader = req.reader(),
.prog_node = pkg_prog_node,
.unit = if (req.response.content_length) |content_length| unit: {
const kib = content_length / 1024;
const mib = kib / 1024;
if (mib > 0) {
pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
pkg_prog_node.setUnit("MiB");
break :unit .mib;
} else {
pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
pkg_prog_node.setUnit("KiB");
break :unit .kib;
}
} else .any,
};
pkg_prog_node.context.refresh();

if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
{
// I observed the gzip stream to read 1 byte at a time, so I am using a
// buffered reader on the front of it.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
// I have not checked what buffer sizes the xz decompression implementation uses
// by default, so the same logic applies for buffering the reader as for gzip.
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz);
} else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
if (isTarAttachment(content_disposition)) {
try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
} else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
} else {
return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});
}
}

fn fetchGit(
http_client: *std.http.Client,
tmp_directory: Compilation.Directory,
dep: Manifest.Dependency,
uri: std.Uri,
report: Report,
pkg_prog_node: *std.Progress.Node,
) !void {
const gpa = http_client.allocator;
var transport_uri = uri;
transport_uri.scheme = uri.scheme["git+".len..];
var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
try session.discoverCapabilities(gpa);

const want_oid = want_oid: {
const want_ref = uri.fragment orelse "HEAD";
if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {}

const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref});
defer gpa.free(want_ref_head);
const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref});
defer gpa.free(want_ref_tag);

var ref_iterator = try session.listRefs(gpa, .{
.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
.include_peeled = true,
});
defer ref_iterator.deinit();
while (try ref_iterator.next()) |ref| {
if (mem.eql(u8, ref.name, want_ref) or
mem.eql(u8, ref.name, want_ref_head) or
mem.eql(u8, ref.name, want_ref_tag))
{
break :want_oid ref.peeled orelse ref.oid;
}
}
return report.fail(dep.url_tok, "ref not found: {s}", .{want_ref});
};
if (uri.fragment == null) {
const file_path = try report.directory.join(gpa, &.{Manifest.basename});
defer gpa.free(file_path);

const eb = report.error_bundle;
const notes_len = 1;
try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
.tok = dep.url_tok,
.off = 0,
.msg = "url field is missing an explicit ref",
});
const notes_start = try eb.reserveNotes(notes_len);
eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
}));
return error.PackageFetchFailed;
}

// The .git directory is used to store the packfile and associated index, but
// we do not attempt to replicate the exact structure of a real .git
// directory, since that isn't relevant for fetching a package.
{
var pack_dir = try tmp_directory.handle.makeOpenPath(".git", .{});
defer pack_dir.close();
var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
defer pack_file.close();
{
var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable;
var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});
defer fetch_stream.deinit();

var fetch_prog_node = pkg_prog_node.start("Fetch", 0);
defer fetch_prog_node.end();
fetch_prog_node.activate();
fetch_prog_node.context.refresh();
var prog_reader: ProgressReader(git.Session.FetchStream.Reader) = .{
.child_reader = fetch_stream.reader(),
.prog_node = pkg_prog_node,
.unit = .any,
};
pkg_prog_node.context.refresh();

var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
try fifo.pump(prog_reader.reader(), pack_file.writer());
try pack_file.sync();
}

var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
defer index_file.close();
{
var index_prog_node = pkg_prog_node.start("Index pack", 0);
defer index_prog_node.end();
index_prog_node.activate();
index_prog_node.context.refresh();
var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
try index_buffered_writer.flush();
try index_file.sync();
}

{
var checkout_prog_node = pkg_prog_node.start("Checkout", 0);
defer checkout_prog_node.end();
checkout_prog_node.activate();
checkout_prog_node.context.refresh();
var repository = try git.Repository.init(gpa, pack_file, index_file);
defer repository.deinit();
try repository.checkout(tmp_directory.handle, want_oid);
}
}

try tmp_directory.handle.deleteTree(".git");
}

fn unpackTarball(
gpa: Allocator,
req_reader: anytype,
Expand Down
Loading

0 comments on commit e2b64ae

Please sign in to comment.