Merge pull request #15579 from squeek502/mem-delimiters

Split `std.mem.split` and `tokenize` into `sequence`, `any`, and `scalar` versions

build.zig

@@ -235,7 +235,7 @@ pub fn build(b: *std.Build) !void {
  },
  2 => {
  // Untagged development build (e.g. 0.10.0-dev.2025+ecf0050a9).
- var it = mem.split(u8, git_describe, "-");
+ var it = mem.splitScalar(u8, git_describe, '-');
  const tagged_ancestor = it.first();
  const commit_height = it.next().?;
  const commit_id = it.next().?;
@@ -280,7 +280,7 @@ pub fn build(b: *std.Build) !void {
  // That means we also have to rely on stage1 compiled c++ files. We parse config.h to find
  // the information passed on to us from cmake.
  if (cfg.cmake_prefix_path.len > 0) {
- var it = mem.tokenize(u8, cfg.cmake_prefix_path, ";");
+ var it = mem.tokenizeScalar(u8, cfg.cmake_prefix_path, ';');
  while (it.next()) |path| {
  b.addSearchPrefix(path);
  }
@@ -682,7 +682,7 @@ fn addCxxKnownPath(
  if (!std.process.can_spawn)
  return error.RequiredLibraryNotFound;
  const path_padded = b.exec(&.{ ctx.cxx_compiler, b.fmt("-print-file-name={s}", .{objname}) });
- var tokenizer = mem.tokenize(u8, path_padded, "\r\n");
+ var tokenizer = mem.tokenizeAny(u8, path_padded, "\r\n");
  const path_unpadded = tokenizer.next().?;
  if (mem.eql(u8, path_unpadded, objname)) {
  if (errtxt) |msg| {
@@ -705,7 +705,7 @@ fn addCxxKnownPath(
 }
 
 fn addCMakeLibraryList(exe: *std.Build.Step.Compile, list: []const u8) void {
- var it = mem.tokenize(u8, list, ";");
+ var it = mem.tokenizeScalar(u8, list, ';');
  while (it.next()) |lib| {
  if (mem.startsWith(u8, lib, "-l")) {
  exe.linkSystemLibrary(lib["-l".len..]);
@@ -850,18 +850,18 @@ fn parseConfigH(b: *std.Build, config_h_text: []const u8) ?CMakeConfig {
  // .prefix = ZIG_LLVM_LINK_MODE parsed manually below
  };
 
- var lines_it = mem.tokenize(u8, config_h_text, "\r\n");
+ var lines_it = mem.tokenizeAny(u8, config_h_text, "\r\n");
  while (lines_it.next()) |line| {
  inline for (mappings) |mapping| {
  if (mem.startsWith(u8, line, mapping.prefix)) {
- var it = mem.split(u8, line, "\"");
+ var it = mem.splitScalar(u8, line, '"');
  _ = it.first(); // skip the stuff before the quote
  const quoted = it.next().?; // the stuff inside the quote
  @field(ctx, mapping.field) = toNativePathSep(b, quoted);
  }
  }
  if (mem.startsWith(u8, line, "#define ZIG_LLVM_LINK_MODE ")) {
- var it = mem.split(u8, line, "\"");
+ var it = mem.splitScalar(u8, line, '"');
  _ = it.next().?; // skip the stuff before the quote
  const quoted = it.next().?; // the stuff inside the quote
  ctx.llvm_linkage = if (mem.eql(u8, quoted, "shared")) .dynamic else .static;

doc/docgen.zig

@@ -1223,7 +1223,7 @@ fn printShell(out: anytype, shell_content: []const u8, escape: bool) !void {
  const trimmed_shell_content = mem.trim(u8, shell_content, " \n");
  try out.writeAll("<figure><figcaption class=\"shell-cap\">Shell</figcaption><pre><samp>");
  var cmd_cont: bool = false;
- var iter = std.mem.split(u8, trimmed_shell_content, "\n");
+ var iter = std.mem.splitScalar(u8, trimmed_shell_content, '\n');
  while (iter.next()) |orig_line| {
  const line = mem.trimRight(u8, orig_line, " ");
  if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') {

lib/std/Build.zig

@@ -1388,7 +1388,7 @@ pub fn findProgram(self: *Build, names: []const []const u8, paths: []const []con
  if (fs.path.isAbsolute(name)) {
  return name;
  }
- var it = mem.tokenize(u8, PATH, &[_]u8{fs.path.delimiter});
+ var it = mem.tokenizeScalar(u8, PATH, fs.path.delimiter);
  while (it.next()) |path| {
  const full_path = self.pathJoin(&.{
  path,

lib/std/Build/Cache.zig

@@ -438,7 +438,7 @@ pub const Manifest = struct {
 
  const input_file_count = self.files.items.len;
  var any_file_changed = false;
- var line_iter = mem.tokenize(u8, file_contents, "\n");
+ var line_iter = mem.tokenizeScalar(u8, file_contents, '\n');
  var idx: usize = 0;
  if (if (line_iter.next()) |line| !std.mem.eql(u8, line, manifest_header) else true) {
  if (try self.upgradeToExclusiveLock()) continue;
@@ -467,7 +467,7 @@ pub const Manifest = struct {
  break :blk new;
  };
 
- var iter = mem.tokenize(u8, line, " ");
+ var iter = mem.tokenizeScalar(u8, line, ' ');
  const size = iter.next() orelse return error.InvalidFormat;
  const inode = iter.next() orelse return error.InvalidFormat;
  const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;

lib/std/Build/Step/CheckObject.zig

@@ -103,8 +103,8 @@ const Action = struct {
  assert(act.tag == .match or act.tag == .not_present);
  const phrase = act.phrase.resolve(b, step);
  var candidate_var: ?struct { name: []const u8, value: u64 } = null;
- var hay_it = mem.tokenize(u8, mem.trim(u8, haystack, " "), " ");
- var needle_it = mem.tokenize(u8, mem.trim(u8, phrase, " "), " ");
+ var hay_it = mem.tokenizeScalar(u8, mem.trim(u8, haystack, " "), ' ');
+ var needle_it = mem.tokenizeScalar(u8, mem.trim(u8, phrase, " "), ' ');
 
  while (needle_it.next()) |needle_tok| {
  const hay_tok = hay_it.next() orelse return false;
@@ -155,7 +155,7 @@ const Action = struct {
  var op_stack = std.ArrayList(enum { add, sub, mod, mul }).init(gpa);
  var values = std.ArrayList(u64).init(gpa);
 
- var it = mem.tokenize(u8, phrase, " ");
+ var it = mem.tokenizeScalar(u8, phrase, ' ');
  while (it.next()) |next| {
  if (mem.eql(u8, next, "+")) {
  try op_stack.append(.add);
@@ -365,7 +365,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
  var vars = std.StringHashMap(u64).init(gpa);
 
  for (self.checks.items) |chk| {
- var it = mem.tokenize(u8, output, "\r\n");
+ var it = mem.tokenizeAny(u8, output, "\r\n");
  for (chk.actions.items) |act| {
  switch (act.tag) {
  .match => {

lib/std/Build/Step/Compile.zig

@@ -853,7 +853,7 @@ fn runPkgConfig(self: *Compile, lib_name: []const u8) ![]const []const u8 {
  var zig_args = ArrayList([]const u8).init(b.allocator);
  defer zig_args.deinit();
 
- var it = mem.tokenize(u8, stdout, " \r\n\t");
+ var it = mem.tokenizeAny(u8, stdout, " \r\n\t");
  while (it.next()) |tok| {
  if (mem.eql(u8, tok, "-I")) {
  const dir = it.next() orelse return error.PkgConfigInvalidOutput;
@@ -2101,10 +2101,10 @@ fn execPkgConfigList(self: *std.Build, out_code: *u8) (PkgConfigError || ExecErr
  const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);
  var list = ArrayList(PkgConfigPkg).init(self.allocator);
  errdefer list.deinit();
- var line_it = mem.tokenize(u8, stdout, "\r\n");
+ var line_it = mem.tokenizeAny(u8, stdout, "\r\n");
  while (line_it.next()) |line| {
  if (mem.trim(u8, line, " \t").len == 0) continue;
- var tok_it = mem.tokenize(u8, line, " \t");
+ var tok_it = mem.tokenizeAny(u8, line, " \t");
  try list.append(PkgConfigPkg{
  .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
  .desc = tok_it.rest(),
@@ -2224,7 +2224,7 @@ fn checkCompileErrors(self: *Compile) !void {
  // Render the expected lines into a string that we can compare verbatim.
  var expected_generated = std.ArrayList(u8).init(arena);
 
- var actual_line_it = mem.split(u8, actual_stderr, "\n");
+ var actual_line_it = mem.splitScalar(u8, actual_stderr, '\n');
  for (self.expect_errors) |expect_line| {
  const actual_line = actual_line_it.next() orelse {
  try expected_generated.appendSlice(expect_line);

lib/std/Build/Step/ConfigHeader.zig

@@ -250,14 +250,14 @@ fn render_autoconf(
 
  var any_errors = false;
  var line_index: u32 = 0;
- var line_it = std.mem.split(u8, contents, "\n");
+ var line_it = std.mem.splitScalar(u8, contents, '\n');
  while (line_it.next()) |line| : (line_index += 1) {
  if (!std.mem.startsWith(u8, line, "#")) {
  try output.appendSlice(line);
  try output.appendSlice("\n");
  continue;
  }
- var it = std.mem.tokenize(u8, line[1..], " \t\r");
+ var it = std.mem.tokenizeAny(u8, line[1..], " \t\r");
  const undef = it.next().?;
  if (!std.mem.eql(u8, undef, "undef")) {
  try output.appendSlice(line);
@@ -297,14 +297,14 @@ fn render_cmake(
 
  var any_errors = false;
  var line_index: u32 = 0;
- var line_it = std.mem.split(u8, contents, "\n");
+ var line_it = std.mem.splitScalar(u8, contents, '\n');
  while (line_it.next()) |line| : (line_index += 1) {
  if (!std.mem.startsWith(u8, line, "#")) {
  try output.appendSlice(line);
  try output.appendSlice("\n");
  continue;
  }
- var it = std.mem.tokenize(u8, line[1..], " \t\r");
+ var it = std.mem.tokenizeAny(u8, line[1..], " \t\r");
  const cmakedefine = it.next().?;
  if (!std.mem.eql(u8, cmakedefine, "cmakedefine") and
  !std.mem.eql(u8, cmakedefine, "cmakedefine01"))

lib/std/SemanticVersion.zig

@@ -42,8 +42,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {
  if (lhs.pre == null and rhs.pre != null) return .gt;
 
  // Iterate over pre-release identifiers until a difference is found.
- var lhs_pre_it = std.mem.split(u8, lhs.pre.?, ".");
- var rhs_pre_it = std.mem.split(u8, rhs.pre.?, ".");
+ var lhs_pre_it = std.mem.splitScalar(u8, lhs.pre.?, '.');
+ var rhs_pre_it = std.mem.splitScalar(u8, rhs.pre.?, '.');
  while (true) {
  const next_lid = lhs_pre_it.next();
  const next_rid = rhs_pre_it.next();
@@ -86,7 +86,7 @@ pub fn parse(text: []const u8) !Version {
  // Parse the required major, minor, and patch numbers.
  const extra_index = std.mem.indexOfAny(u8, text, "-+");
  const required = text[0..(extra_index orelse text.len)];
- var it = std.mem.split(u8, required, ".");
+ var it = std.mem.splitScalar(u8, required, '.');
  var ver = Version{
  .major = try parseNum(it.first()),
  .minor = try parseNum(it.next() orelse return error.InvalidVersion),
@@ -108,7 +108,7 @@ pub fn parse(text: []const u8) !Version {
  // Check validity of optional pre-release identifiers.
  // See: https://semver.org/#spec-item-9
  if (ver.pre) |pre| {
- it = std.mem.split(u8, pre, ".");
+ it = std.mem.splitScalar(u8, pre, '.');
  while (it.next()) |id| {
  // Identifiers MUST NOT be empty.
  if (id.len == 0) return error.InvalidVersion;
@@ -127,7 +127,7 @@ pub fn parse(text: []const u8) !Version {
  // Check validity of optional build metadata identifiers.
  // See: https://semver.org/#spec-item-10
  if (ver.build) |build| {
- it = std.mem.split(u8, build, ".");
+ it = std.mem.splitScalar(u8, build, '.');
  while (it.next()) |id| {
  // Identifiers MUST NOT be empty.
  if (id.len == 0) return error.InvalidVersion;

lib/std/builtin.zig

@@ -531,7 +531,7 @@ pub const Version = struct {
  // found no digits or '.' before unexpected character
  if (end == 0) return error.InvalidVersion;
 
- var it = std.mem.split(u8, text[0..end], ".");
+ var it = std.mem.splitScalar(u8, text[0..end], '.');
  // substring is not empty, first call will succeed
  const major = it.first();
  if (major.len == 0) return error.InvalidVersion;

lib/std/child_process.zig

@@ -850,7 +850,7 @@ pub const ChildProcess = struct {
  return original_err;
  }
 
- var it = mem.tokenize(u16, PATH, &[_]u16{';'});
+ var it = mem.tokenizeScalar(u16, PATH, ';');
  while (it.next()) |search_path| {
  dir_buf.clearRetainingCapacity();
  try dir_buf.appendSlice(self.allocator, search_path);
@@ -1064,7 +1064,7 @@ fn windowsCreateProcessPathExt(
  // Now we know that at least *a* file matching the wildcard exists, we can loop
  // through PATHEXT in order and exec any that exist
 
- var ext_it = mem.tokenize(u16, pathext, &[_]u16{';'});
+ var ext_it = mem.tokenizeScalar(u16, pathext, ';');
  while (ext_it.next()) |ext| {
  if (!windowsCreateProcessSupportsExtension(ext)) continue;
 

lib/std/crypto/Certificate.zig

@@ -337,8 +337,8 @@ pub const Parsed = struct {
  return true; // exact match
  }
 
- var it_host = std.mem.split(u8, host_name, ".");
- var it_dns = std.mem.split(u8, dns_name, ".");
+ var it_host = std.mem.splitScalar(u8, host_name, '.');
+ var it_dns = std.mem.splitScalar(u8, dns_name, '.');
 
  const len_match = while (true) {
  const host = it_host.next();

lib/std/crypto/phc_encoding.zig

@@ -7,9 +7,12 @@ const mem = std.mem;
 const meta = std.meta;
 
 const fields_delimiter = "$";
+const fields_delimiter_scalar = '$';
 const version_param_name = "v";
 const params_delimiter = ",";
+const params_delimiter_scalar = ',';
 const kv_delimiter = "=";
+const kv_delimiter_scalar = '=';
 
 pub const Error = std.crypto.errors.EncodingError || error{NoSpaceLeft};
 
@@ -73,7 +76,7 @@ pub fn BinValue(comptime max_len: usize) type {
 /// Other fields will also be deserialized from the function parameters section.
 pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult {
  var out = mem.zeroes(HashResult);
- var it = mem.split(u8, str, fields_delimiter);
+ var it = mem.splitScalar(u8, str, fields_delimiter_scalar);
  var set_fields: usize = 0;
 
  while (true) {
@@ -104,7 +107,7 @@ pub fn deserialize(comptime HashResult: type, str: []const u8) Error!HashResult
 
  // Read optional parameters
  var has_params = false;
- var it_params = mem.split(u8, field, params_delimiter);
+ var it_params = mem.splitScalar(u8, field, params_delimiter_scalar);
  while (it_params.next()) |params| {
  const param = kvSplit(params) catch break;
  var found = false;
@@ -252,7 +255,7 @@ fn serializeTo(params: anytype, out: anytype) !void {
 
 // Split a `key=value` string into `key` and `value`
 fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
- var it = mem.split(u8, str, kv_delimiter);
+ var it = mem.splitScalar(u8, str, kv_delimiter_scalar);
  const key = it.first();
  const value = it.next() orelse return Error.InvalidEncoding;
  const ret = .{ .key = key, .value = value };

lib/std/crypto/scrypt.zig

@@ -287,7 +287,7 @@ const crypt_format = struct {
  out.r = try Codec.intDecode(u30, str[4..9]);
  out.p = try Codec.intDecode(u30, str[9..14]);
 
- var it = mem.split(u8, str[14..], "$");
+ var it = mem.splitScalar(u8, str[14..], '$');
 
  const salt = it.first();
  if (@hasField(T, "salt")) out.salt = salt;

lib/std/fs.zig

@@ -3021,7 +3021,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {
  } else if (argv0.len != 0) {
  // argv[0] is not empty (and not a path): search it inside PATH
  const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;
- var path_it = mem.tokenize(u8, PATH, &[_]u8{path.delimiter});
+ var path_it = mem.tokenizeScalar(u8, PATH, path.delimiter);
  while (path_it.next()) |a_path| {
  var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;
  const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{