From 94ebda6684f28ad6f1d7509c5ec88caad3b90deb Mon Sep 17 00:00:00 2001 From: Travis Staloch Date: Sat, 20 Nov 2021 17:25:17 -0800 Subject: [PATCH 1/3] std.json: support field aliases - this is a rework of #8987. rebasing that old pr proved difficult so it was easier to just start over. - this patch has the same api and tests as #8987 but uses an enums.EnumMap rather than StringArrayHashMap. this allows `field_alias_map` to be made entirely on the stack and requires no allocator (not even a FixedBufferAllocator which the previous used for re-indexing its map). --- lib/std/json.zig | 109 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index 2997a4085413..0a521016b158 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1489,6 +1489,17 @@ pub const ParseOptions = struct { ignore_unknown_fields: bool = false, allow_trailing_data: bool = false, + + /// Provide if struct field names are different from json key names + field_aliases: []const FieldAlias = &.{}, +}; + +pub const FieldAlias = struct { + /// the struct field name + field: []const u8, + + /// the json key name + alias: []const u8, }; const SkipValueError = error{UnexpectedJsonDepth} || TokenStream.Error; @@ -1555,6 +1566,75 @@ test "skipValue" { } } +test "field aliases" { + const S = struct { value: u8 }; + const text = + \\{"__value": 42} + ; + + const s = try parse(S, &TokenStream.init(text), .{ + .field_aliases = &.{.{ .field = "value", .alias = "__value" }}, + }); + try testing.expectEqual(@as(u8, 42), s.value); +} + +test "many field aliases" { + const S = struct { value: u8, a: u8, z: u8 }; + const text = + \\{"__value": 42, "__a": 43, "__z": 44} + ; + + // too many aliases + try testing.expectError( + error.TooManyFieldAliases, + parse(S, &TokenStream.init(text), .{ .field_aliases = &.{ + .{ .field = "value", .alias = "__value" }, + .{ .field = "a", .alias = "__a" }, + .{ .field = "z", .alias = "__z" }, + .{ .field = "oops", .alias = "__oops" }, + } }), + ); + + const s = try parse(S, &TokenStream.init(text), .{ + .field_aliases = &.{ + .{ .field = "value", .alias = "__value" }, + .{ .field = "a", .alias = "__a" }, + .{ .field = "z", .alias = "__z" }, + }, + }); + try testing.expectEqual(@as(u8, 42), s.value); + try testing.expectEqual(@as(u8, 43), s.a); + try testing.expectEqual(@as(u8, 44), s.z); +} + +test "field alias escapes" { + const S = struct { foo: u8 }; + const text = + \\{"__f\u006fo": 42} + ; + + // match alias, mismatched field + try testing.expectError( + error.UnknownField, + parse(S, &TokenStream.init(text), .{ + .field_aliases = &.{.{ .field = "bar", .alias = "__foo" }}, + }), + ); + + // match field, mismatched alias + try testing.expectError( + error.UnknownField, + parse(S, &TokenStream.init(text), .{ + .field_aliases = &.{.{ .field = "foo", .alias = "__bar" }}, + }), + ); + + const s = try parse(S, &TokenStream.init(text), .{ + .field_aliases = &.{.{ .field = "foo", .alias = "__foo" }}, + }); + try testing.expectEqual(@as(u8, 42), s.foo); +} + fn ParseInternalError(comptime T: type) type { // `inferred_types` is used to avoid infinite recursion for recursive type definitions. const inferred_types = [_]type{}; @@ -1597,6 +1677,7 @@ fn ParseInternalErrorImpl(comptime T: type, comptime inferred_types: []const typ UnexpectedValue, UnknownField, MissingField, + TooManyFieldAliases, } || SkipValueError || TokenStream.Error; for (structInfo.fields) |field| { errors = errors || ParseInternalErrorImpl(field.field_type, inferred_types ++ [_]type{T}); @@ -1730,6 +1811,21 @@ fn parseInternal( } } + // support for options.field_aliases + const fields_len = structInfo.fields.len; + if (options.field_aliases.len > fields_len) return error.TooManyFieldAliases; + // construct field_alias_map: field_enum => field_alias.alias + const FieldsEnum = if (fields_len > 0) std.meta.FieldEnum(T) else void; + const field_alias_map = if (fields_len > 0) blk: { + var map = std.enums.EnumMap(FieldsEnum, []const u8){}; + for (options.field_aliases) |field_alias| { + const field_enum = std.meta.stringToEnum(FieldsEnum, field_alias.field) orelse + return error.UnknownField; + map.put(field_enum, field_alias.alias); + } + break :blk map; + } else {}; + while (true) { switch ((try tokens.next()) orelse return error.UnexpectedEndOfJson) { .ObjectEnd => break, @@ -1740,7 +1836,18 @@ fn parseInternal( var found = false; inline for (structInfo.fields) |field, i| { // TODO: using switches here segfault the compiler (#2727?) - if ((stringToken.escapes == .None and mem.eql(u8, field.name, key_source_slice)) or (stringToken.escapes == .Some and (field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)))) { + const field_enum = std.meta.stringToEnum(FieldsEnum, field.name) orelse unreachable; + if ((stringToken.escapes == .None and + // no escapes, no alias + (mem.eql(u8, field.name, key_source_slice) or + // no escapes, yes alias - the key may be an unescaped alias + if (field_alias_map.get(field_enum)) |alias| mem.eql(u8, alias, key_source_slice) else false) or + (stringToken.escapes == .Some and + // yes escapes, no alias + ((field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)) or + // yes escapes, yes alias - the key may be an escaped alias. + if (field_alias_map.get(field_enum)) |alias| alias.len == stringToken.decodedLength() and encodesTo(alias, key_source_slice) else false)))) + { // if (switch (stringToken.escapes) { // .None => mem.eql(u8, field.name, key_source_slice), // .Some => (field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)), From ec6c3eef9bd5c15a003638e42667ca977b11062b Mon Sep 17 00:00:00 2001 From: Travis Staloch Date: Mon, 22 Nov 2021 12:21:18 -0800 Subject: [PATCH 2/3] std.json: move field_aliases to struct decl - this patch moves field_aliases from an `ParseOptions` field to a public decl of T. this makes the aliases available at comptime and only apply to T. now the aliases won't apply to matching fields in child json objects. - interpret `pub const __field_aliases` decl as a `std.enums.EnumFieldStruct` --- lib/std/json.zig | 117 +++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 81 deletions(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index 0a521016b158..0f10586b920b 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1489,17 +1489,6 @@ pub const ParseOptions = struct { ignore_unknown_fields: bool = false, allow_trailing_data: bool = false, - - /// Provide if struct field names are different from json key names - field_aliases: []const FieldAlias = &.{}, -}; - -pub const FieldAlias = struct { - /// the struct field name - field: []const u8, - - /// the json key name - alias: []const u8, }; const SkipValueError = error{UnexpectedJsonDepth} || TokenStream.Error; @@ -1567,72 +1556,51 @@ test "skipValue" { } test "field aliases" { - const S = struct { value: u8 }; - const text = - \\{"__value": 42} - ; - - const s = try parse(S, &TokenStream.init(text), .{ - .field_aliases = &.{.{ .field = "value", .alias = "__value" }}, - }); - try testing.expectEqual(@as(u8, 42), s.value); -} - -test "many field aliases" { - const S = struct { value: u8, a: u8, z: u8 }; + const S = struct { + value: u8, + a: u8, + z: u8, + pub const __field_aliases = .{ + .value = "__value", + .a = "__a", + .z = "__z", + }; + }; const text = \\{"__value": 42, "__a": 43, "__z": 44} ; - // too many aliases - try testing.expectError( - error.TooManyFieldAliases, - parse(S, &TokenStream.init(text), .{ .field_aliases = &.{ - .{ .field = "value", .alias = "__value" }, - .{ .field = "a", .alias = "__a" }, - .{ .field = "z", .alias = "__z" }, - .{ .field = "oops", .alias = "__oops" }, - } }), - ); - - const s = try parse(S, &TokenStream.init(text), .{ - .field_aliases = &.{ - .{ .field = "value", .alias = "__value" }, - .{ .field = "a", .alias = "__a" }, - .{ .field = "z", .alias = "__z" }, - }, - }); + const s = try parse(S, &TokenStream.init(text), .{}); try testing.expectEqual(@as(u8, 42), s.value); try testing.expectEqual(@as(u8, 43), s.a); try testing.expectEqual(@as(u8, 44), s.z); } test "field alias escapes" { - const S = struct { foo: u8 }; const text = \\{"__f\u006fo": 42} ; - // match alias, mismatched field - try testing.expectError( - error.UnknownField, - parse(S, &TokenStream.init(text), .{ - .field_aliases = &.{.{ .field = "bar", .alias = "__foo" }}, - }), - ); - - // match field, mismatched alias - try testing.expectError( - error.UnknownField, - parse(S, &TokenStream.init(text), .{ - .field_aliases = &.{.{ .field = "foo", .alias = "__bar" }}, - }), - ); - - const s = try parse(S, &TokenStream.init(text), .{ - .field_aliases = &.{.{ .field = "foo", .alias = "__foo" }}, - }); - try testing.expectEqual(@as(u8, 42), s.foo); + { + // match field, mismatched alias + const S = struct { + foo: u8, + pub const __field_aliases = .{ .foo = "__bar" }; + }; + try testing.expectError( + error.UnknownField, + parse(S, &TokenStream.init(text), .{}), + ); + } + { + // match field, match alias + const S = struct { + foo: u8, + pub const __field_aliases = .{ .foo = "__foo" }; + }; + const s = try parse(S, &TokenStream.init(text), .{}); + try testing.expectEqual(@as(u8, 42), s.foo); + } } fn ParseInternalError(comptime T: type) type { @@ -1677,7 +1645,6 @@ fn ParseInternalErrorImpl(comptime T: type, comptime inferred_types: []const typ UnexpectedValue, UnknownField, MissingField, - TooManyFieldAliases, } || SkipValueError || TokenStream.Error; for (structInfo.fields) |field| { errors = errors || ParseInternalErrorImpl(field.field_type, inferred_types ++ [_]type{T}); @@ -1811,20 +1778,9 @@ fn parseInternal( } } - // support for options.field_aliases - const fields_len = structInfo.fields.len; - if (options.field_aliases.len > fields_len) return error.TooManyFieldAliases; - // construct field_alias_map: field_enum => field_alias.alias - const FieldsEnum = if (fields_len > 0) std.meta.FieldEnum(T) else void; - const field_alias_map = if (fields_len > 0) blk: { - var map = std.enums.EnumMap(FieldsEnum, []const u8){}; - for (options.field_aliases) |field_alias| { - const field_enum = std.meta.stringToEnum(FieldsEnum, field_alias.field) orelse - return error.UnknownField; - map.put(field_enum, field_alias.alias); - } - break :blk map; - } else {}; + // support for field_aliases + const FieldAliases = std.enums.EnumFieldStruct(T, ?[]const u8, @as(?[]const u8, null)); + const field_aliases: FieldAliases = if (@hasDecl(T, "__field_aliases")) T.__field_aliases else .{}; while (true) { switch ((try tokens.next()) orelse return error.UnexpectedEndOfJson) { @@ -1836,17 +1792,16 @@ fn parseInternal( var found = false; inline for (structInfo.fields) |field, i| { // TODO: using switches here segfault the compiler (#2727?) - const field_enum = std.meta.stringToEnum(FieldsEnum, field.name) orelse unreachable; if ((stringToken.escapes == .None and // no escapes, no alias (mem.eql(u8, field.name, key_source_slice) or // no escapes, yes alias - the key may be an unescaped alias - if (field_alias_map.get(field_enum)) |alias| mem.eql(u8, alias, key_source_slice) else false) or + if (@field(field_aliases, field.name)) |alias| mem.eql(u8, alias, key_source_slice) else false) or (stringToken.escapes == .Some and // yes escapes, no alias ((field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)) or // yes escapes, yes alias - the key may be an escaped alias. - if (field_alias_map.get(field_enum)) |alias| alias.len == stringToken.decodedLength() and encodesTo(alias, key_source_slice) else false)))) + if (@field(field_aliases, field.name)) |alias| alias.len == stringToken.decodedLength() and encodesTo(alias, key_source_slice) else false)))) { // if (switch (stringToken.escapes) { // .None => mem.eql(u8, field.name, key_source_slice), From 3136da2f32672c4620122fd3134f1ec6df43e326 Mon Sep 17 00:00:00 2001 From: Travis Staloch Date: Mon, 22 Nov 2021 14:35:40 -0800 Subject: [PATCH 3/3] std.json: make field_alias a `?[*:0]const u8` - this shrinks each of the the fields from 24 to 8 bytes - remove extra if condition logic --- lib/std/json.zig | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index 0f10586b920b..c17b83d12efd 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -1779,7 +1779,7 @@ fn parseInternal( } // support for field_aliases - const FieldAliases = std.enums.EnumFieldStruct(T, ?[]const u8, @as(?[]const u8, null)); + const FieldAliases = std.enums.EnumFieldStruct(T, ?[*:0]const u8, @as(?[*:0]const u8, null)); const field_aliases: FieldAliases = if (@hasDecl(T, "__field_aliases")) T.__field_aliases else .{}; while (true) { @@ -1791,18 +1791,15 @@ fn parseInternal( child_options.allow_trailing_data = true; var found = false; inline for (structInfo.fields) |field, i| { - // TODO: using switches here segfault the compiler (#2727?) - if ((stringToken.escapes == .None and - // no escapes, no alias - (mem.eql(u8, field.name, key_source_slice) or - // no escapes, yes alias - the key may be an unescaped alias - if (@field(field_aliases, field.name)) |alias| mem.eql(u8, alias, key_source_slice) else false) or - (stringToken.escapes == .Some and - // yes escapes, no alias - ((field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)) or - // yes escapes, yes alias - the key may be an escaped alias. - if (@field(field_aliases, field.name)) |alias| alias.len == stringToken.decodedLength() and encodesTo(alias, key_source_slice) else false)))) + const field_name = comptime if (@field(field_aliases, field.name)) |alias| + mem.span(alias) + else + field.name; + + if ((stringToken.escapes == .None and mem.eql(u8, field_name, key_source_slice)) or + (stringToken.escapes == .Some and ((field_name.len == stringToken.decodedLength() and encodesTo(field_name, key_source_slice))))) { + // TODO: using switches here segfault the compiler (#2727?) // if (switch (stringToken.escapes) { // .None => mem.eql(u8, field.name, key_source_slice), // .Some => (field.name.len == stringToken.decodedLength() and encodesTo(field.name, key_source_slice)),