From fc0d80c82bef2b555a1de10c1db18c4324f5f99a Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Fri, 12 Apr 2024 14:31:44 -0700 Subject: [PATCH] Use a concrete type for JSON pointer WARNING: This commit includes breaking changes. Declare a jsontext.Pointer type as a named string type. This allows us to implement a Tokens method to conveniently iterate over all the reference tokens in the pointer using the upcoming iterators support. The new functionally is currently not tested by CI, but can be manually tested with: GOEXPERIMENT=rangefunc go.tip test ./... --- errors.go | 4 ++-- jsontext/coder_test.go | 34 ++++++++++++++++---------------- jsontext/decode.go | 4 ++-- jsontext/decode_test.go | 4 ++-- jsontext/encode.go | 4 ++-- jsontext/encode_test.go | 4 ++-- jsontext/example_test.go | 2 +- jsontext/pointer.go | 23 ++++++++++++++++++++++ jsontext/pointer_test.go | 42 ++++++++++++++++++++++++++++++++++++++++ jsontext/state.go | 19 ++++++++++++++++++ 10 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 jsontext/pointer.go create mode 100644 jsontext/pointer_test.go diff --git a/errors.go b/errors.go index 8a67bbd..0b42b41 100644 --- a/errors.go +++ b/errors.go @@ -28,7 +28,7 @@ type SemanticError struct { ByteOffset int64 // JSONPointer indicates that an error occurred within this JSON value // as indicated using the JSON Pointer notation (see RFC 6901). - JSONPointer string + JSONPointer jsontext.Pointer // JSONKind is the JSON kind that could not be handled. JSONKind jsontext.Kind // may be zero if unknown @@ -98,7 +98,7 @@ func (e *SemanticError) Error() string { switch { case e.JSONPointer != "": sb.WriteString(" within JSON value at ") - sb.WriteString(strconv.Quote(e.JSONPointer)) + sb.WriteString(strconv.Quote(string(e.JSONPointer))) case e.ByteOffset > 0: sb.WriteString(" after byte offset ") sb.WriteString(strconv.FormatInt(e.ByteOffset, 10)) diff --git a/jsontext/coder_test.go b/jsontext/coder_test.go index 65f2ff9..ecfe1a5 100644 --- a/jsontext/coder_test.go +++ b/jsontext/coder_test.go @@ -38,7 +38,7 @@ type coderTestdataEntry struct { outIndented string // outCompacted if empty; uses " " for indent prefix and "\t" for indent outCanonicalized string // outCompacted if empty tokens []Token - pointers []string + pointers []Pointer } var coderTestdata = []coderTestdataEntry{{ @@ -46,7 +46,7 @@ var coderTestdata = []coderTestdataEntry{{ in: ` null `, outCompacted: `null`, tokens: []Token{Null}, - pointers: []string{""}, + pointers: []Pointer{""}, }, { name: jsontest.Name("False"), in: ` false `, @@ -157,7 +157,7 @@ var coderTestdata = []coderTestdataEntry{{ Int(minInt64), Int(maxInt64), Uint(minUint64), Uint(maxUint64), ArrayEnd, }, - pointers: []string{ + pointers: []Pointer{ "", "/0", "/1", "/2", "/3", "/4", "/5", "/6", "/7", "/8", "/9", "/10", "/11", "/12", "/13", "/14", "/15", "/16", "/17", "", }, }, { @@ -165,7 +165,7 @@ var coderTestdata = []coderTestdataEntry{{ in: ` { } `, outCompacted: `{}`, tokens: []Token{ObjectStart, ObjectEnd}, - pointers: []string{"", ""}, + pointers: []Pointer{"", ""}, }, { name: jsontest.Name("ObjectN1"), in: ` { "0" : 0 } `, @@ -175,7 +175,7 @@ var coderTestdata = []coderTestdataEntry{{ "0": 0 }`, tokens: []Token{ObjectStart, String("0"), Uint(0), ObjectEnd}, - pointers: []string{"", "/0", "/0", ""}, + pointers: []Pointer{"", "/0", "/0", ""}, }, { name: jsontest.Name("ObjectN2"), in: ` { "0" : 0 , "1" : 1 } `, @@ -186,7 +186,7 @@ var coderTestdata = []coderTestdataEntry{{ "1": 1 }`, tokens: []Token{ObjectStart, String("0"), Uint(0), String("1"), Uint(1), ObjectEnd}, - pointers: []string{"", "/0", "/0", "/1", "/1", ""}, + pointers: []Pointer{"", "/0", "/0", "/1", "/1", ""}, }, { name: jsontest.Name("ObjectNested"), in: ` { "0" : { "1" : { "2" : { "3" : { "4" : { } } } } } } `, @@ -204,7 +204,7 @@ var coderTestdata = []coderTestdataEntry{{ } }`, tokens: []Token{ObjectStart, String("0"), ObjectStart, String("1"), ObjectStart, String("2"), ObjectStart, String("3"), ObjectStart, String("4"), ObjectStart, ObjectEnd, ObjectEnd, ObjectEnd, ObjectEnd, ObjectEnd, ObjectEnd}, - pointers: []string{ + pointers: []Pointer{ "", "/0", "/0", "/0/1", "/0/1", @@ -268,7 +268,7 @@ var coderTestdata = []coderTestdataEntry{{ ObjectEnd, ObjectEnd, }, - pointers: []string{ + pointers: []Pointer{ "", "/", "/", "//44444", "//44444", @@ -289,7 +289,7 @@ var coderTestdata = []coderTestdataEntry{{ in: ` [ ] `, outCompacted: `[]`, tokens: []Token{ArrayStart, ArrayEnd}, - pointers: []string{"", ""}, + pointers: []Pointer{"", ""}, }, { name: jsontest.Name("ArrayN1"), in: ` [ 0 ] `, @@ -298,7 +298,7 @@ var coderTestdata = []coderTestdataEntry{{ 0 ]`, tokens: []Token{ArrayStart, Uint(0), ArrayEnd}, - pointers: []string{"", "/0", ""}, + pointers: []Pointer{"", "/0", ""}, }, { name: jsontest.Name("ArrayN2"), in: ` [ 0 , 1 ] `, @@ -322,7 +322,7 @@ var coderTestdata = []coderTestdataEntry{{ ] ]`, tokens: []Token{ArrayStart, ArrayStart, ArrayStart, ArrayStart, ArrayStart, ArrayEnd, ArrayEnd, ArrayEnd, ArrayEnd, ArrayEnd}, - pointers: []string{ + pointers: []Pointer{ "", "/0", "/0/0", @@ -388,7 +388,7 @@ var coderTestdata = []coderTestdataEntry{{ String("objectN2"), ObjectStart, String("0"), Uint(0), String("1"), Uint(1), ObjectEnd, ObjectEnd, }, - pointers: []string{ + pointers: []Pointer{ "", "/literals", "/literals", "/literals/0", @@ -494,8 +494,8 @@ func testCoderInterleaved(t *testing.T, where jsontest.CasePos, modeName string, func TestCoderStackPointer(t *testing.T) { tests := []struct { token Token - wantWithRejectDuplicateNames string - wantWithAllowDuplicateNames string + wantWithRejectDuplicateNames Pointer + wantWithAllowDuplicateNames Pointer }{ {Null, "", ""}, @@ -549,14 +549,14 @@ func TestCoderStackPointer(t *testing.T) { for _, allowDupes := range []bool{false, true} { var name string - var want func(i int) string + var want func(i int) Pointer switch allowDupes { case false: name = "RejectDuplicateNames" - want = func(i int) string { return tests[i].wantWithRejectDuplicateNames } + want = func(i int) Pointer { return tests[i].wantWithRejectDuplicateNames } case true: name = "AllowDuplicateNames" - want = func(i int) string { return tests[i].wantWithAllowDuplicateNames } + want = func(i int) Pointer { return tests[i].wantWithAllowDuplicateNames } } t.Run(name, func(t *testing.T) { diff --git a/jsontext/decode.go b/jsontext/decode.go index d1ef6ae..e12a37b 100644 --- a/jsontext/decode.go +++ b/jsontext/decode.go @@ -1052,7 +1052,7 @@ func (d *Decoder) StackIndex(i int) (Kind, int64) { // StackPointer returns a JSON Pointer (RFC 6901) to the most recently read value. // Object names are only present if [AllowDuplicateNames] is false, otherwise // object members are represented using their index within the object. -func (d *Decoder) StackPointer() string { +func (d *Decoder) StackPointer() Pointer { d.s.Names.copyQuotedBuffer(d.s.buf) - return string(d.s.appendStackPointer(nil)) + return Pointer(d.s.appendStackPointer(nil)) } diff --git a/jsontext/decode_test.go b/jsontext/decode_test.go index fc156ed..11b25b4 100644 --- a/jsontext/decode_test.go +++ b/jsontext/decode_test.go @@ -48,7 +48,7 @@ func testDecoder(t *testing.T, where jsontest.CasePos, typeName string, td coder switch typeName { case "Token": var tokens []Token - var pointers []string + var pointers []Pointer for { tok, err := dec.ReadToken() if err != nil { @@ -176,7 +176,7 @@ type decoderMethodCall struct { wantKind Kind wantOut tokOrVal wantErr error - wantPointer string + wantPointer Pointer } var decoderErrorTestdata = []struct { diff --git a/jsontext/encode.go b/jsontext/encode.go index e5f3965..1035aa9 100644 --- a/jsontext/encode.go +++ b/jsontext/encode.go @@ -917,7 +917,7 @@ func (e *Encoder) StackIndex(i int) (Kind, int64) { // StackPointer returns a JSON Pointer (RFC 6901) to the most recently written value. // Object names are only present if [AllowDuplicateNames] is false, otherwise // object members are represented using their index within the object. -func (e *Encoder) StackPointer() string { +func (e *Encoder) StackPointer() Pointer { e.s.Names.copyQuotedBuffer(e.s.Buf) - return string(e.s.appendStackPointer(nil)) + return Pointer(e.s.appendStackPointer(nil)) } diff --git a/jsontext/encode_test.go b/jsontext/encode_test.go index 376176a..b7a53c7 100644 --- a/jsontext/encode_test.go +++ b/jsontext/encode_test.go @@ -48,7 +48,7 @@ func testEncoder(t *testing.T, where jsontest.CasePos, formatName, typeName stri switch typeName { case "Token": - var pointers []string + var pointers []Pointer for _, tok := range td.tokens { if err := enc.WriteToken(tok); err != nil { t.Fatalf("%s: Encoder.WriteToken error: %v", where, err) @@ -136,7 +136,7 @@ func testFaultyEncoder(t *testing.T, where jsontest.CasePos, typeName string, td type encoderMethodCall struct { in tokOrVal wantErr error - wantPointer string + wantPointer Pointer } var encoderErrorTestdata = []struct { diff --git a/jsontext/example_test.go b/jsontext/example_test.go index c694b4c..3ab3e2d 100644 --- a/jsontext/example_test.go +++ b/jsontext/example_test.go @@ -34,7 +34,7 @@ func Example_stringReplace() { // Using a Decoder and Encoder, we can parse through every token, // check and modify the token if necessary, and // write the token to the output. - var replacements []string + var replacements []jsontext.Pointer in := strings.NewReader(input) dec := jsontext.NewDecoder(in) out := new(bytes.Buffer) diff --git a/jsontext/pointer.go b/jsontext/pointer.go new file mode 100644 index 0000000..27f7651 --- /dev/null +++ b/jsontext/pointer.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.rangefunc + +package jsontext + +import "iter" + +// Tokens returns an iterator for each reference token in the JSON pointer, +// starting from the first token until the last token (unless stopped early). +// A token is either a JSON object name or an index to a JSON array element +// encoded as a base-10 integer value. +func (p Pointer) Tokens() iter.Seq[string] { + return func(yield func(string) bool) { + for len(p) > 0 { + if !yield(p.nextToken()) { + return + } + } + } +} diff --git a/jsontext/pointer_test.go b/jsontext/pointer_test.go new file mode 100644 index 0000000..bd94370 --- /dev/null +++ b/jsontext/pointer_test.go @@ -0,0 +1,42 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build goexperiment.rangefunc + +package jsontext + +import ( + "iter" + "slices" + "testing" +) + +func TestPointerTokens(t *testing.T) { + // TODO(https://go.dev/issue/61899): Use slices.Collect. + collect := func(seq iter.Seq[string]) (x []string) { + for v := range seq { + x = append(x, v) + } + return x + } + + tests := []struct { + in Pointer + want []string + }{ + {in: "", want: nil}, + {in: "a", want: []string{"a"}}, + {in: "~", want: []string{"~"}}, + {in: "/a", want: []string{"a"}}, + {in: "/foo/bar", want: []string{"foo", "bar"}}, + {in: "///", want: []string{"", "", ""}}, + {in: "/~0~1", want: []string{"~/"}}, + } + for _, tt := range tests { + got := collect(tt.in.Tokens()) + if !slices.Equal(got, tt.want) { + t.Errorf("Pointer(%q).Tokens = %q, want %q", tt.in, got, tt.want) + } + } +} diff --git a/jsontext/state.go b/jsontext/state.go index 5c1a55d..1a15937 100644 --- a/jsontext/state.go +++ b/jsontext/state.go @@ -7,6 +7,7 @@ package jsontext import ( "math" "strconv" + "strings" "github.com/go-json-experiment/json/internal/jsonwire" ) @@ -48,6 +49,24 @@ func (s *state) reset() { s.Namespaces.reset() } +// Pointer is a JSON Pointer (RFC 6901) that references a particular JSON value +// relative to the root of the top-level JSON value. +type Pointer string + +// nextToken returns the next token in the pointer, reducing the length of p. +func (p *Pointer) nextToken() (token string) { + *p = Pointer(strings.TrimPrefix(string(*p), "/")) + i := min(uint(strings.IndexByte(string(*p), '/')), uint(len(*p))) + token = string(*p)[:i] + *p = (*p)[i:] + if strings.Contains(token, "~") { + // Per RFC 6901, section 3, unescape '~' and '/' characters. + token = strings.ReplaceAll(token, "~1", "/") + token = strings.ReplaceAll(token, "~0", "~") + } + return token +} + // appendStackPointer appends a JSON Pointer (RFC 6901) to the current value. // The returned pointer is only accurate if s.names is populated, // otherwise it uses the numeric index as the object member name.